.github/tests/test_data/report-on-schedule-2022-02-02.json
{
"RasaHQ/financial-demo": {
"BERT + DIET(bow) + ResponseSelector(bow)": [{
"accelerator_type": "GPU",
"config_repository": "training-data",
"config_repository_branch": "main",
"dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
"dataset_repository_branch": "fix-model-regression-tests",
"entity_prediction": {
"macro avg": {
"f1-score": 0.7333333333333333,
"precision": 0.8,
"recall": 0.7,
"support": 14
},
"micro avg": {
"f1-score": 0.8333333333333333,
"precision": 1.0,
"recall": 0.7142857142857143,
"support": 14
},
"weighted avg": {
"f1-score": 0.738095238095238,
"precision": 0.7857142857142857,
"recall": 0.7142857142857143,
"support": 14
}
},
"external_dataset_repository": true,
"intent_classification": {
"accuracy": 1.0,
"macro avg": {
"f1-score": 1.0,
"precision": 1.0,
"recall": 1.0,
"support": 28
},
"weighted avg": {
"f1-score": 1.0,
"precision": 1.0,
"recall": 1.0,
"support": 28
}
},
"test_run_time": "35s",
"total_run_time": "2m2s",
"train_run_time": "1m28s",
"type": "nlu"
}],
"BERT + DIET(seq) + ResponseSelector(t2t)": [{
"accelerator_type": "GPU",
"config_repository": "training-data",
"config_repository_branch": "main",
"dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
"dataset_repository_branch": "fix-model-regression-tests",
"entity_prediction": {
"macro avg": {
"f1-score": 0.7333333333333333,
"precision": 0.8,
"recall": 0.7,
"support": 14
},
"micro avg": {
"f1-score": 0.8333333333333333,
"precision": 1.0,
"recall": 0.7142857142857143,
"support": 14
},
"weighted avg": {
"f1-score": 0.738095238095238,
"precision": 0.7857142857142857,
"recall": 0.7142857142857143,
"support": 14
}
},
"external_dataset_repository": true,
"intent_classification": {
"accuracy": 1.0,
"macro avg": {
"f1-score": 1.0,
"precision": 1.0,
"recall": 1.0,
"support": 28
},
"weighted avg": {
"f1-score": 1.0,
"precision": 1.0,
"recall": 1.0,
"support": 28
}
},
"test_run_time": "55s",
"total_run_time": "2m8s",
"train_run_time": "1m14s",
"type": "nlu"
}],
"Rules + Memo + TED": [{
"accelerator_type": "GPU",
"config_repository": "training-data",
"config_repository_branch": "main",
"dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
"dataset_repository_branch": "fix-model-regression-tests",
"external_dataset_repository": true,
"story_prediction": {
"accuracy": 1.0,
"conversation_accuracy": {
"accuracy": 1.0,
"correct": 48,
"total": 48,
"with_warnings": 0
},
"macro avg": {
"f1-score": 1.0,
"precision": 1.0,
"recall": 1.0,
"support": 317
},
"weighted avg": {
"f1-score": 1.0,
"precision": 1.0,
"recall": 1.0,
"support": 317
}
},
"test_run_time": "51s",
"total_run_time": "8m15s",
"train_run_time": "7m24s",
"type": "core"
}]
},
"RasaHQ/retail-demo": {
"BERT + DIET(bow) + ResponseSelector(bow)": [{
"accelerator_type": "GPU",
"config_repository": "training-data",
"config_repository_branch": "main",
"dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
"dataset_repository_branch": "fix-model-regression-tests",
"entity_prediction": {
"macro avg": {
"f1-score": 0.25,
"precision": 0.25,
"recall": 0.25,
"support": 6
},
"micro avg": {
"f1-score": 0.2857142857142857,
"precision": 1.0,
"recall": 0.16666666666666666,
"support": 6
},
"weighted avg": {
"f1-score": 0.16666666666666666,
"precision": 0.16666666666666666,
"recall": 0.16666666666666666,
"support": 6
}
},
"external_dataset_repository": true,
"intent_classification": {
"macro avg": {
"f1-score": 0.8,
"precision": 0.8,
"recall": 0.85,
"support": 16
},
"micro avg": {
"f1-score": 0.8387096774193549,
"precision": 0.8666666666666667,
"recall": 0.8125,
"support": 16
},
"weighted avg": {
"f1-score": 0.8125,
"precision": 0.875,
"recall": 0.8125,
"support": 16
}
},
"test_run_time": "29s",
"total_run_time": "1m16s",
"train_run_time": "47s",
"type": "nlu"
}],
"BERT + DIET(seq) + ResponseSelector(t2t)": [{
"accelerator_type": "GPU",
"config_repository": "training-data",
"config_repository_branch": "main",
"dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
"dataset_repository_branch": "fix-model-regression-tests",
"entity_prediction": {
"macro avg": {
"f1-score": 0.25,
"precision": 0.25,
"recall": 0.25,
"support": 6
},
"micro avg": {
"f1-score": 0.2857142857142857,
"precision": 1.0,
"recall": 0.16666666666666666,
"support": 6
},
"weighted avg": {
"f1-score": 0.16666666666666666,
"precision": 0.16666666666666666,
"recall": 0.16666666666666666,
"support": 6
}
},
"external_dataset_repository": true,
"intent_classification": {
"accuracy": 0.875,
"macro avg": {
"f1-score": 0.8300000000000001,
"precision": 0.8166666666666667,
"recall": 0.85,
"support": 16
},
"weighted avg": {
"f1-score": 0.85,
"precision": 0.8333333333333333,
"recall": 0.875,
"support": 16
}
},
"test_run_time": "56s",
"total_run_time": "2m2s",
"train_run_time": "1m6s",
"type": "nlu"
}],
"Rules + Memo": [{
"accelerator_type": "GPU",
"config_repository": "training-data",
"config_repository_branch": "main",
"dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
"dataset_repository_branch": "fix-model-regression-tests",
"external_dataset_repository": true,
"story_prediction": {
"conversation_accuracy": {
"accuracy": 0.8888888888888888,
"correct": 8,
"total": 9,
"with_warnings": 0
},
"macro avg": {
"f1-score": 0.9663698541747322,
"precision": 1.0,
"recall": 0.946007696007696,
"support": 67
},
"micro avg": {
"f1-score": 0.9692307692307692,
"precision": 1.0,
"recall": 0.9402985074626866,
"support": 67
},
"weighted avg": {
"f1-score": 0.9656317714563074,
"precision": 1.0,
"recall": 0.9402985074626866,
"support": 67
}
},
"test_run_time": "10s",
"total_run_time": "19s",
"train_run_time": "10s",
"type": "core"
}],
"Rules + Memo + TED": [{
"accelerator_type": "GPU",
"config_repository": "training-data",
"config_repository_branch": "main",
"dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
"dataset_repository_branch": "fix-model-regression-tests",
"external_dataset_repository": true,
"story_prediction": {
"accuracy": 1.0,
"conversation_accuracy": {
"accuracy": 1.0,
"correct": 9,
"total": 9,
"with_warnings": 0
},
"macro avg": {
"f1-score": 1.0,
"precision": 1.0,
"recall": 1.0,
"support": 67
},
"weighted avg": {
"f1-score": 1.0,
"precision": 1.0,
"recall": 1.0,
"support": 67
}
},
"test_run_time": "31s",
"total_run_time": "4m57s",
"train_run_time": "4m27s",
"type": "core"
}]
}
}