Doctorado-ML/benchmark
1
0
Fork 0
mirror of https://github.com/Doctorado-ML/benchmark.git synced 2025-08-16 07:55:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add be_summary tests

Browse Source
This commit is contained in:
Ricardo Montañana Gómez
2022-05-07 18:35:04 +02:00
parent 31059ea117
commit 40814c6f1f
7 changed files with 194 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
benchmark/scripts/be_summary.py
View File

@@ -7,18 +7,23 @@ def main(args_test=None):
arguments = Arguments()
metrics = list(ALL_METRICS)
metrics.append("all")
arguments.xset("score", choices=metrics).xset("model", required=False)
arguments.xset("score", choices=metrics).xset("model")
args = arguments.parse(args_test)
metrics = ALL_METRICS if args.score == "all" else [args.score]
summary = Summary()
summary.acquire()
for metric in metrics:
title = f"BEST RESULT of {metric} for {args.model}"
best = summary.best_result(
criterion="model", value=args.model, score=metric
)
summary.show_result(data=best, title=title)
summary.show_result(
summary.best_result(score=metric), title=f"BEST RESULT of {metric}"
)
summary.show_top(score=metric, n=10)
try:
best = summary.best_result(
criterion="model", value=args.model, score=metric
)
except ValueError as e:
print(e)
else:
summary.show_result(data=best, title=title)
summary.show_result(
summary.best_result(score=metric),
title=f"BEST RESULT of {metric}",
)
summary.show_top(score=metric, n=10)

2
benchmark/tests/__init__.py
View File

@@ -14,6 +14,7 @@ from .Arguments_test import ArgumentsTest
from .scripts.Be_Pair_check_test import BePairCheckTest
from .scripts.Be_List_test import BeListTest
from .scripts.Be_Report_test import BeReportTest
from .scripts.Be_Summary_test import BeSummaryTest
all = [
"UtilTest",
@@ -32,4 +33,5 @@ all = [
"BePairCheckTest",
"BeListTest",
"BeReportTest",
"BeSummaryTest",
]

34
benchmark/tests/scripts/Be_Summary_test.py Normal file
View File

@@ -0,0 +1,34 @@
import os
from openpyxl import load_workbook
from ...Utils import Folders
from ..TestBase import TestBase
class BeSummaryTest(TestBase):
def setUp(self):
self.prepare_scripts_env()
def tearDown(self) -> None:
pass
def test_be_summary_list_results_model(self):
stdout, stderr = self.execute_script("be_summary", ["-m", "STree"])
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_summary_list_model")
def test_be_summary_list_results_score(self):
stdout, stderr = self.execute_script("be_summary", ["-s", "accuracy"])
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_summary_list_score")
def test_be_summary_list_results_score_all(self):
stdout, stderr = self.execute_script("be_summary", ["-s", "all"])
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_summary_list_score_all")
def test_summary_list_results_model_score(self):
stdout, stderr = self.execute_script(
"be_summary", ["-s", "accuracy", "-m", "ODTE"]
)
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_summary_list_score_model")

35
benchmark/tests/test_files/be_summary_list_model.test Normal file
View File

@@ -0,0 +1,35 @@
*********************************************************************************
* BEST RESULT of accuracy for STree *
*-------------------------------------------------------------------------------*
* *
*  With gridsearched hyperparameters  *
* *
* Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 *
* *
* Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 *
* Stratified: False *
* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json *
* *
*********************************************************************************
*********************************************************************************
* BEST RESULT of accuracy *
*-------------------------------------------------------------------------------*
* *
*  With gridsearched hyperparameters  *
* *
* Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 *
* *
* Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 *
* Stratified: False *
* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json *
* *
*********************************************************************************
Date File Score Time(h) Title
========== =============================================================== ======== ======= ============================================
2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters
2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init
2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A
2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B
2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest

35
benchmark/tests/test_files/be_summary_list_score.test Normal file
View File

@@ -0,0 +1,35 @@
*********************************************************************************
* BEST RESULT of accuracy for ODTE *
*-------------------------------------------------------------------------------*
* *
*  Gridsearched hyperparams v022.1b random_init  *
* *
* Model: ODTE Ver. 0.3.2 Score: accuracy Metric:  0.0434068 *
* *
* Date : 2022-04-20  Time: 10:52:20 Time Spent: 22,591.47 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: Galgo *
* Stratified: False *
* results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json *
* *
*********************************************************************************
*********************************************************************************
* BEST RESULT of accuracy *
*-------------------------------------------------------------------------------*
* *
*  With gridsearched hyperparameters  *
* *
* Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 *
* *
* Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 *
* Stratified: False *
* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json *
* *
*********************************************************************************
Date File Score Time(h) Title
========== =============================================================== ======== ======= ============================================
2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters
2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init
2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A
2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B
2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest

39
benchmark/tests/test_files/be_summary_list_score_all.test Normal file
View File

@@ -0,0 +1,39 @@
*********************************************************************************
* BEST RESULT of accuracy for ODTE *
*-------------------------------------------------------------------------------*
* *
*  Gridsearched hyperparams v022.1b random_init  *
* *
* Model: ODTE Ver. 0.3.2 Score: accuracy Metric:  0.0434068 *
* *
* Date : 2022-04-20  Time: 10:52:20 Time Spent: 22,591.47 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: Galgo *
* Stratified: False *
* results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json *
* *
*********************************************************************************
*********************************************************************************
* BEST RESULT of accuracy *
*-------------------------------------------------------------------------------*
* *
*  With gridsearched hyperparameters  *
* *
* Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 *
* *
* Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 *
* Stratified: False *
* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json *
* *
*********************************************************************************
Date File Score Time(h) Title
========== =============================================================== ======== ======= ============================================
2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters
2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init
2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A
2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B
2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest
** No results found **
** No results found **
** No results found **
** No results found **

35
benchmark/tests/test_files/be_summary_list_score_model.test Normal file
View File

@@ -0,0 +1,35 @@
*********************************************************************************
* BEST RESULT of accuracy for ODTE *
*-------------------------------------------------------------------------------*
* *
*  Gridsearched hyperparams v022.1b random_init  *
* *
* Model: ODTE Ver. 0.3.2 Score: accuracy Metric:  0.0434068 *
* *
* Date : 2022-04-20  Time: 10:52:20 Time Spent: 22,591.47 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: Galgo *
* Stratified: False *
* results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json *
* *
*********************************************************************************
*********************************************************************************
* BEST RESULT of accuracy *
*-------------------------------------------------------------------------------*
* *
*  With gridsearched hyperparameters  *
* *
* Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 *
* *
* Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 *
* Stratified: False *
* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json *
* *
*********************************************************************************
Date File Score Time(h) Title
========== =============================================================== ======== ======= ============================================
2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters
2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init
2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A
2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B
2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest