From 40814c6f1fe4bf52c8a216019f7b345f2f6d73ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Sat, 7 May 2022 18:35:04 +0200 Subject: [PATCH] Add be_summary tests --- benchmark/scripts/be_summary.py | 23 ++++++----- benchmark/tests/__init__.py | 2 + benchmark/tests/scripts/Be_Summary_test.py | 34 ++++++++++++++++ .../test_files/be_summary_list_model.test | 35 +++++++++++++++++ .../test_files/be_summary_list_score.test | 35 +++++++++++++++++ .../test_files/be_summary_list_score_all.test | 39 +++++++++++++++++++ .../be_summary_list_score_model.test | 35 +++++++++++++++++ 7 files changed, 194 insertions(+), 9 deletions(-) create mode 100644 benchmark/tests/scripts/Be_Summary_test.py create mode 100644 benchmark/tests/test_files/be_summary_list_model.test create mode 100644 benchmark/tests/test_files/be_summary_list_score.test create mode 100644 benchmark/tests/test_files/be_summary_list_score_all.test create mode 100644 benchmark/tests/test_files/be_summary_list_score_model.test diff --git a/benchmark/scripts/be_summary.py b/benchmark/scripts/be_summary.py index e5b0ea2..8ce8aed 100755 --- a/benchmark/scripts/be_summary.py +++ b/benchmark/scripts/be_summary.py @@ -7,18 +7,23 @@ def main(args_test=None): arguments = Arguments() metrics = list(ALL_METRICS) metrics.append("all") - arguments.xset("score", choices=metrics).xset("model", required=False) + arguments.xset("score", choices=metrics).xset("model") args = arguments.parse(args_test) metrics = ALL_METRICS if args.score == "all" else [args.score] summary = Summary() summary.acquire() for metric in metrics: title = f"BEST RESULT of {metric} for {args.model}" - best = summary.best_result( - criterion="model", value=args.model, score=metric - ) - summary.show_result(data=best, title=title) - summary.show_result( - summary.best_result(score=metric), title=f"BEST RESULT of {metric}" - ) - summary.show_top(score=metric, n=10) + try: + best = summary.best_result( + criterion="model", value=args.model, score=metric + ) + except ValueError as e: + print(e) + else: + summary.show_result(data=best, title=title) + summary.show_result( + summary.best_result(score=metric), + title=f"BEST RESULT of {metric}", + ) + summary.show_top(score=metric, n=10) diff --git a/benchmark/tests/__init__.py b/benchmark/tests/__init__.py index a358149..d71832c 100644 --- a/benchmark/tests/__init__.py +++ b/benchmark/tests/__init__.py @@ -14,6 +14,7 @@ from .Arguments_test import ArgumentsTest from .scripts.Be_Pair_check_test import BePairCheckTest from .scripts.Be_List_test import BeListTest from .scripts.Be_Report_test import BeReportTest +from .scripts.Be_Summary_test import BeSummaryTest all = [ "UtilTest", @@ -32,4 +33,5 @@ all = [ "BePairCheckTest", "BeListTest", "BeReportTest", + "BeSummaryTest", ] diff --git a/benchmark/tests/scripts/Be_Summary_test.py b/benchmark/tests/scripts/Be_Summary_test.py new file mode 100644 index 0000000..3a15c88 --- /dev/null +++ b/benchmark/tests/scripts/Be_Summary_test.py @@ -0,0 +1,34 @@ +import os +from openpyxl import load_workbook +from ...Utils import Folders +from ..TestBase import TestBase + + +class BeSummaryTest(TestBase): + def setUp(self): + self.prepare_scripts_env() + + def tearDown(self) -> None: + pass + + def test_be_summary_list_results_model(self): + stdout, stderr = self.execute_script("be_summary", ["-m", "STree"]) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_summary_list_model") + + def test_be_summary_list_results_score(self): + stdout, stderr = self.execute_script("be_summary", ["-s", "accuracy"]) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_summary_list_score") + + def test_be_summary_list_results_score_all(self): + stdout, stderr = self.execute_script("be_summary", ["-s", "all"]) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_summary_list_score_all") + + def test_summary_list_results_model_score(self): + stdout, stderr = self.execute_script( + "be_summary", ["-s", "accuracy", "-m", "ODTE"] + ) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_summary_list_score_model") diff --git a/benchmark/tests/test_files/be_summary_list_model.test b/benchmark/tests/test_files/be_summary_list_model.test new file mode 100644 index 0000000..5cb731f --- /dev/null +++ b/benchmark/tests/test_files/be_summary_list_model.test @@ -0,0 +1,35 @@ +********************************************************************************* +* BEST RESULT of accuracy for STree * +*-------------------------------------------------------------------------------* +* * +*  With gridsearched hyperparameters  * +* * +* Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 * +* * +* Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. * +* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 * +* Stratified: False * +* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * +* * +********************************************************************************* +********************************************************************************* +* BEST RESULT of accuracy * +*-------------------------------------------------------------------------------* +* * +*  With gridsearched hyperparameters  * +* * +* Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 * +* * +* Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. * +* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 * +* Stratified: False * +* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * +* * +********************************************************************************* +Date File Score Time(h) Title +========== =============================================================== ======== ======= ============================================ +2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init +2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A +2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B +2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest diff --git a/benchmark/tests/test_files/be_summary_list_score.test b/benchmark/tests/test_files/be_summary_list_score.test new file mode 100644 index 0000000..e11fda5 --- /dev/null +++ b/benchmark/tests/test_files/be_summary_list_score.test @@ -0,0 +1,35 @@ +********************************************************************************* +* BEST RESULT of accuracy for ODTE * +*-------------------------------------------------------------------------------* +* * +*  Gridsearched hyperparams v022.1b random_init  * +* * +* Model: ODTE Ver. 0.3.2 Score: accuracy Metric:  0.0434068 * +* * +* Date : 2022-04-20  Time: 10:52:20 Time Spent: 22,591.47 secs. * +* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: Galgo * +* Stratified: False * +* results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json * +* * +********************************************************************************* +********************************************************************************* +* BEST RESULT of accuracy * +*-------------------------------------------------------------------------------* +* * +*  With gridsearched hyperparameters  * +* * +* Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 * +* * +* Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. * +* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 * +* Stratified: False * +* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * +* * +********************************************************************************* +Date File Score Time(h) Title +========== =============================================================== ======== ======= ============================================ +2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init +2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A +2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B +2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest diff --git a/benchmark/tests/test_files/be_summary_list_score_all.test b/benchmark/tests/test_files/be_summary_list_score_all.test new file mode 100644 index 0000000..c425399 --- /dev/null +++ b/benchmark/tests/test_files/be_summary_list_score_all.test @@ -0,0 +1,39 @@ +********************************************************************************* +* BEST RESULT of accuracy for ODTE * +*-------------------------------------------------------------------------------* +* * +*  Gridsearched hyperparams v022.1b random_init  * +* * +* Model: ODTE Ver. 0.3.2 Score: accuracy Metric:  0.0434068 * +* * +* Date : 2022-04-20  Time: 10:52:20 Time Spent: 22,591.47 secs. * +* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: Galgo * +* Stratified: False * +* results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json * +* * +********************************************************************************* +********************************************************************************* +* BEST RESULT of accuracy * +*-------------------------------------------------------------------------------* +* * +*  With gridsearched hyperparameters  * +* * +* Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 * +* * +* Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. * +* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 * +* Stratified: False * +* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * +* * +********************************************************************************* +Date File Score Time(h) Title +========== =============================================================== ======== ======= ============================================ +2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init +2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A +2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B +2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest +** No results found ** +** No results found ** +** No results found ** +** No results found ** diff --git a/benchmark/tests/test_files/be_summary_list_score_model.test b/benchmark/tests/test_files/be_summary_list_score_model.test new file mode 100644 index 0000000..e11fda5 --- /dev/null +++ b/benchmark/tests/test_files/be_summary_list_score_model.test @@ -0,0 +1,35 @@ +********************************************************************************* +* BEST RESULT of accuracy for ODTE * +*-------------------------------------------------------------------------------* +* * +*  Gridsearched hyperparams v022.1b random_init  * +* * +* Model: ODTE Ver. 0.3.2 Score: accuracy Metric:  0.0434068 * +* * +* Date : 2022-04-20  Time: 10:52:20 Time Spent: 22,591.47 secs. * +* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: Galgo * +* Stratified: False * +* results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json * +* * +********************************************************************************* +********************************************************************************* +* BEST RESULT of accuracy * +*-------------------------------------------------------------------------------* +* * +*  With gridsearched hyperparameters  * +* * +* Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 * +* * +* Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. * +* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 * +* Stratified: False * +* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * +* * +********************************************************************************* +Date File Score Time(h) Title +========== =============================================================== ======== ======= ============================================ +2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init +2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A +2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B +2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest