diff --git a/benchmark/tests/__init__.py b/benchmark/tests/__init__.py index 989abbc..eae59a7 100644 --- a/benchmark/tests/__init__.py +++ b/benchmark/tests/__init__.py @@ -16,6 +16,7 @@ from .scripts.Be_List_test import BeListTest from .scripts.Be_Report_test import BeReportTest from .scripts.Be_Summary_test import BeSummaryTest from .scripts.Be_Grid_test import BeGridTest +from .scripts.Be_Best_test import BeBestTest all = [ "UtilTest", @@ -36,4 +37,5 @@ all = [ "BeReportTest", "BeSummaryTest", "BeGridTest", + "BeBestTest", ] diff --git a/benchmark/tests/scripts/Be_Best_test.py b/benchmark/tests/scripts/Be_Best_test.py new file mode 100644 index 0000000..f8b2bca --- /dev/null +++ b/benchmark/tests/scripts/Be_Best_test.py @@ -0,0 +1,108 @@ +import os +import json +from ...Utils import Folders, Files +from ..TestBase import TestBase + + +class BeBestTest(TestBase): + def setUp(self): + self.prepare_scripts_env() + + def tearDown(self) -> None: + self.remove_files( + [Files.best_results("accuracy", "ODTE")], + Folders.results, + ) + return super().tearDown() + + def test_be_best_all(self): + stdout, stderr = self.execute_script("be_best", ["-s", "all"]) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_best_all") + + def test_be_build_best_error(self): + stdout, _ = self.execute_script( + "be_build_best", ["-s", "accuracy", "-m", "SVC"] + ) + self.assertEqual(stdout.getvalue(), "** No results found **\n") + + def test_be_build_best(self): + self.execute_script("be_build_best", ["-s", "accuracy", "-m", "ODTE"]) + expected_data = { + "balance-scale": [ + 0.96352, + { + "base_estimator__C": 57, + "base_estimator__gamma": 0.1, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr", + "n_estimators": 100, + "n_jobs": -1, + }, + "results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json", + ], + "balloons": [ + 0.785, + { + "base_estimator__C": 5, + "base_estimator__gamma": 0.14, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr", + "n_estimators": 100, + "n_jobs": -1, + }, + "results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json", + ], + } + name = Files.best_results("accuracy", "ODTE") + file_name = os.path.join(Folders.results, name) + with open(file_name, "r") as f: + computed_data = json.load(f) + for computed, expected in zip(computed_data, expected_data): + self.assertEqual(computed, expected) + for key, value in expected_data.items(): + self.assertIn(key, computed_data) + self.assertEqual(computed_data[key][0], value[0]) + self.assertSequenceEqual(computed_data[key][1], value[1]) + + def test_be_build_best_report(self): + stdout, _ = self.execute_script( + "be_build_best", ["-s", "accuracy", "-m", "ODTE", "-r", "1"] + ) + expected_data = { + "balance-scale": [ + 0.96352, + { + "base_estimator__C": 57, + "base_estimator__gamma": 0.1, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr", + "n_estimators": 100, + "n_jobs": -1, + }, + "results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json", + ], + "balloons": [ + 0.785, + { + "base_estimator__C": 5, + "base_estimator__gamma": 0.14, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr", + "n_estimators": 100, + "n_jobs": -1, + }, + "results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json", + ], + } + name = Files.best_results("accuracy", "ODTE") + file_name = os.path.join(Folders.results, name) + with open(file_name, "r") as f: + computed_data = json.load(f) + for computed, expected in zip(computed_data, expected_data): + self.assertEqual(computed, expected) + for key, value in expected_data.items(): + self.assertIn(key, computed_data) + self.assertEqual(computed_data[key][0], value[0]) + self.assertSequenceEqual(computed_data[key][1], value[1]) + self.check_output_file(stdout, "be_build_best_report") diff --git a/benchmark/tests/scripts/Be_Grid_test.py b/benchmark/tests/scripts/Be_Grid_test.py index 257b738..1027f8a 100644 --- a/benchmark/tests/scripts/Be_Grid_test.py +++ b/benchmark/tests/scripts/Be_Grid_test.py @@ -60,7 +60,7 @@ class BeGridTest(TestBase): for key, value in expected_data.items(): self.assertIn(key, computed_data) self.assertEqual(computed_data[key][0], value[0]) - self.assertEqual(computed_data[key][1], value[1]) + self.assertSequenceEqual(computed_data[key][1], value[1]) def test_be_grid_no_input(self): stdout, stderr = self.execute_script( diff --git a/benchmark/tests/test_files/be_best_all.test b/benchmark/tests/test_files/be_best_all.test new file mode 100644 index 0000000..5c02019 --- /dev/null +++ b/benchmark/tests/test_files/be_best_all.test @@ -0,0 +1,60 @@ +balance-scale results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json +---------------------------------------------------------------------------------------------------- + 0.8361600 {} +---------------------------------------------------------------------------------------------------- +Test default paramters with RandomForest +**************************************************************************************************** +balloons results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json +---------------------------------------------------------------------------------------------------- + 0.5566667 {"max_features": "auto", "splitter": "mutual"} +---------------------------------------------------------------------------------------------------- +default B +**************************************************************************************************** +balance-scale +---------------------------------------------------------------------------------------------------- + 1.0000000 "" +---------------------------------------------------------------------------------------------------- + +**************************************************************************************************** +balloons +---------------------------------------------------------------------------------------------------- + 1.0000000 "" +---------------------------------------------------------------------------------------------------- + +**************************************************************************************************** +balance-scale +---------------------------------------------------------------------------------------------------- + 1.0000000 "" +---------------------------------------------------------------------------------------------------- + +**************************************************************************************************** +balloons +---------------------------------------------------------------------------------------------------- + 1.0000000 "" +---------------------------------------------------------------------------------------------------- + +**************************************************************************************************** +balance-scale +---------------------------------------------------------------------------------------------------- + 1.0000000 "" +---------------------------------------------------------------------------------------------------- + +**************************************************************************************************** +balloons +---------------------------------------------------------------------------------------------------- + 1.0000000 "" +---------------------------------------------------------------------------------------------------- + +**************************************************************************************************** +balance-scale +---------------------------------------------------------------------------------------------------- + 1.0000000 "" +---------------------------------------------------------------------------------------------------- + +**************************************************************************************************** +balloons +---------------------------------------------------------------------------------------------------- + 1.0000000 "" +---------------------------------------------------------------------------------------------------- + +**************************************************************************************************** diff --git a/benchmark/tests/test_files/be_build_best_report.test b/benchmark/tests/test_files/be_build_best_report.test new file mode 100644 index 0000000..603f363 --- /dev/null +++ b/benchmark/tests/test_files/be_build_best_report.test @@ -0,0 +1,11 @@ +****************************************************************************************************************************************************************** +* Report Best accuracy Scores with ODTE in any platform * +****************************************************************************************************************************************************************** + +Dataset Score File/Message Hyperparameters +============================== ======== ============================================================================ ============================================= +balance-scale 0.963520 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json {'base_estimator__C': 57, 'base_estimator__gamma': 0.1, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1} +balloons 0.785000 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json {'base_estimator__C': 5, 'base_estimator__gamma': 0.14, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1} +****************************************************************************************************************************************************************** +* Scores compared to stree_default accuracy (liblinear-ovr) .: 0.0434 * +******************************************************************************************************************************************************************