diff --git a/Makefile b/Makefile index 29f3cf9..788af6a 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,6 @@ SHELL := /bin/bash lint: ## Lint and static-check black benchmark flake8 benchmark - mypy benchmark test: ## Run tests python -m unittest -v benchmark.tests diff --git a/benchmark/Experiments.py b/benchmark/Experiments.py index d66f7a9..9a5e26d 100644 --- a/benchmark/Experiments.py +++ b/benchmark/Experiments.py @@ -376,6 +376,9 @@ class GridSearch: self.duration = 0 self._init_data() + def get_output_file(self): + return self.output_file + def _init_data(self): # if result file not exist initialize it try: @@ -399,12 +402,17 @@ class GridSearch: with open(self.output_file, "w") as f: json.dump(data, f, indent=4) + @staticmethod + def _duration_message(duration): + if duration < 60: + return f"{duration:.3f}s" + elif duration < 3600: + return f"{duration/60:.3f}m" + else: + return f"{duration/3600:.3f}h" + def _store_result(self, name, grid, duration): - d_message = f"{duration:.3f} s" - if duration > 3600: - d_message = f"{duration / 3600:.3f} h" - elif duration > 60: - d_message = f"{duration / 60:.3f} min" + d_message = self._duration_message(duration) message = ( f"v. {self.version}, Computed on {self.platform} on " f"{self.date} at {self.time} " @@ -413,7 +421,6 @@ class GridSearch: score = grid.best_score_ hyperparameters = grid.best_params_ self.results[name] = [score, hyperparameters, message] - print(f"{name:30s} {score} {hyperparameters} {message}") def do_gridsearch(self): now = time.time() diff --git a/benchmark/tests/Experiment_test.py b/benchmark/tests/Experiment_test.py index c7a105f..72e0e9b 100644 --- a/benchmark/tests/Experiment_test.py +++ b/benchmark/tests/Experiment_test.py @@ -1,7 +1,6 @@ import os import json import unittest -from ..Models import Models from ..Experiments import Experiment, Datasets @@ -32,7 +31,7 @@ class ExperimentTest(unittest.TestCase): os.remove(self.exp.get_output_file()) return super().tearDown() - def test_build_hyperparams_and_grid_file(self): + def test_build_hyperparams_file(self): expected = { "balance-scale": [ 0.98, @@ -53,7 +52,29 @@ class ExperimentTest(unittest.TestCase): } exp = self.build_exp(hyperparams=True) self.assertSequenceEqual(exp.hyperparameters_dict, expected) + + def test_build_grid_file(self): + expected = { + "balance-scale": [ + 0.9199946751863685, + { + "C": 1.0, + "kernel": "liblinear", + "multiclass_strategy": "ovr", + }, + "", + ], + "balloons": [ + 0.625, + {"C": 1.0, "kernel": "linear", "multiclass_strategy": "ovr"}, + "", + ], + } exp = self.build_exp(grid=True) + computed = exp.hyperparameters_dict + # Remove generation string as it is dynamic through time + for name in ["balance-scale", "balloons"]: + computed[name][2] = "" self.assertSequenceEqual(exp.hyperparameters_dict, expected) def test_get_output_file(self): diff --git a/benchmark/tests/GridSearch_test.py b/benchmark/tests/GridSearch_test.py new file mode 100644 index 0000000..9bd7539 --- /dev/null +++ b/benchmark/tests/GridSearch_test.py @@ -0,0 +1,85 @@ +import os +import json +import unittest +from ..Experiments import GridSearch, Datasets + + +class GridSearchTest(unittest.TestCase): + def __init__(self, *args, **kwargs): + os.chdir(os.path.dirname(os.path.abspath(__file__))) + self.grid = self.build_exp() + super().__init__(*args, **kwargs) + + def tearDown(self) -> None: + grid = self.build_exp() + grid.do_gridsearch() + return super().tearDown() + + def build_exp(self): + params = { + "score_name": "accuracy", + "model_name": "STree", + "stratified": "0", + "datasets": Datasets(), + "progress_bar": False, + "platform": "Test", + "folds": 2, + } + return GridSearch(**params) + + def test_get_output_file(self): + file_name = self.grid.get_output_file() + self.assertEqual(file_name, "results/grid_output_accuracy_STree.json") + + def test_out_file_not_exits(self): + file_name = self.grid.get_output_file() + if os.path.exists(file_name): + os.remove(file_name) + grid = self.build_exp() + # check the output file is initialized + with open(file_name) as f: + data = json.load(f) + expected = { + "balance-scale": [ + 0.0, + {}, + "", + ], + "balloons": [ + 0.0, + {}, + "", + ], + } + self.assertSequenceEqual(data, expected) + + def test_do_gridsearch(self): + self.grid.do_gridsearch() + file_name = self.grid.get_output_file() + with open(file_name) as f: + data = json.load(f) + expected = { + "balance-scale": [ + 0.9199946751863685, + { + "C": 1.0, + "kernel": "liblinear", + "multiclass_strategy": "ovr", + }, + "", + ], + "balloons": [ + 0.625, + {"C": 1.0, "kernel": "linear", "multiclass_strategy": "ovr"}, + "", + ], + } + dt = Datasets() + for dataset in dt: + self.assertEqual(data[dataset][0], expected[dataset][0]) + self.assertSequenceEqual(data[dataset][1], expected[dataset][1]) + + def test_duration_message(self): + expected = ["47.234s", "5.421m", "1.177h"] + for message, duration in zip(expected, [47.234, 325.237, 4237.173]): + self.assertEqual(self.grid._duration_message(duration), message) diff --git a/benchmark/tests/__init__.py b/benchmark/tests/__init__.py index 2783f17..a935e84 100644 --- a/benchmark/tests/__init__.py +++ b/benchmark/tests/__init__.py @@ -3,6 +3,7 @@ from .Models_test import ModelTest from .Dataset_test import DatasetTest from .BestResults_test import BestResultTest from .Experiment_test import ExperimentTest +from .GridSearch_test import GridSearchTest all = [ "UtilTest", @@ -10,4 +11,5 @@ all = [ "DatasetTest", "BestResultTest", "ExperimentTest", + "GridSearchTest", ] diff --git a/benchmark/tests/results/grid_input_accuracy_STree.json b/benchmark/tests/results/grid_input_accuracy_STree.json new file mode 100644 index 0000000..e0678d0 --- /dev/null +++ b/benchmark/tests/results/grid_input_accuracy_STree.json @@ -0,0 +1,7 @@ +[ + { + "C": [1.0], + "kernel": ["linear", "rbf", "liblinear"], + "multiclass_strategy": ["ovr"] + } +] diff --git a/benchmark/tests/results/grid_output_accuracy_STree.json b/benchmark/tests/results/grid_output_accuracy_STree.json index ece4a7a..78c3c0a 100644 --- a/benchmark/tests/results/grid_output_accuracy_STree.json +++ b/benchmark/tests/results/grid_output_accuracy_STree.json @@ -1 +1,20 @@ -{"balance-scale": [0.98, {"splitter": "iwss", "max_features": "auto"}, "results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json"], "balloons": [0.86, {"C": 7, "gamma": 0.1, "kernel": "rbf", "max_iter": 10000.0, "multiclass_strategy": "ovr"}, "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json"]} \ No newline at end of file +{ + "balance-scale": [ + 0.9199946751863685, + { + "C": 1.0, + "kernel": "liblinear", + "multiclass_strategy": "ovr" + }, + "v. 1.2.4, Computed on Test on 2022-04-24 at 00:15:45 took 0.240s" + ], + "balloons": [ + 0.625, + { + "C": 1.0, + "kernel": "linear", + "multiclass_strategy": "ovr" + }, + "v. 1.2.4, Computed on Test on 2022-04-24 at 00:15:45 took 0.253s" + ] +} \ No newline at end of file