100% coverage in Experiments

Makefile

@@ -5,7 +5,6 @@ SHELL := /bin/bash
 lint:  ## Lint and static-check
 	black benchmark
 	flake8 benchmark
-	mypy benchmark
 
 test:  ## Run tests
 	python -m unittest -v benchmark.tests

benchmark/Experiments.py

@@ -376,6 +376,9 @@ class GridSearch:
         self.duration = 0
         self._init_data()
 
+    def get_output_file(self):
+        return self.output_file
+
     def _init_data(self):
         # if result file not exist initialize it
         try:
@@ -399,12 +402,17 @@ class GridSearch:
         with open(self.output_file, "w") as f:
             json.dump(data, f, indent=4)
 
+    @staticmethod
+    def _duration_message(duration):
+        if duration < 60:
+            return f"{duration:.3f}s"
+        elif duration < 3600:
+            return f"{duration/60:.3f}m"
+        else:
+            return f"{duration/3600:.3f}h"
+
     def _store_result(self, name, grid, duration):
-        d_message = f"{duration:.3f} s"
-        if duration > 3600:
-            d_message = f"{duration / 3600:.3f} h"
-        elif duration > 60:
-            d_message = f"{duration / 60:.3f} min"
+        d_message = self._duration_message(duration)
         message = (
             f"v. {self.version}, Computed on {self.platform} on "
             f"{self.date} at {self.time} "
@@ -413,7 +421,6 @@ class GridSearch:
         score = grid.best_score_
         hyperparameters = grid.best_params_
         self.results[name] = [score, hyperparameters, message]
-        print(f"{name:30s} {score} {hyperparameters} {message}")
 
     def do_gridsearch(self):
         now = time.time()

benchmark/tests/Experiment_test.py

@@ -1,7 +1,6 @@
 import os
 import json
 import unittest
-from ..Models import Models
 from ..Experiments import Experiment, Datasets
 
 
@@ -32,7 +31,7 @@ class ExperimentTest(unittest.TestCase):
             os.remove(self.exp.get_output_file())
         return super().tearDown()
 
-    def test_build_hyperparams_and_grid_file(self):
+    def test_build_hyperparams_file(self):
         expected = {
             "balance-scale": [
                 0.98,
@@ -53,7 +52,29 @@ class ExperimentTest(unittest.TestCase):
         }
         exp = self.build_exp(hyperparams=True)
         self.assertSequenceEqual(exp.hyperparameters_dict, expected)
+
+    def test_build_grid_file(self):
+        expected = {
+            "balance-scale": [
+                0.9199946751863685,
+                {
+                    "C": 1.0,
+                    "kernel": "liblinear",
+                    "multiclass_strategy": "ovr",
+                },
+                "",
+            ],
+            "balloons": [
+                0.625,
+                {"C": 1.0, "kernel": "linear", "multiclass_strategy": "ovr"},
+                "",
+            ],
+        }
         exp = self.build_exp(grid=True)
+        computed = exp.hyperparameters_dict
+        # Remove generation string as it is dynamic through time
+        for name in ["balance-scale", "balloons"]:
+            computed[name][2] = ""
         self.assertSequenceEqual(exp.hyperparameters_dict, expected)
 
     def test_get_output_file(self):

benchmark/tests/GridSearch_test.py

@@ -0,0 +1,85 @@
+import os
+import json
+import unittest
+from ..Experiments import GridSearch, Datasets
+
+
+class GridSearchTest(unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        os.chdir(os.path.dirname(os.path.abspath(__file__)))
+        self.grid = self.build_exp()
+        super().__init__(*args, **kwargs)
+
+    def tearDown(self) -> None:
+        grid = self.build_exp()
+        grid.do_gridsearch()
+        return super().tearDown()
+
+    def build_exp(self):
+        params = {
+            "score_name": "accuracy",
+            "model_name": "STree",
+            "stratified": "0",
+            "datasets": Datasets(),
+            "progress_bar": False,
+            "platform": "Test",
+            "folds": 2,
+        }
+        return GridSearch(**params)
+
+    def test_get_output_file(self):
+        file_name = self.grid.get_output_file()
+        self.assertEqual(file_name, "results/grid_output_accuracy_STree.json")
+
+    def test_out_file_not_exits(self):
+        file_name = self.grid.get_output_file()
+        if os.path.exists(file_name):
+            os.remove(file_name)
+        grid = self.build_exp()
+        # check the output file is initialized
+        with open(file_name) as f:
+            data = json.load(f)
+        expected = {
+            "balance-scale": [
+                0.0,
+                {},
+                "",
+            ],
+            "balloons": [
+                0.0,
+                {},
+                "",
+            ],
+        }
+        self.assertSequenceEqual(data, expected)
+
+    def test_do_gridsearch(self):
+        self.grid.do_gridsearch()
+        file_name = self.grid.get_output_file()
+        with open(file_name) as f:
+            data = json.load(f)
+        expected = {
+            "balance-scale": [
+                0.9199946751863685,
+                {
+                    "C": 1.0,
+                    "kernel": "liblinear",
+                    "multiclass_strategy": "ovr",
+                },
+                "",
+            ],
+            "balloons": [
+                0.625,
+                {"C": 1.0, "kernel": "linear", "multiclass_strategy": "ovr"},
+                "",
+            ],
+        }
+        dt = Datasets()
+        for dataset in dt:
+            self.assertEqual(data[dataset][0], expected[dataset][0])
+            self.assertSequenceEqual(data[dataset][1], expected[dataset][1])
+
+    def test_duration_message(self):
+        expected = ["47.234s", "5.421m", "1.177h"]
+        for message, duration in zip(expected, [47.234, 325.237, 4237.173]):
+            self.assertEqual(self.grid._duration_message(duration), message)

benchmark/tests/__init__.py

@@ -3,6 +3,7 @@ from .Models_test import ModelTest
 from .Dataset_test import DatasetTest
 from .BestResults_test import BestResultTest
 from .Experiment_test import ExperimentTest
+from .GridSearch_test import GridSearchTest
 
 all = [
     "UtilTest",
@@ -10,4 +11,5 @@ all = [
     "DatasetTest",
     "BestResultTest",
     "ExperimentTest",
+    "GridSearchTest",
 ]

benchmark/tests/results/grid_input_accuracy_STree.json

@@ -0,0 +1,7 @@
+[
+  {
+    "C": [1.0],
+    "kernel": ["linear", "rbf", "liblinear"],
+    "multiclass_strategy": ["ovr"]
+  }
+] 

benchmark/tests/results/grid_output_accuracy_STree.json

@@ -1 +1,20 @@
-{"balance-scale": [0.98, {"splitter": "iwss", "max_features": "auto"}, "results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json"], "balloons": [0.86, {"C": 7, "gamma": 0.1, "kernel": "rbf", "max_iter": 10000.0, "multiclass_strategy": "ovr"}, "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json"]}
+{
+    "balance-scale": [
+        0.9199946751863685,
+        {
+            "C": 1.0,
+            "kernel": "liblinear",
+            "multiclass_strategy": "ovr"
+        },
+        "v. 1.2.4, Computed on Test on 2022-04-24 at 00:15:45 took 0.240s"
+    ],
+    "balloons": [
+        0.625,
+        {
+            "C": 1.0,
+            "kernel": "linear",
+            "multiclass_strategy": "ovr"
+        },
+        "v. 1.2.4, Computed on Test on 2022-04-24 at 00:15:45 took 0.253s"
+    ]
+}