diff --git a/benchmark/Experiments.py b/benchmark/Experiments.py index cc4e066..f682cda 100644 --- a/benchmark/Experiments.py +++ b/benchmark/Experiments.py @@ -92,7 +92,10 @@ class Datasets: self.data_sets = [dataset_name] def load(self, name): - return self.dataset.load(name) + try: + return self.dataset.load(name) + except FileNotFoundError: + raise ValueError(f"Unknown dataset: {name}") def __iter__(self) -> Diterator: return Diterator(self.data_sets) diff --git a/benchmark/Results.py b/benchmark/Results.py index 63a5ff3..f4ff7ca 100644 --- a/benchmark/Results.py +++ b/benchmark/Results.py @@ -1,4 +1,3 @@ -from multiprocessing.sharedctypes import Value import os from operator import itemgetter import math diff --git a/benchmark/scripts/be_main.py b/benchmark/scripts/be_main.py index b2fc66d..219dad9 100755 --- a/benchmark/scripts/be_main.py +++ b/benchmark/scripts/be_main.py @@ -31,13 +31,17 @@ def main(args_test=None): title=args.title, folds=args.n_folds, ) - job.do_experiment() - if report: - result_file = job.get_output_file() - report = Report(result_file) - report.report() - if args.dataset is not None: - print(f"Partial result file removed: {result_file}") - os.remove(result_file) + try: + job.do_experiment() + except ValueError as e: + print(e) else: - print(f"Results in {job.get_output_file()}") + if report: + result_file = job.get_output_file() + report = Report(result_file) + report.report() + if args.dataset is not None: + print(f"Partial result file removed: {result_file}") + os.remove(result_file) + else: + print(f"Results in {job.get_output_file()}") diff --git a/benchmark/tests/Benchmark_test.py b/benchmark/tests/Benchmark_test.py index d10fef5..71eea04 100644 --- a/benchmark/tests/Benchmark_test.py +++ b/benchmark/tests/Benchmark_test.py @@ -9,7 +9,6 @@ from ..Results import Benchmark class BenchmarkTest(TestBase): def tearDown(self) -> None: - benchmark = Benchmark("accuracy", visualize=False) files = [] for score in ["accuracy", "unknown"]: files.append(Files.exreport(score)) diff --git a/benchmark/tests/Dataset_test.py b/benchmark/tests/Dataset_test.py index 2e1e819..63ffc9c 100644 --- a/benchmark/tests/Dataset_test.py +++ b/benchmark/tests/Dataset_test.py @@ -45,6 +45,18 @@ class DatasetTest(TestBase): self.assertSequenceEqual(computed, value) self.set_env(".env.dist") + def test_load_dataset(self): + dt = Datasets() + X, y = dt.load("balance-scale") + self.assertSequenceEqual(X.shape, (625, 4)) + self.assertSequenceEqual(y.shape, (625,)) + + def test_load_unknown_dataset(self): + dt = Datasets() + with self.assertRaises(ValueError) as msg: + dt.load("unknown") + self.assertEqual(str(msg.exception), "Unknown dataset: unknown") + def test_Datasets_subset(self): test = { ".env.dist": "balloons", diff --git a/benchmark/tests/__init__.py b/benchmark/tests/__init__.py index edb715e..e5e84fb 100644 --- a/benchmark/tests/__init__.py +++ b/benchmark/tests/__init__.py @@ -18,6 +18,7 @@ from .scripts.Be_Summary_test import BeSummaryTest from .scripts.Be_Grid_test import BeGridTest from .scripts.Be_Best_test import BeBestTest from .scripts.Be_Benchmark_test import BeBenchmarkTest +from .scripts.Be_Main_test import BeMainTest all = [ "UtilTest", @@ -40,4 +41,5 @@ all = [ "BeGridTest", "BeBestTest", "BeBenchmarkTest", + "BeMainTest", ] diff --git a/benchmark/tests/scripts/Be_Main_test.py b/benchmark/tests/scripts/Be_Main_test.py new file mode 100644 index 0000000..8b4ec8d --- /dev/null +++ b/benchmark/tests/scripts/Be_Main_test.py @@ -0,0 +1,47 @@ +import os +from ...Utils import Folders +from ..TestBase import TestBase + + +class BeMainTest(TestBase): + def setUp(self): + self.prepare_scripts_env() + self.score = "accuracy" + + def tearDown(self) -> None: + files = [] + + self.remove_files(files, Folders.exreport) + return super().tearDown() + + def test_be_benchmark_dataset(self): + stdout, _ = self.execute_script( + "be_main", + [ + "-s", + self.score, + "-m", + "STree", + "-d", + "balloons", + "--title", + "test", + ], + ) + with open(os.path.join(self.test_files, "be_main_dataset.test")) as f: + expected = f.read() + n_line = 0 + # compare only report lines without date, time, duration... + lines_to_compare = [0, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13] + for expected, computed in zip( + expected.splitlines(), stdout.getvalue().splitlines() + ): + if n_line in lines_to_compare: + self.assertEqual(computed, expected, n_line) + n_line += 1 + + def test_be_benchmark_no_data(self): + stdout, _ = self.execute_script( + "be_main", ["-m", "STree", "-d", "unknown", "--title", "test"] + ) + self.assertEqual(stdout.getvalue(), "Unknown dataset: unknown\n") diff --git a/benchmark/tests/test_files/be_main_dataset.test b/benchmark/tests/test_files/be_main_dataset.test new file mode 100644 index 0000000..d8553d7 --- /dev/null +++ b/benchmark/tests/test_files/be_main_dataset.test @@ -0,0 +1,15 @@ +*********************************************************************************************************************** +* Report STree ver. 1.2.4 with 5 Folds cross validation and 10 random seeds. 2022-05-08 19:38:28 * +* test * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 0.06 seconds, 0.00 hours, on iMac27 * +* Score is accuracy * +*********************************************************************************************************************** + +Dataset Samp Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ===== ===== === ======= ======= ======= =============== ================ =============== +balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000671±0.0001 {} +*********************************************************************************************************************** +* Accuracy compared to stree_default (liblinear-ovr) .: 0.0165 * +*********************************************************************************************************************** +Partial result file removed: results/results_accuracy_STree_iMac27_2022-05-08_19:38:28_0.json