diff --git a/benchmark/Results.py b/benchmark/Results.py index c365118..63a5ff3 100644 --- a/benchmark/Results.py +++ b/benchmark/Results.py @@ -1,3 +1,4 @@ +from multiprocessing.sharedctypes import Value import os from operator import itemgetter import math @@ -641,6 +642,8 @@ class Benchmark: summary = Summary() summary.acquire(given_score=self._score) self._models = summary.get_models() + if self._models == []: + raise ValueError(NO_RESULTS) for model in self._models: best = summary.best_result( criterion="model", value=model, score=self._score diff --git a/benchmark/scripts/be_benchmark.py b/benchmark/scripts/be_benchmark.py index c2e536f..d0768d2 100755 --- a/benchmark/scripts/be_benchmark.py +++ b/benchmark/scripts/be_benchmark.py @@ -19,6 +19,6 @@ def main(args_test=None): benchmark.exreport() if args.excel: benchmark.excel() - Files.open(benchmark.get_excel_file_name()) + Files.open(benchmark.get_excel_file_name(), test=args.quiet) if args.tex_output: print(f"File {benchmark.get_tex_file()} generated") diff --git a/benchmark/tests/Benchmark_test.py b/benchmark/tests/Benchmark_test.py index 8da6056..d10fef5 100644 --- a/benchmark/tests/Benchmark_test.py +++ b/benchmark/tests/Benchmark_test.py @@ -3,19 +3,19 @@ from io import StringIO from unittest.mock import patch from openpyxl import load_workbook from .TestBase import TestBase -from ..Utils import Folders, Files +from ..Utils import Folders, Files, NO_RESULTS from ..Results import Benchmark class BenchmarkTest(TestBase): - def tearDown(self): + def tearDown(self) -> None: benchmark = Benchmark("accuracy", visualize=False) files = [] for score in ["accuracy", "unknown"]: files.append(Files.exreport(score)) files.append(Files.exreport_output(score)) files.append(Files.exreport_err(score)) - files.append(Files.exreport_excel("accuracy")) + files.append(Files.exreport_excel(score)) files.append(Files.exreport_pdf) files.append(Files.tex_output("accuracy")) self.remove_files(files, Folders.exreport) @@ -65,20 +65,20 @@ class BenchmarkTest(TestBase): self.assertFalse(os.path.exists(Folders.report)) def test_exreport_error(self): - benchmark = Benchmark("unknown", visualize=False) + benchmark = Benchmark("accuracy", visualize=False) benchmark.compile_results() benchmark.save_results() + # Make Rscript exreport fail + benchmark._score = "unknown" with patch(self.output, new=StringIO()) as stdout: benchmark.exreport() self.check_output_file(stdout, "exreport_error") def test_exreport_no_data(self): benchmark = Benchmark("f1-weighted", visualize=False) - benchmark.compile_results() - benchmark.save_results() - with patch(self.output, new=StringIO()) as stdout: - benchmark.exreport() - self.check_output_file(stdout, "exreport_error") + with self.assertRaises(ValueError) as msg: + benchmark.compile_results() + self.assertEqual(str(msg.exception), NO_RESULTS) def test_tex_output(self): benchmark = Benchmark("accuracy", visualize=False) diff --git a/benchmark/tests/scripts/Be_Benchmark_test.py b/benchmark/tests/scripts/Be_Benchmark_test.py index db32b13..9df3c69 100644 --- a/benchmark/tests/scripts/Be_Benchmark_test.py +++ b/benchmark/tests/scripts/Be_Benchmark_test.py @@ -1,36 +1,66 @@ import os -import json -from ...Utils import Folders, Files +from openpyxl import load_workbook +from ...Utils import NO_RESULTS, Folders, Files from ..TestBase import TestBase class BeBenchmarkTest(TestBase): def setUp(self): self.prepare_scripts_env() + self.score = "accuracy" def tearDown(self) -> None: files = [] - for score in ["accuracy", "unknown"]: + for score in [self.score, "unknown"]: files.append(Files.exreport(score)) files.append(Files.exreport_output(score)) files.append(Files.exreport_err(score)) - - files.append(Files.exreport_excel("accuracy")) + files.append(Files.exreport_excel(self.score)) files.append(Files.exreport_pdf) - files.append(Files.tex_output("accuracy")) + files.append(Files.tex_output(self.score)) self.remove_files(files, Folders.exreport) self.remove_files(files, ".") return super().tearDown() - def test_be_benchmark(self): + def test_be_benchmark_complete(self): stdout, stderr = self.execute_script( - "be_benchmark", ["-s", "accuracy", "-q", "1", "-t", "1", "-x", "1"] + "be_benchmark", ["-s", self.score, "-q", "1", "-t", "1", "-x", "1"] ) self.assertEqual(stderr.getvalue(), "") # Check output - self.check_output_file(stdout, "exreport_report") + self.check_output_file(stdout, "be_benchmark_complete") # Check csv file - file_name = os.path.join(Folders.exreport, Files.exreport("accuracy")) + file_name = os.path.join(Folders.exreport, Files.exreport(self.score)) self.check_file_file(file_name, "exreport_csv") # Check tex file + file_name = os.path.join( + Folders.exreport, Files.tex_output(self.score) + ) + self.assertTrue(os.path.exists(file_name)) + self.check_file_file(file_name, "exreport_tex") # Check excel file + file_name = os.path.join( + Folders.exreport, Files.exreport_excel(self.score) + ) + book = load_workbook(file_name) + for sheet_name in book.sheetnames: + sheet = book[sheet_name] + self.check_excel_sheet(sheet, f"exreport_excel_{sheet_name}") + + def test_be_benchmark_single(self): + stdout, stderr = self.execute_script( + "be_benchmark", ["-s", self.score, "-q", "1"] + ) + self.assertEqual(stderr.getvalue(), "") + # Check output + self.check_output_file(stdout, "be_benchmark") + # Check csv file + file_name = os.path.join(Folders.exreport, Files.exreport(self.score)) + self.check_file_file(file_name, "exreport_csv") + + def test_be_benchmark_no_data(self): + stdout, stderr = self.execute_script( + "be_benchmark", ["-s", "f1-weighted"] + ) + self.assertEqual(stderr.getvalue(), "") + self.assertEqual(stdout.getvalue(), f"{NO_RESULTS}\n") diff --git a/benchmark/tests/test_files/be_benchmark.test b/benchmark/tests/test_files/be_benchmark.test new file mode 100644 index 0000000..f40ce42 --- /dev/null +++ b/benchmark/tests/test_files/be_benchmark.test @@ -0,0 +1,32 @@ +Dataset ODTE RandomForest STree +============================== ============= ============= ============= +balance-scale 0.96352±0.025 0.83616±0.026 0.97056±0.015 +balloons 0.78500±0.246 0.62500±0.250 0.86000±0.285 + +Model File Name Score +============================== =========================================================================== ======== + ODTE results/results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 + RandomForest results/results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 + STree results/results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 +**************************************************************************************************** +Benchmark Ok +**************************************************************************************************** +--------------------------------------------------------------------- +Friedman test, objetive maximize output variable accuracy. Obtained p-value: 1.3534e-01 +Chi squared with 2 degrees of freedom statistic: 4.0000 +Test accepted: p-value: 1.3534e-01 >= 0.0500 +--------------------------------------------------------------------- +Control post hoc test for output accuracy +Adjust method: Holm + +Control method: STree +p-values: + ODTE 0.3173 + RandomForest 0.0910 +--------------------------------------------------------------------- +$testMultiple + classifier pvalue rank win tie loss +STree STree NA 1 NA NA NA +ODTE ODTE 0.31731051 2 2 0 0 +RandomForest RandomForest 0.09100053 3 2 0 0 + diff --git a/benchmark/tests/test_files/be_benchmark_complete.test b/benchmark/tests/test_files/be_benchmark_complete.test new file mode 100644 index 0000000..6f32ab6 --- /dev/null +++ b/benchmark/tests/test_files/be_benchmark_complete.test @@ -0,0 +1,33 @@ +Dataset ODTE RandomForest STree +============================== ============= ============= ============= +balance-scale 0.96352±0.025 0.83616±0.026 0.97056±0.015 +balloons 0.78500±0.246 0.62500±0.250 0.86000±0.285 + +Model File Name Score +============================== =========================================================================== ======== + ODTE results/results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 + RandomForest results/results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 + STree results/results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 +**************************************************************************************************** +Benchmark Ok +**************************************************************************************************** +--------------------------------------------------------------------- +Friedman test, objetive maximize output variable accuracy. Obtained p-value: 1.3534e-01 +Chi squared with 2 degrees of freedom statistic: 4.0000 +Test accepted: p-value: 1.3534e-01 >= 0.0500 +--------------------------------------------------------------------- +Control post hoc test for output accuracy +Adjust method: Holm + +Control method: STree +p-values: + ODTE 0.3173 + RandomForest 0.0910 +--------------------------------------------------------------------- +$testMultiple + classifier pvalue rank win tie loss +STree STree NA 1 NA NA NA +ODTE ODTE 0.31731051 2 2 0 0 +RandomForest RandomForest 0.09100053 3 2 0 0 + +File exreport/exreport_accuracy.tex generated diff --git a/benchmark/tests/test_files/exreport_error.test b/benchmark/tests/test_files/exreport_error.test index 404e7c7..7d67fd2 100644 --- a/benchmark/tests/test_files/exreport_error.test +++ b/benchmark/tests/test_files/exreport_error.test @@ -1,7 +1,9 @@ **************************************************************************************************** Error computing benchmark **************************************************************************************************** -Error in dim(ordered) <- ns : - dims [producto 1] no coincide con la longitud del objeto [0] -Calls: testMultipleControl -> .doFriedmanTest -> -> cast +Error in file(file, "rt") : no se puede abrir la conexión +Calls: read.csv -> read.table -> file +Además: Warning message: +In file(file, "rt") : + no fue posible abrir el archivo 'exreport/exreport_unknown.csv': No such file or directory Ejecución interrumpida