From dfd4f8179b6d99e404d86450c5f651eeb6614172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana?= Date: Thu, 17 Nov 2022 12:00:30 +0100 Subject: [PATCH] Complete tests adding excel to be_list --- benchmark/Results.py | 29 ++++---- benchmark/scripts/be_list.py | 8 ++- benchmark/tests/.gitignore | 1 + benchmark/tests/Report_test.py | 5 +- benchmark/tests/scripts/Be_Grid_test.py | 4 ++ benchmark/tests/scripts/Be_List_test.py | 66 +++++++++++++++++-- benchmark/tests/test_files/be_list_model.test | 6 ++ .../tests/test_files/be_list_model_2.test | 11 ++++ .../test_files/be_list_model_invalid.test | 7 ++ .../tests/test_files/be_list_report.test | 21 ++++++ .../test_files/be_list_report_excel.test | 21 ++++++ .../test_files/be_list_report_excel_2.test | 36 ++++++++++ benchmark/tests/test_files/excel2.test | 48 ++++++++++++++ 13 files changed, 238 insertions(+), 25 deletions(-) create mode 100644 benchmark/tests/test_files/be_list_model.test create mode 100644 benchmark/tests/test_files/be_list_model_2.test create mode 100644 benchmark/tests/test_files/be_list_model_invalid.test create mode 100644 benchmark/tests/test_files/be_list_report.test create mode 100644 benchmark/tests/test_files/be_list_report_excel.test create mode 100644 benchmark/tests/test_files/be_list_report_excel_2.test create mode 100644 benchmark/tests/test_files/excel2.test diff --git a/benchmark/Results.py b/benchmark/Results.py index ca0353e..b876b63 100644 --- a/benchmark/Results.py +++ b/benchmark/Results.py @@ -22,8 +22,8 @@ from .Utils import ( from ._version import __version__ -def get_input(): - return input() +def get_input(is_test): + return "test" if is_test else input() class BestResultsEver: @@ -1477,31 +1477,29 @@ class Summary: ) ) - def manage_results(self, excel, isTest): + def manage_results(self, excel, is_test): + """Manage results showed in the summary + return True if excel file is created False otherwise + """ num = "" - first = True book = None - while True and not isTest: - if not first: - print(f"Invalid option {num}. Try again!") - first = False + while True: print( "Which result do you want to report? (q to quit, r to list " "again, number to report): ", end="", ) - num = get_input() - if num == "n": - return + num = get_input(is_test) if num == "r": self.list_results() if num == "q": if excel: if book is not None: book.close() - return + return True + return False if num.isdigit() and int(num) < len(self.data) and int(num) >= 0: - rep = Report(self.data[int(num)]["file"], self.hidden) + rep = Report(self.data_filtered[int(num)]["file"], self.hidden) rep.report() if excel and not self.hidden: if book is None: @@ -1510,10 +1508,13 @@ class Summary: file_name, {"nan_inf_to_errors": True} ) excel = Excel( - file_name=self.data[int(num)]["file"], + file_name=self.data_filtered[int(num)]["file"], book=book, ) excel.report() + else: + if num not in ("r", "q"): + print(f"Invalid option {num}. Try again!") def show_result(self, data: dict, title: str = "") -> None: def whites(n: int) -> str: diff --git a/benchmark/scripts/be_list.py b/benchmark/scripts/be_list.py index 740b3df..5b00114 100755 --- a/benchmark/scripts/be_list.py +++ b/benchmark/scripts/be_list.py @@ -24,9 +24,11 @@ def main(args_test=None): number=args.number, ) is_test = args_test is not None - data.manage_results(args.excel, is_test) - if args.excel: - Files.open(Files.be_list_excel, is_test) + if not args.nan: + excel_generated = data.manage_results(args.excel, is_test) + if args.excel and excel_generated: + print(f"Generated file: {Files.be_list_excel}") + Files.open(Files.be_list_excel, is_test) except ValueError as e: print(e) else: diff --git a/benchmark/tests/.gitignore b/benchmark/tests/.gitignore index c56bbf8..1760e2d 100644 --- a/benchmark/tests/.gitignore +++ b/benchmark/tests/.gitignore @@ -1 +1,2 @@ ReportDatasets.xlsx +some_results.xlsx diff --git a/benchmark/tests/Report_test.py b/benchmark/tests/Report_test.py index 0f512ff..5970cdb 100644 --- a/benchmark/tests/Report_test.py +++ b/benchmark/tests/Report_test.py @@ -2,11 +2,14 @@ import os from io import StringIO from unittest.mock import patch from .TestBase import TestBase -from ..Results import Report, BaseReport, ReportBest, ReportDatasets +from ..Results import Report, BaseReport, ReportBest, ReportDatasets, get_input from ..Utils import Symbols class ReportTest(TestBase): + def test_get_input(self): + self.assertEqual(get_input(is_test=True), "test") + def test_BaseReport(self): with patch.multiple(BaseReport, __abstractmethods__=set()): file_name = os.path.join( diff --git a/benchmark/tests/scripts/Be_Grid_test.py b/benchmark/tests/scripts/Be_Grid_test.py index 1027f8a..1adb296 100644 --- a/benchmark/tests/scripts/Be_Grid_test.py +++ b/benchmark/tests/scripts/Be_Grid_test.py @@ -4,6 +4,10 @@ from ...Utils import Folders, Files from ..TestBase import TestBase +def get_test(): + return "hola" + + class BeGridTest(TestBase): def setUp(self): self.prepare_scripts_env() diff --git a/benchmark/tests/scripts/Be_List_test.py b/benchmark/tests/scripts/Be_List_test.py index e23cda9..b4011e9 100644 --- a/benchmark/tests/scripts/Be_List_test.py +++ b/benchmark/tests/scripts/Be_List_test.py @@ -1,7 +1,7 @@ import os from unittest.mock import patch -from ...Utils import Folders, NO_RESULTS -from ...Results import Summary +from openpyxl import load_workbook +from ...Utils import Folders, Files, NO_RESULTS from ..TestBase import TestBase @@ -9,13 +9,64 @@ class BeListTest(TestBase): def setUp(self): self.prepare_scripts_env() - # @patch("...Results.get_input()", return_value="n") - def test_be_list(self): + @patch("benchmark.Results.get_input", return_value="q") + def test_be_list(self, input_data): stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) self.assertEqual(stderr.getvalue(), "") - self.check_output_file(stdout, "summary_list_model") + self.check_output_file(stdout, "be_list_model") - def test_be_list_no_data(self): + @patch("benchmark.Results.get_input", side_effect=iter(["x", "q"])) + def test_be_list_invalid_option(self, input_data): + stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_model_invalid") + + @patch("benchmark.Results.get_input", side_effect=iter(["0", "q"])) + def test_be_list_report(self, input_data): + stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_report") + + @patch("benchmark.Results.get_input", side_effect=iter(["q"])) + def test_be_list_report_excel_none(self, input_data): + stdout, stderr = self.execute_script( + "be_list", ["-m", "STree", "-x", "1"] + ) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_model") + + @patch("benchmark.Results.get_input", side_effect=iter(["r", "q"])) + def test_be_list_twice(self, input_data): + stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_model_2") + + @patch("benchmark.Results.get_input", side_effect=iter(["2", "q"])) + def test_be_list_report_excel(self, input_data): + stdout, stderr = self.execute_script( + "be_list", ["-m", "STree", "-x", "1"] + ) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_report_excel") + book = load_workbook(Files.be_list_excel) + sheet = book["STree"] + self.check_excel_sheet(sheet, "excel") + + @patch("benchmark.Results.get_input", side_effect=iter(["2", "1", "q"])) + def test_be_list_report_excel_twice(self, input_data): + stdout, stderr = self.execute_script( + "be_list", ["-m", "STree", "-x", "1"] + ) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_report_excel_2") + book = load_workbook(Files.be_list_excel) + sheet = book["STree"] + self.check_excel_sheet(sheet, "excel") + sheet = book["STree2"] + self.check_excel_sheet(sheet, "excel2") + + @patch("benchmark.Results.get_input", return_value="q") + def test_be_list_no_data(self, input_data): stdout, stderr = self.execute_script( "be_list", ["-m", "Wodt", "-s", "f1-macro"] ) @@ -44,7 +95,8 @@ class BeListTest(TestBase): swap_files(Folders.results, Folders.hidden_results, file_name) self.fail("test_be_list_nan() should not raise exception") - def test_be_list_nan_no_nan(self): + @patch("benchmark.Results.get_input", return_value="q") + def test_be_list_nan_no_nan(self, input_data): stdout, stderr = self.execute_script("be_list", ["--nan", "1"]) self.assertEqual(stderr.getvalue(), "") self.check_output_file(stdout, "be_list_no_nan") diff --git a/benchmark/tests/test_files/be_list_model.test b/benchmark/tests/test_files/be_list_model.test new file mode 100644 index 0000000..718c1b4 --- /dev/null +++ b/benchmark/tests/test_files/be_list_model.test @@ -0,0 +1,6 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): \ No newline at end of file diff --git a/benchmark/tests/test_files/be_list_model_2.test b/benchmark/tests/test_files/be_list_model_2.test new file mode 100644 index 0000000..ba84afa --- /dev/null +++ b/benchmark/tests/test_files/be_list_model_2.test @@ -0,0 +1,11 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report):  # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): \ No newline at end of file diff --git a/benchmark/tests/test_files/be_list_model_invalid.test b/benchmark/tests/test_files/be_list_model_invalid.test new file mode 100644 index 0000000..df728c9 --- /dev/null +++ b/benchmark/tests/test_files/be_list_model_invalid.test @@ -0,0 +1,7 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): Invalid option x. Try again! +Which result do you want to report? (q to quit, r to list again, number to report): \ No newline at end of file diff --git a/benchmark/tests/test_files/be_list_report.test b/benchmark/tests/test_files/be_list_report.test new file mode 100644 index 0000000..3b7fdff --- /dev/null +++ b/benchmark/tests/test_files/be_list_report.test @@ -0,0 +1,21 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): ************************************************************************************************************************* +* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-11-01 19:17:07 * +* default B * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 4115.04 seconds, 1.14 hours, on macbook-pro * +* Score is accuracy * +************************************************************************************************************************* + +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 18.78 9.88 5.90 0.970000±0.0020 0.233304±0.0481 {'max_features': 'auto', 'splitter': 'mutual'} +balloons 16 4 2 4.72 2.86 2.78 0.556667±0.2941 0.021352±0.0058 {'max_features': 'auto', 'splitter': 'mutual'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0379 * +************************************************************************************************************************* +Which result do you want to report? (q to quit, r to list again, number to report): \ No newline at end of file diff --git a/benchmark/tests/test_files/be_list_report_excel.test b/benchmark/tests/test_files/be_list_report_excel.test new file mode 100644 index 0000000..e494061 --- /dev/null +++ b/benchmark/tests/test_files/be_list_report_excel.test @@ -0,0 +1,21 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): ************************************************************************************************************************* +* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 * +* With gridsearched hyperparameters * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 624.25 seconds, 0.17 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* + +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 7.00 4.00 3.00 0.970560±0.0150 0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 * +************************************************************************************************************************* +Which result do you want to report? (q to quit, r to list again, number to report): Generated file: some_results.xlsx diff --git a/benchmark/tests/test_files/be_list_report_excel_2.test b/benchmark/tests/test_files/be_list_report_excel_2.test new file mode 100644 index 0000000..15822a8 --- /dev/null +++ b/benchmark/tests/test_files/be_list_report_excel_2.test @@ -0,0 +1,36 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): ************************************************************************************************************************* +* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 * +* With gridsearched hyperparameters * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 624.25 seconds, 0.17 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* + +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 7.00 4.00 3.00 0.970560±0.0150 0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 * +************************************************************************************************************************* +Which result do you want to report? (q to quit, r to list again, number to report): ************************************************************************************************************************* +* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-10-27 09:40:40 * +* default A * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 3395.01 seconds, 0.94 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* + +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 11.08 5.90 5.90 0.980000±0.0010 0.285207±0.0603 {'splitter': 'best', 'max_features': 'auto'} +balloons 16 4 2 4.12 2.56 2.56 0.695000±0.2757 0.021201±0.0035 {'splitter': 'best', 'max_features': 'auto'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0416 * +************************************************************************************************************************* +Which result do you want to report? (q to quit, r to list again, number to report): Generated file: some_results.xlsx diff --git a/benchmark/tests/test_files/excel2.test b/benchmark/tests/test_files/excel2.test new file mode 100644 index 0000000..c586deb --- /dev/null +++ b/benchmark/tests/test_files/excel2.test @@ -0,0 +1,48 @@ +1;1;" STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-10-27 09:40:40" +2;1;" default A" +3;1;" Score is accuracy" +3;2;" Execution time" +3;5;"3,395.01 s" +3;7;" " +3;8;"Platform" +3;9;"iMac27" +3;10;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]" +4;5;" 0.94 h" +4;10;"Stratified: False" +6;1;"Dataset" +6;2;"Samples" +6;3;"Features" +6;4;"Classes" +6;5;"Nodes" +6;6;"Leaves" +6;7;"Depth" +6;8;"Score" +6;9;"Score Std." +6;10;"Time" +6;11;"Time Std." +6;12;"Hyperparameters" +7;1;"balance-scale" +7;2;"625" +7;3;"4" +7;4;"3" +7;5;"11.08" +7;6;"5.9" +7;7;"5.9" +7;8;"0.98" +7;9;"0.001" +7;10;"0.2852065515518188" +7;11;"0.06031593282605064" +7;12;"{'splitter': 'best', 'max_features': 'auto'}" +8;1;"balloons" +8;2;"16" +8;3;"4" +8;4;"2" +8;5;"4.12" +8;6;"2.56" +8;7;"2.56" +8;8;"0.695" +8;9;"0.2756860130252853" +8;10;"0.02120100021362305" +8;11;"0.003526023309468471" +8;12;"{'splitter': 'best', 'max_features': 'auto'}" +10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0416"