From 71a11110bdf0aacd76ef3cb33a47d21a15c32c0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana?= Date: Tue, 22 Nov 2022 23:32:28 +0100 Subject: [PATCH] Update tests --- benchmark/Arguments.py | 2 +- benchmark/Results.py | 8 +-- benchmark/scripts/be_list.py | 7 +- benchmark/scripts/be_report.py | 7 +- benchmark/tests/scripts/Be_List_test.py | 64 ++++++++++++++++--- benchmark/tests/scripts/Be_Report_test.py | 14 ++-- .../test_files/be_list_already_hidden.test | 5 ++ ..._list_no_nan.test => be_list_default.test} | 0 .../tests/test_files/be_list_delete.test | 16 +++++ .../tests/test_files/be_list_hidden.test | 4 ++ .../tests/test_files/be_list_hidden_nan.test | 3 + .../{be_list_nan.test => be_list_hide.test} | 13 ++-- .../test_files/report_unknown_subcommand.test | 2 + .../test_files/report_without_subcommand.test | 12 ++++ 14 files changed, 124 insertions(+), 33 deletions(-) create mode 100644 benchmark/tests/test_files/be_list_already_hidden.test rename benchmark/tests/test_files/{be_list_no_nan.test => be_list_default.test} (100%) create mode 100644 benchmark/tests/test_files/be_list_delete.test create mode 100644 benchmark/tests/test_files/be_list_hidden.test create mode 100644 benchmark/tests/test_files/be_list_hidden_nan.test rename benchmark/tests/test_files/{be_list_nan.test => be_list_hide.test} (57%) create mode 100644 benchmark/tests/test_files/report_unknown_subcommand.test create mode 100644 benchmark/tests/test_files/report_without_subcommand.test diff --git a/benchmark/Arguments.py b/benchmark/Arguments.py index 816b6f0..ffc4023 100644 --- a/benchmark/Arguments.py +++ b/benchmark/Arguments.py @@ -176,7 +176,7 @@ class Arguments(argparse.ArgumentParser): "action": "store_true", "required": False, "default": False, - "help": "Move nan results to hidden folder", + "help": "List nan results to hidden folder", }, ], "number": [ diff --git a/benchmark/Results.py b/benchmark/Results.py index a6be3e7..28376b0 100644 --- a/benchmark/Results.py +++ b/benchmark/Results.py @@ -23,7 +23,7 @@ from .Utils import ( from ._version import __version__ -def get_input(is_test=False, message=""): +def get_input(message="", is_test=False): return "test" if is_test else input(message) @@ -1476,7 +1476,7 @@ class Summary: ) ) - def manage_results(self, is_test): + def manage_results(self): """Manage results showed in the summary return True if excel file is created False otherwise """ @@ -1497,7 +1497,7 @@ class Summary: TextColor.RED + f"Are you sure to {verb1} {file_name_result} (y/n)? " ) - confirm = get_input(message=conf_message, is_test=is_test) + confirm = get_input(message=conf_message) if confirm == "y": print(TextColor.YELLOW + f"{verb2} {file_name_result}") if command == cmd.delete: @@ -1522,7 +1522,7 @@ class Summary: book = None max_value = len(self.data) while True: - match get_input(message=message, is_test=is_test).split(): + match get_input(message=message).split(): case [cmd.relist]: self.list_results() case [cmd.quit]: diff --git a/benchmark/scripts/be_list.py b/benchmark/scripts/be_list.py index 7a7030f..1415767 100755 --- a/benchmark/scripts/be_list.py +++ b/benchmark/scripts/be_list.py @@ -8,10 +8,10 @@ from benchmark.Arguments import Arguments def main(args_test=None): - is_test = args_test is not None arguments = Arguments(prog="be_list") arguments.xset("number").xset("model", required=False).xset("key") arguments.xset("score", required=False).xset("compare").xset("hidden") + arguments.xset("nan") args = arguments.parse(args_test) data = Summary(hidden=args.hidden, compare=args.compare) data.acquire() @@ -21,11 +21,12 @@ def main(args_test=None): model=args.model, sort_key=args.key, number=args.number, + nan=args.nan, ) except ValueError as e: print(e) return - excel_generated = data.manage_results(is_test=is_test) + excel_generated = data.manage_results() if excel_generated: print(f"Generated file: {Files.be_list_excel}") - Files.open(Files.be_list_excel, is_test) + Files.open(Files.be_list_excel, test=args_test is not None) diff --git a/benchmark/scripts/be_report.py b/benchmark/scripts/be_report.py index 9fc6c9f..3e54c63 100755 --- a/benchmark/scripts/be_report.py +++ b/benchmark/scripts/be_report.py @@ -5,12 +5,13 @@ from benchmark.Arguments import Arguments """Build report on screen of a result file, optionally generate excel and sql -file, and can compare results of report with best results obtained by model +file, and can compare results of report wibth best results obtained by model If no argument is set, displays the datasets and its characteristics """ def main(args_test=None): + is_test = args_test is not None arguments = Arguments(prog="be_report") arguments.add_subparser() arguments.add_subparsers_options( @@ -72,13 +73,11 @@ def main(args_test=None): compare=args.compare, ) excel.report() - is_test = args_test is not None Files.open(excel.get_file_name(), is_test) case "datasets": report = ReportDatasets(args.excel) report.report() if args.excel: - is_test = args_test is not None Files.open(report.get_file_name(), is_test) case _: - arguments.parse(["-h"]) + arguments.print_help() diff --git a/benchmark/tests/scripts/Be_List_test.py b/benchmark/tests/scripts/Be_List_test.py index cffff4c..f9e8c83 100644 --- a/benchmark/tests/scripts/Be_List_test.py +++ b/benchmark/tests/scripts/Be_List_test.py @@ -1,4 +1,5 @@ import os +import shutil from unittest.mock import patch from openpyxl import load_workbook from ...Utils import Folders, Files, NO_RESULTS @@ -63,7 +64,35 @@ class BeListTest(TestBase): self.assertEqual(stderr.getvalue(), "") self.assertEqual(stdout.getvalue(), f"{NO_RESULTS}\n") - @patch("benchmark.Results.get_input", side_effect=iter(["h 0", "y", "q"])) + @patch( + "benchmark.Results.get_input", side_effect=iter(["d 0", "y", "", "q"]) + ) + # @patch("benchmark.Results.get_input", side_effect=iter(["q"])) + def test_be_list_delete(self, input_data): + def copy_files(source_folder, target_folder, file_name): + source = os.path.join(source_folder, file_name) + target = os.path.join(target_folder, file_name) + shutil.copyfile(source, target) + + file_name = ( + "results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:" + "35_0.json" + ) + # move nan result from hidden to results + copy_files(Folders.hidden_results, Folders.results, file_name) + try: + # list and delete result + stdout, stderr = self.execute_script("be_list", "") + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_delete") + except Exception: + # delete the result copied if be_list couldn't + os.unlink(os.path.join(Folders.results, file_name)) + self.fail("test_be_list_delete() should not raise exception") + + @patch( + "benchmark.Results.get_input", side_effect=iter(["h 0", "y", "", "q"]) + ) def test_be_list_hide(self, input_data): def swap_files(source_folder, target_folder, file_name): source = os.path.join(source_folder, file_name) @@ -78,20 +107,37 @@ class BeListTest(TestBase): swap_files(Folders.hidden_results, Folders.results, file_name) try: # list and move nan result to hidden again - stdout, stderr = self.execute_script("be_list", []) + stdout, stderr = self.execute_script("be_list", "") self.assertEqual(stderr.getvalue(), "") - print(stdout.getvalue()) - # self.check_output_file(stdout, "be_list_nan") + self.check_output_file(stdout, "be_list_hide") except Exception: - # move back nan result file if be_list couldn't + # delete the result copied if be_list couldn't swap_files(Folders.results, Folders.hidden_results, file_name) self.fail("test_be_list_hide() should not raise exception") - @patch("benchmark.Results.get_input", return_value="q") - def test_be_list_nan_no_nan(self, input_data): - stdout, stderr = self.execute_script("be_list", ["--nan"]) + @patch("benchmark.Results.get_input", side_effect=iter(["h 0", "q"])) + def test_be_list_already_hidden(self, input_data): + stdout, stderr = self.execute_script("be_list", ["--hidden"]) self.assertEqual(stderr.getvalue(), "") - self.check_output_file(stdout, "be_list_no_nan") + self.check_output_file(stdout, "be_list_already_hidden") + + @patch("benchmark.Results.get_input", side_effect=iter(["h 0", "n", "q"])) + def test_be_list_dont_hide(self, input_data): + stdout, stderr = self.execute_script("be_list", "") + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_default") + + @patch("benchmark.Results.get_input", side_effect=iter(["q"])) + def test_be_list_hidden_nan(self, input_data): + stdout, stderr = self.execute_script("be_list", ["--hidden", "--nan"]) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_hidden_nan") + + @patch("benchmark.Results.get_input", side_effect=iter(["q"])) + def test_be_list_hidden(self, input_data): + stdout, stderr = self.execute_script("be_list", ["--hidden"]) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_hidden") def test_be_no_env(self): path = os.getcwd() diff --git a/benchmark/tests/scripts/Be_Report_test.py b/benchmark/tests/scripts/Be_Report_test.py index b371906..cc03a8d 100644 --- a/benchmark/tests/scripts/Be_Report_test.py +++ b/benchmark/tests/scripts/Be_Report_test.py @@ -104,14 +104,14 @@ class BeReportTest(TestBase): def test_be_report_unknown_subcommand(self, stderr): with self.assertRaises(SystemExit) as msg: module = self.search_script("be_report") - module.main(["unknown", "accuracy", "-m", "STree"]) + module.main(["unknown"]) self.assertEqual(msg.exception.code, 2) - self.assertEqual( - stderr.getvalue(), - "usage: be_report [-h] {best,grid,file,datasets} ...\n" - "be_report: error: argument subcommand: invalid choice: " - "'unknown' (choose from 'best', 'grid', 'file', 'datasets')\n", - ) + self.check_output_file(stderr, "report_unknown_subcommand") + + def test_be_report_without_subcommand(self): + stdout, stderr = self.execute_script("be_report", "") + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "report_without_subcommand") def test_be_report_excel_compared(self): file_name = "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json" diff --git a/benchmark/tests/test_files/be_list_already_hidden.test b/benchmark/tests/test_files/be_list_already_hidden.test new file mode 100644 index 0000000..efb2f7e --- /dev/null +++ b/benchmark/tests/test_files/be_list_already_hidden.test @@ -0,0 +1,5 @@ + # Date File Score Time(h) Title +=== ========== ================================================================ ======== ======= ======================= + 0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters + 1 2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json 0.97446 0.098 default +Already hidden diff --git a/benchmark/tests/test_files/be_list_no_nan.test b/benchmark/tests/test_files/be_list_default.test similarity index 100% rename from benchmark/tests/test_files/be_list_no_nan.test rename to benchmark/tests/test_files/be_list_default.test diff --git a/benchmark/tests/test_files/be_list_delete.test b/benchmark/tests/test_files/be_list_delete.test new file mode 100644 index 0000000..f8da2f7 --- /dev/null +++ b/benchmark/tests/test_files/be_list_delete.test @@ -0,0 +1,16 @@ + # Date File Score Time(h) Title +=== ========== ================================================================ ======== ======= ============================================ + 0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters + 1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 2 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + 3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 4 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 5 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Deleting results/results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + 2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 3 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 4 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters diff --git a/benchmark/tests/test_files/be_list_hidden.test b/benchmark/tests/test_files/be_list_hidden.test new file mode 100644 index 0000000..f98eb9f --- /dev/null +++ b/benchmark/tests/test_files/be_list_hidden.test @@ -0,0 +1,4 @@ + # Date File Score Time(h) Title +=== ========== ================================================================ ======== ======= ======================= + 0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters + 1 2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json 0.97446 0.098 default diff --git a/benchmark/tests/test_files/be_list_hidden_nan.test b/benchmark/tests/test_files/be_list_hidden_nan.test new file mode 100644 index 0000000..c73194f --- /dev/null +++ b/benchmark/tests/test_files/be_list_hidden_nan.test @@ -0,0 +1,3 @@ + # Date File Score Time(h) Title +=== ========== ================================================================ ======== ======= ======================= + 0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters diff --git a/benchmark/tests/test_files/be_list_nan.test b/benchmark/tests/test_files/be_list_hide.test similarity index 57% rename from benchmark/tests/test_files/be_list_nan.test rename to benchmark/tests/test_files/be_list_hide.test index f6a799d..6c2678a 100644 --- a/benchmark/tests/test_files/be_list_nan.test +++ b/benchmark/tests/test_files/be_list_hide.test @@ -6,8 +6,11 @@  3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B  4 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A  5 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters - -****************************** Results with nan moved to hidden ****************************** - # Date File Score Time(h) Title -=== ========== ================================================================ ======== ======= ======================= - 0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters +Hiding results/results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + 2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 3 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 4 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters diff --git a/benchmark/tests/test_files/report_unknown_subcommand.test b/benchmark/tests/test_files/report_unknown_subcommand.test new file mode 100644 index 0000000..0df03e4 --- /dev/null +++ b/benchmark/tests/test_files/report_unknown_subcommand.test @@ -0,0 +1,2 @@ +usage: be_report [-h] {best,grid,file,datasets} ... +be_report: error: argument subcommand: invalid choice: 'unknown' (choose from 'best', 'grid', 'file', 'datasets') diff --git a/benchmark/tests/test_files/report_without_subcommand.test b/benchmark/tests/test_files/report_without_subcommand.test new file mode 100644 index 0000000..57152d7 --- /dev/null +++ b/benchmark/tests/test_files/report_without_subcommand.test @@ -0,0 +1,12 @@ +usage: be_report [-h] {best,grid,file,datasets} ... + +positional arguments: + {best,grid,file,datasets} + help for subcommand + best Report best results obtained by any model/score. See be_build_best + grid Report grid results obtained by any model/score. See be_build_grid + file Report file results + datasets Report datasets information + +options: + -h, --help show this help message and exit