Update tests

benchmark/Arguments.py

@@ -176,7 +176,7 @@ class Arguments(argparse.ArgumentParser):
                     "action": "store_true",
                     "required": False,
                     "default": False,
-                    "help": "Move nan results to hidden folder",
+                    "help": "List nan results to hidden folder",
                 },
             ],
             "number": [

benchmark/Results.py

@@ -23,7 +23,7 @@ from .Utils import (
 from ._version import __version__
 
 
-def get_input(is_test=False, message=""):
+def get_input(message="", is_test=False):
     return "test" if is_test else input(message)
 
 
@@ -1476,7 +1476,7 @@ class Summary:
             )
         )
 
-    def manage_results(self, is_test):
+    def manage_results(self):
         """Manage results showed in the summary
         return True if excel file is created False otherwise
         """
@@ -1497,7 +1497,7 @@ class Summary:
                 TextColor.RED
                 + f"Are you sure to {verb1} {file_name_result} (y/n)? "
             )
-            confirm = get_input(message=conf_message, is_test=is_test)
+            confirm = get_input(message=conf_message)
             if confirm == "y":
                 print(TextColor.YELLOW + f"{verb2} {file_name_result}")
                 if command == cmd.delete:
@@ -1522,7 +1522,7 @@ class Summary:
         book = None
         max_value = len(self.data)
         while True:
-            match get_input(message=message, is_test=is_test).split():
+            match get_input(message=message).split():
                 case [cmd.relist]:
                     self.list_results()
                 case [cmd.quit]:

benchmark/scripts/be_list.py

@@ -8,10 +8,10 @@ from benchmark.Arguments import Arguments
 
 
 def main(args_test=None):
-    is_test = args_test is not None
     arguments = Arguments(prog="be_list")
     arguments.xset("number").xset("model", required=False).xset("key")
     arguments.xset("score", required=False).xset("compare").xset("hidden")
+    arguments.xset("nan")
     args = arguments.parse(args_test)
     data = Summary(hidden=args.hidden, compare=args.compare)
     data.acquire()
@@ -21,11 +21,12 @@ def main(args_test=None):
             model=args.model,
             sort_key=args.key,
             number=args.number,
+            nan=args.nan,
         )
     except ValueError as e:
         print(e)
         return
-    excel_generated = data.manage_results(is_test=is_test)
+    excel_generated = data.manage_results()
     if excel_generated:
         print(f"Generated file: {Files.be_list_excel}")
-        Files.open(Files.be_list_excel, is_test)
+        Files.open(Files.be_list_excel, test=args_test is not None)

benchmark/scripts/be_report.py

@@ -5,12 +5,13 @@ from benchmark.Arguments import Arguments
 
 
 """Build report on screen of a result file, optionally generate excel and sql
-file, and can compare results of report with best results obtained by model
+file, and can compare results of report wibth best results obtained by model
 If no argument is set, displays the datasets and its characteristics
 """
 
 
 def main(args_test=None):
+    is_test = args_test is not None
     arguments = Arguments(prog="be_report")
     arguments.add_subparser()
     arguments.add_subparsers_options(
@@ -72,13 +73,11 @@ def main(args_test=None):
                     compare=args.compare,
                 )
                 excel.report()
-                is_test = args_test is not None
                 Files.open(excel.get_file_name(), is_test)
         case "datasets":
             report = ReportDatasets(args.excel)
             report.report()
             if args.excel:
-                is_test = args_test is not None
                 Files.open(report.get_file_name(), is_test)
         case _:
-            arguments.parse(["-h"])
+            arguments.print_help()

benchmark/tests/scripts/Be_List_test.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 from unittest.mock import patch
 from openpyxl import load_workbook
 from ...Utils import Folders, Files, NO_RESULTS
@@ -63,7 +64,35 @@ class BeListTest(TestBase):
         self.assertEqual(stderr.getvalue(), "")
         self.assertEqual(stdout.getvalue(), f"{NO_RESULTS}\n")
 
-    @patch("benchmark.Results.get_input", side_effect=iter(["h 0", "y", "q"]))
+    @patch(
+        "benchmark.Results.get_input", side_effect=iter(["d 0", "y", "", "q"])
+    )
+    # @patch("benchmark.Results.get_input", side_effect=iter(["q"]))
+    def test_be_list_delete(self, input_data):
+        def copy_files(source_folder, target_folder, file_name):
+            source = os.path.join(source_folder, file_name)
+            target = os.path.join(target_folder, file_name)
+            shutil.copyfile(source, target)
+
+        file_name = (
+            "results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:"
+            "35_0.json"
+        )
+        # move nan result from hidden to results
+        copy_files(Folders.hidden_results, Folders.results, file_name)
+        try:
+            # list and delete result
+            stdout, stderr = self.execute_script("be_list", "")
+            self.assertEqual(stderr.getvalue(), "")
+            self.check_output_file(stdout, "be_list_delete")
+        except Exception:
+            # delete the result copied if be_list couldn't
+            os.unlink(os.path.join(Folders.results, file_name))
+            self.fail("test_be_list_delete() should not raise exception")
+
+    @patch(
+        "benchmark.Results.get_input", side_effect=iter(["h 0", "y", "", "q"])
+    )
     def test_be_list_hide(self, input_data):
         def swap_files(source_folder, target_folder, file_name):
             source = os.path.join(source_folder, file_name)
@@ -78,20 +107,37 @@ class BeListTest(TestBase):
         swap_files(Folders.hidden_results, Folders.results, file_name)
         try:
             # list and move nan result to hidden again
-            stdout, stderr = self.execute_script("be_list", [])
+            stdout, stderr = self.execute_script("be_list", "")
             self.assertEqual(stderr.getvalue(), "")
-            print(stdout.getvalue())
+            self.check_output_file(stdout, "be_list_hide")
-            # self.check_output_file(stdout, "be_list_nan")
         except Exception:
-            # move back nan result file if be_list couldn't
+            # delete the result copied if be_list couldn't
             swap_files(Folders.results, Folders.hidden_results, file_name)
             self.fail("test_be_list_hide() should not raise exception")
 
-    @patch("benchmark.Results.get_input", return_value="q")
+    @patch("benchmark.Results.get_input", side_effect=iter(["h 0", "q"]))
-    def test_be_list_nan_no_nan(self, input_data):
+    def test_be_list_already_hidden(self, input_data):
-        stdout, stderr = self.execute_script("be_list", ["--nan"])
+        stdout, stderr = self.execute_script("be_list", ["--hidden"])
         self.assertEqual(stderr.getvalue(), "")
-        self.check_output_file(stdout, "be_list_no_nan")
+        self.check_output_file(stdout, "be_list_already_hidden")
+
+    @patch("benchmark.Results.get_input", side_effect=iter(["h 0", "n", "q"]))
+    def test_be_list_dont_hide(self, input_data):
+        stdout, stderr = self.execute_script("be_list", "")
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_default")
+
+    @patch("benchmark.Results.get_input", side_effect=iter(["q"]))
+    def test_be_list_hidden_nan(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["--hidden", "--nan"])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_hidden_nan")
+
+    @patch("benchmark.Results.get_input", side_effect=iter(["q"]))
+    def test_be_list_hidden(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["--hidden"])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_hidden")
 
     def test_be_no_env(self):
         path = os.getcwd()

benchmark/tests/scripts/Be_Report_test.py

@@ -104,14 +104,14 @@ class BeReportTest(TestBase):
     def test_be_report_unknown_subcommand(self, stderr):
         with self.assertRaises(SystemExit) as msg:
             module = self.search_script("be_report")
-            module.main(["unknown", "accuracy", "-m", "STree"])
+            module.main(["unknown"])
         self.assertEqual(msg.exception.code, 2)
-        self.assertEqual(
+        self.check_output_file(stderr, "report_unknown_subcommand")
-            stderr.getvalue(),
+
-            "usage: be_report [-h] {best,grid,file,datasets} ...\n"
+    def test_be_report_without_subcommand(self):
-            "be_report: error: argument subcommand: invalid choice: "
+        stdout, stderr = self.execute_script("be_report", "")
-            "'unknown' (choose from 'best', 'grid', 'file', 'datasets')\n",
+        self.assertEqual(stderr.getvalue(), "")
-        )
+        self.check_output_file(stdout, "report_without_subcommand")
 
     def test_be_report_excel_compared(self):
         file_name = "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json"

benchmark/tests/test_files/be_list_already_hidden.test

@@ -0,0 +1,5 @@
+ #  Date       File                                                             Score    Time(h) Title
+=== ========== ================================================================ ======== ======= =======================
+  0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json      nan   3.091 Default hyperparameters
+  1 2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json          0.97446   0.098 default
+Already hidden

benchmark/tests/test_files/be_list_delete.test

@@ -0,0 +1,16 @@
+ #  Date       File                                                             Score    Time(h) Title
+=== ========== ================================================================ ======== ======= ============================================
+  0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json      nan   3.091 Default hyperparameters
+  1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json            0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  2 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json   0.03627   0.076 Test default paramters with RandomForest
+  3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json     0.03790   1.143 default B
+  4 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json          0.04158   0.943 default A
+  5 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json          0.04544   0.173 With gridsearched hyperparameters
+Deleting results/results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json
+ #  Date       File                                                            Score    Time(h) Title
+=== ========== =============================================================== ======== ======= ============================================
+  0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
+  2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
+  3 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
+  4 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters

benchmark/tests/test_files/be_list_hidden.test

@@ -0,0 +1,4 @@
+ #  Date       File                                                             Score    Time(h) Title
+=== ========== ================================================================ ======== ======= =======================
+  0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json      nan   3.091 Default hyperparameters
+  1 2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json          0.97446   0.098 default

benchmark/tests/test_files/be_list_hidden_nan.test

@@ -0,0 +1,3 @@
+ #  Date       File                                                             Score    Time(h) Title
+=== ========== ================================================================ ======== ======= =======================
+  0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json      nan   3.091 Default hyperparameters

benchmark/tests/test_files/be_list_hide.test

@@ -6,8 +6,11 @@
   3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json     0.03790   1.143 default B
   4 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json          0.04158   0.943 default A
   5 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json          0.04544   0.173 With gridsearched hyperparameters
-
+Hiding results/results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json
-****************************** Results with nan moved to hidden ******************************
+ #  Date       File                                                            Score    Time(h) Title
- #  Date       File                                                             Score    Time(h) Title
+=== ========== =============================================================== ======== ======= ============================================
-=== ========== ================================================================ ======== ======= =======================
+  0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
-  0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json      nan   3.091 Default hyperparameters
+  1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
+  2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
+  3 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
+  4 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters

benchmark/tests/test_files/report_unknown_subcommand.test

@@ -0,0 +1,2 @@
+usage: be_report [-h] {best,grid,file,datasets} ...
+be_report: error: argument subcommand: invalid choice: 'unknown' (choose from 'best', 'grid', 'file', 'datasets')

benchmark/tests/test_files/report_without_subcommand.test

@@ -0,0 +1,12 @@
+usage: be_report [-h] {best,grid,file,datasets} ...
+
+positional arguments:
+  {best,grid,file,datasets}
+                        help for subcommand
+    best                Report best results obtained by any model/score. See be_build_best
+    grid                Report grid results obtained by any model/score. See be_build_grid
+    file                Report file results
+    datasets            Report datasets information
+
+options:
+  -h, --help            show this help message and exit