Complete Summary tests

benchmark/scripts/be_summary

@@ -24,19 +24,10 @@ def parse_arguments():
         required=True,
         help="score name {accuracy, f1_micro, f1_macro, all}",
     )
-    ap.add_argument(
-        "-l",
-        "--list",
-        type=bool,
-        required=False,
-        default=False,
-        help="List all results",
-    )
     args = ap.parse_args()
     return (
         args.score,
         args.model,
-        args.list,
     )
 
 
@@ -44,7 +35,6 @@ if __name__ == "__main__":
     (
         score,
         model,
-        list_results,
     ) = parse_arguments()
     all_metrics = ["accuracy", "f1-macro", "f1-micro"]
     metrics = all_metrics if score == "all" else [score]
@@ -60,5 +50,3 @@ if __name__ == "__main__":
             summary.best_result(score=metric), title=f"BEST RESULT of {metric}"
         )
         summary.show_top(score=metric, n=10)
-    if list_results:
-        summary.list_results()

benchmark/tests/Benchmark_test.py

@@ -80,7 +80,8 @@ class BenchmarkTest(unittest.TestCase):
         benchmark = Benchmark("accuracy", visualize=False)
         benchmark.compile_results()
         benchmark.save_results()
-        benchmark.exreport()
+        with patch("sys.stdout", new=StringIO()) as fake_out:
+            benchmark.exreport()
         self.assertFalse(os.path.exists(Files.exreport_pdf))
         self.assertFalse(os.path.exists(Folders.report))
 

benchmark/tests/PairCheck_test.py

@@ -0,0 +1,120 @@
+import os
+import unittest
+from io import StringIO
+from unittest.mock import patch
+from ..Results import Summary
+from ..Utils import Symbols
+
+
+class PairCheckTest(unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        os.chdir(os.path.dirname(os.path.abspath(__file__)))
+        super().__init__(*args, **kwargs)
+
+    def test_summary_list_results_model(self):
+        report = Summary()
+        report.acquire()
+        with patch("sys.stdout", new=StringIO()) as fake_out:
+            report.list_results(model="STree")
+        computed = fake_out.getvalue()
+        with open(
+            os.path.join("test_files", "summary_list_model.test"), "r"
+        ) as f:
+            expected = f.read()
+        self.assertEqual(computed, expected)
+
+    def test_summary_list_results_score(self):
+        report = Summary()
+        report.acquire()
+        with patch("sys.stdout", new=StringIO()) as fake_out:
+            report.list_results(score="accuracy")
+        computed = fake_out.getvalue()
+        with open(
+            os.path.join("test_files", "summary_list_score.test"), "r"
+        ) as f:
+            expected = f.read()
+        self.assertEqual(computed, expected)
+
+    def test_summary_list_results_n(self):
+        report = Summary()
+        report.acquire()
+        with patch("sys.stdout", new=StringIO()) as fake_out:
+            report.list_results(score="accuracy", number=3)
+        computed = fake_out.getvalue()
+        with open(os.path.join("test_files", "summary_list_n.test"), "r") as f:
+            expected = f.read()
+        self.assertEqual(computed, expected)
+
+    def test_summary_list_hiden(self):
+        report = Summary(hidden=True)
+        report.acquire()
+        with patch("sys.stdout", new=StringIO()) as fake_out:
+            report.list_results(score="accuracy")
+        computed = fake_out.getvalue()
+        with open(
+            os.path.join("test_files", "summary_list_hidden.test"), "r"
+        ) as f:
+            expected = f.read()
+        self.assertEqual(computed, expected)
+
+    def test_show_result_no_title(self):
+        report = Summary()
+        report.acquire()
+        with patch("sys.stdout", new=StringIO()) as fake_out:
+            title = ""
+            best = report.best_result(
+                criterion="model", value="STree", score="accuracy"
+            )
+            report.show_result(data=best, title=title)
+        computed = fake_out.getvalue()
+        with open(
+            os.path.join("test_files", "summary_show_results.test"), "r"
+        ) as f:
+            expected = f.read()
+        self.assertEqual(computed, expected)
+
+    def test_show_result_title(self):
+        report = Summary()
+        report.acquire()
+        with patch("sys.stdout", new=StringIO()) as fake_out:
+            title = "**Title**"
+            best = report.best_result(
+                criterion="model", value="STree", score="accuracy"
+            )
+            report.show_result(data=best, title=title)
+        computed = fake_out.getvalue()
+        with open(
+            os.path.join("test_files", "summary_show_results_title.test"), "r"
+        ) as f:
+            expected = f.read()
+        self.assertEqual(computed, expected)
+
+    def test_show_result_no_data(self):
+        report = Summary()
+        report.acquire()
+        with patch("sys.stdout", new=StringIO()) as fake_out:
+            title = "**Test**"
+            report.show_result(data={}, title=title)
+        computed = fake_out.getvalue()
+        expected = f"** **Test** has No data **\n"
+        self.assertEqual(computed, expected)
+
+    def test_best_results_datasets(self):
+        report = Summary()
+        report.acquire()
+        computed = report.best_results_datasets()
+        expected = {
+            "balance-scale": (
+                0.83616,
+                {},
+                "results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json",
+                "Test default paramters with RandomForest",
+            ),
+            "balloons": (
+                0.5566666666666668,
+                {"max_features": "auto", "splitter": "mutual"},
+                "results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json",
+                "default B",
+            ),
+        }
+        self.assertSequenceEqual(computed, expected)

benchmark/tests/Summary_test.py

@@ -0,0 +1,234 @@
+import os
+import unittest
+from io import StringIO
+from unittest.mock import patch
+from ..Results import Summary
+from ..Utils import Symbols
+
+
+class SummaryTest(unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        os.chdir(os.path.dirname(os.path.abspath(__file__)))
+        super().__init__(*args, **kwargs)
+
+    def test_summary_without_model(self):
+        report = Summary()
+        report.acquire()
+        computed = report.best_results(score="accuracy")
+        expected = [
+            {
+                "score": "accuracy",
+                "model": "STree",
+                "title": "With gridsearched hyperparameters",
+                "platform": "iMac27",
+                "date": "2021-09-30",
+                "time": "11:42:07",
+                "stratified": "0",
+                "file": "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json",
+                "metric": 0.04544339345094904,
+                "duration": 624.2505249977112,
+            },
+            {
+                "score": "accuracy",
+                "model": "ODTE",
+                "title": "Gridsearched hyperparams v022.1b random_init",
+                "platform": "Galgo",
+                "date": "2022-04-20",
+                "time": "10:52:20",
+                "stratified": "0",
+                "file": "results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json",
+                "metric": 0.04340676203831255,
+                "duration": 22591.471411943436,
+            },
+            {
+                "score": "accuracy",
+                "model": "STree",
+                "title": "default A",
+                "platform": "iMac27",
+                "date": "2021-10-27",
+                "time": "09:40:40",
+                "stratified": "0",
+                "file": "results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json",
+                "metric": 0.04158163842230773,
+                "duration": 3395.009148836136,
+            },
+            {
+                "score": "accuracy",
+                "model": "STree",
+                "title": "default B",
+                "platform": "macbook-pro",
+                "date": "2021-11-01",
+                "time": "19:17:07",
+                "stratified": "0",
+                "file": "results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json",
+                "metric": 0.03789928437296904,
+                "duration": 4115.042420864105,
+            },
+            {
+                "score": "accuracy",
+                "model": "RandomForest",
+                "title": "Test default paramters with RandomForest",
+                "platform": "iMac27",
+                "date": "2022-01-14",
+                "time": "12:39:30",
+                "stratified": "0",
+                "file": "results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json",
+                "metric": 0.03627309062515771,
+                "duration": 272.7363500595093,
+            },
+        ]
+        self.assertListEqual(computed, expected)
+
+    def test_summary_with_model(self):
+        report = Summary()
+        report.acquire()
+        computed = report.best_results(
+            criterion="model", value="STree", score="accuracy"
+        )
+        expected = [
+            {
+                "score": "accuracy",
+                "model": "STree",
+                "title": "With gridsearched hyperparameters",
+                "platform": "iMac27",
+                "date": "2021-09-30",
+                "time": "11:42:07",
+                "stratified": "0",
+                "file": "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json",
+                "metric": 0.04544339345094904,
+                "duration": 624.2505249977112,
+            },
+            {
+                "score": "accuracy",
+                "model": "STree",
+                "title": "default A",
+                "platform": "iMac27",
+                "date": "2021-10-27",
+                "time": "09:40:40",
+                "stratified": "0",
+                "file": "results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json",
+                "metric": 0.04158163842230773,
+                "duration": 3395.009148836136,
+            },
+            {
+                "score": "accuracy",
+                "model": "STree",
+                "title": "default B",
+                "platform": "macbook-pro",
+                "date": "2021-11-01",
+                "time": "19:17:07",
+                "stratified": "0",
+                "file": "results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json",
+                "metric": 0.03789928437296904,
+                "duration": 4115.042420864105,
+            },
+        ]
+        self.assertListEqual(computed, expected)
+
+    def test_summary_list_results_model(self):
+        report = Summary()
+        report.acquire()
+        with patch("sys.stdout", new=StringIO()) as fake_out:
+            report.list_results(model="STree")
+        computed = fake_out.getvalue()
+        with open(
+            os.path.join("test_files", "summary_list_model.test"), "r"
+        ) as f:
+            expected = f.read()
+        self.assertEqual(computed, expected)
+
+    def test_summary_list_results_score(self):
+        report = Summary()
+        report.acquire()
+        with patch("sys.stdout", new=StringIO()) as fake_out:
+            report.list_results(score="accuracy")
+        computed = fake_out.getvalue()
+        with open(
+            os.path.join("test_files", "summary_list_score.test"), "r"
+        ) as f:
+            expected = f.read()
+        self.assertEqual(computed, expected)
+
+    def test_summary_list_results_n(self):
+        report = Summary()
+        report.acquire()
+        with patch("sys.stdout", new=StringIO()) as fake_out:
+            report.list_results(score="accuracy", number=3)
+        computed = fake_out.getvalue()
+        with open(os.path.join("test_files", "summary_list_n.test"), "r") as f:
+            expected = f.read()
+        self.assertEqual(computed, expected)
+
+    def test_summary_list_hiden(self):
+        report = Summary(hidden=True)
+        report.acquire()
+        with patch("sys.stdout", new=StringIO()) as fake_out:
+            report.list_results(score="accuracy")
+        computed = fake_out.getvalue()
+        with open(
+            os.path.join("test_files", "summary_list_hidden.test"), "r"
+        ) as f:
+            expected = f.read()
+        self.assertEqual(computed, expected)
+
+    def test_show_result_no_title(self):
+        report = Summary()
+        report.acquire()
+        with patch("sys.stdout", new=StringIO()) as fake_out:
+            title = ""
+            best = report.best_result(
+                criterion="model", value="STree", score="accuracy"
+            )
+            report.show_result(data=best, title=title)
+        computed = fake_out.getvalue()
+        with open(
+            os.path.join("test_files", "summary_show_results.test"), "r"
+        ) as f:
+            expected = f.read()
+        self.assertEqual(computed, expected)
+
+    def test_show_result_title(self):
+        report = Summary()
+        report.acquire()
+        with patch("sys.stdout", new=StringIO()) as fake_out:
+            title = "**Title**"
+            best = report.best_result(
+                criterion="model", value="STree", score="accuracy"
+            )
+            report.show_result(data=best, title=title)
+        computed = fake_out.getvalue()
+        with open(
+            os.path.join("test_files", "summary_show_results_title.test"), "r"
+        ) as f:
+            expected = f.read()
+        self.assertEqual(computed, expected)
+
+    def test_show_result_no_data(self):
+        report = Summary()
+        report.acquire()
+        with patch("sys.stdout", new=StringIO()) as fake_out:
+            title = "**Test**"
+            report.show_result(data={}, title=title)
+        computed = fake_out.getvalue()
+        expected = f"** **Test** has No data **\n"
+        self.assertEqual(computed, expected)
+
+    def test_best_results_datasets(self):
+        report = Summary()
+        report.acquire()
+        computed = report.best_results_datasets()
+        expected = {
+            "balance-scale": (
+                0.83616,
+                {},
+                "results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json",
+                "Test default paramters with RandomForest",
+            ),
+            "balloons": (
+                0.5566666666666668,
+                {"max_features": "auto", "splitter": "mutual"},
+                "results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json",
+                "default B",
+            ),
+        }
+        self.assertSequenceEqual(computed, expected)

benchmark/tests/__init__.py

@@ -8,6 +8,8 @@ from .Report_test import ReportTest
 from .Excel_test import ExcelTest
 from .SQL_test import SQLTest
 from .Benchmark_test import BenchmarkTest
+from .Summary_test import SummaryTest
+from .PairCheck_test import PairCheckTest
 
 all = [
     "UtilTest",
@@ -20,4 +22,6 @@ all = [
     "ExcelTest",
     "SQLTest",
     "BenchmarkTest",
+    "SummaryTest",
+    "PairCheckTest",
 ]

benchmark/tests/test_files/summary_list_hidden.test

@@ -0,0 +1,3 @@
+Date       File                                                     Score   Time(h) Title
+========== ======================================================== ======== ======= =======
+2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json  0.97446   0.098 default

benchmark/tests/test_files/summary_list_model.test

@@ -0,0 +1,5 @@
+Date       File                                                          Score   Time(h) Title
+========== ============================================================= ======== ======= =================================
+2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json  0.03790   1.143 default B
+2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json       0.04158   0.943 default A
+2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json       0.04544   0.173 With gridsearched hyperparameters

benchmark/tests/test_files/summary_list_n.test

@@ -0,0 +1,5 @@
+Date       File                                                            Score   Time(h) Title
+========== =============================================================== ======== ======= ============================================
+2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
+2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B

benchmark/tests/test_files/summary_list_score.test

@@ -0,0 +1,7 @@
+Date       File                                                            Score   Time(h) Title
+========== =============================================================== ======== ======= ============================================
+2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
+2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
+2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
+2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters

benchmark/tests/test_files/summary_show_results.test

@@ -0,0 +1,12 @@
+*********************************************************************************
+*                                                                               *
+*                       With gridsearched hyperparameters                       *
+*                                                                               *
+* Model: STree           Ver. 1.2.3      Score: accuracy   Metric:  0.0454434   *
+*                                                                               *
+* Date : 2021-09-30      Time: 11:42:07           Time Spent:    624.25 secs.   *
+* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27              *
+* Stratified: False                                                             *
+* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json                      *
+*                                                                               *
+*********************************************************************************

benchmark/tests/test_files/summary_show_results_title.test

@@ -0,0 +1,14 @@
+*********************************************************************************
+*                                   **Title**                                   *
+*-------------------------------------------------------------------------------*
+*                                                                               *
+*                       With gridsearched hyperparameters                       *
+*                                                                               *
+* Model: STree           Ver. 1.2.3      Score: accuracy   Metric:  0.0454434   *
+*                                                                               *
+* Date : 2021-09-30      Time: 11:42:07           Time Spent:    624.25 secs.   *
+* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27              *
+* Stratified: False                                                             *
+* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json                      *
+*                                                                               *
+*********************************************************************************