From 974227166ca529742a762dfa6f6e31c0dbe0d169 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana?= Date: Thu, 17 Nov 2022 01:36:19 +0100 Subject: [PATCH 1/7] Add excel to be_list --- benchmark/Results.py | 82 ++++++++++++++++--- benchmark/Utils.py | 1 + benchmark/scripts/be_list.py | 8 +- benchmark/tests/scripts/Be_List_test.py | 3 + benchmark/tests/test_files/be_list_nan.test | 22 ++--- .../tests/test_files/be_list_no_nan.test | 14 ++-- .../test_files/be_summary_list_model.test | 14 ++-- .../test_files/be_summary_list_score.test | 14 ++-- .../test_files/be_summary_list_score_all.test | 14 ++-- .../be_summary_list_score_model.test | 14 ++-- .../tests/test_files/summary_list_hidden.test | 8 +- .../tests/test_files/summary_list_model.test | 10 +-- .../tests/test_files/summary_list_n.test | 10 +-- .../tests/test_files/summary_list_score.test | 14 ++-- .../tests/test_files/summary_show_top.test | 14 ++-- 15 files changed, 154 insertions(+), 88 deletions(-) diff --git a/benchmark/Results.py b/benchmark/Results.py index ccf833c..655e709 100644 --- a/benchmark/Results.py +++ b/benchmark/Results.py @@ -7,6 +7,7 @@ import abc import shutil import subprocess import xlsxwriter +from xlsxwriter.exceptions import DuplicateWorksheetName import numpy as np from .Experiments import BestResults from .Datasets import Datasets @@ -21,6 +22,10 @@ from .Utils import ( from ._version import __version__ +def get_input(): + return input() + + class BestResultsEver: def __init__(self): self.data = {} @@ -328,7 +333,17 @@ class Excel(BaseReport): else: self.book = book self.close = False - self.sheet = self.book.add_worksheet(self.data["model"]) + suffix = "" + num = 1 + while True: + try: + self.sheet = self.book.add_worksheet( + self.data["model"] + suffix + ) + break + except DuplicateWorksheetName: + num += 1 + suffix = str(num) self.max_hyper_width = 0 self.col_hyperparams = 0 @@ -1341,6 +1356,7 @@ class Summary: def __init__(self, hidden=False) -> None: self.results = Files().get_all_results(hidden=hidden) self.data = [] + self.data_filtered = [] self.datasets = {} self.models = set() self.hidden = hidden @@ -1417,13 +1433,14 @@ class Summary: number=0, ) -> None: """Print the list of results""" - data = self.get_results_criteria( - score, model, input_data, sort_key, number - ) - if data == []: + if self.data_filtered == []: + self.data_filtered = self.get_results_criteria( + score, model, input_data, sort_key, number + ) + if self.data_filtered == []: raise ValueError(NO_RESULTS) - max_file = max(len(x["file"]) for x in data) - max_title = max(len(x["title"]) for x in data) + max_file = max(len(x["file"]) for x in self.data_filtered) + max_title = max(len(x["title"]) for x in self.data_filtered) if self.hidden: color1 = TextColor.GREEN color2 = TextColor.YELLOW @@ -1432,10 +1449,11 @@ class Summary: color2 = TextColor.LINE2 print(color1, end="") print( - f"{'Date':10s} {'File':{max_file}s} {'Score':8s} {'Time(h)':7s} " - f"{'Title':s}" + f" # {'Date':10s} {'File':{max_file}s} {'Score':8s} " + f" {'Time(h)':7s} {'Title':s}" ) print( + "===", "=" * 10 + " " + "=" * max_file @@ -1444,21 +1462,59 @@ class Summary: + " " + "=" * 7 + " " - + "=" * max_title + + "=" * max_title, ) print( "\n".join( [ - (color2 if n % 2 == 0 else color1) - + f"{x['date']} {x['file']:{max_file}s} " + (color2 if n % 2 == 0 else color1) + f"{n:3d} " + f"{x['date']} {x['file']:{max_file}s} " f"{x['metric']:8.5f} " f"{x['duration']/3600:7.3f} " f"{x['title']}" - for n, x in enumerate(data) + for n, x in enumerate(self.data_filtered) ] ) ) + def manage_results(self, excel, isTest): + num = "" + first = True + book = None + while True and not isTest: + if not first: + print(f"Invalid option {num}. Try again!") + first = False + print( + "Which result do you want to report? (q to quit, r to list " + "again, number to report): ", + end="", + ) + num = self.get_input() + if num == "n": + return + if num == "r": + self.list_results() + if num == "q": + if excel: + if book is not None: + book.close() + return + if num.isdigit() and int(num) < len(self.data) and int(num) >= 0: + rep = Report(self.data[int(num)]["file"], self.hidden) + rep.report() + if excel and not self.hidden: + if book is None: + file_name = Files.be_list_excel + book = xlsxwriter.Workbook( + file_name, {"nan_inf_to_errors": True} + ) + excel = Excel( + file_name=self.data[int(num)]["file"], + book=book, + ) + excel.report() + def show_result(self, data: dict, title: str = "") -> None: def whites(n: int) -> str: return " " * n + color1 + "*" diff --git a/benchmark/Utils.py b/benchmark/Utils.py index 9663086..fbf7102 100644 --- a/benchmark/Utils.py +++ b/benchmark/Utils.py @@ -28,6 +28,7 @@ class Files: benchmark_r = "benchmark.r" dot_env = ".env" datasets_report_excel = "ReportDatasets.xlsx" + be_list_excel = "some_results.xlsx" @staticmethod def exreport_output(score): diff --git a/benchmark/scripts/be_list.py b/benchmark/scripts/be_list.py index cb31cb0..740b3df 100755 --- a/benchmark/scripts/be_list.py +++ b/benchmark/scripts/be_list.py @@ -1,7 +1,7 @@ #! /usr/bin/env python import os from benchmark.Results import Summary -from benchmark.Utils import Folders +from benchmark.Utils import Folders, Files from benchmark.Arguments import Arguments """List experiments of a model @@ -12,6 +12,7 @@ def main(args_test=None): arguments = Arguments() arguments.xset("number").xset("model", required=False).xset("key") arguments.xset("hidden").xset("nan").xset("score", required=False) + arguments.xset("excel") args = arguments.parse(args_test) data = Summary(hidden=args.hidden) data.acquire() @@ -22,6 +23,10 @@ def main(args_test=None): sort_key=args.key, number=args.number, ) + is_test = args_test is not None + data.manage_results(args.excel, is_test) + if args.excel: + Files.open(Files.be_list_excel, is_test) except ValueError as e: print(e) else: @@ -44,6 +49,7 @@ def main(args_test=None): + " Results with nan moved to hidden " + "*" * 30 ) + data.data_filtered = [] data.list_results(input_data=results_nan) for result in results_nan: name = result["file"] diff --git a/benchmark/tests/scripts/Be_List_test.py b/benchmark/tests/scripts/Be_List_test.py index 25bed40..e23cda9 100644 --- a/benchmark/tests/scripts/Be_List_test.py +++ b/benchmark/tests/scripts/Be_List_test.py @@ -1,5 +1,7 @@ import os +from unittest.mock import patch from ...Utils import Folders, NO_RESULTS +from ...Results import Summary from ..TestBase import TestBase @@ -7,6 +9,7 @@ class BeListTest(TestBase): def setUp(self): self.prepare_scripts_env() + # @patch("...Results.get_input()", return_value="n") def test_be_list(self): stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) self.assertEqual(stderr.getvalue(), "") diff --git a/benchmark/tests/test_files/be_list_nan.test b/benchmark/tests/test_files/be_list_nan.test index ed62daf..58ed8bc 100644 --- a/benchmark/tests/test_files/be_list_nan.test +++ b/benchmark/tests/test_files/be_list_nan.test @@ -1,13 +1,13 @@ -Date File Score Time(h) Title -========== ================================================================ ======== ======= ============================================ -2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + # Date File Score Time(h) Title +=== ========== ================================================================ ======== ======= ============================================ + 0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters + 1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 2 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + 3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 4 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 5 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters ****************************** Results with nan moved to hidden ****************************** -Date File Score Time(h) Title -========== ================================================================ ======== ======= ======================= -2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters + # Date File Score Time(h) Title +=== ========== ================================================================ ======== ======= ======================= + 0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters diff --git a/benchmark/tests/test_files/be_list_no_nan.test b/benchmark/tests/test_files/be_list_no_nan.test index e1e5dd7..720b46c 100644 --- a/benchmark/tests/test_files/be_list_no_nan.test +++ b/benchmark/tests/test_files/be_list_no_nan.test @@ -1,7 +1,7 @@ -Date File Score Time(h) Title -========== =============================================================== ======== ======= ============================================ -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + 2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 3 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 4 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters diff --git a/benchmark/tests/test_files/be_summary_list_model.test b/benchmark/tests/test_files/be_summary_list_model.test index 5cb731f..05ed964 100644 --- a/benchmark/tests/test_files/be_summary_list_model.test +++ b/benchmark/tests/test_files/be_summary_list_model.test @@ -26,10 +26,10 @@ * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * * * ********************************************************************************* -Date File Score Time(h) Title -========== =============================================================== ======== ======= ============================================ -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + 1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 2 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 4 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest diff --git a/benchmark/tests/test_files/be_summary_list_score.test b/benchmark/tests/test_files/be_summary_list_score.test index e11fda5..816cab0 100644 --- a/benchmark/tests/test_files/be_summary_list_score.test +++ b/benchmark/tests/test_files/be_summary_list_score.test @@ -26,10 +26,10 @@ * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * * * ********************************************************************************* -Date File Score Time(h) Title -========== =============================================================== ======== ======= ============================================ -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + 1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 2 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 4 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest diff --git a/benchmark/tests/test_files/be_summary_list_score_all.test b/benchmark/tests/test_files/be_summary_list_score_all.test index c425399..57d8260 100644 --- a/benchmark/tests/test_files/be_summary_list_score_all.test +++ b/benchmark/tests/test_files/be_summary_list_score_all.test @@ -26,13 +26,13 @@ * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * * * ********************************************************************************* -Date File Score Time(h) Title -========== =============================================================== ======== ======= ============================================ -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + 1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 2 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 4 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest ** No results found ** ** No results found ** ** No results found ** diff --git a/benchmark/tests/test_files/be_summary_list_score_model.test b/benchmark/tests/test_files/be_summary_list_score_model.test index e11fda5..816cab0 100644 --- a/benchmark/tests/test_files/be_summary_list_score_model.test +++ b/benchmark/tests/test_files/be_summary_list_score_model.test @@ -26,10 +26,10 @@ * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * * * ********************************************************************************* -Date File Score Time(h) Title -========== =============================================================== ======== ======= ============================================ -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + 1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 2 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 4 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest diff --git a/benchmark/tests/test_files/summary_list_hidden.test b/benchmark/tests/test_files/summary_list_hidden.test index d34b5bd..b663821 100644 --- a/benchmark/tests/test_files/summary_list_hidden.test +++ b/benchmark/tests/test_files/summary_list_hidden.test @@ -1,4 +1,4 @@ -Date File Score Time(h) Title -========== ================================================================ ======== ======= ======================= -2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters -2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json 0.97446 0.098 default + # Date File Score Time(h) Title +=== ========== ================================================================ ======== ======= ======================= + 0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters + 1 2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json 0.97446 0.098 default diff --git a/benchmark/tests/test_files/summary_list_model.test b/benchmark/tests/test_files/summary_list_model.test index 3a5b441..636ebe7 100644 --- a/benchmark/tests/test_files/summary_list_model.test +++ b/benchmark/tests/test_files/summary_list_model.test @@ -1,5 +1,5 @@ -Date File Score Time(h) Title -========== ============================================================= ======== ======= ================================= -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters diff --git a/benchmark/tests/test_files/summary_list_n.test b/benchmark/tests/test_files/summary_list_n.test index ac56095..d4453ec 100644 --- a/benchmark/tests/test_files/summary_list_n.test +++ b/benchmark/tests/test_files/summary_list_n.test @@ -1,5 +1,5 @@ -Date File Score Time(h) Title -========== =============================================================== ======== ======= ============================================ -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + 2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B diff --git a/benchmark/tests/test_files/summary_list_score.test b/benchmark/tests/test_files/summary_list_score.test index e1e5dd7..720b46c 100644 --- a/benchmark/tests/test_files/summary_list_score.test +++ b/benchmark/tests/test_files/summary_list_score.test @@ -1,7 +1,7 @@ -Date File Score Time(h) Title -========== =============================================================== ======== ======= ============================================ -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + 2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 3 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 4 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters diff --git a/benchmark/tests/test_files/summary_show_top.test b/benchmark/tests/test_files/summary_show_top.test index dc6a8e0..5b42449 100644 --- a/benchmark/tests/test_files/summary_show_top.test +++ b/benchmark/tests/test_files/summary_show_top.test @@ -1,7 +1,7 @@ -Date File Score Time(h) Title -========== =============================================================== ======== ======= ============================================ -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + 1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 2 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 4 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest From 8a9342c97b52762582ec9b592718643b5e49408a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana?= Date: Thu, 17 Nov 2022 09:41:17 +0100 Subject: [PATCH 2/7] Add space to time column in report --- benchmark/Results.py | 6 ++-- benchmark/tests/test_files/be_main_best.test | 28 ++++++++--------- .../tests/test_files/be_main_complete.test | 28 ++++++++--------- .../tests/test_files/be_main_dataset.test | 26 ++++++++-------- benchmark/tests/test_files/be_main_grid.test | 28 ++++++++--------- benchmark/tests/test_files/report.test | 28 ++++++++--------- .../tests/test_files/report_compared.test | 30 +++++++++---------- 7 files changed, 87 insertions(+), 87 deletions(-) diff --git a/benchmark/Results.py b/benchmark/Results.py index 655e709..ca0353e 100644 --- a/benchmark/Results.py +++ b/benchmark/Results.py @@ -130,7 +130,7 @@ class BaseReport(abc.ABC): class Report(BaseReport): - header_lengths = [30, 6, 5, 3, 7, 7, 7, 15, 16, 15] + header_lengths = [30, 6, 5, 3, 7, 7, 7, 15, 17, 15] header_cols = [ "Dataset", "Sampl.", @@ -189,7 +189,7 @@ class Report(BaseReport): ) i += 1 print( - f"{result['time']:9.6f}±{result['time_std']:6.4f} ", + f"{result['time']:10.6f}±{result['time_std']:6.4f} ", end="", ) i += 1 @@ -1490,7 +1490,7 @@ class Summary: "again, number to report): ", end="", ) - num = self.get_input() + num = get_input() if num == "n": return if num == "r": diff --git a/benchmark/tests/test_files/be_main_best.test b/benchmark/tests/test_files/be_main_best.test index adecb4c..a18372b 100644 --- a/benchmark/tests/test_files/be_main_best.test +++ b/benchmark/tests/test_files/be_main_best.test @@ -1,16 +1,16 @@ -************************************************************************************************************************ -* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-09 00:15:25 * -* test * -* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * -* Execution took 0.80 seconds, 0.00 hours, on iMac27 * -* Score is accuracy * -************************************************************************************************************************ +************************************************************************************************************************* +* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-09 00:15:25 * +* test * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 0.80 seconds, 0.00 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* -Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters -============================== ====== ===== === ======= ======= ======= =============== ================ =============== -balance-scale 625 4 3 23.32 12.16 6.44 0.840160±0.0304 0.013745±0.0019 {'splitter': 'best', 'max_features': 'auto'} -balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000388±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} -************************************************************************************************************************ -* accuracy compared to STree_default (liblinear-ovr) .: 0.0422 * -************************************************************************************************************************ +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 23.32 12.16 6.44 0.840160±0.0304 0.013745±0.0019 {'splitter': 'best', 'max_features': 'auto'} +balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000388±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0422 * +************************************************************************************************************************* Results in results/results_accuracy_STree_iMac27_2022-05-09_00:15:25_0.json diff --git a/benchmark/tests/test_files/be_main_complete.test b/benchmark/tests/test_files/be_main_complete.test index 8f36fe6..edbfead 100644 --- a/benchmark/tests/test_files/be_main_complete.test +++ b/benchmark/tests/test_files/be_main_complete.test @@ -1,16 +1,16 @@ -************************************************************************************************************************ -* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-08 20:14:43 * -* test * -* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * -* Execution took 0.48 seconds, 0.00 hours, on iMac27 * -* Score is accuracy * -************************************************************************************************************************ +************************************************************************************************************************* +* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-08 20:14:43 * +* test * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 0.48 seconds, 0.00 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* -Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters -============================== ====== ===== === ======= ======= ======= =============== ================ =============== -balance-scale 625 4 3 17.36 9.18 6.18 0.908480±0.0247 0.007388±0.0013 {} -balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000664±0.0002 {} -************************************************************************************************************************ -* accuracy compared to STree_default (liblinear-ovr) .: 0.0390 * -************************************************************************************************************************ +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 17.36 9.18 6.18 0.908480±0.0247 0.007388±0.0013 {} +balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000664±0.0002 {} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0390 * +************************************************************************************************************************* Results in results/results_accuracy_STree_iMac27_2022-05-08_20:14:43_0.json diff --git a/benchmark/tests/test_files/be_main_dataset.test b/benchmark/tests/test_files/be_main_dataset.test index 46e50e3..3fedf46 100644 --- a/benchmark/tests/test_files/be_main_dataset.test +++ b/benchmark/tests/test_files/be_main_dataset.test @@ -1,15 +1,15 @@ -************************************************************************************************************************ -* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-08 19:38:28 * -* test * -* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * -* Execution took 0.06 seconds, 0.00 hours, on iMac27 * -* Score is accuracy * -************************************************************************************************************************ +************************************************************************************************************************* +* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-08 19:38:28 * +* test * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 0.06 seconds, 0.00 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* -Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters -============================== ====== ===== === ======= ======= ======= =============== ================ =============== -balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000671±0.0001 {} -************************************************************************************************************************ -* accuracy compared to STree_default (liblinear-ovr) .: 0.0165 * -************************************************************************************************************************ +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000671±0.0001 {} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0165 * +************************************************************************************************************************* Partial result file removed: results/results_accuracy_STree_iMac27_2022-05-08_19:38:28_0.json diff --git a/benchmark/tests/test_files/be_main_grid.test b/benchmark/tests/test_files/be_main_grid.test index c29ec32..adcf79a 100644 --- a/benchmark/tests/test_files/be_main_grid.test +++ b/benchmark/tests/test_files/be_main_grid.test @@ -1,16 +1,16 @@ -************************************************************************************************************************ -* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-09 00:21:06 * -* test * -* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * -* Execution took 0.89 seconds, 0.00 hours, on iMac27 * -* Score is accuracy * -************************************************************************************************************************ +************************************************************************************************************************* +* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-09 00:21:06 * +* test * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 0.89 seconds, 0.00 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* -Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters -============================== ====== ===== === ======= ======= ======= =============== ================ =============== -balance-scale 625 4 3 26.12 13.56 7.94 0.910720±0.0249 0.015852±0.0027 {'C': 1.0, 'kernel': 'liblinear', 'multiclass_strategy': 'ovr'} -balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000640±0.0001 {'C': 1.0, 'kernel': 'linear', 'multiclass_strategy': 'ovr'} -************************************************************************************************************************ -* accuracy compared to STree_default (liblinear-ovr) .: 0.0391 * -************************************************************************************************************************ +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 26.12 13.56 7.94 0.910720±0.0249 0.015852±0.0027 {'C': 1.0, 'kernel': 'liblinear', 'multiclass_strategy': 'ovr'} +balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000640±0.0001 {'C': 1.0, 'kernel': 'linear', 'multiclass_strategy': 'ovr'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0391 * +************************************************************************************************************************* Results in results/results_accuracy_STree_iMac27_2022-05-09_00:21:06_0.json diff --git a/benchmark/tests/test_files/report.test b/benchmark/tests/test_files/report.test index 47d4a1a..0e5ffa9 100644 --- a/benchmark/tests/test_files/report.test +++ b/benchmark/tests/test_files/report.test @@ -1,15 +1,15 @@ -************************************************************************************************************************ -* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 * -* With gridsearched hyperparameters * -* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * -* Execution took 624.25 seconds, 0.17 hours, on iMac27 * -* Score is accuracy * -************************************************************************************************************************ +************************************************************************************************************************* +* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 * +* With gridsearched hyperparameters * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 624.25 seconds, 0.17 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* -Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters -============================== ====== ===== === ======= ======= ======= =============== ================ =============== -balance-scale 625 4 3 7.00 4.00 3.00 0.970560±0.0150 0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} -balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} -************************************************************************************************************************ -* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 * -************************************************************************************************************************ +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 7.00 4.00 3.00 0.970560±0.0150 0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 * +************************************************************************************************************************* diff --git a/benchmark/tests/test_files/report_compared.test b/benchmark/tests/test_files/report_compared.test index 018e3ee..103fb5d 100644 --- a/benchmark/tests/test_files/report_compared.test +++ b/benchmark/tests/test_files/report_compared.test @@ -1,16 +1,16 @@ -************************************************************************************************************************ -* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 * -* With gridsearched hyperparameters * -* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * -* Execution took 624.25 seconds, 0.17 hours, on iMac27 * -* Score is accuracy * -************************************************************************************************************************ +************************************************************************************************************************* +* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 * +* With gridsearched hyperparameters * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 624.25 seconds, 0.17 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* -Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters -============================== ====== ===== === ======= ======= ======= =============== ================ =============== -balance-scale 625 4 3 7.00 4.00 3.00 0.970560±0.0150 0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} -balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850✔ 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} -************************************************************************************************************************ -* ✔ Equal to best .....: 1 * -* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 * -************************************************************************************************************************ +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 7.00 4.00 3.00 0.970560±0.0150 0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850✔ 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +************************************************************************************************************************* +* ✔ Equal to best .....: 1 * +* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 * +************************************************************************************************************************* From dfd4f8179b6d99e404d86450c5f651eeb6614172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana?= Date: Thu, 17 Nov 2022 12:00:30 +0100 Subject: [PATCH 3/7] Complete tests adding excel to be_list --- benchmark/Results.py | 29 ++++---- benchmark/scripts/be_list.py | 8 ++- benchmark/tests/.gitignore | 1 + benchmark/tests/Report_test.py | 5 +- benchmark/tests/scripts/Be_Grid_test.py | 4 ++ benchmark/tests/scripts/Be_List_test.py | 66 +++++++++++++++++-- benchmark/tests/test_files/be_list_model.test | 6 ++ .../tests/test_files/be_list_model_2.test | 11 ++++ .../test_files/be_list_model_invalid.test | 7 ++ .../tests/test_files/be_list_report.test | 21 ++++++ .../test_files/be_list_report_excel.test | 21 ++++++ .../test_files/be_list_report_excel_2.test | 36 ++++++++++ benchmark/tests/test_files/excel2.test | 48 ++++++++++++++ 13 files changed, 238 insertions(+), 25 deletions(-) create mode 100644 benchmark/tests/test_files/be_list_model.test create mode 100644 benchmark/tests/test_files/be_list_model_2.test create mode 100644 benchmark/tests/test_files/be_list_model_invalid.test create mode 100644 benchmark/tests/test_files/be_list_report.test create mode 100644 benchmark/tests/test_files/be_list_report_excel.test create mode 100644 benchmark/tests/test_files/be_list_report_excel_2.test create mode 100644 benchmark/tests/test_files/excel2.test diff --git a/benchmark/Results.py b/benchmark/Results.py index ca0353e..b876b63 100644 --- a/benchmark/Results.py +++ b/benchmark/Results.py @@ -22,8 +22,8 @@ from .Utils import ( from ._version import __version__ -def get_input(): - return input() +def get_input(is_test): + return "test" if is_test else input() class BestResultsEver: @@ -1477,31 +1477,29 @@ class Summary: ) ) - def manage_results(self, excel, isTest): + def manage_results(self, excel, is_test): + """Manage results showed in the summary + return True if excel file is created False otherwise + """ num = "" - first = True book = None - while True and not isTest: - if not first: - print(f"Invalid option {num}. Try again!") - first = False + while True: print( "Which result do you want to report? (q to quit, r to list " "again, number to report): ", end="", ) - num = get_input() - if num == "n": - return + num = get_input(is_test) if num == "r": self.list_results() if num == "q": if excel: if book is not None: book.close() - return + return True + return False if num.isdigit() and int(num) < len(self.data) and int(num) >= 0: - rep = Report(self.data[int(num)]["file"], self.hidden) + rep = Report(self.data_filtered[int(num)]["file"], self.hidden) rep.report() if excel and not self.hidden: if book is None: @@ -1510,10 +1508,13 @@ class Summary: file_name, {"nan_inf_to_errors": True} ) excel = Excel( - file_name=self.data[int(num)]["file"], + file_name=self.data_filtered[int(num)]["file"], book=book, ) excel.report() + else: + if num not in ("r", "q"): + print(f"Invalid option {num}. Try again!") def show_result(self, data: dict, title: str = "") -> None: def whites(n: int) -> str: diff --git a/benchmark/scripts/be_list.py b/benchmark/scripts/be_list.py index 740b3df..5b00114 100755 --- a/benchmark/scripts/be_list.py +++ b/benchmark/scripts/be_list.py @@ -24,9 +24,11 @@ def main(args_test=None): number=args.number, ) is_test = args_test is not None - data.manage_results(args.excel, is_test) - if args.excel: - Files.open(Files.be_list_excel, is_test) + if not args.nan: + excel_generated = data.manage_results(args.excel, is_test) + if args.excel and excel_generated: + print(f"Generated file: {Files.be_list_excel}") + Files.open(Files.be_list_excel, is_test) except ValueError as e: print(e) else: diff --git a/benchmark/tests/.gitignore b/benchmark/tests/.gitignore index c56bbf8..1760e2d 100644 --- a/benchmark/tests/.gitignore +++ b/benchmark/tests/.gitignore @@ -1 +1,2 @@ ReportDatasets.xlsx +some_results.xlsx diff --git a/benchmark/tests/Report_test.py b/benchmark/tests/Report_test.py index 0f512ff..5970cdb 100644 --- a/benchmark/tests/Report_test.py +++ b/benchmark/tests/Report_test.py @@ -2,11 +2,14 @@ import os from io import StringIO from unittest.mock import patch from .TestBase import TestBase -from ..Results import Report, BaseReport, ReportBest, ReportDatasets +from ..Results import Report, BaseReport, ReportBest, ReportDatasets, get_input from ..Utils import Symbols class ReportTest(TestBase): + def test_get_input(self): + self.assertEqual(get_input(is_test=True), "test") + def test_BaseReport(self): with patch.multiple(BaseReport, __abstractmethods__=set()): file_name = os.path.join( diff --git a/benchmark/tests/scripts/Be_Grid_test.py b/benchmark/tests/scripts/Be_Grid_test.py index 1027f8a..1adb296 100644 --- a/benchmark/tests/scripts/Be_Grid_test.py +++ b/benchmark/tests/scripts/Be_Grid_test.py @@ -4,6 +4,10 @@ from ...Utils import Folders, Files from ..TestBase import TestBase +def get_test(): + return "hola" + + class BeGridTest(TestBase): def setUp(self): self.prepare_scripts_env() diff --git a/benchmark/tests/scripts/Be_List_test.py b/benchmark/tests/scripts/Be_List_test.py index e23cda9..b4011e9 100644 --- a/benchmark/tests/scripts/Be_List_test.py +++ b/benchmark/tests/scripts/Be_List_test.py @@ -1,7 +1,7 @@ import os from unittest.mock import patch -from ...Utils import Folders, NO_RESULTS -from ...Results import Summary +from openpyxl import load_workbook +from ...Utils import Folders, Files, NO_RESULTS from ..TestBase import TestBase @@ -9,13 +9,64 @@ class BeListTest(TestBase): def setUp(self): self.prepare_scripts_env() - # @patch("...Results.get_input()", return_value="n") - def test_be_list(self): + @patch("benchmark.Results.get_input", return_value="q") + def test_be_list(self, input_data): stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) self.assertEqual(stderr.getvalue(), "") - self.check_output_file(stdout, "summary_list_model") + self.check_output_file(stdout, "be_list_model") - def test_be_list_no_data(self): + @patch("benchmark.Results.get_input", side_effect=iter(["x", "q"])) + def test_be_list_invalid_option(self, input_data): + stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_model_invalid") + + @patch("benchmark.Results.get_input", side_effect=iter(["0", "q"])) + def test_be_list_report(self, input_data): + stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_report") + + @patch("benchmark.Results.get_input", side_effect=iter(["q"])) + def test_be_list_report_excel_none(self, input_data): + stdout, stderr = self.execute_script( + "be_list", ["-m", "STree", "-x", "1"] + ) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_model") + + @patch("benchmark.Results.get_input", side_effect=iter(["r", "q"])) + def test_be_list_twice(self, input_data): + stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_model_2") + + @patch("benchmark.Results.get_input", side_effect=iter(["2", "q"])) + def test_be_list_report_excel(self, input_data): + stdout, stderr = self.execute_script( + "be_list", ["-m", "STree", "-x", "1"] + ) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_report_excel") + book = load_workbook(Files.be_list_excel) + sheet = book["STree"] + self.check_excel_sheet(sheet, "excel") + + @patch("benchmark.Results.get_input", side_effect=iter(["2", "1", "q"])) + def test_be_list_report_excel_twice(self, input_data): + stdout, stderr = self.execute_script( + "be_list", ["-m", "STree", "-x", "1"] + ) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_report_excel_2") + book = load_workbook(Files.be_list_excel) + sheet = book["STree"] + self.check_excel_sheet(sheet, "excel") + sheet = book["STree2"] + self.check_excel_sheet(sheet, "excel2") + + @patch("benchmark.Results.get_input", return_value="q") + def test_be_list_no_data(self, input_data): stdout, stderr = self.execute_script( "be_list", ["-m", "Wodt", "-s", "f1-macro"] ) @@ -44,7 +95,8 @@ class BeListTest(TestBase): swap_files(Folders.results, Folders.hidden_results, file_name) self.fail("test_be_list_nan() should not raise exception") - def test_be_list_nan_no_nan(self): + @patch("benchmark.Results.get_input", return_value="q") + def test_be_list_nan_no_nan(self, input_data): stdout, stderr = self.execute_script("be_list", ["--nan", "1"]) self.assertEqual(stderr.getvalue(), "") self.check_output_file(stdout, "be_list_no_nan") diff --git a/benchmark/tests/test_files/be_list_model.test b/benchmark/tests/test_files/be_list_model.test new file mode 100644 index 0000000..718c1b4 --- /dev/null +++ b/benchmark/tests/test_files/be_list_model.test @@ -0,0 +1,6 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): \ No newline at end of file diff --git a/benchmark/tests/test_files/be_list_model_2.test b/benchmark/tests/test_files/be_list_model_2.test new file mode 100644 index 0000000..ba84afa --- /dev/null +++ b/benchmark/tests/test_files/be_list_model_2.test @@ -0,0 +1,11 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report):  # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): \ No newline at end of file diff --git a/benchmark/tests/test_files/be_list_model_invalid.test b/benchmark/tests/test_files/be_list_model_invalid.test new file mode 100644 index 0000000..df728c9 --- /dev/null +++ b/benchmark/tests/test_files/be_list_model_invalid.test @@ -0,0 +1,7 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): Invalid option x. Try again! +Which result do you want to report? (q to quit, r to list again, number to report): \ No newline at end of file diff --git a/benchmark/tests/test_files/be_list_report.test b/benchmark/tests/test_files/be_list_report.test new file mode 100644 index 0000000..3b7fdff --- /dev/null +++ b/benchmark/tests/test_files/be_list_report.test @@ -0,0 +1,21 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): ************************************************************************************************************************* +* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-11-01 19:17:07 * +* default B * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 4115.04 seconds, 1.14 hours, on macbook-pro * +* Score is accuracy * +************************************************************************************************************************* + +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 18.78 9.88 5.90 0.970000±0.0020 0.233304±0.0481 {'max_features': 'auto', 'splitter': 'mutual'} +balloons 16 4 2 4.72 2.86 2.78 0.556667±0.2941 0.021352±0.0058 {'max_features': 'auto', 'splitter': 'mutual'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0379 * +************************************************************************************************************************* +Which result do you want to report? (q to quit, r to list again, number to report): \ No newline at end of file diff --git a/benchmark/tests/test_files/be_list_report_excel.test b/benchmark/tests/test_files/be_list_report_excel.test new file mode 100644 index 0000000..e494061 --- /dev/null +++ b/benchmark/tests/test_files/be_list_report_excel.test @@ -0,0 +1,21 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): ************************************************************************************************************************* +* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 * +* With gridsearched hyperparameters * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 624.25 seconds, 0.17 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* + +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 7.00 4.00 3.00 0.970560±0.0150 0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 * +************************************************************************************************************************* +Which result do you want to report? (q to quit, r to list again, number to report): Generated file: some_results.xlsx diff --git a/benchmark/tests/test_files/be_list_report_excel_2.test b/benchmark/tests/test_files/be_list_report_excel_2.test new file mode 100644 index 0000000..15822a8 --- /dev/null +++ b/benchmark/tests/test_files/be_list_report_excel_2.test @@ -0,0 +1,36 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): ************************************************************************************************************************* +* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 * +* With gridsearched hyperparameters * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 624.25 seconds, 0.17 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* + +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 7.00 4.00 3.00 0.970560±0.0150 0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 * +************************************************************************************************************************* +Which result do you want to report? (q to quit, r to list again, number to report): ************************************************************************************************************************* +* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-10-27 09:40:40 * +* default A * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 3395.01 seconds, 0.94 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* + +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 11.08 5.90 5.90 0.980000±0.0010 0.285207±0.0603 {'splitter': 'best', 'max_features': 'auto'} +balloons 16 4 2 4.12 2.56 2.56 0.695000±0.2757 0.021201±0.0035 {'splitter': 'best', 'max_features': 'auto'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0416 * +************************************************************************************************************************* +Which result do you want to report? (q to quit, r to list again, number to report): Generated file: some_results.xlsx diff --git a/benchmark/tests/test_files/excel2.test b/benchmark/tests/test_files/excel2.test new file mode 100644 index 0000000..c586deb --- /dev/null +++ b/benchmark/tests/test_files/excel2.test @@ -0,0 +1,48 @@ +1;1;" STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-10-27 09:40:40" +2;1;" default A" +3;1;" Score is accuracy" +3;2;" Execution time" +3;5;"3,395.01 s" +3;7;" " +3;8;"Platform" +3;9;"iMac27" +3;10;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]" +4;5;" 0.94 h" +4;10;"Stratified: False" +6;1;"Dataset" +6;2;"Samples" +6;3;"Features" +6;4;"Classes" +6;5;"Nodes" +6;6;"Leaves" +6;7;"Depth" +6;8;"Score" +6;9;"Score Std." +6;10;"Time" +6;11;"Time Std." +6;12;"Hyperparameters" +7;1;"balance-scale" +7;2;"625" +7;3;"4" +7;4;"3" +7;5;"11.08" +7;6;"5.9" +7;7;"5.9" +7;8;"0.98" +7;9;"0.001" +7;10;"0.2852065515518188" +7;11;"0.06031593282605064" +7;12;"{'splitter': 'best', 'max_features': 'auto'}" +8;1;"balloons" +8;2;"16" +8;3;"4" +8;4;"2" +8;5;"4.12" +8;6;"2.56" +8;7;"2.56" +8;8;"0.695" +8;9;"0.2756860130252853" +8;10;"0.02120100021362305" +8;11;"0.003526023309468471" +8;12;"{'splitter': 'best', 'max_features': 'auto'}" +10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0416" From 38667d61f7e144fdef202df81c781b7031b5f0af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana?= Date: Thu, 17 Nov 2022 12:09:02 +0100 Subject: [PATCH 4/7] Refactor be_list --- benchmark/scripts/be_list.py | 52 ++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/benchmark/scripts/be_list.py b/benchmark/scripts/be_list.py index 5b00114..4f34502 100755 --- a/benchmark/scripts/be_list.py +++ b/benchmark/scripts/be_list.py @@ -31,31 +31,31 @@ def main(args_test=None): Files.open(Files.be_list_excel, is_test) except ValueError as e: print(e) - else: - if args.nan: - results_nan = [] - results = data.get_results_criteria( - score=args.score, - model=args.model, - input_data=None, - sort_key=args.key, - number=args.number, + return + if args.nan: + results_nan = [] + results = data.get_results_criteria( + score=args.score, + model=args.model, + input_data=None, + sort_key=args.key, + number=args.number, + ) + for result in results: + if result["metric"] != result["metric"]: + results_nan.append(result) + if results_nan != []: + print( + "\n" + + "*" * 30 + + " Results with nan moved to hidden " + + "*" * 30 ) - for result in results: - if result["metric"] != result["metric"]: - results_nan.append(result) - if results_nan != []: - print( - "\n" - + "*" * 30 - + " Results with nan moved to hidden " - + "*" * 30 + data.data_filtered = [] + data.list_results(input_data=results_nan) + for result in results_nan: + name = result["file"] + os.rename( + os.path.join(Folders.results, name), + os.path.join(Folders.hidden_results, name), ) - data.data_filtered = [] - data.list_results(input_data=results_nan) - for result in results_nan: - name = result["file"] - os.rename( - os.path.join(Folders.results, name), - os.path.join(Folders.hidden_results, name), - ) From d68fb4768898c6e30ba5a46078756996669ae8e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana?= Date: Thu, 17 Nov 2022 13:42:27 +0100 Subject: [PATCH 5/7] Remove extra space in report header --- benchmark/Results.py | 2 +- benchmark/tests/test_files/be_list_model.test | 2 +- benchmark/tests/test_files/be_list_model_2.test | 4 ++-- benchmark/tests/test_files/be_list_model_invalid.test | 2 +- benchmark/tests/test_files/be_list_nan.test | 4 ++-- benchmark/tests/test_files/be_list_no_nan.test | 2 +- benchmark/tests/test_files/be_list_report.test | 2 +- benchmark/tests/test_files/be_list_report_excel.test | 2 +- benchmark/tests/test_files/be_list_report_excel_2.test | 2 +- benchmark/tests/test_files/be_summary_list_model.test | 2 +- benchmark/tests/test_files/be_summary_list_score.test | 2 +- benchmark/tests/test_files/be_summary_list_score_all.test | 2 +- benchmark/tests/test_files/be_summary_list_score_model.test | 2 +- benchmark/tests/test_files/summary_list_hidden.test | 2 +- benchmark/tests/test_files/summary_list_model.test | 2 +- benchmark/tests/test_files/summary_list_n.test | 2 +- benchmark/tests/test_files/summary_list_score.test | 2 +- benchmark/tests/test_files/summary_show_top.test | 2 +- 18 files changed, 20 insertions(+), 20 deletions(-) diff --git a/benchmark/Results.py b/benchmark/Results.py index b876b63..1124dec 100644 --- a/benchmark/Results.py +++ b/benchmark/Results.py @@ -1450,7 +1450,7 @@ class Summary: print(color1, end="") print( f" # {'Date':10s} {'File':{max_file}s} {'Score':8s} " - f" {'Time(h)':7s} {'Title':s}" + f"{'Time(h)':7s} {'Title':s}" ) print( "===", diff --git a/benchmark/tests/test_files/be_list_model.test b/benchmark/tests/test_files/be_list_model.test index 718c1b4..bcdaec8 100644 --- a/benchmark/tests/test_files/be_list_model.test +++ b/benchmark/tests/test_files/be_list_model.test @@ -1,4 +1,4 @@ - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== ============================================================= ======== ======= =================================  0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B  1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A diff --git a/benchmark/tests/test_files/be_list_model_2.test b/benchmark/tests/test_files/be_list_model_2.test index ba84afa..74fb2cf 100644 --- a/benchmark/tests/test_files/be_list_model_2.test +++ b/benchmark/tests/test_files/be_list_model_2.test @@ -1,9 +1,9 @@ - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== ============================================================= ======== ======= =================================  0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B  1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A  2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters -Which result do you want to report? (q to quit, r to list again, number to report):  # Date File Score Time(h) Title +Which result do you want to report? (q to quit, r to list again, number to report):  # Date File Score Time(h) Title === ========== ============================================================= ======== ======= =================================  0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B  1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A diff --git a/benchmark/tests/test_files/be_list_model_invalid.test b/benchmark/tests/test_files/be_list_model_invalid.test index df728c9..66d66b9 100644 --- a/benchmark/tests/test_files/be_list_model_invalid.test +++ b/benchmark/tests/test_files/be_list_model_invalid.test @@ -1,4 +1,4 @@ - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== ============================================================= ======== ======= =================================  0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B  1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A diff --git a/benchmark/tests/test_files/be_list_nan.test b/benchmark/tests/test_files/be_list_nan.test index 58ed8bc..f6a799d 100644 --- a/benchmark/tests/test_files/be_list_nan.test +++ b/benchmark/tests/test_files/be_list_nan.test @@ -1,4 +1,4 @@ - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== ================================================================ ======== ======= ============================================  0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters  1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init @@ -8,6 +8,6 @@  5 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters ****************************** Results with nan moved to hidden ****************************** - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== ================================================================ ======== ======= =======================  0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters diff --git a/benchmark/tests/test_files/be_list_no_nan.test b/benchmark/tests/test_files/be_list_no_nan.test index 720b46c..ed99d08 100644 --- a/benchmark/tests/test_files/be_list_no_nan.test +++ b/benchmark/tests/test_files/be_list_no_nan.test @@ -1,4 +1,4 @@ - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== =============================================================== ======== ======= ============================================  0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init  1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest diff --git a/benchmark/tests/test_files/be_list_report.test b/benchmark/tests/test_files/be_list_report.test index 3b7fdff..f955490 100644 --- a/benchmark/tests/test_files/be_list_report.test +++ b/benchmark/tests/test_files/be_list_report.test @@ -1,4 +1,4 @@ - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== ============================================================= ======== ======= =================================  0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B  1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A diff --git a/benchmark/tests/test_files/be_list_report_excel.test b/benchmark/tests/test_files/be_list_report_excel.test index e494061..1b64fe8 100644 --- a/benchmark/tests/test_files/be_list_report_excel.test +++ b/benchmark/tests/test_files/be_list_report_excel.test @@ -1,4 +1,4 @@ - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== ============================================================= ======== ======= =================================  0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B  1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A diff --git a/benchmark/tests/test_files/be_list_report_excel_2.test b/benchmark/tests/test_files/be_list_report_excel_2.test index 15822a8..457df27 100644 --- a/benchmark/tests/test_files/be_list_report_excel_2.test +++ b/benchmark/tests/test_files/be_list_report_excel_2.test @@ -1,4 +1,4 @@ - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== ============================================================= ======== ======= =================================  0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B  1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A diff --git a/benchmark/tests/test_files/be_summary_list_model.test b/benchmark/tests/test_files/be_summary_list_model.test index 05ed964..35d9894 100644 --- a/benchmark/tests/test_files/be_summary_list_model.test +++ b/benchmark/tests/test_files/be_summary_list_model.test @@ -26,7 +26,7 @@ * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * * * ********************************************************************************* - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== =============================================================== ======== ======= ============================================  0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters  1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init diff --git a/benchmark/tests/test_files/be_summary_list_score.test b/benchmark/tests/test_files/be_summary_list_score.test index 816cab0..8b655ad 100644 --- a/benchmark/tests/test_files/be_summary_list_score.test +++ b/benchmark/tests/test_files/be_summary_list_score.test @@ -26,7 +26,7 @@ * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * * * ********************************************************************************* - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== =============================================================== ======== ======= ============================================  0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters  1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init diff --git a/benchmark/tests/test_files/be_summary_list_score_all.test b/benchmark/tests/test_files/be_summary_list_score_all.test index 57d8260..23fd89c 100644 --- a/benchmark/tests/test_files/be_summary_list_score_all.test +++ b/benchmark/tests/test_files/be_summary_list_score_all.test @@ -26,7 +26,7 @@ * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * * * ********************************************************************************* - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== =============================================================== ======== ======= ============================================  0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters  1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init diff --git a/benchmark/tests/test_files/be_summary_list_score_model.test b/benchmark/tests/test_files/be_summary_list_score_model.test index 816cab0..8b655ad 100644 --- a/benchmark/tests/test_files/be_summary_list_score_model.test +++ b/benchmark/tests/test_files/be_summary_list_score_model.test @@ -26,7 +26,7 @@ * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * * * ********************************************************************************* - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== =============================================================== ======== ======= ============================================  0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters  1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init diff --git a/benchmark/tests/test_files/summary_list_hidden.test b/benchmark/tests/test_files/summary_list_hidden.test index b663821..f98eb9f 100644 --- a/benchmark/tests/test_files/summary_list_hidden.test +++ b/benchmark/tests/test_files/summary_list_hidden.test @@ -1,4 +1,4 @@ - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== ================================================================ ======== ======= =======================  0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters  1 2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json 0.97446 0.098 default diff --git a/benchmark/tests/test_files/summary_list_model.test b/benchmark/tests/test_files/summary_list_model.test index 636ebe7..2de8e68 100644 --- a/benchmark/tests/test_files/summary_list_model.test +++ b/benchmark/tests/test_files/summary_list_model.test @@ -1,4 +1,4 @@ - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== ============================================================= ======== ======= =================================  0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B  1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A diff --git a/benchmark/tests/test_files/summary_list_n.test b/benchmark/tests/test_files/summary_list_n.test index d4453ec..754d36e 100644 --- a/benchmark/tests/test_files/summary_list_n.test +++ b/benchmark/tests/test_files/summary_list_n.test @@ -1,4 +1,4 @@ - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== =============================================================== ======== ======= ============================================  0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init  1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest diff --git a/benchmark/tests/test_files/summary_list_score.test b/benchmark/tests/test_files/summary_list_score.test index 720b46c..ed99d08 100644 --- a/benchmark/tests/test_files/summary_list_score.test +++ b/benchmark/tests/test_files/summary_list_score.test @@ -1,4 +1,4 @@ - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== =============================================================== ======== ======= ============================================  0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init  1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest diff --git a/benchmark/tests/test_files/summary_show_top.test b/benchmark/tests/test_files/summary_show_top.test index 5b42449..5fc9f82 100644 --- a/benchmark/tests/test_files/summary_show_top.test +++ b/benchmark/tests/test_files/summary_show_top.test @@ -1,4 +1,4 @@ - # Date File Score Time(h) Title + # Date File Score Time(h) Title === ========== =============================================================== ======== ======= ============================================  0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters  1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init From 58c52849d8a4c914676678103415434ea9db886d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana?= Date: Fri, 18 Nov 2022 23:33:41 +0100 Subject: [PATCH 6/7] Add AODE to models --- benchmark/Models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmark/Models.py b/benchmark/Models.py index d9109ca..cee25ba 100644 --- a/benchmark/Models.py +++ b/benchmark/Models.py @@ -8,7 +8,7 @@ from sklearn.ensemble import ( ) from sklearn.svm import SVC from stree import Stree -from bayesclass import TAN, KDB +from bayesclass import TAN, KDB, AODE from wodt import Wodt from odte import Odte from xgboost import XGBClassifier @@ -23,6 +23,7 @@ class Models: "STree": Stree(random_state=random_state), "TAN": TAN(random_state=random_state), "KDB": KDB(k=3), + "AODE": AODE(random_state=random_state), "Cart": DecisionTreeClassifier(random_state=random_state), "ExtraTree": ExtraTreeClassifier(random_state=random_state), "Wodt": Wodt(random_state=random_state), From c8124be1196a6a17843b7bf89e2bfa37a0d9103d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana?= Date: Fri, 18 Nov 2022 23:36:43 +0100 Subject: [PATCH 7/7] Update version info --- benchmark/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/_version.py b/benchmark/_version.py index 493f741..6a9beea 100644 --- a/benchmark/_version.py +++ b/benchmark/_version.py @@ -1 +1 @@ -__version__ = "0.3.0" +__version__ = "0.4.0"