diff --git a/benchmark/Models.py b/benchmark/Models.py index d9109ca..cee25ba 100644 --- a/benchmark/Models.py +++ b/benchmark/Models.py @@ -8,7 +8,7 @@ from sklearn.ensemble import ( ) from sklearn.svm import SVC from stree import Stree -from bayesclass import TAN, KDB +from bayesclass import TAN, KDB, AODE from wodt import Wodt from odte import Odte from xgboost import XGBClassifier @@ -23,6 +23,7 @@ class Models: "STree": Stree(random_state=random_state), "TAN": TAN(random_state=random_state), "KDB": KDB(k=3), + "AODE": AODE(random_state=random_state), "Cart": DecisionTreeClassifier(random_state=random_state), "ExtraTree": ExtraTreeClassifier(random_state=random_state), "Wodt": Wodt(random_state=random_state), diff --git a/benchmark/Results.py b/benchmark/Results.py index ccf833c..1124dec 100644 --- a/benchmark/Results.py +++ b/benchmark/Results.py @@ -7,6 +7,7 @@ import abc import shutil import subprocess import xlsxwriter +from xlsxwriter.exceptions import DuplicateWorksheetName import numpy as np from .Experiments import BestResults from .Datasets import Datasets @@ -21,6 +22,10 @@ from .Utils import ( from ._version import __version__ +def get_input(is_test): + return "test" if is_test else input() + + class BestResultsEver: def __init__(self): self.data = {} @@ -125,7 +130,7 @@ class BaseReport(abc.ABC): class Report(BaseReport): - header_lengths = [30, 6, 5, 3, 7, 7, 7, 15, 16, 15] + header_lengths = [30, 6, 5, 3, 7, 7, 7, 15, 17, 15] header_cols = [ "Dataset", "Sampl.", @@ -184,7 +189,7 @@ class Report(BaseReport): ) i += 1 print( - f"{result['time']:9.6f}±{result['time_std']:6.4f} ", + f"{result['time']:10.6f}±{result['time_std']:6.4f} ", end="", ) i += 1 @@ -328,7 +333,17 @@ class Excel(BaseReport): else: self.book = book self.close = False - self.sheet = self.book.add_worksheet(self.data["model"]) + suffix = "" + num = 1 + while True: + try: + self.sheet = self.book.add_worksheet( + self.data["model"] + suffix + ) + break + except DuplicateWorksheetName: + num += 1 + suffix = str(num) self.max_hyper_width = 0 self.col_hyperparams = 0 @@ -1341,6 +1356,7 @@ class Summary: def __init__(self, hidden=False) -> None: self.results = Files().get_all_results(hidden=hidden) self.data = [] + self.data_filtered = [] self.datasets = {} self.models = set() self.hidden = hidden @@ -1417,13 +1433,14 @@ class Summary: number=0, ) -> None: """Print the list of results""" - data = self.get_results_criteria( - score, model, input_data, sort_key, number - ) - if data == []: + if self.data_filtered == []: + self.data_filtered = self.get_results_criteria( + score, model, input_data, sort_key, number + ) + if self.data_filtered == []: raise ValueError(NO_RESULTS) - max_file = max(len(x["file"]) for x in data) - max_title = max(len(x["title"]) for x in data) + max_file = max(len(x["file"]) for x in self.data_filtered) + max_title = max(len(x["title"]) for x in self.data_filtered) if self.hidden: color1 = TextColor.GREEN color2 = TextColor.YELLOW @@ -1432,10 +1449,11 @@ class Summary: color2 = TextColor.LINE2 print(color1, end="") print( - f"{'Date':10s} {'File':{max_file}s} {'Score':8s} {'Time(h)':7s} " - f"{'Title':s}" + f" # {'Date':10s} {'File':{max_file}s} {'Score':8s} " + f"{'Time(h)':7s} {'Title':s}" ) print( + "===", "=" * 10 + " " + "=" * max_file @@ -1444,21 +1462,60 @@ class Summary: + " " + "=" * 7 + " " - + "=" * max_title + + "=" * max_title, ) print( "\n".join( [ - (color2 if n % 2 == 0 else color1) - + f"{x['date']} {x['file']:{max_file}s} " + (color2 if n % 2 == 0 else color1) + f"{n:3d} " + f"{x['date']} {x['file']:{max_file}s} " f"{x['metric']:8.5f} " f"{x['duration']/3600:7.3f} " f"{x['title']}" - for n, x in enumerate(data) + for n, x in enumerate(self.data_filtered) ] ) ) + def manage_results(self, excel, is_test): + """Manage results showed in the summary + return True if excel file is created False otherwise + """ + num = "" + book = None + while True: + print( + "Which result do you want to report? (q to quit, r to list " + "again, number to report): ", + end="", + ) + num = get_input(is_test) + if num == "r": + self.list_results() + if num == "q": + if excel: + if book is not None: + book.close() + return True + return False + if num.isdigit() and int(num) < len(self.data) and int(num) >= 0: + rep = Report(self.data_filtered[int(num)]["file"], self.hidden) + rep.report() + if excel and not self.hidden: + if book is None: + file_name = Files.be_list_excel + book = xlsxwriter.Workbook( + file_name, {"nan_inf_to_errors": True} + ) + excel = Excel( + file_name=self.data_filtered[int(num)]["file"], + book=book, + ) + excel.report() + else: + if num not in ("r", "q"): + print(f"Invalid option {num}. Try again!") + def show_result(self, data: dict, title: str = "") -> None: def whites(n: int) -> str: return " " * n + color1 + "*" diff --git a/benchmark/Utils.py b/benchmark/Utils.py index 9663086..fbf7102 100644 --- a/benchmark/Utils.py +++ b/benchmark/Utils.py @@ -28,6 +28,7 @@ class Files: benchmark_r = "benchmark.r" dot_env = ".env" datasets_report_excel = "ReportDatasets.xlsx" + be_list_excel = "some_results.xlsx" @staticmethod def exreport_output(score): diff --git a/benchmark/_version.py b/benchmark/_version.py index 493f741..6a9beea 100644 --- a/benchmark/_version.py +++ b/benchmark/_version.py @@ -1 +1 @@ -__version__ = "0.3.0" +__version__ = "0.4.0" diff --git a/benchmark/scripts/be_list.py b/benchmark/scripts/be_list.py index cb31cb0..4f34502 100755 --- a/benchmark/scripts/be_list.py +++ b/benchmark/scripts/be_list.py @@ -1,7 +1,7 @@ #! /usr/bin/env python import os from benchmark.Results import Summary -from benchmark.Utils import Folders +from benchmark.Utils import Folders, Files from benchmark.Arguments import Arguments """List experiments of a model @@ -12,6 +12,7 @@ def main(args_test=None): arguments = Arguments() arguments.xset("number").xset("model", required=False).xset("key") arguments.xset("hidden").xset("nan").xset("score", required=False) + arguments.xset("excel") args = arguments.parse(args_test) data = Summary(hidden=args.hidden) data.acquire() @@ -22,32 +23,39 @@ def main(args_test=None): sort_key=args.key, number=args.number, ) + is_test = args_test is not None + if not args.nan: + excel_generated = data.manage_results(args.excel, is_test) + if args.excel and excel_generated: + print(f"Generated file: {Files.be_list_excel}") + Files.open(Files.be_list_excel, is_test) except ValueError as e: print(e) - else: - if args.nan: - results_nan = [] - results = data.get_results_criteria( - score=args.score, - model=args.model, - input_data=None, - sort_key=args.key, - number=args.number, + return + if args.nan: + results_nan = [] + results = data.get_results_criteria( + score=args.score, + model=args.model, + input_data=None, + sort_key=args.key, + number=args.number, + ) + for result in results: + if result["metric"] != result["metric"]: + results_nan.append(result) + if results_nan != []: + print( + "\n" + + "*" * 30 + + " Results with nan moved to hidden " + + "*" * 30 ) - for result in results: - if result["metric"] != result["metric"]: - results_nan.append(result) - if results_nan != []: - print( - "\n" - + "*" * 30 - + " Results with nan moved to hidden " - + "*" * 30 + data.data_filtered = [] + data.list_results(input_data=results_nan) + for result in results_nan: + name = result["file"] + os.rename( + os.path.join(Folders.results, name), + os.path.join(Folders.hidden_results, name), ) - data.list_results(input_data=results_nan) - for result in results_nan: - name = result["file"] - os.rename( - os.path.join(Folders.results, name), - os.path.join(Folders.hidden_results, name), - ) diff --git a/benchmark/tests/.gitignore b/benchmark/tests/.gitignore index c56bbf8..1760e2d 100644 --- a/benchmark/tests/.gitignore +++ b/benchmark/tests/.gitignore @@ -1 +1,2 @@ ReportDatasets.xlsx +some_results.xlsx diff --git a/benchmark/tests/Report_test.py b/benchmark/tests/Report_test.py index 0f512ff..5970cdb 100644 --- a/benchmark/tests/Report_test.py +++ b/benchmark/tests/Report_test.py @@ -2,11 +2,14 @@ import os from io import StringIO from unittest.mock import patch from .TestBase import TestBase -from ..Results import Report, BaseReport, ReportBest, ReportDatasets +from ..Results import Report, BaseReport, ReportBest, ReportDatasets, get_input from ..Utils import Symbols class ReportTest(TestBase): + def test_get_input(self): + self.assertEqual(get_input(is_test=True), "test") + def test_BaseReport(self): with patch.multiple(BaseReport, __abstractmethods__=set()): file_name = os.path.join( diff --git a/benchmark/tests/scripts/Be_Grid_test.py b/benchmark/tests/scripts/Be_Grid_test.py index 1027f8a..1adb296 100644 --- a/benchmark/tests/scripts/Be_Grid_test.py +++ b/benchmark/tests/scripts/Be_Grid_test.py @@ -4,6 +4,10 @@ from ...Utils import Folders, Files from ..TestBase import TestBase +def get_test(): + return "hola" + + class BeGridTest(TestBase): def setUp(self): self.prepare_scripts_env() diff --git a/benchmark/tests/scripts/Be_List_test.py b/benchmark/tests/scripts/Be_List_test.py index 25bed40..b4011e9 100644 --- a/benchmark/tests/scripts/Be_List_test.py +++ b/benchmark/tests/scripts/Be_List_test.py @@ -1,5 +1,7 @@ import os -from ...Utils import Folders, NO_RESULTS +from unittest.mock import patch +from openpyxl import load_workbook +from ...Utils import Folders, Files, NO_RESULTS from ..TestBase import TestBase @@ -7,12 +9,64 @@ class BeListTest(TestBase): def setUp(self): self.prepare_scripts_env() - def test_be_list(self): + @patch("benchmark.Results.get_input", return_value="q") + def test_be_list(self, input_data): stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) self.assertEqual(stderr.getvalue(), "") - self.check_output_file(stdout, "summary_list_model") + self.check_output_file(stdout, "be_list_model") - def test_be_list_no_data(self): + @patch("benchmark.Results.get_input", side_effect=iter(["x", "q"])) + def test_be_list_invalid_option(self, input_data): + stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_model_invalid") + + @patch("benchmark.Results.get_input", side_effect=iter(["0", "q"])) + def test_be_list_report(self, input_data): + stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_report") + + @patch("benchmark.Results.get_input", side_effect=iter(["q"])) + def test_be_list_report_excel_none(self, input_data): + stdout, stderr = self.execute_script( + "be_list", ["-m", "STree", "-x", "1"] + ) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_model") + + @patch("benchmark.Results.get_input", side_effect=iter(["r", "q"])) + def test_be_list_twice(self, input_data): + stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_model_2") + + @patch("benchmark.Results.get_input", side_effect=iter(["2", "q"])) + def test_be_list_report_excel(self, input_data): + stdout, stderr = self.execute_script( + "be_list", ["-m", "STree", "-x", "1"] + ) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_report_excel") + book = load_workbook(Files.be_list_excel) + sheet = book["STree"] + self.check_excel_sheet(sheet, "excel") + + @patch("benchmark.Results.get_input", side_effect=iter(["2", "1", "q"])) + def test_be_list_report_excel_twice(self, input_data): + stdout, stderr = self.execute_script( + "be_list", ["-m", "STree", "-x", "1"] + ) + self.assertEqual(stderr.getvalue(), "") + self.check_output_file(stdout, "be_list_report_excel_2") + book = load_workbook(Files.be_list_excel) + sheet = book["STree"] + self.check_excel_sheet(sheet, "excel") + sheet = book["STree2"] + self.check_excel_sheet(sheet, "excel2") + + @patch("benchmark.Results.get_input", return_value="q") + def test_be_list_no_data(self, input_data): stdout, stderr = self.execute_script( "be_list", ["-m", "Wodt", "-s", "f1-macro"] ) @@ -41,7 +95,8 @@ class BeListTest(TestBase): swap_files(Folders.results, Folders.hidden_results, file_name) self.fail("test_be_list_nan() should not raise exception") - def test_be_list_nan_no_nan(self): + @patch("benchmark.Results.get_input", return_value="q") + def test_be_list_nan_no_nan(self, input_data): stdout, stderr = self.execute_script("be_list", ["--nan", "1"]) self.assertEqual(stderr.getvalue(), "") self.check_output_file(stdout, "be_list_no_nan") diff --git a/benchmark/tests/test_files/be_list_model.test b/benchmark/tests/test_files/be_list_model.test new file mode 100644 index 0000000..bcdaec8 --- /dev/null +++ b/benchmark/tests/test_files/be_list_model.test @@ -0,0 +1,6 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): \ No newline at end of file diff --git a/benchmark/tests/test_files/be_list_model_2.test b/benchmark/tests/test_files/be_list_model_2.test new file mode 100644 index 0000000..74fb2cf --- /dev/null +++ b/benchmark/tests/test_files/be_list_model_2.test @@ -0,0 +1,11 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report):  # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): \ No newline at end of file diff --git a/benchmark/tests/test_files/be_list_model_invalid.test b/benchmark/tests/test_files/be_list_model_invalid.test new file mode 100644 index 0000000..66d66b9 --- /dev/null +++ b/benchmark/tests/test_files/be_list_model_invalid.test @@ -0,0 +1,7 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): Invalid option x. Try again! +Which result do you want to report? (q to quit, r to list again, number to report): \ No newline at end of file diff --git a/benchmark/tests/test_files/be_list_nan.test b/benchmark/tests/test_files/be_list_nan.test index ed62daf..f6a799d 100644 --- a/benchmark/tests/test_files/be_list_nan.test +++ b/benchmark/tests/test_files/be_list_nan.test @@ -1,13 +1,13 @@ -Date File Score Time(h) Title -========== ================================================================ ======== ======= ============================================ -2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + # Date File Score Time(h) Title +=== ========== ================================================================ ======== ======= ============================================ + 0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters + 1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 2 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + 3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 4 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 5 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters ****************************** Results with nan moved to hidden ****************************** -Date File Score Time(h) Title -========== ================================================================ ======== ======= ======================= -2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters + # Date File Score Time(h) Title +=== ========== ================================================================ ======== ======= ======================= + 0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters diff --git a/benchmark/tests/test_files/be_list_no_nan.test b/benchmark/tests/test_files/be_list_no_nan.test index e1e5dd7..ed99d08 100644 --- a/benchmark/tests/test_files/be_list_no_nan.test +++ b/benchmark/tests/test_files/be_list_no_nan.test @@ -1,7 +1,7 @@ -Date File Score Time(h) Title -========== =============================================================== ======== ======= ============================================ -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + 2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 3 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 4 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters diff --git a/benchmark/tests/test_files/be_list_report.test b/benchmark/tests/test_files/be_list_report.test new file mode 100644 index 0000000..f955490 --- /dev/null +++ b/benchmark/tests/test_files/be_list_report.test @@ -0,0 +1,21 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): ************************************************************************************************************************* +* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-11-01 19:17:07 * +* default B * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 4115.04 seconds, 1.14 hours, on macbook-pro * +* Score is accuracy * +************************************************************************************************************************* + +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 18.78 9.88 5.90 0.970000±0.0020 0.233304±0.0481 {'max_features': 'auto', 'splitter': 'mutual'} +balloons 16 4 2 4.72 2.86 2.78 0.556667±0.2941 0.021352±0.0058 {'max_features': 'auto', 'splitter': 'mutual'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0379 * +************************************************************************************************************************* +Which result do you want to report? (q to quit, r to list again, number to report): \ No newline at end of file diff --git a/benchmark/tests/test_files/be_list_report_excel.test b/benchmark/tests/test_files/be_list_report_excel.test new file mode 100644 index 0000000..1b64fe8 --- /dev/null +++ b/benchmark/tests/test_files/be_list_report_excel.test @@ -0,0 +1,21 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): ************************************************************************************************************************* +* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 * +* With gridsearched hyperparameters * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 624.25 seconds, 0.17 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* + +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 7.00 4.00 3.00 0.970560±0.0150 0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 * +************************************************************************************************************************* +Which result do you want to report? (q to quit, r to list again, number to report): Generated file: some_results.xlsx diff --git a/benchmark/tests/test_files/be_list_report_excel_2.test b/benchmark/tests/test_files/be_list_report_excel_2.test new file mode 100644 index 0000000..457df27 --- /dev/null +++ b/benchmark/tests/test_files/be_list_report_excel_2.test @@ -0,0 +1,36 @@ + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters +Which result do you want to report? (q to quit, r to list again, number to report): ************************************************************************************************************************* +* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 * +* With gridsearched hyperparameters * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 624.25 seconds, 0.17 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* + +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 7.00 4.00 3.00 0.970560±0.0150 0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 * +************************************************************************************************************************* +Which result do you want to report? (q to quit, r to list again, number to report): ************************************************************************************************************************* +* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-10-27 09:40:40 * +* default A * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 3395.01 seconds, 0.94 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* + +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 11.08 5.90 5.90 0.980000±0.0010 0.285207±0.0603 {'splitter': 'best', 'max_features': 'auto'} +balloons 16 4 2 4.12 2.56 2.56 0.695000±0.2757 0.021201±0.0035 {'splitter': 'best', 'max_features': 'auto'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0416 * +************************************************************************************************************************* +Which result do you want to report? (q to quit, r to list again, number to report): Generated file: some_results.xlsx diff --git a/benchmark/tests/test_files/be_main_best.test b/benchmark/tests/test_files/be_main_best.test index adecb4c..a18372b 100644 --- a/benchmark/tests/test_files/be_main_best.test +++ b/benchmark/tests/test_files/be_main_best.test @@ -1,16 +1,16 @@ -************************************************************************************************************************ -* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-09 00:15:25 * -* test * -* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * -* Execution took 0.80 seconds, 0.00 hours, on iMac27 * -* Score is accuracy * -************************************************************************************************************************ +************************************************************************************************************************* +* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-09 00:15:25 * +* test * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 0.80 seconds, 0.00 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* -Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters -============================== ====== ===== === ======= ======= ======= =============== ================ =============== -balance-scale 625 4 3 23.32 12.16 6.44 0.840160±0.0304 0.013745±0.0019 {'splitter': 'best', 'max_features': 'auto'} -balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000388±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} -************************************************************************************************************************ -* accuracy compared to STree_default (liblinear-ovr) .: 0.0422 * -************************************************************************************************************************ +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 23.32 12.16 6.44 0.840160±0.0304 0.013745±0.0019 {'splitter': 'best', 'max_features': 'auto'} +balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000388±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0422 * +************************************************************************************************************************* Results in results/results_accuracy_STree_iMac27_2022-05-09_00:15:25_0.json diff --git a/benchmark/tests/test_files/be_main_complete.test b/benchmark/tests/test_files/be_main_complete.test index 8f36fe6..edbfead 100644 --- a/benchmark/tests/test_files/be_main_complete.test +++ b/benchmark/tests/test_files/be_main_complete.test @@ -1,16 +1,16 @@ -************************************************************************************************************************ -* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-08 20:14:43 * -* test * -* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * -* Execution took 0.48 seconds, 0.00 hours, on iMac27 * -* Score is accuracy * -************************************************************************************************************************ +************************************************************************************************************************* +* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-08 20:14:43 * +* test * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 0.48 seconds, 0.00 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* -Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters -============================== ====== ===== === ======= ======= ======= =============== ================ =============== -balance-scale 625 4 3 17.36 9.18 6.18 0.908480±0.0247 0.007388±0.0013 {} -balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000664±0.0002 {} -************************************************************************************************************************ -* accuracy compared to STree_default (liblinear-ovr) .: 0.0390 * -************************************************************************************************************************ +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 17.36 9.18 6.18 0.908480±0.0247 0.007388±0.0013 {} +balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000664±0.0002 {} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0390 * +************************************************************************************************************************* Results in results/results_accuracy_STree_iMac27_2022-05-08_20:14:43_0.json diff --git a/benchmark/tests/test_files/be_main_dataset.test b/benchmark/tests/test_files/be_main_dataset.test index 46e50e3..3fedf46 100644 --- a/benchmark/tests/test_files/be_main_dataset.test +++ b/benchmark/tests/test_files/be_main_dataset.test @@ -1,15 +1,15 @@ -************************************************************************************************************************ -* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-08 19:38:28 * -* test * -* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * -* Execution took 0.06 seconds, 0.00 hours, on iMac27 * -* Score is accuracy * -************************************************************************************************************************ +************************************************************************************************************************* +* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-08 19:38:28 * +* test * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 0.06 seconds, 0.00 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* -Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters -============================== ====== ===== === ======= ======= ======= =============== ================ =============== -balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000671±0.0001 {} -************************************************************************************************************************ -* accuracy compared to STree_default (liblinear-ovr) .: 0.0165 * -************************************************************************************************************************ +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000671±0.0001 {} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0165 * +************************************************************************************************************************* Partial result file removed: results/results_accuracy_STree_iMac27_2022-05-08_19:38:28_0.json diff --git a/benchmark/tests/test_files/be_main_grid.test b/benchmark/tests/test_files/be_main_grid.test index c29ec32..adcf79a 100644 --- a/benchmark/tests/test_files/be_main_grid.test +++ b/benchmark/tests/test_files/be_main_grid.test @@ -1,16 +1,16 @@ -************************************************************************************************************************ -* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-09 00:21:06 * -* test * -* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * -* Execution took 0.89 seconds, 0.00 hours, on iMac27 * -* Score is accuracy * -************************************************************************************************************************ +************************************************************************************************************************* +* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-09 00:21:06 * +* test * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 0.89 seconds, 0.00 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* -Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters -============================== ====== ===== === ======= ======= ======= =============== ================ =============== -balance-scale 625 4 3 26.12 13.56 7.94 0.910720±0.0249 0.015852±0.0027 {'C': 1.0, 'kernel': 'liblinear', 'multiclass_strategy': 'ovr'} -balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000640±0.0001 {'C': 1.0, 'kernel': 'linear', 'multiclass_strategy': 'ovr'} -************************************************************************************************************************ -* accuracy compared to STree_default (liblinear-ovr) .: 0.0391 * -************************************************************************************************************************ +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 26.12 13.56 7.94 0.910720±0.0249 0.015852±0.0027 {'C': 1.0, 'kernel': 'liblinear', 'multiclass_strategy': 'ovr'} +balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000640±0.0001 {'C': 1.0, 'kernel': 'linear', 'multiclass_strategy': 'ovr'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0391 * +************************************************************************************************************************* Results in results/results_accuracy_STree_iMac27_2022-05-09_00:21:06_0.json diff --git a/benchmark/tests/test_files/be_summary_list_model.test b/benchmark/tests/test_files/be_summary_list_model.test index 5cb731f..35d9894 100644 --- a/benchmark/tests/test_files/be_summary_list_model.test +++ b/benchmark/tests/test_files/be_summary_list_model.test @@ -26,10 +26,10 @@ * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * * * ********************************************************************************* -Date File Score Time(h) Title -========== =============================================================== ======== ======= ============================================ -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + 1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 2 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 4 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest diff --git a/benchmark/tests/test_files/be_summary_list_score.test b/benchmark/tests/test_files/be_summary_list_score.test index e11fda5..8b655ad 100644 --- a/benchmark/tests/test_files/be_summary_list_score.test +++ b/benchmark/tests/test_files/be_summary_list_score.test @@ -26,10 +26,10 @@ * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * * * ********************************************************************************* -Date File Score Time(h) Title -========== =============================================================== ======== ======= ============================================ -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + 1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 2 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 4 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest diff --git a/benchmark/tests/test_files/be_summary_list_score_all.test b/benchmark/tests/test_files/be_summary_list_score_all.test index c425399..23fd89c 100644 --- a/benchmark/tests/test_files/be_summary_list_score_all.test +++ b/benchmark/tests/test_files/be_summary_list_score_all.test @@ -26,13 +26,13 @@ * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * * * ********************************************************************************* -Date File Score Time(h) Title -========== =============================================================== ======== ======= ============================================ -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + 1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 2 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 4 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest ** No results found ** ** No results found ** ** No results found ** diff --git a/benchmark/tests/test_files/be_summary_list_score_model.test b/benchmark/tests/test_files/be_summary_list_score_model.test index e11fda5..8b655ad 100644 --- a/benchmark/tests/test_files/be_summary_list_score_model.test +++ b/benchmark/tests/test_files/be_summary_list_score_model.test @@ -26,10 +26,10 @@ * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * * * ********************************************************************************* -Date File Score Time(h) Title -========== =============================================================== ======== ======= ============================================ -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + 1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 2 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 4 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest diff --git a/benchmark/tests/test_files/excel2.test b/benchmark/tests/test_files/excel2.test new file mode 100644 index 0000000..c586deb --- /dev/null +++ b/benchmark/tests/test_files/excel2.test @@ -0,0 +1,48 @@ +1;1;" STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-10-27 09:40:40" +2;1;" default A" +3;1;" Score is accuracy" +3;2;" Execution time" +3;5;"3,395.01 s" +3;7;" " +3;8;"Platform" +3;9;"iMac27" +3;10;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]" +4;5;" 0.94 h" +4;10;"Stratified: False" +6;1;"Dataset" +6;2;"Samples" +6;3;"Features" +6;4;"Classes" +6;5;"Nodes" +6;6;"Leaves" +6;7;"Depth" +6;8;"Score" +6;9;"Score Std." +6;10;"Time" +6;11;"Time Std." +6;12;"Hyperparameters" +7;1;"balance-scale" +7;2;"625" +7;3;"4" +7;4;"3" +7;5;"11.08" +7;6;"5.9" +7;7;"5.9" +7;8;"0.98" +7;9;"0.001" +7;10;"0.2852065515518188" +7;11;"0.06031593282605064" +7;12;"{'splitter': 'best', 'max_features': 'auto'}" +8;1;"balloons" +8;2;"16" +8;3;"4" +8;4;"2" +8;5;"4.12" +8;6;"2.56" +8;7;"2.56" +8;8;"0.695" +8;9;"0.2756860130252853" +8;10;"0.02120100021362305" +8;11;"0.003526023309468471" +8;12;"{'splitter': 'best', 'max_features': 'auto'}" +10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0416" diff --git a/benchmark/tests/test_files/report.test b/benchmark/tests/test_files/report.test index 47d4a1a..0e5ffa9 100644 --- a/benchmark/tests/test_files/report.test +++ b/benchmark/tests/test_files/report.test @@ -1,15 +1,15 @@ -************************************************************************************************************************ -* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 * -* With gridsearched hyperparameters * -* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * -* Execution took 624.25 seconds, 0.17 hours, on iMac27 * -* Score is accuracy * -************************************************************************************************************************ +************************************************************************************************************************* +* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 * +* With gridsearched hyperparameters * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 624.25 seconds, 0.17 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* -Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters -============================== ====== ===== === ======= ======= ======= =============== ================ =============== -balance-scale 625 4 3 7.00 4.00 3.00 0.970560±0.0150 0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} -balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} -************************************************************************************************************************ -* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 * -************************************************************************************************************************ +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 7.00 4.00 3.00 0.970560±0.0150 0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +************************************************************************************************************************* +* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 * +************************************************************************************************************************* diff --git a/benchmark/tests/test_files/report_compared.test b/benchmark/tests/test_files/report_compared.test index 018e3ee..103fb5d 100644 --- a/benchmark/tests/test_files/report_compared.test +++ b/benchmark/tests/test_files/report_compared.test @@ -1,16 +1,16 @@ -************************************************************************************************************************ -* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 * -* With gridsearched hyperparameters * -* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * -* Execution took 624.25 seconds, 0.17 hours, on iMac27 * -* Score is accuracy * -************************************************************************************************************************ +************************************************************************************************************************* +* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 * +* With gridsearched hyperparameters * +* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False * +* Execution took 624.25 seconds, 0.17 hours, on iMac27 * +* Score is accuracy * +************************************************************************************************************************* -Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters -============================== ====== ===== === ======= ======= ======= =============== ================ =============== -balance-scale 625 4 3 7.00 4.00 3.00 0.970560±0.0150 0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} -balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850✔ 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} -************************************************************************************************************************ -* ✔ Equal to best .....: 1 * -* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 * -************************************************************************************************************************ +Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters +============================== ====== ===== === ======= ======= ======= =============== ================= =============== +balance-scale 625 4 3 7.00 4.00 3.00 0.970560±0.0150 0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850✔ 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} +************************************************************************************************************************* +* ✔ Equal to best .....: 1 * +* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 * +************************************************************************************************************************* diff --git a/benchmark/tests/test_files/summary_list_hidden.test b/benchmark/tests/test_files/summary_list_hidden.test index d34b5bd..f98eb9f 100644 --- a/benchmark/tests/test_files/summary_list_hidden.test +++ b/benchmark/tests/test_files/summary_list_hidden.test @@ -1,4 +1,4 @@ -Date File Score Time(h) Title -========== ================================================================ ======== ======= ======================= -2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters -2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json 0.97446 0.098 default + # Date File Score Time(h) Title +=== ========== ================================================================ ======== ======= ======================= + 0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters + 1 2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json 0.97446 0.098 default diff --git a/benchmark/tests/test_files/summary_list_model.test b/benchmark/tests/test_files/summary_list_model.test index 3a5b441..2de8e68 100644 --- a/benchmark/tests/test_files/summary_list_model.test +++ b/benchmark/tests/test_files/summary_list_model.test @@ -1,5 +1,5 @@ -Date File Score Time(h) Title -========== ============================================================= ======== ======= ================================= -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + # Date File Score Time(h) Title +=== ========== ============================================================= ======== ======= ================================= + 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters diff --git a/benchmark/tests/test_files/summary_list_n.test b/benchmark/tests/test_files/summary_list_n.test index ac56095..754d36e 100644 --- a/benchmark/tests/test_files/summary_list_n.test +++ b/benchmark/tests/test_files/summary_list_n.test @@ -1,5 +1,5 @@ -Date File Score Time(h) Title -========== =============================================================== ======== ======= ============================================ -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + 2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B diff --git a/benchmark/tests/test_files/summary_list_score.test b/benchmark/tests/test_files/summary_list_score.test index e1e5dd7..ed99d08 100644 --- a/benchmark/tests/test_files/summary_list_score.test +++ b/benchmark/tests/test_files/summary_list_score.test @@ -1,7 +1,7 @@ -Date File Score Time(h) Title -========== =============================================================== ======== ======= ============================================ -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + 2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 3 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 4 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters diff --git a/benchmark/tests/test_files/summary_show_top.test b/benchmark/tests/test_files/summary_show_top.test index dc6a8e0..5fc9f82 100644 --- a/benchmark/tests/test_files/summary_show_top.test +++ b/benchmark/tests/test_files/summary_show_top.test @@ -1,7 +1,7 @@ -Date File Score Time(h) Title -========== =============================================================== ======== ======= ============================================ -2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters -2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init -2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A -2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B -2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest + # Date File Score Time(h) Title +=== ========== =============================================================== ======== ======= ============================================ + 0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters + 1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init + 2 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A + 3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B + 4 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest