diff --git a/benchmark/Experiments.py b/benchmark/Experiments.py index 597c627..0b289e8 100644 --- a/benchmark/Experiments.py +++ b/benchmark/Experiments.py @@ -156,7 +156,7 @@ class BestResults: self._process_datafile(results, data, name) found = True if not found: - raise ValueError(f"No results found") + raise ValueError("** No results found **") # Build best results json file output = {} datasets = Datasets() diff --git a/benchmark/Results.py b/benchmark/Results.py index 9c7cc9e..4ef9bb5 100644 --- a/benchmark/Results.py +++ b/benchmark/Results.py @@ -521,6 +521,34 @@ class Excel(BaseReport): self.book.close() +class ReportDatasets: + @staticmethod + def report(): + data_sets = Datasets() + color_line = TextColor.LINE1 + print(color_line, end="") + print(f"{'Dataset':30s} Samp. Feat. Cls Balance") + print("=" * 30 + " ===== ===== === " + "=" * 40) + for dataset in data_sets: + X, y = data_sets.load(dataset) + color_line = ( + TextColor.LINE2 + if color_line == TextColor.LINE1 + else TextColor.LINE1 + ) + values, counts = np.unique(y, return_counts=True) + comp = "" + sep = "" + for count in counts: + comp += f"{sep}{count/sum(counts)*100:5.2f}%" + sep = "/ " + print(color_line, end="") + print( + f"{dataset:30s} {X.shape[0]:5,d} {X.shape[1]:5,d} " + f"{len(np.unique(y)):3d} {comp:40s}" + ) + + class SQL(BaseReport): table_name = "results" @@ -1112,8 +1140,7 @@ class Summary: score, model, input_data, sort_key, number ) if data == []: - print("*No results found*") - exit(1) + raise ValueError("** No results found **") max_file = max(len(x["file"]) for x in data) max_title = max(len(x["title"]) for x in data) if self.hidden: @@ -1285,11 +1312,14 @@ class Summary: return best_results def show_top(self, score="accuracy", n=10): - self.list_results( - score=score, - input_data=self.best_results(score=score, n=n), - sort_key="metric", - ) + try: + self.list_results( + score=score, + input_data=self.best_results(score=score, n=n), + sort_key="metric", + ) + except ValueError as e: + print(e) class PairCheck: diff --git a/benchmark/Utils.py b/benchmark/Utils.py index 7d860cf..842099b 100644 --- a/benchmark/Utils.py +++ b/benchmark/Utils.py @@ -9,6 +9,7 @@ class Folders: hidden_results = "hidden_results" exreport = "exreport" report = os.path.join(exreport, "exreport_output") + img = "img" @staticmethod def src(): diff --git a/benchmark/scripts/be_build_best.py b/benchmark/scripts/be_build_best.py index 45eeb16..21942c4 100755 --- a/benchmark/scripts/be_build_best.py +++ b/benchmark/scripts/be_build_best.py @@ -9,14 +9,15 @@ from benchmark.Arguments import Arguments def main(): arguments = Arguments() - arguments.xset("score").xset("report").xset("model") + arguments.xset("score", mandatory=True).xset("report") + arguments.xset("model", mandatory=True) args = arguments.parse() datasets = Datasets() best = BestResults(args.score, args.model, datasets) try: best.build() - except ValueError: - print("** No results found **") + except ValueError as e: + print(e) else: if args.report: report = ReportBest(args.score, args.model, best=True, grid=False) diff --git a/benchmark/scripts/be_grid.py b/benchmark/scripts/be_grid.py index 5de2f05..ea2fe78 100755 --- a/benchmark/scripts/be_grid.py +++ b/benchmark/scripts/be_grid.py @@ -8,7 +8,7 @@ from benchmark.Arguments import Arguments def main(): arguments = Arguments() - arguments.xset("score").xset("platform").xset("model") + arguments.xset("score").xset("platform").xset("model", mandatory=True) arguments.xset("quiet").xset("stratified").xset("dataset").xset("n_folds") args = arguments.parse() if not args.quiet: diff --git a/benchmark/scripts/be_list.py b/benchmark/scripts/be_list.py index 2382288..3552d5c 100755 --- a/benchmark/scripts/be_list.py +++ b/benchmark/scripts/be_list.py @@ -15,35 +15,39 @@ def main(): args = arguments.parse() data = Summary(hidden=args.hidden) data.acquire() - data.list_results( - score=args.score, - model=args.model, - sort_key=args.key, - number=args.number, - ) - if args.nan: - results_nan = [] - results = data.get_results_criteria( + try: + data.list_results( score=args.score, model=args.model, - input_data=None, sort_key=args.key, number=args.number, ) - for result in results: - if result["metric"] != result["metric"]: - results_nan.append(result) - if results_nan != []: - print( - "\n" - + "*" * 30 - + " Results with nan moved to hidden " - + "*" * 30 + except ValueError as e: + print(e) + else: + if args.nan: + results_nan = [] + results = data.get_results_criteria( + score=args.score, + model=args.model, + input_data=None, + sort_key=args.key, + number=args.number, ) - data.list_results(input_data=results_nan) - for result in results_nan: - name = result["file"] - os.rename( - os.path.join(Folders.results, name), - os.path.join(Folders.hidden_results, name), + for result in results: + if result["metric"] != result["metric"]: + results_nan.append(result) + if results_nan != []: + print( + "\n" + + "*" * 30 + + " Results with nan moved to hidden " + + "*" * 30 ) + data.list_results(input_data=results_nan) + for result in results_nan: + name = result["file"] + os.rename( + os.path.join(Folders.results, name), + os.path.join(Folders.hidden_results, name), + ) diff --git a/benchmark/scripts/be_main.py b/benchmark/scripts/be_main.py index 34ccc95..7cbd496 100755 --- a/benchmark/scripts/be_main.py +++ b/benchmark/scripts/be_main.py @@ -10,10 +10,10 @@ from benchmark.Arguments import Arguments def main(): arguments = Arguments() - arguments.xset("stratified").xset("score").xset("model").xset("dataset") + arguments.xset("stratified").xset("score").xset("model", mandatory=True) arguments.xset("n_folds").xset("platform").xset("quiet").xset("title") arguments.xset("hyperparameters").xset("paramfile").xset("report") - arguments.xset("grid_paramfile") + arguments.xset("grid_paramfile").xset("dataset") args = arguments.parse() report = args.report or args.dataset is not None if args.grid_paramfile: diff --git a/benchmark/scripts/be_print_strees.py b/benchmark/scripts/be_print_strees.py index a359287..bae32cf 100755 --- a/benchmark/scripts/be_print_strees.py +++ b/benchmark/scripts/be_print_strees.py @@ -62,7 +62,6 @@ def add_color(source): def print_stree(clf, dataset, X, y, color, quiet): - output_folder = "img" samples, features = X.shape classes = max(y) + 1 accuracy = clf.score(X, y) @@ -72,7 +71,7 @@ def print_stree(clf, dataset, X, y, color, quiet): if color: dot_source = add_color(dot_source) grp = Source(dot_source) - file_name = os.path.join(output_folder, f"stree_{dataset}") + file_name = os.path.join(Folders.img, f"stree_{dataset}") grp.render(format="png", filename=f"{file_name}") os.remove(f"{file_name}") print(f"File {file_name}.png generated") diff --git a/benchmark/scripts/be_report.py b/benchmark/scripts/be_report.py index 29ee983..f726a52 100755 --- a/benchmark/scripts/be_report.py +++ b/benchmark/scripts/be_report.py @@ -1,11 +1,6 @@ #!/usr/bin/env python -import numpy as np -from benchmark.Experiments import Datasets -from benchmark.Results import Report, Excel, SQL, ReportBest -from benchmark.Utils import ( - Files, - TextColor, -) +from benchmark.Results import Report, Excel, SQL, ReportBest, ReportDatasets +from benchmark.Utils import Files from benchmark.Arguments import Arguments @@ -15,32 +10,6 @@ If no argument is set, displays the datasets and its characteristics """ -def default_report(): - sets = Datasets() - color_line = TextColor.LINE1 - print(color_line, end="") - print(f"{'Dataset':30s} Samp. Feat Cls Balance") - print("=" * 30 + " ===== ==== === " + "=" * 40) - for line in sets: - X, y = sets.load(line) - color_line = ( - TextColor.LINE2 - if color_line == TextColor.LINE1 - else TextColor.LINE1 - ) - values, counts = np.unique(y, return_counts=True) - comp = "" - sep = "" - for value, count in zip(values, counts): - comp += f"{sep}{count/sum(counts)*100:5.2f}%" - sep = "/ " - print(color_line, end="") - print( - f"{line:30s} {X.shape[0]:5,d} {X.shape[1]:4d} " - f"{len(np.unique(y)):3d} {comp:40s}" - ) - - def main(): arguments = Arguments() arguments.xset("file").xset("excel").xset("sql").xset("compare") @@ -48,11 +17,10 @@ def main(): "score" ) args = arguments.parse() - if args.grid: args.best = False if args.file is None and args.best is None: - default_report() + ReportDatasets.report() else: if args.best is not None or args.grid is not None: report = ReportBest(args.score, args.model, args.best, args.grid) diff --git a/benchmark/tests/GridSearch_test.py b/benchmark/tests/GridSearch_test.py index 6016bbf..4cfb0f6 100644 --- a/benchmark/tests/GridSearch_test.py +++ b/benchmark/tests/GridSearch_test.py @@ -37,7 +37,8 @@ class GridSearchTest(TestBase): ], ".", ) - _ = self.build_exp() + grid = self.build_exp() + grid._init_data() # check the output file is initialized with open(file_name) as f: data = json.load(f) diff --git a/benchmark/tests/test_files/summary_list_hidden.test b/benchmark/tests/test_files/summary_list_hidden.test index 11f94d5..c883589 100644 --- a/benchmark/tests/test_files/summary_list_hidden.test +++ b/benchmark/tests/test_files/summary_list_hidden.test @@ -1,3 +1,3 @@ -Date File Score Time(h) Title +Date File Score Time(h) Title ========== ======================================================== ======== ======= ======= 2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json 0.97446 0.098 default diff --git a/benchmark/tests/test_files/summary_list_model.test b/benchmark/tests/test_files/summary_list_model.test index 1b56e3b..3a5b441 100644 --- a/benchmark/tests/test_files/summary_list_model.test +++ b/benchmark/tests/test_files/summary_list_model.test @@ -1,4 +1,4 @@ -Date File Score Time(h) Title +Date File Score Time(h) Title ========== ============================================================= ======== ======= ================================= 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A diff --git a/benchmark/tests/test_files/summary_list_n.test b/benchmark/tests/test_files/summary_list_n.test index b4c2270..ac56095 100644 --- a/benchmark/tests/test_files/summary_list_n.test +++ b/benchmark/tests/test_files/summary_list_n.test @@ -1,4 +1,4 @@ -Date File Score Time(h) Title +Date File Score Time(h) Title ========== =============================================================== ======== ======= ============================================ 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest diff --git a/benchmark/tests/test_files/summary_list_score.test b/benchmark/tests/test_files/summary_list_score.test index c449be5..e1e5dd7 100644 --- a/benchmark/tests/test_files/summary_list_score.test +++ b/benchmark/tests/test_files/summary_list_score.test @@ -1,4 +1,4 @@ -Date File Score Time(h) Title +Date File Score Time(h) Title ========== =============================================================== ======== ======= ============================================ 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest diff --git a/benchmark/tests/test_files/summary_show_results.test b/benchmark/tests/test_files/summary_show_results.test index ddb37bd..a2e8005 100644 --- a/benchmark/tests/test_files/summary_show_results.test +++ b/benchmark/tests/test_files/summary_show_results.test @@ -1,12 +1,12 @@ -********************************************************************************* -* * -* With gridsearched hyperparameters * -* * -* Model: STree Ver. 1.2.3 Score: accuracy Metric: 0.0454434 * -* * -* Date : 2021-09-30 Time: 11:42:07 Time Spent: 624.25 secs. * -* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 * -* Stratified: False * -* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * -* * -********************************************************************************* +********************************************************************************* +* * +*  With gridsearched hyperparameters  * +* * +* Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 * +* * +* Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. * +* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 * +* Stratified: False * +* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * +* * +********************************************************************************* diff --git a/benchmark/tests/test_files/summary_show_results_title.test b/benchmark/tests/test_files/summary_show_results_title.test index b043127..0604f18 100644 --- a/benchmark/tests/test_files/summary_show_results_title.test +++ b/benchmark/tests/test_files/summary_show_results_title.test @@ -1,14 +1,14 @@ -********************************************************************************* -* **Title** * +********************************************************************************* +* **Title** * *-------------------------------------------------------------------------------* -* * -* With gridsearched hyperparameters * -* * -* Model: STree Ver. 1.2.3 Score: accuracy Metric: 0.0454434 * -* * -* Date : 2021-09-30 Time: 11:42:07 Time Spent: 624.25 secs. * -* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 * -* Stratified: False * -* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * -* * -********************************************************************************* +* * +*  With gridsearched hyperparameters  * +* * +* Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 * +* * +* Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. * +* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 * +* Stratified: False * +* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * +* * +********************************************************************************* diff --git a/benchmark/tests/test_files/summary_show_top.test b/benchmark/tests/test_files/summary_show_top.test index 7dc47f9..dc6a8e0 100644 --- a/benchmark/tests/test_files/summary_show_top.test +++ b/benchmark/tests/test_files/summary_show_top.test @@ -1,4 +1,4 @@ -Date File Score Time(h) Title +Date File Score Time(h) Title ========== =============================================================== ======== ======= ============================================ 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init