From 31a70fbd699dd3077c67cbcd8985697cdc70343b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Thu, 10 Mar 2022 18:58:11 +0100 Subject: [PATCH] Add report of grid search results and fix best results report --- results/grid_output_accuracy_ODTE.json | 468 ++++++++++++++++++++++++- src/Results.py | 9 +- src/report.py | 37 +- 3 files changed, 501 insertions(+), 13 deletions(-) diff --git a/results/grid_output_accuracy_ODTE.json b/results/grid_output_accuracy_ODTE.json index 10d1002..341a33b 100644 --- a/results/grid_output_accuracy_ODTE.json +++ b/results/grid_output_accuracy_ODTE.json @@ -1 +1,467 @@ -{"balance-scale": [0.0, {}, ""], "balloons": [0.9, {"base_estimator__C": 5, "base_estimator__gamma": 0.14, "base_estimator__kernel": "rbf", "base_estimator__multiclass_strategy": "ovr"}, "v. 0.3.2, Computed on iMac27 on 2022-03-09 at 14:02:32 took 8.661 s"], "breast-cancer-wisc-diag": [0.0, {}, ""], "breast-cancer-wisc-prog": [0.0, {}, ""], "breast-cancer-wisc": [0.0, {}, ""], "breast-cancer": [0.0, {}, ""], "cardiotocography-10clases": [0.0, {}, ""], "cardiotocography-3clases": [0.0, {}, ""], "conn-bench-sonar-mines-rocks": [0.0, {}, ""], "cylinder-bands": [0.0, {}, ""], "dermatology": [0.0, {}, ""], "echocardiogram": [0.0, {}, ""], "fertility": [0.0, {}, ""], "haberman-survival": [0.0, {}, ""], "heart-hungarian": [0.0, {}, ""], "hepatitis": [0.0, {}, ""], "ilpd-indian-liver": [0.0, {}, ""], "ionosphere": [0.0, {}, ""], "iris": [0.0, {}, ""], "led-display": [0.0, {}, ""], "libras": [0.0, {}, ""], "low-res-spect": [0.0, {}, ""], "lymphography": [0.0, {}, ""], "mammographic": [0.0, {}, ""], "molec-biol-promoter": [0.0, {}, ""], "musk-1": [0.0, {}, ""], "oocytes_merluccius_nucleus_4d": [0.0, {}, ""], "oocytes_merluccius_states_2f": [0.0, {}, ""], "oocytes_trisopterus_nucleus_2f": [0.0, {}, ""], "oocytes_trisopterus_states_5b": [0.0, {}, ""], "parkinsons": [0.0, {}, ""], "pima": [0.0, {}, ""], "pittsburg-bridges-MATERIAL": [0.0, {}, ""], "pittsburg-bridges-REL-L": [0.0, {}, ""], "pittsburg-bridges-SPAN": [0.0, {}, ""], "pittsburg-bridges-T-OR-D": [0.0, {}, ""], "planning": [0.0, {}, ""], "post-operative": [0.0, {}, ""], "seeds": [0.0, {}, ""], "statlog-australian-credit": [0.0, {}, ""], "statlog-german-credit": [0.0, {}, ""], "statlog-heart": [0.0, {}, ""], "statlog-image": [0.0, {}, ""], "statlog-vehicle": [0.0, {}, ""], "synthetic-control": [0.0, {}, ""], "tic-tac-toe": [0.0, {}, ""], "vertebral-column-2clases": [0.0, {}, ""], "wine": [0.0, {}, ""], "zoo": [0.0, {}, ""]} \ No newline at end of file +{ + "balance-scale": [ + 0.9728, + { + "n_jobs": -1,"base_estimator__C": 57, + "base_estimator__gamma": 0.1, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:21:53 took 14.341 min" + ], + "balloons": [ + 0.9, + { + "n_jobs": -1,"base_estimator__C": 5, + "base_estimator__gamma": 0.14, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:21:53 took 20.007 s" + ], + "breast-cancer-wisc-diag": [ + 0.9788852662630028, + { + "n_jobs": -1,"base_estimator__C": 0.0275, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:21:53 took 1.487 min" + ], + "breast-cancer-wisc-prog": [ + 0.8283333333333334, + { + "n_jobs": -1,"base_estimator__C": 57, + "base_estimator__gamma": 0.001, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:21:56 took 1.617 min" + ], + "breast-cancer-wisc": [ + 0.9713977389516959, + { + "n_jobs": -1,"base_estimator__C": 0.95, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:22:25 took 2.696 min" + ], + "breast-cancer": [ + 0.7411978221415608, + { + "n_jobs": -1,"base_estimator__C": 0.2, + "base_estimator__gamma": 0.1, + "base_estimator__kernel": "poly", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:22:26 took 8.269 min" + ], + "cardiotocography-10clases": [ + 0.8462038111019055, + { + "n_jobs": -1,"base_estimator__C": 10000.0, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:22:26 took 10.859 h" + ], + "cardiotocography-3clases": [ + 0.9261552057442696, + { + "n_jobs": -1,"base_estimator__C": 8.25, + "base_estimator__gamma": 0.1, + "base_estimator__kernel": "poly", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:22:50 took 1.453 h" + ], + "conn-bench-sonar-mines-rocks": [ + 0.8750290360046458, + { + "n_jobs": -1,"base_estimator__C": 1.05, + "base_estimator__gamma": "scale", + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:22:56 took 48.216 s" + ], + "cylinder-bands": [ + 0.8064534551684751, + { + "n_jobs": -1,"base_estimator__C": 5, + "base_estimator__gamma": "auto", + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:22:49 took 21.097 min" + ], + "dermatology": [ + 0.9753424657534246, + { + "n_jobs": -1,"base_estimator__C": 10000.0, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:32:25 took 2.718 min" + ], + "echocardiogram": [ + 0.8552706552706553, + { + "n_jobs": -1,"base_estimator__C": 5, + "base_estimator__gamma": 0.001, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:32:25 took 1.331 min" + ], + "fertility": [ + 0.89, + { + "n_jobs": -1,"base_estimator__C": 2, + "base_estimator__gamma": 0.1, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:32:26 took 1.028 min" + ], + "haberman-survival": [ + 0.7481755684822845, + { + "n_jobs": -1,"base_estimator__C": 0.001, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:32:25 took 7.450 min" + ], + "heart-hungarian": [ + 0.8333722969023963, + { + "n_jobs": -1,"base_estimator__C": 10000.0, + "base_estimator__gamma": 0.001, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:32:29 took 5.355 min" + ], + "hepatitis": [ + 0.8451612903225806, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__gamma": "auto", + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:32:29 took 46.302 s" + ], + "ilpd-indian-liver": [ + 0.7374742116121427, + { + "n_jobs": -1,"base_estimator__C": 1.75, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:32:46 took 21.048 min" + ], + "ionosphere": [ + 0.9544869215291751, + { + "n_jobs": -1,"base_estimator__C": 2, + "base_estimator__gamma": 0.1, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:33:04 took 1.232 min" + ], + "iris": [ + 0.9733333333333334, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__kernel": "linear", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:32:46 took 1.259 min" + ], + "led-display": [ + 0.7209999999999999, + { + "n_jobs": -1,"base_estimator__C": 0.001, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:33:08 took 2.524 h" + ], + "libras": [ + 0.861111111111111, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__kernel": "linear", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:15 took 2.297 h" + ], + "low-res-spect": [ + 0.9058014459530946, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__kernel": "linear", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:23 took 28.283 min" + ], + "lymphography": [ + 0.8577011494252874, + { + "n_jobs": -1,"base_estimator__C": 0.001, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:15 took 1.425 min" + ], + "mammographic": [ + 0.8387305699481866, + { + "n_jobs": -1,"base_estimator__C": 7, + "base_estimator__gamma": 0.1, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:15 took 25.759 min" + ], + "molec-biol-promoter": [ + 0.8406926406926407, + { + "n_jobs": -1,"base_estimator__C": 0.0275, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:14 took 24.426 s" + ], + "musk-1": [ + 0.9391008771929824, + { + "n_jobs": -1,"base_estimator__C": 7, + "base_estimator__gamma": "auto", + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:14 took 7.178 min" + ], + "oocytes_merluccius_nucleus_4d": [ + 0.8600669536107126, + { + "n_jobs": -1,"base_estimator__C": 10000.0, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:14 took 1.841 h" + ], + "oocytes_merluccius_states_2f": [ + 0.9314968914395025, + { + "n_jobs": -1,"base_estimator__C": 10000.0, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:14 took 42.120 min" + ], + "oocytes_trisopterus_nucleus_2f": [ + 0.862919594067135, + { + "n_jobs": -1,"base_estimator__C": 57, + "base_estimator__gamma": "auto", + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:14 took 35.631 min" + ], + "oocytes_trisopterus_states_5b": [ + 0.9342460817870654, + { + "n_jobs": -1,"base_estimator__C": 7, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:13 took 29.433 min" + ], + "parkinsons": [ + 0.9435897435897436, + { + "n_jobs": -1,"base_estimator__C": 2.8, + "base_estimator__gamma": 0.14, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 1.380 min" + ], + "pima": [ + 0.7760122230710466, + { + "n_jobs": -1,"base_estimator__C": 0.95, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 25.392 min" + ], + "pittsburg-bridges-MATERIAL": [ + 0.8770562770562771, + { + "n_jobs": -1,"base_estimator__C": 2.8, + "base_estimator__gamma": 0.14, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 1.160 min" + ], + "pittsburg-bridges-REL-L": [ + 0.67, + { + "n_jobs": -1,"base_estimator__C": 0.2, + "base_estimator__gamma": "scale", + "base_estimator__kernel": "poly", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 3.985 min" + ], + "pittsburg-bridges-SPAN": [ + 0.7076023391812866, + { + "n_jobs": -1,"base_estimator__C": 0.05, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 3.171 min" + ], + "pittsburg-bridges-T-OR-D": [ + 0.8823809523809523, + { + "n_jobs": -1,"base_estimator__C": 0.08, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 50.029 s" + ], + "planning": [ + 0.7358858858858859, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__gamma": 10.0, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 5.112 min" + ], + "post-operative": [ + 0.7111111111111111, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__kernel": "linear", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 2.260 min" + ], + "seeds": [ + 0.9571428571428571, + { + "n_jobs": -1,"base_estimator__C": 7, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 49.530 s" + ], + "statlog-australian-credit": [ + 0.6782608695652174, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__kernel": "linear", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 34.814 min" + ], + "statlog-german-credit": [ + 0.764, + { + "n_jobs": -1,"base_estimator__C": 7, + "base_estimator__gamma": 0.001, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 50.042 min" + ], + "statlog-heart": [ + 0.8518518518518519, + { + "n_jobs": -1,"base_estimator__C": 2, + "base_estimator__gamma": 0.001, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 3.803 min" + ], + "statlog-image": [ + 0.974025974025974, + { + "n_jobs": -1,"base_estimator__C": 10000.0, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 5.191 h" + ], + "statlog-vehicle": [ + 0.8203271841280891, + { + "n_jobs": -1,"base_estimator__C": 7, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 1.030 h" + ], + "synthetic-control": [ + 0.9866666666666667, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__kernel": "linear", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 32.053 min" + ], + "tic-tac-toe": [ + 0.9979112129144851, + { + "n_jobs": -1,"base_estimator__C": 10000.0, + "base_estimator__gamma": 0.001, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 7.476 min" + ], + "vertebral-column-2clases": [ + 0.864516129032258, + { + "n_jobs": -1,"base_estimator__C": 0.95, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 4.015 min" + ], + "wine": [ + 0.9887301587301588, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__gamma": "auto", + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 44.207 s" + ], + "zoo": [ + 0.96, + { + "n_jobs": -1,"base_estimator__C": 8.25, + "base_estimator__gamma": 0.1, + "base_estimator__kernel": "poly", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 52.683 s" + ] +} diff --git a/src/Results.py b/src/Results.py index 4e6ccf4..2999ee5 100644 --- a/src/Results.py +++ b/src/Results.py @@ -191,10 +191,13 @@ class ReportBest(BaseReport): "Hyperparameters", ] - def __init__(self, score, model): - file_name = os.path.join( - Folders.results, Files.best_results(score, model) + def __init__(self, score, model, best, grid): + name = ( + Files.best_results(score, model) + if best + else Files.grid_output(score, model) ) + file_name = os.path.join(Folders.results, name) super().__init__(file_name, best_file=True) self.compare = False self.score_name = score diff --git a/src/report.py b/src/report.py index 6c0db17..34be96c 100755 --- a/src/report.py +++ b/src/report.py @@ -3,7 +3,8 @@ import argparse import numpy as np from Experiments import Datasets from Results import Report, Excel, SQL, ReportBest -from Utils import Files, TextColor +from Utils import Files, TextColor, EnvDefault + """Build report on screen of a result file, optionally generate excel and sql file, and can compare results of report with best results obtained by model @@ -49,12 +50,27 @@ def parse_arguments(): help="best results of models", ) ap.add_argument( - "-s", - "--score", + "-g", + "--grid", type=str, required=False, - default="accuracy", - help="score used in best results model", + help="grid results of model", + ) + ap.add_argument( + "-m", + "--model", + type=str, + required=True, + help="model name", + ) + ap.add_argument( + "-s", + "--score", + action=EnvDefault, + envvar="score", + type=str, + required=True, + help="score name {accuracy, f1_macro, ...}", ) args = ap.parse_args() @@ -64,7 +80,9 @@ def parse_arguments(): args.sql, args.compare, args.best, + args.grid, args.score, + args.model, ) @@ -88,13 +106,14 @@ def default_report(): ) -(file, excel, sql, compare, best, score) = parse_arguments() - +(file, excel, sql, compare, best, grid, score, model) = parse_arguments() +if grid: + best = False if file is None and best is None: default_report() else: - if best is not None: - report = ReportBest(score, best) + if best is not None or grid is not None: + report = ReportBest(score, model, best, grid) report.report() else: report = Report(file, compare)