From 32a82ceb3c4a23b3a2c762d7be00725c64e7b910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Thu, 10 Mar 2022 12:07:09 +0100 Subject: [PATCH 1/7] Add njobs and nestimators to build_grid --- results/grid_input_accuracy_ODTE.json | 24 ++++++++++++++++++++++++ src/build_grid.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/results/grid_input_accuracy_ODTE.json b/results/grid_input_accuracy_ODTE.json index 879927a..332abc1 100644 --- a/results/grid_input_accuracy_ODTE.json +++ b/results/grid_input_accuracy_ODTE.json @@ -1,5 +1,11 @@ [ { + "n_jobs": [ + -1 + ], + "n_estimators": [ + 100 + ], "base_estimator__C": [ 1.0 ], @@ -11,6 +17,12 @@ ] }, { + "n_jobs": [ + -1 + ], + "n_estimators": [ + 100 + ], "base_estimator__C": [ 0.001, 0.0275, @@ -32,6 +44,12 @@ ] }, { + "n_jobs": [ + -1 + ], + "n_estimators": [ + 100 + ], "base_estimator__C": [ 0.05, 1.0, @@ -60,6 +78,12 @@ ] }, { + "n_jobs": [ + -1 + ], + "n_estimators": [ + 100 + ], "base_estimator__C": [ 0.05, 0.2, diff --git a/src/build_grid.py b/src/build_grid.py index a9a2f96..b53c00c 100755 --- a/src/build_grid.py +++ b/src/build_grid.py @@ -81,7 +81,7 @@ for kernel in kernels: results[kernel]["C"].append(1.0) for item in results: - results_tmp = {} + results_tmp = {"n_jobs": [-1], "n_estimators": [100]} for key, value in results[item].items(): new_key = f"base_estimator__{key}" try: From 0d405a8556d3d1acb457f48a27ac7c93b35d4932 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Thu, 10 Mar 2022 12:34:11 +0100 Subject: [PATCH 2/7] fix build_grid style issues --- src/build_grid.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/build_grid.py b/src/build_grid.py index b53c00c..d85927f 100755 --- a/src/build_grid.py +++ b/src/build_grid.py @@ -14,7 +14,8 @@ data = [ '{"kernel": "rbf"}', '{"C": 1.05, "gamma": "auto","kernel": "rbf"}', '{"splitter": "random", "max_features": "auto"}', - '{"C": 0.05, "max_features": "auto", "kernel": "liblinear", "multiclass_strategy": "ovr"}', + '{"C": 0.05, "max_features": "auto", "kernel": "liblinear", ' + '"multiclass_strategy": "ovr"}', '{"kernel": "rbf", "C": 0.05}', '{"C": 0.05, "kernel": "liblinear", "multiclass_strategy": "ovr"}', '{"C": 7, "gamma": 0.1, "kernel": "rbf"}', @@ -27,13 +28,15 @@ data = [ '{"C": 2.8, "kernel": "rbf", "gamma": "auto"}', '{"kernel": "rbf"}', '{"C": 0.05, "gamma": 0.1, "kernel": "poly"}', - '{"C": 8.25, "gamma": 0.1, "kernel": "poly", "multiclass_strategy": "ovr"}', + '{"C": 8.25, "gamma": 0.1, "kernel": "poly", "multiclass_strategy": ' + '"ovr"}', '{"kernel": "liblinear", "multiclass_strategy": "ovr"}', '{"C": 1.75, "kernel": "liblinear", "multiclass_strategy": "ovr"}', '{"C":57, "kernel": "rbf"}', '{"C": 7, "gamma": 0.1, "kernel": "rbf", "multiclass_strategy": "ovr"}', '{"C": 5, "kernel": "rbf", "gamma": "auto"}', - '{"C": 0.05, "max_iter": 10000.0, "kernel": "liblinear", "multiclass_strategy": "ovr"}', + '{"C": 0.05, "max_iter": 10000.0, "kernel": "liblinear", ' + '"multiclass_strategy": "ovr"}', '{"C":0.0275, "kernel": "liblinear", "multiclass_strategy": "ovr"}', '{"C": 7, "gamma": 10.0, "kernel": "rbf", "multiclass_strategy": "ovr"}', '{"kernel": "rbf", "gamma": 0.001}', @@ -59,16 +62,14 @@ for kernel in kernels: for item in hyper: results[kernel][item] = [] # load data -for item in data: - line = json.loads(item) +for sample in data: + line = json.loads(sample) if "kernel" not in line: line["kernel"] = "linear" kernel = line["kernel"] for item in hyper: - if item in line: - results[kernel][item].append(line[item]) if line[ - item - ] not in results[kernel][item] else None + if item in line and line[item] not in results[kernel][item]: + results[kernel][item].append(line[item]) # Add default values and remove inconsistent values results["linear"]["multiclass_strategy"] = ["ovo"] @@ -86,7 +87,7 @@ for item in results: new_key = f"base_estimator__{key}" try: results_tmp[new_key] = sorted(value) - except: + except TypeError: t1 = sorted( [ x From 31a70fbd699dd3077c67cbcd8985697cdc70343b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Thu, 10 Mar 2022 18:58:11 +0100 Subject: [PATCH 3/7] Add report of grid search results and fix best results report --- results/grid_output_accuracy_ODTE.json | 468 ++++++++++++++++++++++++- src/Results.py | 9 +- src/report.py | 37 +- 3 files changed, 501 insertions(+), 13 deletions(-) diff --git a/results/grid_output_accuracy_ODTE.json b/results/grid_output_accuracy_ODTE.json index 10d1002..341a33b 100644 --- a/results/grid_output_accuracy_ODTE.json +++ b/results/grid_output_accuracy_ODTE.json @@ -1 +1,467 @@ -{"balance-scale": [0.0, {}, ""], "balloons": [0.9, {"base_estimator__C": 5, "base_estimator__gamma": 0.14, "base_estimator__kernel": "rbf", "base_estimator__multiclass_strategy": "ovr"}, "v. 0.3.2, Computed on iMac27 on 2022-03-09 at 14:02:32 took 8.661 s"], "breast-cancer-wisc-diag": [0.0, {}, ""], "breast-cancer-wisc-prog": [0.0, {}, ""], "breast-cancer-wisc": [0.0, {}, ""], "breast-cancer": [0.0, {}, ""], "cardiotocography-10clases": [0.0, {}, ""], "cardiotocography-3clases": [0.0, {}, ""], "conn-bench-sonar-mines-rocks": [0.0, {}, ""], "cylinder-bands": [0.0, {}, ""], "dermatology": [0.0, {}, ""], "echocardiogram": [0.0, {}, ""], "fertility": [0.0, {}, ""], "haberman-survival": [0.0, {}, ""], "heart-hungarian": [0.0, {}, ""], "hepatitis": [0.0, {}, ""], "ilpd-indian-liver": [0.0, {}, ""], "ionosphere": [0.0, {}, ""], "iris": [0.0, {}, ""], "led-display": [0.0, {}, ""], "libras": [0.0, {}, ""], "low-res-spect": [0.0, {}, ""], "lymphography": [0.0, {}, ""], "mammographic": [0.0, {}, ""], "molec-biol-promoter": [0.0, {}, ""], "musk-1": [0.0, {}, ""], "oocytes_merluccius_nucleus_4d": [0.0, {}, ""], "oocytes_merluccius_states_2f": [0.0, {}, ""], "oocytes_trisopterus_nucleus_2f": [0.0, {}, ""], "oocytes_trisopterus_states_5b": [0.0, {}, ""], "parkinsons": [0.0, {}, ""], "pima": [0.0, {}, ""], "pittsburg-bridges-MATERIAL": [0.0, {}, ""], "pittsburg-bridges-REL-L": [0.0, {}, ""], "pittsburg-bridges-SPAN": [0.0, {}, ""], "pittsburg-bridges-T-OR-D": [0.0, {}, ""], "planning": [0.0, {}, ""], "post-operative": [0.0, {}, ""], "seeds": [0.0, {}, ""], "statlog-australian-credit": [0.0, {}, ""], "statlog-german-credit": [0.0, {}, ""], "statlog-heart": [0.0, {}, ""], "statlog-image": [0.0, {}, ""], "statlog-vehicle": [0.0, {}, ""], "synthetic-control": [0.0, {}, ""], "tic-tac-toe": [0.0, {}, ""], "vertebral-column-2clases": [0.0, {}, ""], "wine": [0.0, {}, ""], "zoo": [0.0, {}, ""]} \ No newline at end of file +{ + "balance-scale": [ + 0.9728, + { + "n_jobs": -1,"base_estimator__C": 57, + "base_estimator__gamma": 0.1, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:21:53 took 14.341 min" + ], + "balloons": [ + 0.9, + { + "n_jobs": -1,"base_estimator__C": 5, + "base_estimator__gamma": 0.14, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:21:53 took 20.007 s" + ], + "breast-cancer-wisc-diag": [ + 0.9788852662630028, + { + "n_jobs": -1,"base_estimator__C": 0.0275, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:21:53 took 1.487 min" + ], + "breast-cancer-wisc-prog": [ + 0.8283333333333334, + { + "n_jobs": -1,"base_estimator__C": 57, + "base_estimator__gamma": 0.001, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:21:56 took 1.617 min" + ], + "breast-cancer-wisc": [ + 0.9713977389516959, + { + "n_jobs": -1,"base_estimator__C": 0.95, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:22:25 took 2.696 min" + ], + "breast-cancer": [ + 0.7411978221415608, + { + "n_jobs": -1,"base_estimator__C": 0.2, + "base_estimator__gamma": 0.1, + "base_estimator__kernel": "poly", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:22:26 took 8.269 min" + ], + "cardiotocography-10clases": [ + 0.8462038111019055, + { + "n_jobs": -1,"base_estimator__C": 10000.0, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:22:26 took 10.859 h" + ], + "cardiotocography-3clases": [ + 0.9261552057442696, + { + "n_jobs": -1,"base_estimator__C": 8.25, + "base_estimator__gamma": 0.1, + "base_estimator__kernel": "poly", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:22:50 took 1.453 h" + ], + "conn-bench-sonar-mines-rocks": [ + 0.8750290360046458, + { + "n_jobs": -1,"base_estimator__C": 1.05, + "base_estimator__gamma": "scale", + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:22:56 took 48.216 s" + ], + "cylinder-bands": [ + 0.8064534551684751, + { + "n_jobs": -1,"base_estimator__C": 5, + "base_estimator__gamma": "auto", + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:22:49 took 21.097 min" + ], + "dermatology": [ + 0.9753424657534246, + { + "n_jobs": -1,"base_estimator__C": 10000.0, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:32:25 took 2.718 min" + ], + "echocardiogram": [ + 0.8552706552706553, + { + "n_jobs": -1,"base_estimator__C": 5, + "base_estimator__gamma": 0.001, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:32:25 took 1.331 min" + ], + "fertility": [ + 0.89, + { + "n_jobs": -1,"base_estimator__C": 2, + "base_estimator__gamma": 0.1, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:32:26 took 1.028 min" + ], + "haberman-survival": [ + 0.7481755684822845, + { + "n_jobs": -1,"base_estimator__C": 0.001, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:32:25 took 7.450 min" + ], + "heart-hungarian": [ + 0.8333722969023963, + { + "n_jobs": -1,"base_estimator__C": 10000.0, + "base_estimator__gamma": 0.001, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:32:29 took 5.355 min" + ], + "hepatitis": [ + 0.8451612903225806, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__gamma": "auto", + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:32:29 took 46.302 s" + ], + "ilpd-indian-liver": [ + 0.7374742116121427, + { + "n_jobs": -1,"base_estimator__C": 1.75, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:32:46 took 21.048 min" + ], + "ionosphere": [ + 0.9544869215291751, + { + "n_jobs": -1,"base_estimator__C": 2, + "base_estimator__gamma": 0.1, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:33:04 took 1.232 min" + ], + "iris": [ + 0.9733333333333334, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__kernel": "linear", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:32:46 took 1.259 min" + ], + "led-display": [ + 0.7209999999999999, + { + "n_jobs": -1,"base_estimator__C": 0.001, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 17:33:08 took 2.524 h" + ], + "libras": [ + 0.861111111111111, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__kernel": "linear", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:15 took 2.297 h" + ], + "low-res-spect": [ + 0.9058014459530946, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__kernel": "linear", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:23 took 28.283 min" + ], + "lymphography": [ + 0.8577011494252874, + { + "n_jobs": -1,"base_estimator__C": 0.001, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:15 took 1.425 min" + ], + "mammographic": [ + 0.8387305699481866, + { + "n_jobs": -1,"base_estimator__C": 7, + "base_estimator__gamma": 0.1, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:15 took 25.759 min" + ], + "molec-biol-promoter": [ + 0.8406926406926407, + { + "n_jobs": -1,"base_estimator__C": 0.0275, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:14 took 24.426 s" + ], + "musk-1": [ + 0.9391008771929824, + { + "n_jobs": -1,"base_estimator__C": 7, + "base_estimator__gamma": "auto", + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:14 took 7.178 min" + ], + "oocytes_merluccius_nucleus_4d": [ + 0.8600669536107126, + { + "n_jobs": -1,"base_estimator__C": 10000.0, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:14 took 1.841 h" + ], + "oocytes_merluccius_states_2f": [ + 0.9314968914395025, + { + "n_jobs": -1,"base_estimator__C": 10000.0, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:14 took 42.120 min" + ], + "oocytes_trisopterus_nucleus_2f": [ + 0.862919594067135, + { + "n_jobs": -1,"base_estimator__C": 57, + "base_estimator__gamma": "auto", + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:14 took 35.631 min" + ], + "oocytes_trisopterus_states_5b": [ + 0.9342460817870654, + { + "n_jobs": -1,"base_estimator__C": 7, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:06:13 took 29.433 min" + ], + "parkinsons": [ + 0.9435897435897436, + { + "n_jobs": -1,"base_estimator__C": 2.8, + "base_estimator__gamma": 0.14, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 1.380 min" + ], + "pima": [ + 0.7760122230710466, + { + "n_jobs": -1,"base_estimator__C": 0.95, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 25.392 min" + ], + "pittsburg-bridges-MATERIAL": [ + 0.8770562770562771, + { + "n_jobs": -1,"base_estimator__C": 2.8, + "base_estimator__gamma": 0.14, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 1.160 min" + ], + "pittsburg-bridges-REL-L": [ + 0.67, + { + "n_jobs": -1,"base_estimator__C": 0.2, + "base_estimator__gamma": "scale", + "base_estimator__kernel": "poly", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 3.985 min" + ], + "pittsburg-bridges-SPAN": [ + 0.7076023391812866, + { + "n_jobs": -1,"base_estimator__C": 0.05, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 3.171 min" + ], + "pittsburg-bridges-T-OR-D": [ + 0.8823809523809523, + { + "n_jobs": -1,"base_estimator__C": 0.08, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 50.029 s" + ], + "planning": [ + 0.7358858858858859, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__gamma": 10.0, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 5.112 min" + ], + "post-operative": [ + 0.7111111111111111, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__kernel": "linear", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 2.260 min" + ], + "seeds": [ + 0.9571428571428571, + { + "n_jobs": -1,"base_estimator__C": 7, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 49.530 s" + ], + "statlog-australian-credit": [ + 0.6782608695652174, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__kernel": "linear", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 19:46:02 took 34.814 min" + ], + "statlog-german-credit": [ + 0.764, + { + "n_jobs": -1,"base_estimator__C": 7, + "base_estimator__gamma": 0.001, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 50.042 min" + ], + "statlog-heart": [ + 0.8518518518518519, + { + "n_jobs": -1,"base_estimator__C": 2, + "base_estimator__gamma": 0.001, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 3.803 min" + ], + "statlog-image": [ + 0.974025974025974, + { + "n_jobs": -1,"base_estimator__C": 10000.0, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 5.191 h" + ], + "statlog-vehicle": [ + 0.8203271841280891, + { + "n_jobs": -1,"base_estimator__C": 7, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 1.030 h" + ], + "synthetic-control": [ + 0.9866666666666667, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__kernel": "linear", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 32.053 min" + ], + "tic-tac-toe": [ + 0.9979112129144851, + { + "n_jobs": -1,"base_estimator__C": 10000.0, + "base_estimator__gamma": 0.001, + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 7.476 min" + ], + "vertebral-column-2clases": [ + 0.864516129032258, + { + "n_jobs": -1,"base_estimator__C": 0.95, + "base_estimator__kernel": "liblinear", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 4.015 min" + ], + "wine": [ + 0.9887301587301588, + { + "n_jobs": -1,"base_estimator__C": 1.0, + "base_estimator__gamma": "auto", + "base_estimator__kernel": "rbf", + "base_estimator__multiclass_strategy": "ovr" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 44.207 s" + ], + "zoo": [ + 0.96, + { + "n_jobs": -1,"base_estimator__C": 8.25, + "base_estimator__gamma": 0.1, + "base_estimator__kernel": "poly", + "base_estimator__multiclass_strategy": "ovo" + }, + "v. 0.3.2, Computed on Galgo on 2022-03-09 at 20:03:50 took 52.683 s" + ] +} diff --git a/src/Results.py b/src/Results.py index 4e6ccf4..2999ee5 100644 --- a/src/Results.py +++ b/src/Results.py @@ -191,10 +191,13 @@ class ReportBest(BaseReport): "Hyperparameters", ] - def __init__(self, score, model): - file_name = os.path.join( - Folders.results, Files.best_results(score, model) + def __init__(self, score, model, best, grid): + name = ( + Files.best_results(score, model) + if best + else Files.grid_output(score, model) ) + file_name = os.path.join(Folders.results, name) super().__init__(file_name, best_file=True) self.compare = False self.score_name = score diff --git a/src/report.py b/src/report.py index 6c0db17..34be96c 100755 --- a/src/report.py +++ b/src/report.py @@ -3,7 +3,8 @@ import argparse import numpy as np from Experiments import Datasets from Results import Report, Excel, SQL, ReportBest -from Utils import Files, TextColor +from Utils import Files, TextColor, EnvDefault + """Build report on screen of a result file, optionally generate excel and sql file, and can compare results of report with best results obtained by model @@ -49,12 +50,27 @@ def parse_arguments(): help="best results of models", ) ap.add_argument( - "-s", - "--score", + "-g", + "--grid", type=str, required=False, - default="accuracy", - help="score used in best results model", + help="grid results of model", + ) + ap.add_argument( + "-m", + "--model", + type=str, + required=True, + help="model name", + ) + ap.add_argument( + "-s", + "--score", + action=EnvDefault, + envvar="score", + type=str, + required=True, + help="score name {accuracy, f1_macro, ...}", ) args = ap.parse_args() @@ -64,7 +80,9 @@ def parse_arguments(): args.sql, args.compare, args.best, + args.grid, args.score, + args.model, ) @@ -88,13 +106,14 @@ def default_report(): ) -(file, excel, sql, compare, best, score) = parse_arguments() - +(file, excel, sql, compare, best, grid, score, model) = parse_arguments() +if grid: + best = False if file is None and best is None: default_report() else: - if best is not None: - report = ReportBest(score, best) + if best is not None or grid is not None: + report = ReportBest(score, model, best, grid) report.report() else: report = Report(file, compare) From aed72e9c14bdbb4fa2f6024fbb2c79910f8deccd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana?= Date: Thu, 10 Mar 2022 19:55:51 +0100 Subject: [PATCH 4/7] Fix best report and grid report --- src/Results.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Results.py b/src/Results.py index 2999ee5..203e563 100644 --- a/src/Results.py +++ b/src/Results.py @@ -183,11 +183,11 @@ class Report(BaseReport): class ReportBest(BaseReport): - header_lengths = [30, 8, 50, 35] + header_lengths = [30, 8, 76, 45] header_cols = [ "Dataset", "Score", - "File", + "File/Message", "Hyperparameters", ] @@ -197,6 +197,8 @@ class ReportBest(BaseReport): if best else Files.grid_output(score, model) ) + self.best = best + self.grid = grid file_name = os.path.join(Folders.results, name) super().__init__(file_name, best_file=True) self.compare = False @@ -225,8 +227,9 @@ class ReportBest(BaseReport): def header(self): self.header_line("*") + kind = "Best" if self.best else "Grid" self.header_line( - f" Report Best {self.score_name} Scores with {self.model} in any " + f" Report {kind} {self.score_name} Scores with {self.model} in any " "platform" ) self.header_line("*") From 33accba9ba61fdafa0edc47ac902b22c92b62abe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana?= Date: Thu, 10 Mar 2022 20:03:16 +0100 Subject: [PATCH 5/7] Fix style issue --- src/Results.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Results.py b/src/Results.py index 203e563..8ccf571 100644 --- a/src/Results.py +++ b/src/Results.py @@ -229,8 +229,8 @@ class ReportBest(BaseReport): self.header_line("*") kind = "Best" if self.best else "Grid" self.header_line( - f" Report {kind} {self.score_name} Scores with {self.model} in any " - "platform" + f" Report {kind} {self.score_name} Scores with {self.model} " + "in any platform" ) self.header_line("*") print("") From d9f1f4b5722a53f51ea02b40122d0dab895e078d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Fri, 11 Mar 2022 11:30:30 +0100 Subject: [PATCH 6/7] Add writing results to disk as they're computed in experiments --- src/Experiments.py | 2 ++ src/report.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/Experiments.py b/src/Experiments.py index 9c05b57..7cc744a 100644 --- a/src/Experiments.py +++ b/src/Experiments.py @@ -287,6 +287,7 @@ class Experiment: position=0, disable=not self.progress_bar, ) + self.duration = 0.0 for name in loop: loop.set_description(f"{name:30s}") X, y = self.datasets.load(name) @@ -296,6 +297,7 @@ class Experiment: self._init_experiment() self._n_fold_crossval(X, y, hyperparameters) self._add_results(name, hyperparameters, samp, feat, n_classes) + self._output_results() self.duration = time.time() - now self._output_results() if self.progress_bar: diff --git a/src/report.py b/src/report.py index 34be96c..4bb6140 100755 --- a/src/report.py +++ b/src/report.py @@ -59,6 +59,8 @@ def parse_arguments(): ap.add_argument( "-m", "--model", + action=EnvDefault, + envvar="model", type=str, required=True, help="model name", From 9628bb38897440a96722c6d22347dc762ddaf3a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Fri, 11 Mar 2022 11:50:29 +0100 Subject: [PATCH 7/7] Add flush to write experiments results --- src/Experiments.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Experiments.py b/src/Experiments.py index 7cc744a..791a034 100644 --- a/src/Experiments.py +++ b/src/Experiments.py @@ -279,6 +279,7 @@ class Experiment: output["results"] = self.results with open(self.output_file, "w") as f: json.dump(output, f) + f.flush() def do_experiment(self): now = time.time()