diff --git a/results/grid_input_accuracy_ODTE.json b/results/grid_input_accuracy_ODTE.json index ff0d47d..879927a 100644 --- a/results/grid_input_accuracy_ODTE.json +++ b/results/grid_input_accuracy_ODTE.json @@ -1,26 +1,81 @@ [ - { - "base_estimator__C": [1.0], - "base_estimator__kernel": ["linear"], - "base_estimator__multiclass_strategy": ["ovo"] - }, - { - "base_estimator__C": [ - 0.001, 0.0275, 0.05, 0.08, 0.2, 0.25, 0.95, 1.0, 1.75, 7, 10000.0 - ], - "base_estimator__kernel": ["liblinear"], - "base_estimator__multiclass_strategy": ["ovr"] - }, - { - "base_estimator__C": [0.05, 1.0, 1.05, 2, 2.8, 2.83, 5, 7, 57, 10000.0], - "base_estimator__gamma": [0.001, 0.1, 0.14, 10.0, "auto", "scale"], - "base_estimator__kernel": ["rbf"], - "base_estimator__multiclass_strategy": ["ovr"] - }, - { - "base_estimator__C": [0.05, 0.2, 1.0, 8.25], - "base_estimator__gamma": [0.1], - "base_estimator__kernel": ["poly"], - "base_estimator__multiclass_strategy": ["ovo", "ovr"] - } -] + { + "base_estimator__C": [ + 1.0 + ], + "base_estimator__kernel": [ + "linear" + ], + "base_estimator__multiclass_strategy": [ + "ovo" + ] + }, + { + "base_estimator__C": [ + 0.001, + 0.0275, + 0.05, + 0.08, + 0.2, + 0.25, + 0.95, + 1.0, + 1.75, + 7, + 10000.0 + ], + "base_estimator__kernel": [ + "liblinear" + ], + "base_estimator__multiclass_strategy": [ + "ovr" + ] + }, + { + "base_estimator__C": [ + 0.05, + 1.0, + 1.05, + 2, + 2.8, + 2.83, + 5, + 7, + 57, + 10000.0 + ], + "base_estimator__gamma": [ + 0.001, + 0.1, + 0.14, + 10.0, + "auto", + "scale" + ], + "base_estimator__kernel": [ + "rbf" + ], + "base_estimator__multiclass_strategy": [ + "ovr" + ] + }, + { + "base_estimator__C": [ + 0.05, + 0.2, + 1.0, + 8.25 + ], + "base_estimator__gamma": [ + 0.1, + "scale" + ], + "base_estimator__kernel": [ + "poly" + ], + "base_estimator__multiclass_strategy": [ + "ovo", + "ovr" + ] + } +] \ No newline at end of file diff --git a/results/grid_output_accuracy_ODTE.json b/results/grid_output_accuracy_ODTE.json index 3261f31..10d1002 100644 --- a/results/grid_output_accuracy_ODTE.json +++ b/results/grid_output_accuracy_ODTE.json @@ -1,60 +1 @@ -{ - "balance-scale": [0.0, {}, ""], - "balloons": [ - 0.9, - { - "base_estimator__C": 5, - "base_estimator__gamma": 0.14, - "base_estimator__kernel": "rbf", - "base_estimator__multiclass_strategy": "ovr" - }, - "v. 0.3.2, Computed on iMac27 on 2022-03-09 at 13:39:26 took 7.610 s" - ], - "breast-cancer-wisc-diag": [0.0, {}, ""], - "breast-cancer-wisc-prog": [0.0, {}, ""], - "breast-cancer-wisc": [0.0, {}, ""], - "breast-cancer": [0.0, {}, ""], - "cardiotocography-10clases": [0.0, {}, ""], - "cardiotocography-3clases": [0.0, {}, ""], - "conn-bench-sonar-mines-rocks": [0.0, {}, ""], - "cylinder-bands": [0.0, {}, ""], - "dermatology": [0.0, {}, ""], - "echocardiogram": [0.0, {}, ""], - "fertility": [0.0, {}, ""], - "haberman-survival": [0.0, {}, ""], - "heart-hungarian": [0.0, {}, ""], - "hepatitis": [0.0, {}, ""], - "ilpd-indian-liver": [0.0, {}, ""], - "ionosphere": [0.0, {}, ""], - "iris": [0.0, {}, ""], - "led-display": [0.0, {}, ""], - "libras": [0.0, {}, ""], - "low-res-spect": [0.0, {}, ""], - "lymphography": [0.0, {}, ""], - "mammographic": [0.0, {}, ""], - "molec-biol-promoter": [0.0, {}, ""], - "musk-1": [0.0, {}, ""], - "oocytes_merluccius_nucleus_4d": [0.0, {}, ""], - "oocytes_merluccius_states_2f": [0.0, {}, ""], - "oocytes_trisopterus_nucleus_2f": [0.0, {}, ""], - "oocytes_trisopterus_states_5b": [0.0, {}, ""], - "parkinsons": [0.0, {}, ""], - "pima": [0.0, {}, ""], - "pittsburg-bridges-MATERIAL": [0.0, {}, ""], - "pittsburg-bridges-REL-L": [0.0, {}, ""], - "pittsburg-bridges-SPAN": [0.0, {}, ""], - "pittsburg-bridges-T-OR-D": [0.0, {}, ""], - "planning": [0.0, {}, ""], - "post-operative": [0.0, {}, ""], - "seeds": [0.0, {}, ""], - "statlog-australian-credit": [0.0, {}, ""], - "statlog-german-credit": [0.0, {}, ""], - "statlog-heart": [0.0, {}, ""], - "statlog-image": [0.0, {}, ""], - "statlog-vehicle": [0.0, {}, ""], - "synthetic-control": [0.0, {}, ""], - "tic-tac-toe": [0.0, {}, ""], - "vertebral-column-2clases": [0.0, {}, ""], - "wine": [0.0, {}, ""], - "zoo": [0.0, {}, ""] -} +{"balance-scale": [0.0, {}, ""], "balloons": [0.9, {"base_estimator__C": 5, "base_estimator__gamma": 0.14, "base_estimator__kernel": "rbf", "base_estimator__multiclass_strategy": "ovr"}, "v. 0.3.2, Computed on iMac27 on 2022-03-09 at 14:02:32 took 8.661 s"], "breast-cancer-wisc-diag": [0.0, {}, ""], "breast-cancer-wisc-prog": [0.0, {}, ""], "breast-cancer-wisc": [0.0, {}, ""], "breast-cancer": [0.0, {}, ""], "cardiotocography-10clases": [0.0, {}, ""], "cardiotocography-3clases": [0.0, {}, ""], "conn-bench-sonar-mines-rocks": [0.0, {}, ""], "cylinder-bands": [0.0, {}, ""], "dermatology": [0.0, {}, ""], "echocardiogram": [0.0, {}, ""], "fertility": [0.0, {}, ""], "haberman-survival": [0.0, {}, ""], "heart-hungarian": [0.0, {}, ""], "hepatitis": [0.0, {}, ""], "ilpd-indian-liver": [0.0, {}, ""], "ionosphere": [0.0, {}, ""], "iris": [0.0, {}, ""], "led-display": [0.0, {}, ""], "libras": [0.0, {}, ""], "low-res-spect": [0.0, {}, ""], "lymphography": [0.0, {}, ""], "mammographic": [0.0, {}, ""], "molec-biol-promoter": [0.0, {}, ""], "musk-1": [0.0, {}, ""], "oocytes_merluccius_nucleus_4d": [0.0, {}, ""], "oocytes_merluccius_states_2f": [0.0, {}, ""], "oocytes_trisopterus_nucleus_2f": [0.0, {}, ""], "oocytes_trisopterus_states_5b": [0.0, {}, ""], "parkinsons": [0.0, {}, ""], "pima": [0.0, {}, ""], "pittsburg-bridges-MATERIAL": [0.0, {}, ""], "pittsburg-bridges-REL-L": [0.0, {}, ""], "pittsburg-bridges-SPAN": [0.0, {}, ""], "pittsburg-bridges-T-OR-D": [0.0, {}, ""], "planning": [0.0, {}, ""], "post-operative": [0.0, {}, ""], "seeds": [0.0, {}, ""], "statlog-australian-credit": [0.0, {}, ""], "statlog-german-credit": [0.0, {}, ""], "statlog-heart": [0.0, {}, ""], "statlog-image": [0.0, {}, ""], "statlog-vehicle": [0.0, {}, ""], "synthetic-control": [0.0, {}, ""], "tic-tac-toe": [0.0, {}, ""], "vertebral-column-2clases": [0.0, {}, ""], "wine": [0.0, {}, ""], "zoo": [0.0, {}, ""]} \ No newline at end of file diff --git a/src/Experiments.py b/src/Experiments.py index 029ddcb..5ca1c23 100644 --- a/src/Experiments.py +++ b/src/Experiments.py @@ -140,6 +140,7 @@ class Experiment: datasets, hyperparams_dict, hyperparams_file, + grid_paramfile, platform, title, progress_bar=True, @@ -173,6 +174,12 @@ class Experiment: self.hyperparameters_dict = hyper.load( dictionary=dictionary, ) + elif grid_paramfile: + grid_file = os.path.join( + Folders.results, Files.grid_output(score_name, model_name) + ) + with open(grid_file) as f: + self.hyperparameters_dict = json.load(f) else: self.hyperparameters_dict = hyper.fill( dictionary=dictionary, @@ -354,7 +361,7 @@ class GridSearch: for item in self.datasets: data[item] = self.results[item] with open(self.output_file, "w") as f: - json.dump(data, f) + json.dump(data, f, indent=4) def _store_result(self, name, grid, duration): d_message = f"{duration:.3f} s" diff --git a/src/build_grid.py b/src/build_grid.py index c0fc166..a9a2f96 100755 --- a/src/build_grid.py +++ b/src/build_grid.py @@ -75,6 +75,7 @@ results["linear"]["multiclass_strategy"] = ["ovo"] del results["linear"]["gamma"] del results["liblinear"]["gamma"] results["rbf"]["gamma"].append("scale") +results["poly"]["gamma"].append("scale") results["poly"]["multiclass_strategy"].append("ovo") for kernel in kernels: results[kernel]["C"].append(1.0) @@ -101,5 +102,5 @@ for item in results: file_name = Files.grid_input("accuracy", "ODTE") file_output = os.path.join(Folders.results, file_name) with open(file_output, "w") as f: - json.dump(output, f) + json.dump(output, f, indent=4) print(f"Grid values saved to {file_output}") diff --git a/src/main.py b/src/main.py index 2964bcb..9c2e701 100755 --- a/src/main.py +++ b/src/main.py @@ -49,7 +49,20 @@ def parse_arguments(): "-p", "--hyperparameters", type=str, required=False, default="{}" ) ap.add_argument( - "-f", "--paramfile", type=bool, required=False, default=False + "-f", + "--paramfile", + type=bool, + required=False, + default=False, + help="Use best hyperparams file?", + ) + ap.add_argument( + "-g", + "--grid_paramfile", + type=bool, + required=False, + default=False, + help="Use grid searched hyperparams file?", ) ap.add_argument( "--title", type=str, required=True, help="experiment title" @@ -97,6 +110,7 @@ def parse_arguments(): args.quiet, args.hyperparameters, args.paramfile, + args.grid_paramfile, args.report, args.title, args.dataset, @@ -112,11 +126,14 @@ def parse_arguments(): quiet, hyperparameters, paramfile, + grid_paramfile, report, experiment_title, dataset, ) = parse_arguments() report = report or dataset is not None +if grid_paramfile: + paramfile = False job = Experiment( score_name=score, model_name=model, @@ -124,6 +141,7 @@ job = Experiment( datasets=Datasets(dataset=dataset), hyperparams_dict=hyperparameters, hyperparams_file=paramfile, + grid_paramfile=grid_paramfile, progress_bar=not quiet, platform=platform, title=experiment_title,