Add grid file param to cross validation experiment

results/grid_input_accuracy_ODTE.json

@@ -1,26 +1,81 @@
 [
-  {
+    {
-    "base_estimator__C": [1.0],
+        "base_estimator__C": [
-    "base_estimator__kernel": ["linear"],
+            1.0
-    "base_estimator__multiclass_strategy": ["ovo"]
+        ],
-  },
+        "base_estimator__kernel": [
-  {
+            "linear"
-    "base_estimator__C": [
+        ],
-      0.001, 0.0275, 0.05, 0.08, 0.2, 0.25, 0.95, 1.0, 1.75, 7, 10000.0
+        "base_estimator__multiclass_strategy": [
-    ],
+            "ovo"
-    "base_estimator__kernel": ["liblinear"],
+        ]
-    "base_estimator__multiclass_strategy": ["ovr"]
+    },
-  },
+    {
-  {
+        "base_estimator__C": [
-    "base_estimator__C": [0.05, 1.0, 1.05, 2, 2.8, 2.83, 5, 7, 57, 10000.0],
+            0.001,
-    "base_estimator__gamma": [0.001, 0.1, 0.14, 10.0, "auto", "scale"],
+            0.0275,
-    "base_estimator__kernel": ["rbf"],
+            0.05,
-    "base_estimator__multiclass_strategy": ["ovr"]
+            0.08,
-  },
+            0.2,
-  {
+            0.25,
-    "base_estimator__C": [0.05, 0.2, 1.0, 8.25],
+            0.95,
-    "base_estimator__gamma": [0.1],
+            1.0,
-    "base_estimator__kernel": ["poly"],
+            1.75,
-    "base_estimator__multiclass_strategy": ["ovo", "ovr"]
+            7,
-  }
+            10000.0
-]
+        ],
+        "base_estimator__kernel": [
+            "liblinear"
+        ],
+        "base_estimator__multiclass_strategy": [
+            "ovr"
+        ]
+    },
+    {
+        "base_estimator__C": [
+            0.05,
+            1.0,
+            1.05,
+            2,
+            2.8,
+            2.83,
+            5,
+            7,
+            57,
+            10000.0
+        ],
+        "base_estimator__gamma": [
+            0.001,
+            0.1,
+            0.14,
+            10.0,
+            "auto",
+            "scale"
+        ],
+        "base_estimator__kernel": [
+            "rbf"
+        ],
+        "base_estimator__multiclass_strategy": [
+            "ovr"
+        ]
+    },
+    {
+        "base_estimator__C": [
+            0.05,
+            0.2,
+            1.0,
+            8.25
+        ],
+        "base_estimator__gamma": [
+            0.1,
+            "scale"
+        ],
+        "base_estimator__kernel": [
+            "poly"
+        ],
+        "base_estimator__multiclass_strategy": [
+            "ovo",
+            "ovr"
+        ]
+    }
+]

results/grid_output_accuracy_ODTE.json

@@ -1,60 +1 @@
-{
+{"balance-scale": [0.0, {}, ""], "balloons": [0.9, {"base_estimator__C": 5, "base_estimator__gamma": 0.14, "base_estimator__kernel": "rbf", "base_estimator__multiclass_strategy": "ovr"}, "v. 0.3.2, Computed on iMac27 on 2022-03-09 at 14:02:32 took 8.661 s"], "breast-cancer-wisc-diag": [0.0, {}, ""], "breast-cancer-wisc-prog": [0.0, {}, ""], "breast-cancer-wisc": [0.0, {}, ""], "breast-cancer": [0.0, {}, ""], "cardiotocography-10clases": [0.0, {}, ""], "cardiotocography-3clases": [0.0, {}, ""], "conn-bench-sonar-mines-rocks": [0.0, {}, ""], "cylinder-bands": [0.0, {}, ""], "dermatology": [0.0, {}, ""], "echocardiogram": [0.0, {}, ""], "fertility": [0.0, {}, ""], "haberman-survival": [0.0, {}, ""], "heart-hungarian": [0.0, {}, ""], "hepatitis": [0.0, {}, ""], "ilpd-indian-liver": [0.0, {}, ""], "ionosphere": [0.0, {}, ""], "iris": [0.0, {}, ""], "led-display": [0.0, {}, ""], "libras": [0.0, {}, ""], "low-res-spect": [0.0, {}, ""], "lymphography": [0.0, {}, ""], "mammographic": [0.0, {}, ""], "molec-biol-promoter": [0.0, {}, ""], "musk-1": [0.0, {}, ""], "oocytes_merluccius_nucleus_4d": [0.0, {}, ""], "oocytes_merluccius_states_2f": [0.0, {}, ""], "oocytes_trisopterus_nucleus_2f": [0.0, {}, ""], "oocytes_trisopterus_states_5b": [0.0, {}, ""], "parkinsons": [0.0, {}, ""], "pima": [0.0, {}, ""], "pittsburg-bridges-MATERIAL": [0.0, {}, ""], "pittsburg-bridges-REL-L": [0.0, {}, ""], "pittsburg-bridges-SPAN": [0.0, {}, ""], "pittsburg-bridges-T-OR-D": [0.0, {}, ""], "planning": [0.0, {}, ""], "post-operative": [0.0, {}, ""], "seeds": [0.0, {}, ""], "statlog-australian-credit": [0.0, {}, ""], "statlog-german-credit": [0.0, {}, ""], "statlog-heart": [0.0, {}, ""], "statlog-image": [0.0, {}, ""], "statlog-vehicle": [0.0, {}, ""], "synthetic-control": [0.0, {}, ""], "tic-tac-toe": [0.0, {}, ""], "vertebral-column-2clases": [0.0, {}, ""], "wine": [0.0, {}, ""], "zoo": [0.0, {}, ""]}
-  "balance-scale": [0.0, {}, ""],
-  "balloons": [
-    0.9,
-    {
-      "base_estimator__C": 5,
-      "base_estimator__gamma": 0.14,
-      "base_estimator__kernel": "rbf",
-      "base_estimator__multiclass_strategy": "ovr"
-    },
-    "v. 0.3.2, Computed on iMac27 on 2022-03-09 at 13:39:26 took 7.610 s"
-  ],
-  "breast-cancer-wisc-diag": [0.0, {}, ""],
-  "breast-cancer-wisc-prog": [0.0, {}, ""],
-  "breast-cancer-wisc": [0.0, {}, ""],
-  "breast-cancer": [0.0, {}, ""],
-  "cardiotocography-10clases": [0.0, {}, ""],
-  "cardiotocography-3clases": [0.0, {}, ""],
-  "conn-bench-sonar-mines-rocks": [0.0, {}, ""],
-  "cylinder-bands": [0.0, {}, ""],
-  "dermatology": [0.0, {}, ""],
-  "echocardiogram": [0.0, {}, ""],
-  "fertility": [0.0, {}, ""],
-  "haberman-survival": [0.0, {}, ""],
-  "heart-hungarian": [0.0, {}, ""],
-  "hepatitis": [0.0, {}, ""],
-  "ilpd-indian-liver": [0.0, {}, ""],
-  "ionosphere": [0.0, {}, ""],
-  "iris": [0.0, {}, ""],
-  "led-display": [0.0, {}, ""],
-  "libras": [0.0, {}, ""],
-  "low-res-spect": [0.0, {}, ""],
-  "lymphography": [0.0, {}, ""],
-  "mammographic": [0.0, {}, ""],
-  "molec-biol-promoter": [0.0, {}, ""],
-  "musk-1": [0.0, {}, ""],
-  "oocytes_merluccius_nucleus_4d": [0.0, {}, ""],
-  "oocytes_merluccius_states_2f": [0.0, {}, ""],
-  "oocytes_trisopterus_nucleus_2f": [0.0, {}, ""],
-  "oocytes_trisopterus_states_5b": [0.0, {}, ""],
-  "parkinsons": [0.0, {}, ""],
-  "pima": [0.0, {}, ""],
-  "pittsburg-bridges-MATERIAL": [0.0, {}, ""],
-  "pittsburg-bridges-REL-L": [0.0, {}, ""],
-  "pittsburg-bridges-SPAN": [0.0, {}, ""],
-  "pittsburg-bridges-T-OR-D": [0.0, {}, ""],
-  "planning": [0.0, {}, ""],
-  "post-operative": [0.0, {}, ""],
-  "seeds": [0.0, {}, ""],
-  "statlog-australian-credit": [0.0, {}, ""],
-  "statlog-german-credit": [0.0, {}, ""],
-  "statlog-heart": [0.0, {}, ""],
-  "statlog-image": [0.0, {}, ""],
-  "statlog-vehicle": [0.0, {}, ""],
-  "synthetic-control": [0.0, {}, ""],
-  "tic-tac-toe": [0.0, {}, ""],
-  "vertebral-column-2clases": [0.0, {}, ""],
-  "wine": [0.0, {}, ""],
-  "zoo": [0.0, {}, ""]
-}

src/Experiments.py

@@ -140,6 +140,7 @@ class Experiment:
         datasets,
         hyperparams_dict,
         hyperparams_file,
+        grid_paramfile,
         platform,
         title,
         progress_bar=True,
@@ -173,6 +174,12 @@ class Experiment:
             self.hyperparameters_dict = hyper.load(
                 dictionary=dictionary,
             )
+        elif grid_paramfile:
+            grid_file = os.path.join(
+                Folders.results, Files.grid_output(score_name, model_name)
+            )
+            with open(grid_file) as f:
+                self.hyperparameters_dict = json.load(f)
         else:
             self.hyperparameters_dict = hyper.fill(
                 dictionary=dictionary,
@@ -354,7 +361,7 @@ class GridSearch:
         for item in self.datasets:
             data[item] = self.results[item]
         with open(self.output_file, "w") as f:
-            json.dump(data, f)
+            json.dump(data, f, indent=4)
 
     def _store_result(self, name, grid, duration):
         d_message = f"{duration:.3f} s"

src/build_grid.py

@@ -75,6 +75,7 @@ results["linear"]["multiclass_strategy"] = ["ovo"]
 del results["linear"]["gamma"]
 del results["liblinear"]["gamma"]
 results["rbf"]["gamma"].append("scale")
+results["poly"]["gamma"].append("scale")
 results["poly"]["multiclass_strategy"].append("ovo")
 for kernel in kernels:
     results[kernel]["C"].append(1.0)
@@ -101,5 +102,5 @@ for item in results:
 file_name = Files.grid_input("accuracy", "ODTE")
 file_output = os.path.join(Folders.results, file_name)
 with open(file_output, "w") as f:
-    json.dump(output, f)
+    json.dump(output, f, indent=4)
 print(f"Grid values saved to {file_output}")

src/main.py

@@ -49,7 +49,20 @@ def parse_arguments():
         "-p", "--hyperparameters", type=str, required=False, default="{}"
     )
     ap.add_argument(
-        "-f", "--paramfile", type=bool, required=False, default=False
+        "-f",
+        "--paramfile",
+        type=bool,
+        required=False,
+        default=False,
+        help="Use best hyperparams file?",
+    )
+    ap.add_argument(
+        "-g",
+        "--grid_paramfile",
+        type=bool,
+        required=False,
+        default=False,
+        help="Use grid searched hyperparams file?",
     )
     ap.add_argument(
         "--title", type=str, required=True, help="experiment title"
@@ -97,6 +110,7 @@ def parse_arguments():
         args.quiet,
         args.hyperparameters,
         args.paramfile,
+        args.grid_paramfile,
         args.report,
         args.title,
         args.dataset,
@@ -112,11 +126,14 @@ def parse_arguments():
     quiet,
     hyperparameters,
     paramfile,
+    grid_paramfile,
     report,
     experiment_title,
     dataset,
 ) = parse_arguments()
 report = report or dataset is not None
+if grid_paramfile:
+    paramfile = False
 job = Experiment(
     score_name=score,
     model_name=model,
@@ -124,6 +141,7 @@ job = Experiment(
     datasets=Datasets(dataset=dataset),
     hyperparams_dict=hyperparameters,
     hyperparams_file=paramfile,
+    grid_paramfile=grid_paramfile,
     progress_bar=not quiet,
     platform=platform,
     title=experiment_title,