From 3e72ba613d084e54dae4ef1523b9b3c552aeef84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= <rmontanana@gmail.com>
Date: Wed, 9 Mar 2022 17:11:49 +0100
Subject: [PATCH] Add build grid experiments

---
 experiments/build_grid_experiments.py | 81 +++++++++++++++++++++++++++
 experiments/grid/.gitignore           |  2 +
 experiments/script_grid.txt           |  2 +
 src/Experiments.py                    |  1 +
 4 files changed, 86 insertions(+)
 create mode 100755 experiments/build_grid_experiments.py
 create mode 100644 experiments/grid/.gitignore
 create mode 100644 experiments/script_grid.txt

diff --git a/experiments/build_grid_experiments.py b/experiments/build_grid_experiments.py
new file mode 100755
index 0000000..8c73287
--- /dev/null
+++ b/experiments/build_grid_experiments.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+import os
+import argparse
+import datetime
+
+
+def parse_arguments():
+    ap = argparse.ArgumentParser()
+    ap.add_argument(
+        "-s",
+        "--score",
+        type=str,
+        required=False,
+        default="accuracy",
+        help="score used in gridsearch experiment",
+    )
+    ap.add_argument(
+        "-p",
+        "--platform",
+        type=str,
+        required=True,
+        choices=["pbs", "slurm"],
+        help="Platform used to run the gridsearch experiments {pbs, slurm}",
+    )
+    ap.add_argument(
+        "-m",
+        "--model",
+        type=str,
+        required=True,
+        help="model to use",
+    )
+    args = ap.parse_args()
+
+    return (args.score, args.platform, args.model)
+
+
+def content(file_name):
+    with open(file_name) as f:
+        return f.read().splitlines()
+
+
+def generate_experiment(dataset, model, score, platform):
+    path = content("path.txt")[0]
+    file_name = "experiment.pbs" if platform == "pbs" else "experiment.slurm"
+    lines = content(file_name)
+    lines.extend(content("script_grid.txt"))
+    day = (
+        f"{datetime.datetime.now().month:02d}{datetime.datetime.now().day:02d}"
+    )
+    file_name = f"grid_{model}_{platform}_{day}_{dataset}"
+    output_file_name = os.path.join("grid", f"{file_name}.sh")
+    strings = [
+        ("<date>", day),
+        ("<folder>", path),
+        ("<score>", score),
+        ("<model>", model),
+        ("<file_name>", file_name),
+        ("<dataset>", dataset),
+    ]
+    data = lines.copy()
+    for item, value in strings:
+        data = [line.replace(item, value) for line in data]
+    with open(output_file_name, "w") as f:
+        f.write("\n".join(data))
+    return output_file_name
+
+
+(
+    score,
+    platform,
+    model,
+) = parse_arguments()
+
+with open(os.path.join("..", "data", "all.txt")) as f:
+    lines = f.read().splitlines()
+    for dataset in lines:
+        if dataset.startswith("#") or dataset.strip() == "":
+            continue
+        else:
+            file_name = generate_experiment(dataset, model, score, platform)
+            print(f"Generated {file_name}")
diff --git a/experiments/grid/.gitignore b/experiments/grid/.gitignore
new file mode 100644
index 0000000..c96a04f
--- /dev/null
+++ b/experiments/grid/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
\ No newline at end of file
diff --git a/experiments/script_grid.txt b/experiments/script_grid.txt
new file mode 100644
index 0000000..0600e7f
--- /dev/null
+++ b/experiments/script_grid.txt
@@ -0,0 +1,2 @@
+cd <folder>
+python src/grid.py -q 1 -m <model> -s <score> -d <dataset>
\ No newline at end of file
diff --git a/src/Experiments.py b/src/Experiments.py
index 5ca1c23..9c05b57 100644
--- a/src/Experiments.py
+++ b/src/Experiments.py
@@ -377,6 +377,7 @@ class GridSearch:
         score = grid.best_score_
         hyperparameters = grid.best_params_
         self.results[name] = [score, hyperparameters, message]
+        print(f"{name:30s} {score} {hyperparameters} {message}")
 
     def do_gridsearch(self):
         now = time.time()