Add experiment script generation

experiments/.gitignore

@@ -0,0 +1,2 @@
+path.txt
+*.sh

experiments/build_experiments.py

@@ -0,0 +1,82 @@
+import argparse
+import datetime
+
+
+def parse_arguments():
+    ap = argparse.ArgumentParser()
+    ap.add_argument(
+        "-s",
+        "--score",
+        type=str,
+        required=False,
+        default="accuracy",
+        help="score used in experiment",
+    )
+    ap.add_argument(
+        "-p",
+        "--platform",
+        type=str,
+        required=True,
+        choices=["pbs", "slurm"],
+        help="Platform used to run the experiments {pbs, slurm}",
+    )
+    args = ap.parse_args()
+
+    return (
+        args.score,
+        args.platform,
+    )
+
+
+def content(file_name):
+    with open(file_name) as f:
+        return f.read().splitlines()
+
+
+def generate_experiment(data, score, platform):
+    generate_experiment.idx += 1
+    path = content("path.txt")[0]
+    file_name = "experiment.pbs" if platform == "pbs" else "experiment.slurm"
+    lines = content(file_name)
+    lines.extend(content("script.txt"))
+    day = (
+        f"{datetime.datetime.now().month:02d}{datetime.datetime.now().day:02d}"
+    )
+    (model, title, parameters) = data.split("&")
+    strings = [
+        ("<date>", day),
+        ("<folder>", path),
+        ("<score>", score),
+        ("<model>", model),
+        ("<title>", title),
+        ("<parameters>", parameters),
+    ]
+    output_file_name = (
+        f"{model}_{score}_{platform}_{day}_{generate_experiment.idx}.sh"
+    )
+    data = lines.copy()
+    for item, value in strings:
+        data = [line.replace(item, value) for line in data]
+    with open(output_file_name, "w") as f:
+        f.write("\n".join(data))
+    return output_file_name
+
+
+(
+    score,
+    platform,
+) = parse_arguments()
+
+generate_experiment.idx = 0
+output = ""
+with open("experiments.txt") as f:
+    lines = f.read().splitlines()
+    for line in lines:
+        if line.startswith("#") or line.strip() == "":
+            output += line + "\n"
+        else:
+            file_name = generate_experiment(line, score, platform)
+            output += f"#{file_name}, {line}\n"
+            print(f"Generated {file_name}")
+with open("experiments.txt", "w") as f:
+    print(output, file=f)

experiments/experiment.pbs

@@ -0,0 +1,18 @@
+#!/bin/bash
+### Nombre de trabajo
+#PBS -N <model>-<score>-<date>
+### Seleccion de cola de trabajos
+#PBS -q workq
+### mezcla errores con la salida principal
+#PBS -j oe
+### Recursos
+#PBS -l select=1:ncpus=16:mem=56Gb
+#PBS -l place=exclhost
+### Esportar variables de entorno
+#PBS -V
+### Send email on end
+#PBS -m e
+### Specify mail recipient
+#PBS -M ricardo.montanana@alu.uclm.es
+
+### Ejecutable con sus parametros

experiments/experiment.slurm

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+#SBATCH --partition=qdata
+#SBATCH --exclusive
+#SBATCH --ntasks=10   # number of processor cores (i.e. tasks)
+#SBATCH --nodes=1   # number of nodes
+#SBATCH --job-name="<model>-<score>-<date>"
+#SBATCH --mail-user=ricardo.montanana@alu.uclm.es   # email address
+#SBATCH --mail-type=END
+#SBATCH -w nodo-7-33
+# LOAD MODULES, INSERT CODE, AND RUN YOUR PROGRAMS HERE

experiments/experiments.txt

@@ -0,0 +1,13 @@
+# Experiments to do:
+# Bagging
+# Random Forest
+# Odte
+# Random Forest
+
+#ODTE_accuracy_slurm_0202_1.sh, ODTE&ODTE Random Forest rbf-best&{"n_jobs": 10, "n_estimators": 100, "max_features": "sqrt", "max_samples": 0.75, "be_hyperparams":  "{\"kernel\": \"rbf\", \"splitter\": \"best\", \"max_features\": \"sqrt\"}"}
+#ODTE_accuracy_slurm_0202_2.sh, ODTE&ODTE Random Forest linear-best&{"n_jobs": 10, "n_estimators": 100, "max_features": "sqrt", "max_samples": 0.75, "be_hyperparams":  "{\"kernel\": \"linear\", \"splitter\": \"best\", \"max_features\": \"sqrt\"}"}
+#ODTE_accuracy_slurm_0202_3.sh, ODTE&ODTE Random Forest rbf-random&{"n_jobs": 10, "n_estimators": 100, "max_features": "sqrt", "max_samples": 0.75, "be_hyperparams":  "{\"kernel\": \"rbf\", \"splitter\": \"random\", \"max_features\": \"sqrt\"}"}
+#ODTE_accuracy_slurm_0202_4.sh, ODTE&ODTE Random Forest linear-random&{"n_jobs": 10, "n_estimators": 100, "max_features": "sqrt", "max_samples": 0.75, "be_hyperparams":  "{\"kernel\": \"linear\", \"splitter\": \"random\", \"max_features\": \"sqrt\"}"}
+#ODTE_accuracy_slurm_0202_5.sh, ODTE&ODTE Random Forest rbf-trandom&{"n_jobs": 10, "n_estimators": 100, "max_features": "sqrt", "max_samples": 0.75, "be_hyperparams":  "{\"kernel\": \"rbf\", \"splitter\": \"trandom\", \"max_features\": \"sqrt\"}"}
+#ODTE_accuracy_slurm_0202_6.sh, ODTE&ODTE Random Forest linear-trandom&{"n_jobs": 10, "n_estimators": 100, "max_features": "sqrt", "max_samples": 0.75, "be_hyperparams":  "{\"kernel\": \"linear\", \"splitter\": \"trandom\", \"max_features\": \"sqrt\"}"}
+

experiments/script.txt

@@ -0,0 +1,2 @@
+cd <folder>
+python src/main.py -m <model> -s <score> --title "<title>" -p '<parameters>'

src/experiments.txt

@@ -1,5 +0,0 @@
-Experiments to do:
-Bagging
-
-Odte
-Random Forest