diff --git a/experiments/.gitignore b/experiments/.gitignore new file mode 100644 index 0000000..1b123c8 --- /dev/null +++ b/experiments/.gitignore @@ -0,0 +1,2 @@ +path.txt +*.sh \ No newline at end of file diff --git a/experiments/build_experiments.py b/experiments/build_experiments.py new file mode 100644 index 0000000..03d8b57 --- /dev/null +++ b/experiments/build_experiments.py @@ -0,0 +1,82 @@ +import argparse +import datetime + + +def parse_arguments(): + ap = argparse.ArgumentParser() + ap.add_argument( + "-s", + "--score", + type=str, + required=False, + default="accuracy", + help="score used in experiment", + ) + ap.add_argument( + "-p", + "--platform", + type=str, + required=True, + choices=["pbs", "slurm"], + help="Platform used to run the experiments {pbs, slurm}", + ) + args = ap.parse_args() + + return ( + args.score, + args.platform, + ) + + +def content(file_name): + with open(file_name) as f: + return f.read().splitlines() + + +def generate_experiment(data, score, platform): + generate_experiment.idx += 1 + path = content("path.txt")[0] + file_name = "experiment.pbs" if platform == "pbs" else "experiment.slurm" + lines = content(file_name) + lines.extend(content("script.txt")) + day = ( + f"{datetime.datetime.now().month:02d}{datetime.datetime.now().day:02d}" + ) + (model, title, parameters) = data.split("&") + strings = [ + ("", day), + ("", path), + ("", score), + ("", model), + ("", title), + ("<parameters>", parameters), + ] + output_file_name = ( + f"{model}_{score}_{platform}_{day}_{generate_experiment.idx}.sh" + ) + data = lines.copy() + for item, value in strings: + data = [line.replace(item, value) for line in data] + with open(output_file_name, "w") as f: + f.write("\n".join(data)) + return output_file_name + + +( + score, + platform, +) = parse_arguments() + +generate_experiment.idx = 0 +output = "" +with open("experiments.txt") as f: + lines = f.read().splitlines() + for line in lines: + if line.startswith("#") or line.strip() == "": + output += line + "\n" + else: + file_name = generate_experiment(line, score, platform) + output += f"#{file_name}, {line}\n" + print(f"Generated {file_name}") +with open("experiments.txt", "w") as f: + print(output, file=f) diff --git a/experiments/experiment.pbs b/experiments/experiment.pbs new file mode 100644 index 0000000..cf14618 --- /dev/null +++ b/experiments/experiment.pbs @@ -0,0 +1,18 @@ +#!/bin/bash +### Nombre de trabajo +#PBS -N <model>-<score>-<date> +### Seleccion de cola de trabajos +#PBS -q workq +### mezcla errores con la salida principal +#PBS -j oe +### Recursos +#PBS -l select=1:ncpus=16:mem=56Gb +#PBS -l place=exclhost +### Esportar variables de entorno +#PBS -V +### Send email on end +#PBS -m e +### Specify mail recipient +#PBS -M ricardo.montanana@alu.uclm.es + +### Ejecutable con sus parametros diff --git a/experiments/experiment.slurm b/experiments/experiment.slurm new file mode 100644 index 0000000..4faa4af --- /dev/null +++ b/experiments/experiment.slurm @@ -0,0 +1,11 @@ +#!/bin/bash + +#SBATCH --partition=qdata +#SBATCH --exclusive +#SBATCH --ntasks=10 # number of processor cores (i.e. tasks) +#SBATCH --nodes=1 # number of nodes +#SBATCH --job-name="<model>-<score>-<date>" +#SBATCH --mail-user=ricardo.montanana@alu.uclm.es # email address +#SBATCH --mail-type=END +#SBATCH -w nodo-7-33 +# LOAD MODULES, INSERT CODE, AND RUN YOUR PROGRAMS HERE \ No newline at end of file diff --git a/experiments/experiments.txt b/experiments/experiments.txt new file mode 100644 index 0000000..d07470d --- /dev/null +++ b/experiments/experiments.txt @@ -0,0 +1,13 @@ +# Experiments to do: +# Bagging +# Random Forest +# Odte +# Random Forest + +#ODTE_accuracy_slurm_0202_1.sh, ODTE&ODTE Random Forest rbf-best&{"n_jobs": 10, "n_estimators": 100, "max_features": "sqrt", "max_samples": 0.75, "be_hyperparams": "{\"kernel\": \"rbf\", \"splitter\": \"best\", \"max_features\": \"sqrt\"}"} +#ODTE_accuracy_slurm_0202_2.sh, ODTE&ODTE Random Forest linear-best&{"n_jobs": 10, "n_estimators": 100, "max_features": "sqrt", "max_samples": 0.75, "be_hyperparams": "{\"kernel\": \"linear\", \"splitter\": \"best\", \"max_features\": \"sqrt\"}"} +#ODTE_accuracy_slurm_0202_3.sh, ODTE&ODTE Random Forest rbf-random&{"n_jobs": 10, "n_estimators": 100, "max_features": "sqrt", "max_samples": 0.75, "be_hyperparams": "{\"kernel\": \"rbf\", \"splitter\": \"random\", \"max_features\": \"sqrt\"}"} +#ODTE_accuracy_slurm_0202_4.sh, ODTE&ODTE Random Forest linear-random&{"n_jobs": 10, "n_estimators": 100, "max_features": "sqrt", "max_samples": 0.75, "be_hyperparams": "{\"kernel\": \"linear\", \"splitter\": \"random\", \"max_features\": \"sqrt\"}"} +#ODTE_accuracy_slurm_0202_5.sh, ODTE&ODTE Random Forest rbf-trandom&{"n_jobs": 10, "n_estimators": 100, "max_features": "sqrt", "max_samples": 0.75, "be_hyperparams": "{\"kernel\": \"rbf\", \"splitter\": \"trandom\", \"max_features\": \"sqrt\"}"} +#ODTE_accuracy_slurm_0202_6.sh, ODTE&ODTE Random Forest linear-trandom&{"n_jobs": 10, "n_estimators": 100, "max_features": "sqrt", "max_samples": 0.75, "be_hyperparams": "{\"kernel\": \"linear\", \"splitter\": \"trandom\", \"max_features\": \"sqrt\"}"} + diff --git a/experiments/hyperparams.xlsx b/experiments/hyperparams.xlsx index 809c79b..d2c11b4 100644 Binary files a/experiments/hyperparams.xlsx and b/experiments/hyperparams.xlsx differ diff --git a/experiments/script.txt b/experiments/script.txt new file mode 100644 index 0000000..2e65217 --- /dev/null +++ b/experiments/script.txt @@ -0,0 +1,2 @@ +cd <folder> +python src/main.py -m <model> -s <score> --title "<title>" -p '<parameters>' \ No newline at end of file diff --git a/src/experiments.txt b/src/experiments.txt deleted file mode 100644 index 5e13183..0000000 --- a/src/experiments.txt +++ /dev/null @@ -1,5 +0,0 @@ -Experiments to do: -Bagging - -Odte -Random Forest