Add # threads as argument to experimentation

experiment.py

@@ -92,6 +92,14 @@ def parse_arguments() -> Tuple[str, str, str, str, str, bool, bool, dict]:
         action="store_true",
         help="Exclude parameters in reports",
     )
+    ap.add_argument(
+        "-t",
+        "--threads",
+        default=-1,
+        type=int,
+        required=False,
+        help="Number of threads to use or -1 for available cores",
+    )
     args = ap.parse_args()
     return (
         args.host,
@@ -103,6 +111,7 @@ def parse_arguments() -> Tuple[str, str, str, str, str, bool, bool, dict]:
         args.standardize,
         args.excludeparams,
         args.kernel,
+        args.threads,
     )
 
 
@@ -116,6 +125,7 @@ def parse_arguments() -> Tuple[str, str, str, str, str, bool, bool, dict]:
     standardize,
     exclude_params,
     kernel,
+    threads,
 ) = parse_arguments()
 
 experiment = Experiment(
@@ -124,6 +134,7 @@ experiment = Experiment(
     host=host,
     set_of_files=set_of_files,
     kernel=kernel,
+    threads=threads,
 )
 if experiment_type[0:6] == "report":
     bd = (

experimentation/Experiments.py

@@ -18,6 +18,7 @@ class Experiment:
         host: str,
         set_of_files: str,
         kernel: str,
+        threads: int = -1,
     ) -> None:
         self._random_state = random_state
         self._set_model(model)
@@ -28,6 +29,7 @@ class Experiment:
         # base class or gridsearch these hyperparams as well
         self._base_params = "any"
         self._kernel = kernel
+        self._threads = threads
 
     def set_base_params(self, base_params: str) -> None:
         self._base_params = base_params
@@ -78,7 +80,7 @@ class Experiment:
             # Also affect subprocesses
             os.environ["PYTHONWARNINGS"] = "ignore"
             results = cross_validate(
-                model, X, y, return_train_score=True, n_jobs=-1
+                model, X, y, return_train_score=True, n_jobs=self._threads
             )
         outcomes = Outcomes(host=self._host, model=self._model_name)
         parameters = json.dumps(parameters, sort_keys=True)
@@ -139,7 +141,7 @@ class Experiment:
                 model,
                 return_train_score=True,
                 param_grid=hyperparameters,
-                n_jobs=-1,
+                n_jobs=self._threads,
                 verbose=1,
             )
             start_time = time.time()

scripts/experiment.pbs

@@ -1,16 +1,14 @@
 #!/bin/bash
 ### Nombre de trabajo
 #PBS -N <experiment>-<data>-<model>-<kernel>
-### Tiempo máximo de ejecución del trabajo
-#PBS -l walltime=96:00:00
 ### Seleccion de cola de trabajos
 #PBS -q workq
 ### mezcla errores con la salida principal
 #PBS -j oe
 ### Recursos
-#PBS -l select=2:ncpus=4:mem=16Gb
+#PBS -l select=1:ncpus=1:mem=16Gb
 ### Esportar variables de entorno
 #PBS -V
 ### Ejecutable con sus parametros
 cd <folder>
-python experiment.py -H galgo -e <experiment> -m <model> -d <data> -S tanveer -k <kernel> -n 1
+python experiment.py -H galgo -e <experiment> -m <model> -d <data> -S tanveer -k <kernel> -n 1 -t 4

scripts/experiment.slurm

@@ -9,4 +9,4 @@
 
 # LOAD MODULES, INSERT CODE, AND RUN YOUR PROGRAMS HERE
 cd <folder>
-python experiment.py -H galgo -e <experiment> -m <model> -d <data> -S tanveer -k <kernel> -n 1
+python experiment.py -H galgo -e <experiment> -m <model> -d <data> -S tanveer -k <kernel> -n 1 -t 4