benchmark/src/main

#!/usr/bin/env python
import os
import argparse
from Experiments import Experiment, Datasets
from Results import Report
from Utils import EnvDefault

"""Do experiment and build result file, optionally print report with results
"""


def parse_arguments():
    ap = argparse.ArgumentParser()
    ap.add_argument(
        "-s",
        "--score",
        action=EnvDefault,
        envvar="score",
        type=str,
        required=True,
        help="score name {accuracy, f1_macro, ...}",
    )
    ap.add_argument(
        "-P",
        "--platform",
        action=EnvDefault,
        envvar="platform",
        type=str,
        required=True,
        help="Platform where the test is run",
    )
    ap.add_argument(
        "-m",
        "--model",
        type=str,
        required=True,
        help="model name",
    )
    ap.add_argument(
        "-n",
        "--n_folds",
        action=EnvDefault,
        envvar="n_folds",
        type=int,
        required=True,
        help="number of folds",
    )
    ap.add_argument(
        "-p", "--hyperparameters", type=str, required=False, default="{}"
    )
    ap.add_argument(
        "-f",
        "--paramfile",
        type=bool,
        required=False,
        default=False,
        help="Use best hyperparams file?",
    )
    ap.add_argument(
        "-g",
        "--grid_paramfile",
        type=bool,
        required=False,
        default=False,
        help="Use grid searched hyperparams file?",
    )
    ap.add_argument(
        "--title", type=str, required=True, help="experiment title"
    )
    ap.add_argument(
        "-q",
        "--quiet",
        type=bool,
        default=False,
        required=False,
        help="Wether to show progress bar or not",
    )
    ap.add_argument(
        "-r",
        "--report",
        type=bool,
        default=False,
        required=False,
        help="Report results",
    )
    ap.add_argument(
        "-t",
        "--stratified",
        action=EnvDefault,
        envvar="stratified",
        type=str,
        required=True,
        help="Stratified",
    )
    ap.add_argument(
        "-d",
        "--dataset",
        type=str,
        required=False,
        default=None,
        help="Experiment with only this dataset",
    )
    args = ap.parse_args()
    return (
        args.stratified,
        args.score,
        args.model,
        args.n_folds,
        args.platform,
        args.quiet,
        args.hyperparameters,
        args.paramfile,
        args.grid_paramfile,
        args.report,
        args.title,
        args.dataset,
    )


(
    stratified,
    score,
    model,
    folds,
    platform,
    quiet,
    hyperparameters,
    paramfile,
    grid_paramfile,
    report,
    experiment_title,
    dataset,
) = parse_arguments()
report = report or dataset is not None
if grid_paramfile:
    paramfile = False
job = Experiment(
    score_name=score,
    model_name=model,
    stratified=stratified,
    datasets=Datasets(dataset=dataset),
    hyperparams_dict=hyperparameters,
    hyperparams_file=paramfile,
    grid_paramfile=grid_paramfile,
    progress_bar=not quiet,
    platform=platform,
    title=experiment_title,
    folds=folds,
)
job.do_experiment()
if report:
    result_file = job.get_output_file()
    report = Report(result_file)
    report.report()
if dataset is not None:
    print(f"Partial result file removed: {result_file}")
    os.remove(result_file)