From 90c895b616ee013ebd76562b96cbe20ce7f57856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Wed, 4 May 2022 01:43:36 +0200 Subject: [PATCH] Begin refactor arguments --- benchmark/Arguments.py | 147 +++++++++++++++++++++++++++++++++ benchmark/Experiments.py | 3 +- benchmark/Utils.py | 28 ------- benchmark/__init__.py | 2 +- benchmark/scripts/be_benchmark | 46 ++--------- benchmark/scripts/be_best | 26 ++---- benchmark/scripts/be_report | 110 +++++------------------- benchmark/tests/Util_test.py | 5 ++ 8 files changed, 191 insertions(+), 176 deletions(-) create mode 100644 benchmark/Arguments.py diff --git a/benchmark/Arguments.py b/benchmark/Arguments.py new file mode 100644 index 0000000..3c3e91b --- /dev/null +++ b/benchmark/Arguments.py @@ -0,0 +1,147 @@ +import argparse +from .Experiments import Models +from .Utils import Files + +ALL_METRICS = ( + "accuracy", + "f1-macro", + "f1-micro", + "f1-weighted", + "roc-auc-ovr", +) + + +class EnvData: + @staticmethod + def load(): + args = {} + with open(Files.dot_env) as f: + for line in f.read().splitlines(): + if line == "" or line.startswith("#"): + continue + key, value = line.split("=") + args[key] = value + return args + + +class EnvDefault(argparse.Action): + # Thanks to https://stackoverflow.com/users/445507/russell-heilling + def __init__(self, envvar, required=True, default=None, **kwargs): + self._args = EnvData.load() + default = self._args[envvar] + required = False + super(EnvDefault, self).__init__( + default=default, required=required, **kwargs + ) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, values) + + +class Arguments: + def __init__(self): + self.ap = argparse.ArgumentParser() + models_data = Models.define_models(random_state=0) + models = "{" + ", ".join(models_data) + "}" + self.parameters = { + "best": [ + ("-b", "--best"), + { + "type": str, + "required": False, + "help": "best results of models", + }, + ], + "color": [], + "compare": [ + ("-c", "--compare"), + { + "type": bool, + "required": False, + "help": "Compare accuracy with best results", + }, + ], + "dataset": [], + "excel": [ + ("-x", "--excel"), + { + "type": bool, + "required": False, + "default": False, + "help": "Generate Excel File", + }, + ], + "file": [ + ("-f", "--file"), + {"type": str, "required": False, "help": "Result file"}, + ], + "grid": [ + ("-g", "--grid"), + { + "type": str, + "required": False, + "help": "grid results of model", + }, + ], + "grid_paramfile": [], + "hidden": [], + "hyperparameters": [], + "key": [], + "lose": [], + "model": [ + ("-m", "--model"), + { + "type": str, + "required": True, + "choices": list(models_data), + "help": f"model name: {models}", + }, + ], + "model1": [], + "model2": [], + "nan": [], + "number": [], + "n_folds": [], + "paramfile": [], + "platform": [], + "quiet": [], + "report": [], + "score": [ + ("-s", "--score"), + { + "action": EnvDefault, + "envvar": "score", + "type": str, + "required": True, + "choices": ALL_METRICS, + }, + ], + "sql": [ + ("-q", "--sql"), + {"type": bool, "required": False, "help": "Generate SQL File"}, + ], + "stratified": [], + "tex_output": [ + ("-t", "--tex-output"), + { + "type": bool, + "required": False, + "default": False, + "help": "Generate Tex file with the table", + }, + ], + "title": [], + "win": [], + } + + def xset(self, *arg_name, **kwargs): + print("parameters", arg_name[0]) + names, default = self.parameters[arg_name[0]] + self.ap.add_argument( + *names, + **{**default, **kwargs}, + ) + return self + + def parse(self): + return self.ap.parse_args() diff --git a/benchmark/Experiments.py b/benchmark/Experiments.py index 90d3f30..2247602 100644 --- a/benchmark/Experiments.py +++ b/benchmark/Experiments.py @@ -13,7 +13,8 @@ from sklearn.model_selection import ( GridSearchCV, cross_validate, ) -from .Utils import Folders, Files, EnvData +from .Utils import Folders, Files +from .Arguments import EnvData from .Models import Models diff --git a/benchmark/Utils.py b/benchmark/Utils.py index b67f8e9..3d202ba 100644 --- a/benchmark/Utils.py +++ b/benchmark/Utils.py @@ -1,6 +1,5 @@ import os import subprocess -import argparse BEST_ACCURACY_STREE = 40.282203 ALL_METRICS = ( @@ -132,33 +131,6 @@ class Symbols: better_best = black_star -class EnvData: - @staticmethod - def load(): - args = {} - with open(Files.dot_env) as f: - for line in f.read().splitlines(): - if line == "" or line.startswith("#"): - continue - key, value = line.split("=") - args[key] = value - return args - - -class EnvDefault(argparse.Action): - # Thanks to https://stackoverflow.com/users/445507/russell-heilling - def __init__(self, envvar, required=True, default=None, **kwargs): - self._args = EnvData.load() - default = self._args[envvar] - required = False - super(EnvDefault, self).__init__( - default=default, required=required, **kwargs - ) - - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, values) - - class TextColor: BLUE = "\033[94m" CYAN = "\033[96m" diff --git a/benchmark/__init__.py b/benchmark/__init__.py index 7c1740b..6d699da 100644 --- a/benchmark/__init__.py +++ b/benchmark/__init__.py @@ -1,6 +1,6 @@ from .Experiments import Experiment, Datasets, DatasetsSurcov, DatasetsTanveer from .Results import Report, Summary -from .Utils import EnvDefault +from .Arguments import EnvDefault __author__ = "Ricardo Montañana Gómez" __copyright__ = "Copyright 2020-2022, Ricardo Montañana Gómez" diff --git a/benchmark/scripts/be_benchmark b/benchmark/scripts/be_benchmark index 12b0fd6..219c7fa 100755 --- a/benchmark/scripts/be_benchmark +++ b/benchmark/scripts/be_benchmark @@ -1,47 +1,19 @@ #!/usr/bin/env python from benchmark.Results import Benchmark -from benchmark.Utils import ALL_METRICS, Files, EnvDefault -import argparse +from benchmark.Utils import Files +from benchmark.Arguments import Arguments -def parse_arguments(): - ap = argparse.ArgumentParser() - ap.add_argument( - "-s", - "--score", - action=EnvDefault, - envvar="score", - type=str, - required=True, - choices=ALL_METRICS, - help="score name {accuracy, f1_macro, ...}", - ) - ap.add_argument( - "-x", - "--excel", - type=bool, - required=False, - help="Generate Excel File", - ) - ap.add_argument( - "-t", - "--tex-output", - type=bool, - required=False, - default=False, - ) - args = ap.parse_args() - return (args.score, args.excel, args.tex_output) - - -(score, excel, tex_output) = parse_arguments() -benchmark = Benchmark(score=score, visualize=True) +arguments = Arguments() +arguments.xset("score").xset("excel").xset("tex_output") +ar = arguments.parse() +benchmark = Benchmark(score=ar.score, visualize=True) benchmark.compile_results() benchmark.save_results() -benchmark.report(tex_output) +benchmark.report(ar.tex_output) benchmark.exreport() -if excel: +if ar.excel: benchmark.excel() Files.open(benchmark.get_excel_file_name()) -if tex_output: +if ar.tex_output: print(f"File {benchmark.get_tex_file()} generated") diff --git a/benchmark/scripts/be_best b/benchmark/scripts/be_best index 2e32aed..283f182 100755 --- a/benchmark/scripts/be_best +++ b/benchmark/scripts/be_best @@ -1,29 +1,17 @@ #!/usr/bin/env python -import argparse import json from benchmark.Results import Summary -from benchmark.Utils import EnvDefault, ALL_METRICS +from benchmark.Utils import ALL_METRICS, Arguments -def parse_arguments(): - ap = argparse.ArgumentParser() - ap.add_argument( - "-s", - "--score", - type=str, - action=EnvDefault, - envvar="score", - required=True, - choices=ALL_METRICS, - help="score name {accuracy, f1-macro, f1-weighted, roc-auc-ovr}", - ) - args = ap.parse_args() - return (args.score,) +arguments = Arguments() +metrics = list(ALL_METRICS) +metrics.append("all") +arguments.xset("score", choices=metrics) +args = arguments.parse() -(score,) = parse_arguments() - -metrics = ALL_METRICS if score == "all" else [score] +metrics = ALL_METRICS if args.score == "all" else [args.score] summary = Summary() summary.acquire() diff --git a/benchmark/scripts/be_report b/benchmark/scripts/be_report index a523472..dda69c6 100755 --- a/benchmark/scripts/be_report +++ b/benchmark/scripts/be_report @@ -1,9 +1,12 @@ #!/usr/bin/env python -import argparse import numpy as np from benchmark.Experiments import Datasets from benchmark.Results import Report, Excel, SQL, ReportBest -from benchmark.Utils import ALL_METRICS, Files, TextColor, EnvDefault +from benchmark.Utils import ( + Files, + TextColor, +) +from benchmark.Arguments import Arguments """Build report on screen of a result file, optionally generate excel and sql @@ -12,83 +15,6 @@ If no argument is set, displays the datasets and its characteristics """ -def parse_arguments(): - ap = argparse.ArgumentParser() - ap.add_argument( - "-f", - "--file", - type=str, - required=False, - help="Result file", - ) - ap.add_argument( - "-x", - "--excel", - type=bool, - required=False, - help="Generate Excel file", - ) - ap.add_argument( - "-q", - "--sql", - type=bool, - required=False, - help="Generate sql file", - ) - ap.add_argument( - "-c", - "--compare", - type=bool, - required=False, - help="Compare accuracy with best results", - ) - ap.add_argument( - "-b", - "--best", - type=str, - required=False, - help="best results of models", - ) - ap.add_argument( - "-g", - "--grid", - type=str, - required=False, - help="grid results of model", - ) - ap.add_argument( - "-m", - "--model", - action=EnvDefault, - envvar="model", - type=str, - required=True, - help="model name", - ) - ap.add_argument( - "-s", - "--score", - action=EnvDefault, - envvar="score", - type=str, - required=True, - choices=ALL_METRICS, - help="score name {accuracy, f1_macro, ...}", - ) - args = ap.parse_args() - - return ( - args.file, - args.excel, - args.sql, - args.compare, - args.best, - args.grid, - args.score, - args.model, - ) - - def default_report(): sets = Datasets() color_line = TextColor.LINE1 @@ -116,22 +42,26 @@ def default_report(): if __name__ == "__main__": - (file, excel, sql, compare, best, grid, score, model) = parse_arguments() - if grid: - best = False - if file is None and best is None: + arguments = Arguments() + arguments.xset("file").xset("excel").xset("sql").xset("compare") + arguments.xset("best").xset("grid").xset("model").xset("score") + args = arguments.parse() + + if args.grid: + args.best = False + if args.file is None and args.best is None: default_report() else: - if best is not None or grid is not None: - report = ReportBest(score, model, best, grid) + if args.best is not None or args.grid is not None: + report = ReportBest(args.score, args.model, args.best, args.grid) report.report() else: - report = Report(file, compare) + report = Report(args.file, args.compare) report.report() - if excel: - excel = Excel(file, compare) + if args.excel: + excel = Excel(args.file, args.compare) excel.report() Files.open(excel.get_file_name()) - if sql: - sql = SQL(file) + if args.sql: + sql = SQL(args.file) sql.report() diff --git a/benchmark/tests/Util_test.py b/benchmark/tests/Util_test.py index b3d87f1..e26a101 100644 --- a/benchmark/tests/Util_test.py +++ b/benchmark/tests/Util_test.py @@ -266,3 +266,8 @@ class UtilTest(TestBase): self.assertEqual(TextColor.ENDC, "\033[0m") self.assertEqual(TextColor.BOLD, "\033[1m") self.assertEqual(TextColor.UNDERLINE, "\033[4m") + + def test_Arguments(self): + arguments = Arguments() + arg_list = ["score", "excel", "tex_output"] + arguments.set_arguments(arg_list)