From de903749601cb4c258bd42988ac774891a34646f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Sat, 9 Apr 2022 19:12:47 +0200 Subject: [PATCH] Begin refactor main scripts --- src/Results.py | 53 ++++++++++++++++++++++++++++++ src/list | 10 +++--- src/main | 77 +++++++++++++++++++++---------------------- src/pair_check | 88 +++++++++++++++++++++++++------------------------- src/report | 37 ++++++++++----------- src/summary | 45 +++++++++++++------------- 6 files changed, 182 insertions(+), 128 deletions(-) diff --git a/src/Results.py b/src/Results.py index e3bc18e..ce63f78 100644 --- a/src/Results.py +++ b/src/Results.py @@ -1159,3 +1159,56 @@ class Summary: input_data=self.best_results(score=score, n=n), sort_key="metric", ) + + +class PairCheck: + def __init__(self, score, model_a, model_b, winners=False, loosers=False): + self.score = score + self.model_a = model_a + self.model_b = model_b + self.winners = winners + self.loosers = loosers + self.winners_data = [] + self.loosers_data = [] + self.tie_data = [] + + def compute(self): + summary = Summary() + summary.acquire() + best_a = summary.best_result( + criterion="model", value=self.model_a, score=self.score + ) + best_b = summary.best_result( + criterion="model", value=self.model_b, score=self.score + ) + report_a = StubReport(os.path.join(Folders.results, best_a["file"])) + report_a.report() + report_b = StubReport(os.path.join(Folders.results, best_b["file"])) + report_b.report() + for result_a, result_b in zip(report_a.lines, report_b.lines): + result = result_a["score"] - result_b["score"] + if result > 0: + self.winners_data.append(result_a["dataset"]) + elif result < 0: + self.loosers_data.append(result_a["dataset"]) + else: + self.tie_data.append(result_a["dataset"]) + + def print(self): + print(f"{'Model':<20} {'File':<70} {'Score':<10} Win Tie Loose") + print("=" * 20 + " " + "=" * 70 + " " + "=" * 10 + " === === =====") + print( + f"{self.model_a:<20} {self.best_a['file']:<70} {report_1.score:10.5f}" + ) + print( + f"{model2:<20} {best_2['file']:<70} " + f"{report_2.score:10.5f} " + f"{TextColor.GREEN}{win:3d} {TextColor.YELLOW}{tie:3d} " + f"{TextColor.RED}{loose:5d}" + ) + if win_results: + print(TextColor.GREEN + "Winners:") + print(winners) + if loose_results: + print(TextColor.RED + "Loosers:") + print(loosers) diff --git a/src/list b/src/list index 6bcfacb..6ca7efb 100755 --- a/src/list +++ b/src/list @@ -64,8 +64,8 @@ def parse_arguments(): ) -(excel, score, model, key, number, hidden) = parse_arguments() - -data = Summary(hidden=hidden) -data.acquire() -data.list_results(score=score, model=model, sort_key=key, number=number) +if __name__ == "__main__": + (excel, score, model, key, number, hidden) = parse_arguments() + data = Summary(hidden=hidden) + data.acquire() + data.list_results(score=score, model=model, sort_key=key, number=number) diff --git a/src/main b/src/main index 9c2e701..a48a070 100755 --- a/src/main +++ b/src/main @@ -117,41 +117,42 @@ def parse_arguments(): ) -( - stratified, - score, - model, - folds, - platform, - quiet, - hyperparameters, - paramfile, - grid_paramfile, - report, - experiment_title, - dataset, -) = parse_arguments() -report = report or dataset is not None -if grid_paramfile: - paramfile = False -job = Experiment( - score_name=score, - model_name=model, - stratified=stratified, - datasets=Datasets(dataset=dataset), - hyperparams_dict=hyperparameters, - hyperparams_file=paramfile, - grid_paramfile=grid_paramfile, - progress_bar=not quiet, - platform=platform, - title=experiment_title, - folds=folds, -) -job.do_experiment() -if report: - result_file = job.get_output_file() - report = Report(result_file) - report.report() -if dataset is not None: - print(f"Partial result file removed: {result_file}") - os.remove(result_file) +if __name__ == "__main__": + ( + stratified, + score, + model, + folds, + platform, + quiet, + hyperparameters, + paramfile, + grid_paramfile, + report, + experiment_title, + dataset, + ) = parse_arguments() + report = report or dataset is not None + if grid_paramfile: + paramfile = False + job = Experiment( + score_name=score, + model_name=model, + stratified=stratified, + datasets=Datasets(dataset=dataset), + hyperparams_dict=hyperparameters, + hyperparams_file=paramfile, + grid_paramfile=grid_paramfile, + progress_bar=not quiet, + platform=platform, + title=experiment_title, + folds=folds, + ) + job.do_experiment() + if report: + result_file = job.get_output_file() + report = Report(result_file) + report.report() + if dataset is not None: + print(f"Partial result file removed: {result_file}") + os.remove(result_file) diff --git a/src/pair_check b/src/pair_check index 5557dde..a06b360 100755 --- a/src/pair_check +++ b/src/pair_check @@ -59,47 +59,47 @@ def parse_arguments(): ) -( - score, - model1, - model2, - win_results, - loose_results, -) = parse_arguments() - -summary = Summary() -summary.acquire() -win = tie = loose = 0 -winners = [] -loosers = [] -best_1 = summary.best_result(criterion="model", value=model1, score=score) -best_2 = summary.best_result(criterion="model", value=model2, score=score) -report_1 = StubReport(os.path.join(Folders.results, best_1["file"])) -report_1.report() -report_2 = StubReport(os.path.join(Folders.results, best_2["file"])) -report_2.report() -for result1, result2 in zip(report_1.lines, report_2.lines): - result = result1["score"] - result2["score"] - if result > 0: - win += 1 - winners.append(result1["dataset"]) - elif result < 0: - loose += 1 - loosers.append(result1["dataset"]) - else: - tie += 1 -print(f"{'Model':<20} {'File':<70} {'Score':<10} Win Tie Loose") -print("=" * 20 + " " + "=" * 70 + " " + "=" * 10 + " === === =====") -print(f"{model1:<20} {best_1['file']:<70} {report_1.score:10.5f}") -print( - f"{model2:<20} {best_2['file']:<70} " - f"{report_2.score:10.5f} " - f"{TextColor.GREEN}{win:3d} {TextColor.YELLOW}{tie:3d} " - f"{TextColor.RED}{loose:5d}" -) -if win_results: - print(TextColor.GREEN + "Winners:") - print(winners) -if loose_results: - print(TextColor.RED + "Loosers:") - print(loosers) +if __name__ == "__main__": + ( + score, + model1, + model2, + win_results, + loose_results, + ) = parse_arguments() + summary = Summary() + summary.acquire() + win = tie = loose = 0 + winners = [] + loosers = [] + best_1 = summary.best_result(criterion="model", value=model1, score=score) + best_2 = summary.best_result(criterion="model", value=model2, score=score) + report_1 = StubReport(os.path.join(Folders.results, best_1["file"])) + report_1.report() + report_2 = StubReport(os.path.join(Folders.results, best_2["file"])) + report_2.report() + for result1, result2 in zip(report_1.lines, report_2.lines): + result = result1["score"] - result2["score"] + if result > 0: + win += 1 + winners.append(result1["dataset"]) + elif result < 0: + loose += 1 + loosers.append(result1["dataset"]) + else: + tie += 1 + print(f"{'Model':<20} {'File':<70} {'Score':<10} Win Tie Loose") + print("=" * 20 + " " + "=" * 70 + " " + "=" * 10 + " === === =====") + print(f"{model1:<20} {best_1['file']:<70} {report_1.score:10.5f}") + print( + f"{model2:<20} {best_2['file']:<70} " + f"{report_2.score:10.5f} " + f"{TextColor.GREEN}{win:3d} {TextColor.YELLOW}{tie:3d} " + f"{TextColor.RED}{loose:5d}" + ) + if win_results: + print(TextColor.GREEN + "Winners:") + print(winners) + if loose_results: + print(TextColor.RED + "Loosers:") + print(loosers) diff --git a/src/report b/src/report index 8b9e316..569ae5c 100755 --- a/src/report +++ b/src/report @@ -114,22 +114,23 @@ def default_report(): ) -(file, excel, sql, compare, best, grid, score, model) = parse_arguments() -if grid: - best = False -if file is None and best is None: - default_report() -else: - if best is not None or grid is not None: - report = ReportBest(score, model, best, grid) - report.report() +if __name__ == "__main__": + (file, excel, sql, compare, best, grid, score, model) = parse_arguments() + if grid: + best = False + if file is None and best is None: + default_report() else: - report = Report(file, compare) - report.report() - if excel: - excel = Excel(file, compare) - excel.report() - Files.open(excel.get_file_name()) - if sql: - sql = SQL(file) - sql.report() + if best is not None or grid is not None: + report = ReportBest(score, model, best, grid) + report.report() + else: + report = Report(file, compare) + report.report() + if excel: + excel = Excel(file, compare) + excel.report() + Files.open(excel.get_file_name()) + if sql: + sql = SQL(file) + sql.report() diff --git a/src/summary b/src/summary index e78408e..32df71a 100755 --- a/src/summary +++ b/src/summary @@ -40,26 +40,25 @@ def parse_arguments(): ) -( - score, - model, - list_results, -) = parse_arguments() - -all_metrics = ["accuracy", "f1-macro", "f1-micro"] - -metrics = all_metrics if score == "all" else [score] - -summary = Summary() -summary.acquire() - -for metric in metrics: - title = f"BEST RESULT of {metric} for {model}" - best = summary.best_result(criterion="model", value=model, score=metric) - summary.show_result(data=best, title=title) - summary.show_result( - summary.best_result(score=metric), title=f"BEST RESULT of {metric}" - ) - summary.show_top(score=metric, n=10) -if list_results: - summary.list_results() +if __name__ == "__main__": + ( + score, + model, + list_results, + ) = parse_arguments() + all_metrics = ["accuracy", "f1-macro", "f1-micro"] + metrics = all_metrics if score == "all" else [score] + summary = Summary() + summary.acquire() + for metric in metrics: + title = f"BEST RESULT of {metric} for {model}" + best = summary.best_result( + criterion="model", value=model, score=metric + ) + summary.show_result(data=best, title=title) + summary.show_result( + summary.best_result(score=metric), title=f"BEST RESULT of {metric}" + ) + summary.show_top(score=metric, n=10) + if list_results: + summary.list_results()