From b6fc9096a13fde5a976208ed70247f9d3fb81e13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Mon, 22 May 2023 10:07:28 +0200 Subject: [PATCH] Fix tests --- benchmark/Manager.py | 120 ++++++ benchmark/Results.py | 381 +----------------- benchmark/ResultsBase.py | 278 ++++++++++++- benchmark/ResultsFiles.py | 2 +- benchmark/__init__.py | 3 +- benchmark/scripts/be_benchmark.py | 2 +- benchmark/scripts/be_best.py | 2 +- benchmark/scripts/be_list.py | 6 +- benchmark/scripts/be_summary.py | 2 +- benchmark/tests/Benchmark_test.py | 9 - benchmark/tests/Report_test.py | 3 +- benchmark/tests/Summary_test.py | 2 +- benchmark/tests/scripts/Be_List_test.py | 26 +- benchmark/tests/scripts/Be_Main_test.py | 6 +- benchmark/tests/test_files/be_main_best.test | 1 + .../tests/test_files/be_main_complete.test | 1 + .../tests/test_files/be_main_dataset.test | 3 +- benchmark/tests/test_files/be_main_grid.test | 1 + .../tests/test_files/exreport_excel_ODTE.test | 40 +- .../exreport_excel_RandomForest.test | 40 +- .../test_files/exreport_excel_STree.test | 40 +- 21 files changed, 500 insertions(+), 468 deletions(-) create mode 100644 benchmark/Manager.py diff --git a/benchmark/Manager.py b/benchmark/Manager.py new file mode 100644 index 0000000..cf0f453 --- /dev/null +++ b/benchmark/Manager.py @@ -0,0 +1,120 @@ +import os +from types import SimpleNamespace + +import xlsxwriter + +from benchmark.Results import Report +from benchmark.ResultsFiles import Excel +from benchmark.Utils import Files, Folders, TextColor + + +def get_input(message="", is_test=False): + return "test" if is_test else input(message) + + +class Manage: + def __init__(self, summary): + self.summary = summary + + def manage_results(self): + """Manage results showed in the summary + return True if excel file is created False otherwise + """ + + def process_file(num, command, path): + num = int(num) + name = self.summary.data_filtered[num]["file"] + file_name_result = os.path.join(path, name) + verb1, verb2 = ( + ("delete", "Deleting") + if command == cmd.delete + else ( + "hide", + "Hiding", + ) + ) + conf_message = ( + TextColor.RED + + f"Are you sure to {verb1} {file_name_result} (y/n)? " + ) + confirm = get_input(message=conf_message) + if confirm == "y": + print(TextColor.YELLOW + f"{verb2} {file_name_result}") + if command == cmd.delete: + os.unlink(file_name_result) + else: + os.rename( + os.path.join(Folders.results, name), + os.path.join(Folders.hidden_results, name), + ) + self.summary.data_filtered.pop(num) + get_input(message="Press enter to continue") + self.summary.list_results() + + cmd = SimpleNamespace( + quit="q", relist="r", delete="d", hide="h", excel="e" + ) + message = ( + TextColor.ENDC + + f"Choose option {str(cmd).replace('namespace', '')}: " + ) + path = ( + Folders.hidden_results if self.summary.hidden else Folders.results + ) + book = None + max_value = len(self.summary.data_filtered) + while True: + match get_input(message=message).split(): + case [cmd.relist]: + self.summary.list_results() + case [cmd.quit]: + if book is not None: + book.close() + return True + return False + case [cmd.hide, num] if num.isdigit() and int(num) < max_value: + if self.summary.hidden: + print("Already hidden") + else: + process_file(num, path=path, command=cmd.hide) + case [cmd.delete, num] if num.isdigit() and int( + num + ) < max_value: + process_file(num=num, path=path, command=cmd.delete) + case [cmd.excel, num] if num.isdigit() and int( + num + ) < max_value: + # Add to excel file result #num + num = int(num) + file_name_result = os.path.join( + path, self.summary.data_filtered[num]["file"] + ) + if book is None: + file_name = os.path.join( + Folders.excel, Files.be_list_excel + ) + book = xlsxwriter.Workbook( + file_name, {"nan_inf_to_errors": True} + ) + excel = Excel( + file_name=file_name_result, + book=book, + compare=self.summary.compare, + ) + excel.report() + print(f"Added {file_name_result} to {Files.be_list_excel}") + case [num] if num.isdigit() and int(num) < max_value: + # Report the result #num + num = int(num) + file_name_result = os.path.join( + path, self.summary.data_filtered[num]["file"] + ) + try: + rep = Report( + file_name_result, compare=self.summary.compare + ) + rep.report() + except ValueError as e: + print(e) + case _: + print("Invalid option. Try again!") diff --git a/benchmark/Results.py b/benchmark/Results.py index 30042c5..5669268 100644 --- a/benchmark/Results.py +++ b/benchmark/Results.py @@ -1,14 +1,6 @@ -import math import os -from operator import itemgetter -from types import SimpleNamespace - -import xlsxwriter - -from .Datasets import Datasets -from .ResultsBase import BaseReport, StubReport, get_input -from .ResultsFiles import Excel -from .Utils import NO_RESULTS, Files, Folders, TextColor +from .ResultsBase import BaseReport, StubReport, Summary +from .Utils import Files, Folders, TextColor class Report(BaseReport): @@ -189,375 +181,6 @@ class ReportBest(BaseReport): self.header_line("*") -class Summary: - def __init__(self, hidden=False, compare=False) -> None: - self.results = Files().get_all_results(hidden=hidden) - self.data = [] - self.data_filtered = [] - self.datasets = {} - self.models = set() - self.hidden = hidden - self.compare = compare - - def get_models(self): - return sorted(self.models) - - def acquire(self, given_score="any") -> None: - """Get all results""" - for result in self.results: - ( - score, - model, - platform, - date, - time, - stratified, - ) = Files().split_file_name(result) - if given_score in ("any", score): - self.models.add(model) - report = StubReport( - os.path.join( - Folders.hidden_results - if self.hidden - else Folders.results, - result, - ) - ) - report.report() - entry = dict( - score=score, - model=model, - title=report.title, - platform=platform, - date=date, - time=time, - stratified=stratified, - file=result, - metric=report.score, - duration=report.duration, - ) - self.datasets[result] = report.lines - self.data.append(entry) - - def get_results_criteria( - self, score, model, input_data, sort_key, number, nan=False - ): - data = self.data.copy() if input_data is None else input_data - if score: - data = [x for x in data if x["score"] == score] - if model: - data = [x for x in data if x["model"] == model] - if nan: - data = [x for x in data if x["metric"] != x["metric"]] - keys = ( - itemgetter(sort_key, "time") - if sort_key == "date" - else itemgetter(sort_key, "date", "time") - ) - data = sorted(data, key=keys, reverse=True) - if number > 0: - data = data[:number] - return data - - def list_results( - self, - score=None, - model=None, - input_data=None, - sort_key="date", - number=0, - nan=False, - ) -> None: - """Print the list of results""" - if self.data_filtered == []: - self.data_filtered = self.get_results_criteria( - score, model, input_data, sort_key, number, nan=nan - ) - if self.data_filtered == []: - raise ValueError(NO_RESULTS) - max_file = max(len(x["file"]) for x in self.data_filtered) - max_title = max(len(x["title"]) for x in self.data_filtered) - if self.hidden: - color1 = TextColor.GREEN - color2 = TextColor.YELLOW - else: - color1 = TextColor.LINE1 - color2 = TextColor.LINE2 - print(color1, end="") - print( - f" # {'Date':10s} {'File':{max_file}s} {'Score':8s} " - f"{'Time(h)':7s} {'Title':s}" - ) - print( - "===", - "=" * 10 - + " " - + "=" * max_file - + " " - + "=" * 8 - + " " - + "=" * 7 - + " " - + "=" * max_title, - ) - print( - "\n".join( - [ - (color2 if n % 2 == 0 else color1) + f"{n:3d} " - f"{x['date']} {x['file']:{max_file}s} " - f"{x['metric']:8.5f} " - f"{x['duration']/3600:7.3f} " - f"{x['title']}" - for n, x in enumerate(self.data_filtered) - ] - ) - ) - - def manage_results(self): - """Manage results showed in the summary - return True if excel file is created False otherwise - """ - - def process_file(num, command, path): - num = int(num) - name = self.data_filtered[num]["file"] - file_name_result = os.path.join(path, name) - verb1, verb2 = ( - ("delete", "Deleting") - if command == cmd.delete - else ( - "hide", - "Hiding", - ) - ) - conf_message = ( - TextColor.RED - + f"Are you sure to {verb1} {file_name_result} (y/n)? " - ) - confirm = get_input(message=conf_message) - if confirm == "y": - print(TextColor.YELLOW + f"{verb2} {file_name_result}") - if command == cmd.delete: - os.unlink(file_name_result) - else: - os.rename( - os.path.join(Folders.results, name), - os.path.join(Folders.hidden_results, name), - ) - self.data_filtered.pop(num) - get_input(message="Press enter to continue") - self.list_results() - - cmd = SimpleNamespace( - quit="q", relist="r", delete="d", hide="h", excel="e" - ) - message = ( - TextColor.ENDC - + f"Choose option {str(cmd).replace('namespace', '')}: " - ) - path = Folders.hidden_results if self.hidden else Folders.results - book = None - max_value = len(self.data_filtered) - while True: - match get_input(message=message).split(): - case [cmd.relist]: - self.list_results() - case [cmd.quit]: - if book is not None: - book.close() - return True - return False - case [cmd.hide, num] if num.isdigit() and int(num) < max_value: - if self.hidden: - print("Already hidden") - else: - process_file(num, path=path, command=cmd.hide) - case [cmd.delete, num] if num.isdigit() and int( - num - ) < max_value: - process_file(num=num, path=path, command=cmd.delete) - case [cmd.excel, num] if num.isdigit() and int( - num - ) < max_value: - # Add to excel file result #num - num = int(num) - file_name_result = os.path.join( - path, self.data_filtered[num]["file"] - ) - if book is None: - file_name = os.path.join( - Folders.excel, Files.be_list_excel - ) - book = xlsxwriter.Workbook( - file_name, {"nan_inf_to_errors": True} - ) - excel = Excel( - file_name=file_name_result, - book=book, - compare=self.compare, - ) - excel.report() - print(f"Added {file_name_result} to {Files.be_list_excel}") - case [num] if num.isdigit() and int(num) < max_value: - # Report the result #num - num = int(num) - file_name_result = os.path.join( - path, self.data_filtered[num]["file"] - ) - try: - rep = Report(file_name_result, compare=self.compare) - rep.report() - except ValueError as e: - print(e) - case _: - print("Invalid option. Try again!") - - def show_result(self, data: dict, title: str = "") -> None: - def whites(n: int) -> str: - return " " * n + color1 + "*" - - if data == {}: - print(f"** {title} has No data **") - return - color1 = TextColor.CYAN - color2 = TextColor.YELLOW - file_name = data["file"] - metric = data["metric"] - result = StubReport(os.path.join(Folders.results, file_name)) - length = 81 - print(color1 + "*" * length) - if title != "": - print( - "*" - + color2 - + TextColor.BOLD - + f"{title:^{length - 2}s}" - + TextColor.ENDC - + color1 - + "*" - ) - print("*" + "-" * (length - 2) + "*") - print("*" + whites(length - 2)) - print( - "* " - + color2 - + f"{result.data['title']:^{length - 4}}" - + color1 - + " *" - ) - print("*" + whites(length - 2)) - print( - "* Model: " - + color2 - + f"{result.data['model']:15s} " - + color1 - + "Ver. " - + color2 - + f"{result.data['version']:10s} " - + color1 - + "Score: " - + color2 - + f"{result.data['score_name']:10s} " - + color1 - + "Metric: " - + color2 - + f"{metric:10.7f}" - + whites(length - 78) - ) - print(color1 + "*" + whites(length - 2)) - print( - "* Date : " - + color2 - + f"{result.data['date']:15s}" - + color1 - + " Time: " - + color2 - + f"{result.data['time']:18s} " - + color1 - + "Time Spent: " - + color2 - + f"{result.data['duration']:9,.2f}" - + color1 - + " secs." - + whites(length - 78) - ) - seeds = str(result.data["seeds"]) - seeds_len = len(seeds) - print( - "* Seeds: " - + color2 - + f"{seeds:{seeds_len}s} " - + color1 - + "Platform: " - + color2 - + f"{result.data['platform']:17s} " - + whites(length - 79) - ) - print( - "* Stratified: " - + color2 - + f"{str(result.data['stratified']):15s}" - + whites(length - 30) - ) - print("* " + color2 + f"{file_name:60s}" + whites(length - 63)) - print(color1 + "*" + whites(length - 2)) - print(color1 + "*" * length) - - def best_results(self, criterion=None, value=None, score="accuracy", n=10): - # First filter the same score results (accuracy, f1, ...) - haystack = [x for x in self.data if x["score"] == score] - haystack = ( - haystack - if criterion is None or value is None - else [x for x in haystack if x[criterion] == value] - ) - if haystack == []: - raise ValueError(NO_RESULTS) - return ( - sorted( - haystack, - key=lambda x: -1.0 if math.isnan(x["metric"]) else x["metric"], - reverse=True, - )[:n] - if len(haystack) > 0 - else {} - ) - - def best_result( - self, criterion=None, value=None, score="accuracy" - ) -> dict: - return self.best_results(criterion, value, score)[0] - - def best_results_datasets(self, score="accuracy") -> dict: - """Get the best results for each dataset""" - dt = Datasets() - best_results = {} - for dataset in dt: - best_results[dataset] = (1, "", "", "") - haystack = [x for x in self.data if x["score"] == score] - # Search for the best results for each dataset - for entry in haystack: - for dataset in self.datasets[entry["file"]]: - if dataset["score"] < best_results[dataset["dataset"]][0]: - best_results[dataset["dataset"]] = ( - dataset["score"], - dataset["hyperparameters"], - entry["file"], - entry["title"], - ) - return best_results - - def show_top(self, score="accuracy", n=10): - try: - self.list_results( - score=score, - input_data=self.best_results(score=score, n=n), - sort_key="metric", - ) - except ValueError as e: - print(e) - - class PairCheck: def __init__(self, score, model_a, model_b, winners=False, losers=False): self.score = score diff --git a/benchmark/ResultsBase.py b/benchmark/ResultsBase.py index 0a77588..0c3fde3 100644 --- a/benchmark/ResultsBase.py +++ b/benchmark/ResultsBase.py @@ -1,7 +1,11 @@ import abc import json +import math import os +from operator import itemgetter +from benchmark.Datasets import Datasets +from benchmark.Utils import NO_RESULTS, Files, Folders, TextColor from .Arguments import ALL_METRICS, EnvData from .Datasets import Datasets @@ -9,10 +13,6 @@ from .Experiments import BestResults from .Utils import Folders, Symbols -def get_input(message="", is_test=False): - return "test" if is_test else input(message) - - class BestResultsEver: def __init__(self): self.data = {} @@ -161,3 +161,273 @@ class StubReport(BaseReport): def footer(self, accuracy: float) -> None: self.accuracy = accuracy self.score = accuracy / self._get_best_accuracy() + + +class Summary: + def __init__(self, hidden=False, compare=False) -> None: + self.results = Files().get_all_results(hidden=hidden) + self.data = [] + self.data_filtered = [] + self.datasets = {} + self.models = set() + self.hidden = hidden + self.compare = compare + + def get_models(self): + return sorted(self.models) + + def acquire(self, given_score="any") -> None: + """Get all results""" + for result in self.results: + ( + score, + model, + platform, + date, + time, + stratified, + ) = Files().split_file_name(result) + if given_score in ("any", score): + self.models.add(model) + report = StubReport( + os.path.join( + Folders.hidden_results + if self.hidden + else Folders.results, + result, + ) + ) + report.report() + entry = dict( + score=score, + model=model, + title=report.title, + platform=platform, + date=date, + time=time, + stratified=stratified, + file=result, + metric=report.score, + duration=report.duration, + ) + self.datasets[result] = report.lines + self.data.append(entry) + + def get_results_criteria( + self, score, model, input_data, sort_key, number, nan=False + ): + data = self.data.copy() if input_data is None else input_data + if score: + data = [x for x in data if x["score"] == score] + if model: + data = [x for x in data if x["model"] == model] + if nan: + data = [x for x in data if x["metric"] != x["metric"]] + keys = ( + itemgetter(sort_key, "time") + if sort_key == "date" + else itemgetter(sort_key, "date", "time") + ) + data = sorted(data, key=keys, reverse=True) + if number > 0: + data = data[:number] + return data + + def list_results( + self, + score=None, + model=None, + input_data=None, + sort_key="date", + number=0, + nan=False, + ) -> None: + """Print the list of results""" + if self.data_filtered == []: + self.data_filtered = self.get_results_criteria( + score, model, input_data, sort_key, number, nan=nan + ) + if self.data_filtered == []: + raise ValueError(NO_RESULTS) + max_file = max(len(x["file"]) for x in self.data_filtered) + max_title = max(len(x["title"]) for x in self.data_filtered) + if self.hidden: + color1 = TextColor.GREEN + color2 = TextColor.YELLOW + else: + color1 = TextColor.LINE1 + color2 = TextColor.LINE2 + print(color1, end="") + print( + f" # {'Date':10s} {'File':{max_file}s} {'Score':8s} " + f"{'Time(h)':7s} {'Title':s}" + ) + print( + "===", + "=" * 10 + + " " + + "=" * max_file + + " " + + "=" * 8 + + " " + + "=" * 7 + + " " + + "=" * max_title, + ) + print( + "\n".join( + [ + (color2 if n % 2 == 0 else color1) + f"{n:3d} " + f"{x['date']} {x['file']:{max_file}s} " + f"{x['metric']:8.5f} " + f"{x['duration']/3600:7.3f} " + f"{x['title']}" + for n, x in enumerate(self.data_filtered) + ] + ) + ) + + def show_result(self, data: dict, title: str = "") -> None: + def whites(n: int) -> str: + return " " * n + color1 + "*" + + if data == {}: + print(f"** {title} has No data **") + return + color1 = TextColor.CYAN + color2 = TextColor.YELLOW + file_name = data["file"] + metric = data["metric"] + result = StubReport(os.path.join(Folders.results, file_name)) + length = 81 + print(color1 + "*" * length) + if title != "": + print( + "*" + + color2 + + TextColor.BOLD + + f"{title:^{length - 2}s}" + + TextColor.ENDC + + color1 + + "*" + ) + print("*" + "-" * (length - 2) + "*") + print("*" + whites(length - 2)) + print( + "* " + + color2 + + f"{result.data['title']:^{length - 4}}" + + color1 + + " *" + ) + print("*" + whites(length - 2)) + print( + "* Model: " + + color2 + + f"{result.data['model']:15s} " + + color1 + + "Ver. " + + color2 + + f"{result.data['version']:10s} " + + color1 + + "Score: " + + color2 + + f"{result.data['score_name']:10s} " + + color1 + + "Metric: " + + color2 + + f"{metric:10.7f}" + + whites(length - 78) + ) + print(color1 + "*" + whites(length - 2)) + print( + "* Date : " + + color2 + + f"{result.data['date']:15s}" + + color1 + + " Time: " + + color2 + + f"{result.data['time']:18s} " + + color1 + + "Time Spent: " + + color2 + + f"{result.data['duration']:9,.2f}" + + color1 + + " secs." + + whites(length - 78) + ) + seeds = str(result.data["seeds"]) + seeds_len = len(seeds) + print( + "* Seeds: " + + color2 + + f"{seeds:{seeds_len}s} " + + color1 + + "Platform: " + + color2 + + f"{result.data['platform']:17s} " + + whites(length - 79) + ) + print( + "* Stratified: " + + color2 + + f"{str(result.data['stratified']):15s}" + + whites(length - 30) + ) + print("* " + color2 + f"{file_name:60s}" + whites(length - 63)) + print(color1 + "*" + whites(length - 2)) + print(color1 + "*" * length) + + def best_results(self, criterion=None, value=None, score="accuracy", n=10): + # First filter the same score results (accuracy, f1, ...) + haystack = [x for x in self.data if x["score"] == score] + haystack = ( + haystack + if criterion is None or value is None + else [x for x in haystack if x[criterion] == value] + ) + if haystack == []: + raise ValueError(NO_RESULTS) + return ( + sorted( + haystack, + key=lambda x: -1.0 if math.isnan(x["metric"]) else x["metric"], + reverse=True, + )[:n] + if len(haystack) > 0 + else {} + ) + + def best_result( + self, criterion=None, value=None, score="accuracy" + ) -> dict: + return self.best_results(criterion, value, score)[0] + + def best_results_datasets(self, score="accuracy") -> dict: + """Get the best results for each dataset""" + dt = Datasets() + best_results = {} + for dataset in dt: + best_results[dataset] = (1, "", "", "") + haystack = [x for x in self.data if x["score"] == score] + # Search for the best results for each dataset + for entry in haystack: + for dataset in self.datasets[entry["file"]]: + if dataset["score"] < best_results[dataset["dataset"]][0]: + best_results[dataset["dataset"]] = ( + dataset["score"], + dataset["hyperparameters"], + entry["file"], + entry["title"], + ) + return best_results + + def show_top(self, score="accuracy", n=10): + try: + self.list_results( + score=score, + input_data=self.best_results(score=score, n=n), + sort_key="metric", + ) + except ValueError as e: + print(e) diff --git a/benchmark/ResultsFiles.py b/benchmark/ResultsFiles.py index d75e140..7e87d83 100644 --- a/benchmark/ResultsFiles.py +++ b/benchmark/ResultsFiles.py @@ -12,7 +12,7 @@ from xlsxwriter.exceptions import DuplicateWorksheetName from ._version import __version__ from .Arguments import EnvData from .Datasets import Datasets -from .ResultsBase import BaseReport +from .ResultsBase import BaseReport, BestResultsEver, Summary, StubReport from .Utils import NO_RESULTS, Files, Folders, TextColor diff --git a/benchmark/__init__.py b/benchmark/__init__.py index eea7c15..1aff1ed 100644 --- a/benchmark/__init__.py +++ b/benchmark/__init__.py @@ -1,3 +1,4 @@ +from .ResultsBase import Summary from .Datasets import ( Datasets, DatasetsSurcov, @@ -5,7 +6,7 @@ from .Datasets import ( DatasetsArff, ) from .Experiments import Experiment -from .Results import Report, Summary +from .Results import Report from ._version import __version__ __author__ = "Ricardo Montañana Gómez" diff --git a/benchmark/scripts/be_benchmark.py b/benchmark/scripts/be_benchmark.py index 72ed1c1..7a6469c 100755 --- a/benchmark/scripts/be_benchmark.py +++ b/benchmark/scripts/be_benchmark.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from benchmark.Results import Benchmark +from benchmark.ResultsFiles import Benchmark from benchmark.Utils import Files from benchmark.Arguments import Arguments diff --git a/benchmark/scripts/be_best.py b/benchmark/scripts/be_best.py index 919c875..3248598 100755 --- a/benchmark/scripts/be_best.py +++ b/benchmark/scripts/be_best.py @@ -1,6 +1,6 @@ #!/usr/bin/env python import json -from benchmark.Results import Summary +from benchmark.ResultsBase import Summary from benchmark.Arguments import ALL_METRICS, Arguments diff --git a/benchmark/scripts/be_list.py b/benchmark/scripts/be_list.py index 30764d0..c6a8bd6 100755 --- a/benchmark/scripts/be_list.py +++ b/benchmark/scripts/be_list.py @@ -1,8 +1,9 @@ #! /usr/bin/env python import os -from benchmark.Results import Summary +from benchmark.ResultsBase import Summary from benchmark.Utils import Files, Folders from benchmark.Arguments import Arguments +from benchmark.Manager import Manage """List experiments of a model """ @@ -27,7 +28,8 @@ def main(args_test=None): except ValueError as e: print(e) return - excel_generated = data.manage_results() + manager = Manage(data) + excel_generated = manager.manage_results() if excel_generated: name = os.path.join(Folders.excel, Files.be_list_excel) print(f"Generated file: {name}") diff --git a/benchmark/scripts/be_summary.py b/benchmark/scripts/be_summary.py index 8ce8aed..451f1c9 100755 --- a/benchmark/scripts/be_summary.py +++ b/benchmark/scripts/be_summary.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from benchmark.Results import Summary +from benchmark.ResultsBase import Summary from benchmark.Arguments import ALL_METRICS, Arguments diff --git a/benchmark/tests/Benchmark_test.py b/benchmark/tests/Benchmark_test.py index eb207d3..f27f8ca 100644 --- a/benchmark/tests/Benchmark_test.py +++ b/benchmark/tests/Benchmark_test.py @@ -90,15 +90,6 @@ class BenchmarkTest(TestBase): self.assertTrue(os.path.exists(benchmark.get_tex_file())) self.check_file_file(benchmark.get_tex_file(), "exreport_tex") - @staticmethod - def generate_excel_sheet(test, sheet, file_name): - with open(os.path.join("test_files", file_name), "w") as f: - for row in range(1, sheet.max_row + 1): - for col in range(1, sheet.max_column + 1): - value = sheet.cell(row=row, column=col).value - if value is not None: - print(f'{row};{col};"{value}"', file=f) - def test_excel_output(self): benchmark = Benchmark("accuracy", visualize=False) benchmark.compile_results() diff --git a/benchmark/tests/Report_test.py b/benchmark/tests/Report_test.py index 9731e17..3390ca7 100644 --- a/benchmark/tests/Report_test.py +++ b/benchmark/tests/Report_test.py @@ -4,7 +4,8 @@ from unittest.mock import patch from .TestBase import TestBase from ..Results import Report, ReportBest from ..ResultsFiles import ReportDatasets -from ..ResultsBase import BaseReport, get_input +from ..ResultsBase import BaseReport +from ..Manager import get_input from ..Utils import Symbols diff --git a/benchmark/tests/Summary_test.py b/benchmark/tests/Summary_test.py index b900c4d..cbe4128 100644 --- a/benchmark/tests/Summary_test.py +++ b/benchmark/tests/Summary_test.py @@ -1,7 +1,7 @@ from io import StringIO from unittest.mock import patch from .TestBase import TestBase -from ..Results import Summary +from ..ResultsBase import Summary from ..Utils import NO_RESULTS diff --git a/benchmark/tests/scripts/Be_List_test.py b/benchmark/tests/scripts/Be_List_test.py index 51836f6..76f47cc 100644 --- a/benchmark/tests/scripts/Be_List_test.py +++ b/benchmark/tests/scripts/Be_List_test.py @@ -10,31 +10,31 @@ class BeListTest(TestBase): def setUp(self): self.prepare_scripts_env() - @patch("benchmark.Results.get_input", return_value="q") + @patch("benchmark.Manager.get_input", return_value="q") def test_be_list(self, input_data): stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) self.assertEqual(stderr.getvalue(), "") self.check_output_file(stdout, "be_list_model") - @patch("benchmark.Results.get_input", side_effect=iter(["x", "q"])) + @patch("benchmark.Manager.get_input", side_effect=iter(["x", "q"])) def test_be_list_invalid_option(self, input_data): stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) self.assertEqual(stderr.getvalue(), "") self.check_output_file(stdout, "be_list_model_invalid") - @patch("benchmark.Results.get_input", side_effect=iter(["0", "q"])) + @patch("benchmark.Manager.get_input", side_effect=iter(["0", "q"])) def test_be_list_report(self, input_data): stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) self.assertEqual(stderr.getvalue(), "") self.check_output_file(stdout, "be_list_report") - @patch("benchmark.Results.get_input", side_effect=iter(["r", "q"])) + @patch("benchmark.Manager.get_input", side_effect=iter(["r", "q"])) def test_be_list_twice(self, input_data): stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) self.assertEqual(stderr.getvalue(), "") self.check_output_file(stdout, "be_list_model_2") - @patch("benchmark.Results.get_input", side_effect=iter(["e 2", "q"])) + @patch("benchmark.Manager.get_input", side_effect=iter(["e 2", "q"])) def test_be_list_report_excel(self, input_data): stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) self.assertEqual(stderr.getvalue(), "") @@ -45,7 +45,7 @@ class BeListTest(TestBase): self.check_excel_sheet(sheet, "excel") @patch( - "benchmark.Results.get_input", + "benchmark.Manager.get_input", side_effect=iter(["e 2", "e 1", "q"]), ) def test_be_list_report_excel_twice(self, input_data): @@ -58,7 +58,7 @@ class BeListTest(TestBase): sheet = book["STree2"] self.check_excel_sheet(sheet, "excel2") - @patch("benchmark.Results.get_input", return_value="q") + @patch("benchmark.Manager.get_input", return_value="q") def test_be_list_no_data(self, input_data): stdout, stderr = self.execute_script( "be_list", ["-m", "Wodt", "-s", "f1-macro"] @@ -67,7 +67,7 @@ class BeListTest(TestBase): self.assertEqual(stdout.getvalue(), f"{NO_RESULTS}\n") @patch( - "benchmark.Results.get_input", + "benchmark.Manager.get_input", side_effect=iter(["d 0", "y", "", "q"]), ) # @patch("benchmark.ResultsBase.get_input", side_effect=iter(["q"])) @@ -94,7 +94,7 @@ class BeListTest(TestBase): self.fail("test_be_list_delete() should not raise exception") @patch( - "benchmark.Results.get_input", + "benchmark.Manager.get_input", side_effect=iter(["h 0", "y", "", "q"]), ) def test_be_list_hide(self, input_data): @@ -119,25 +119,25 @@ class BeListTest(TestBase): swap_files(Folders.results, Folders.hidden_results, file_name) self.fail("test_be_list_hide() should not raise exception") - @patch("benchmark.Results.get_input", side_effect=iter(["h 0", "q"])) + @patch("benchmark.Manager.get_input", side_effect=iter(["h 0", "q"])) def test_be_list_already_hidden(self, input_data): stdout, stderr = self.execute_script("be_list", ["--hidden"]) self.assertEqual(stderr.getvalue(), "") self.check_output_file(stdout, "be_list_already_hidden") - @patch("benchmark.Results.get_input", side_effect=iter(["h 0", "n", "q"])) + @patch("benchmark.Manager.get_input", side_effect=iter(["h 0", "n", "q"])) def test_be_list_dont_hide(self, input_data): stdout, stderr = self.execute_script("be_list", "") self.assertEqual(stderr.getvalue(), "") self.check_output_file(stdout, "be_list_default") - @patch("benchmark.Results.get_input", side_effect=iter(["q"])) + @patch("benchmark.Manager.get_input", side_effect=iter(["q"])) def test_be_list_hidden_nan(self, input_data): stdout, stderr = self.execute_script("be_list", ["--hidden", "--nan"]) self.assertEqual(stderr.getvalue(), "") self.check_output_file(stdout, "be_list_hidden_nan") - @patch("benchmark.Results.get_input", side_effect=iter(["q"])) + @patch("benchmark.Manager.get_input", side_effect=iter(["q"])) def test_be_list_hidden(self, input_data): stdout, stderr = self.execute_script("be_list", ["--hidden"]) self.assertEqual(stderr.getvalue(), "") diff --git a/benchmark/tests/scripts/Be_Main_test.py b/benchmark/tests/scripts/Be_Main_test.py index fabb305..a8053c5 100644 --- a/benchmark/tests/scripts/Be_Main_test.py +++ b/benchmark/tests/scripts/Be_Main_test.py @@ -25,7 +25,7 @@ class BeMainTest(TestBase): self.check_output_lines( stdout=stdout, file_name="be_main_dataset", - lines_to_compare=[0, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13], + lines_to_compare=[0, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13, 14], ) def test_be_main_complete(self): @@ -37,7 +37,9 @@ class BeMainTest(TestBase): report_name = stdout.getvalue().splitlines()[-1].split("in ")[1] self.files.append(report_name) self.check_output_lines( - stdout, "be_main_complete", [0, 2, 3, 5, 6, 7, 8, 9, 12, 13, 14] + stdout, + "be_main_complete", + [0, 2, 3, 5, 6, 7, 8, 9, 12, 13, 14, 15], ) def test_be_main_no_report(self): diff --git a/benchmark/tests/test_files/be_main_best.test b/benchmark/tests/test_files/be_main_best.test index 4bdcb37..7a6991d 100644 --- a/benchmark/tests/test_files/be_main_best.test +++ b/benchmark/tests/test_files/be_main_best.test @@ -11,6 +11,7 @@ Dataset Sampl. Feat. Cls Nodes Leaves Depth Score balance-scale 625 4 3 23.32 12.16 6.44 0.840160±0.0304 0.013745±0.0019 {'splitter': 'best', 'max_features': 'auto'} balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000388±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000, 'multiclass_strategy': 'ovr'} ************************************************************************************************************************* +* ➶ Better than ZeroR + 10.0%.....: 1 * * accuracy compared to STree_default (liblinear-ovr) .: 0.0422 * ************************************************************************************************************************* Results in results/results_accuracy_STree_iMac27_2022-05-09_00:15:25_0.json diff --git a/benchmark/tests/test_files/be_main_complete.test b/benchmark/tests/test_files/be_main_complete.test index d53a956..531162d 100644 --- a/benchmark/tests/test_files/be_main_complete.test +++ b/benchmark/tests/test_files/be_main_complete.test @@ -11,6 +11,7 @@ Dataset Sampl. Feat. Cls Nodes Leaves Depth Score balance-scale 625 4 3 17.36 9.18 6.18 0.908480±0.0247 0.007388±0.0013 {} balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000664±0.0002 {} ************************************************************************************************************************* +* ➶ Better than ZeroR + 10.0%.....: 1 * * accuracy compared to STree_default (liblinear-ovr) .: 0.0390 * ************************************************************************************************************************* Results in results/results_accuracy_STree_iMac27_2022-05-08_20:14:43_0.json diff --git a/benchmark/tests/test_files/be_main_dataset.test b/benchmark/tests/test_files/be_main_dataset.test index 2d10fb3..abe486b 100644 --- a/benchmark/tests/test_files/be_main_dataset.test +++ b/benchmark/tests/test_files/be_main_dataset.test @@ -8,8 +8,9 @@ Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters ============================== ====== ===== === ======= ======= ======= =============== ================= =============== -balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000671±0.0001 {} +balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009➶ 0.000671±0.0001 {} ************************************************************************************************************************* +* ➶ Better than ZeroR + 10.0%.....: 1 * * accuracy compared to STree_default (liblinear-ovr) .: 0.0165 * ************************************************************************************************************************* Partial result file removed: results/results_accuracy_STree_iMac27_2022-05-08_19:38:28_0.json diff --git a/benchmark/tests/test_files/be_main_grid.test b/benchmark/tests/test_files/be_main_grid.test index 11a63f6..d7aeebd 100644 --- a/benchmark/tests/test_files/be_main_grid.test +++ b/benchmark/tests/test_files/be_main_grid.test @@ -11,6 +11,7 @@ Dataset Sampl. Feat. Cls Nodes Leaves Depth Score balance-scale 625 4 3 26.12 13.56 7.94 0.910720±0.0249 0.015852±0.0027 {'C': 1.0, 'kernel': 'liblinear', 'multiclass_strategy': 'ovr'} balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000640±0.0001 {'C': 1.0, 'kernel': 'linear', 'multiclass_strategy': 'ovr'} ************************************************************************************************************************* +* ➶ Better than ZeroR + 10.0%.....: 1 * * accuracy compared to STree_default (liblinear-ovr) .: 0.0391 * ************************************************************************************************************************* Results in results/results_accuracy_STree_iMac27_2022-05-09_00:21:06_0.json diff --git a/benchmark/tests/test_files/exreport_excel_ODTE.test b/benchmark/tests/test_files/exreport_excel_ODTE.test index f1ea096..716f3f4 100644 --- a/benchmark/tests/test_files/exreport_excel_ODTE.test +++ b/benchmark/tests/test_files/exreport_excel_ODTE.test @@ -3,12 +3,12 @@ 3;1;" Score is accuracy" 3;2;" Execution time" 3;5;"22,591.47 s" -3;7;" " -3;8;"Platform" +3;7;"Platform" 3;9;"Galgo" -3;10;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]" +3;11;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]" 4;5;" 6.28 h" -4;10;"Stratified: False" +4;11;"Stratified: False" +4;13;"Discretized: False" 6;1;"Dataset" 6;2;"Samples" 6;3;"Features" @@ -17,10 +17,11 @@ 6;6;"Leaves" 6;7;"Depth" 6;8;"Score" -6;9;"Score Std." -6;10;"Time" -6;11;"Time Std." -6;12;"Hyperparameters" +6;9;"Stat" +6;10;"Score Std." +6;11;"Time" +6;12;"Time Std." +6;13;"Hyperparameters" 7;1;"balance-scale" 7;2;"625" 7;3;"4" @@ -29,10 +30,11 @@ 7;6;"4.180599999999999" 7;7;"3.536" 7;8;"0.96352" -7;9;"0.02494974148162661" -7;10;"0.3166321754455567" -7;11;"0.1991881389525559" -7;12;"{'base_estimator__C': 57, 'base_estimator__gamma': 0.1, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1}" +7;9;" " +7;10;"0.02494974148162661" +7;11;"0.3166321754455567" +7;12;"0.1991881389525559" +7;13;"{'base_estimator__C': 57, 'base_estimator__gamma': 0.1, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1}" 8;1;"balloons" 8;2;"16" 8;3;"4" @@ -41,8 +43,12 @@ 8;6;"1.9976" 8;7;"1.9976" 8;8;"0.785" -8;9;"0.2461311755051675" -8;10;"0.1156062078475952" -8;11;"0.0127842418285999" -8;12;"{'base_estimator__C': 5, 'base_estimator__gamma': 0.14, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1}" -10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0434" +8;9;"➶" +8;10;"0.2461311755051675" +8;11;"0.1156062078475952" +8;12;"0.0127842418285999" +8;13;"{'base_estimator__C': 5, 'base_estimator__gamma': 0.14, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1}" +11;2;"➶" +11;3;"1" +11;4;"Better than ZeroR + 10.0%" +13;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0434" diff --git a/benchmark/tests/test_files/exreport_excel_RandomForest.test b/benchmark/tests/test_files/exreport_excel_RandomForest.test index aa608d0..6695507 100644 --- a/benchmark/tests/test_files/exreport_excel_RandomForest.test +++ b/benchmark/tests/test_files/exreport_excel_RandomForest.test @@ -3,12 +3,12 @@ 3;1;" Score is accuracy" 3;2;" Execution time" 3;5;" 272.74 s" -3;7;" " -3;8;"Platform" +3;7;"Platform" 3;9;"iMac27" -3;10;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]" +3;11;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]" 4;5;" 0.08 h" -4;10;"Stratified: False" +4;11;"Stratified: False" +4;13;"Discretized: False" 6;1;"Dataset" 6;2;"Samples" 6;3;"Features" @@ -17,10 +17,11 @@ 6;6;"Leaves" 6;7;"Depth" 6;8;"Score" -6;9;"Score Std." -6;10;"Time" -6;11;"Time Std." -6;12;"Hyperparameters" +6;9;"Stat" +6;10;"Score Std." +6;11;"Time" +6;12;"Time Std." +6;13;"Hyperparameters" 7;1;"balance-scale" 7;2;"625" 7;3;"4" @@ -29,10 +30,11 @@ 7;6;"98.42" 7;7;"10.6814" 7;8;"0.83616" -7;9;"0.02649630917694009" -7;10;"0.08222018241882324" -7;11;"0.001302632681512063" -7;12;"{}" +7;9;" " +7;10;"0.02649630917694009" +7;11;"0.08222018241882324" +7;12;"0.001302632681512063" +7;13;"{}" 8;1;"balloons" 8;2;"16" 8;3;"4" @@ -41,8 +43,12 @@ 8;6;"4.58" 8;7;"3.0982" 8;8;"0.625" -8;9;"0.249582985531199" -8;10;"0.07016648769378662" -8;11;"0.002460508923990468" -8;12;"{}" -10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0363" +8;9;"➶" +8;10;"0.249582985531199" +8;11;"0.07016648769378662" +8;12;"0.002460508923990468" +8;13;"{}" +11;2;"➶" +11;3;"1" +11;4;"Better than ZeroR + 10.0%" +13;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0363" diff --git a/benchmark/tests/test_files/exreport_excel_STree.test b/benchmark/tests/test_files/exreport_excel_STree.test index dd1728a..017e1c8 100644 --- a/benchmark/tests/test_files/exreport_excel_STree.test +++ b/benchmark/tests/test_files/exreport_excel_STree.test @@ -3,12 +3,12 @@ 3;1;" Score is accuracy" 3;2;" Execution time" 3;5;" 624.25 s" -3;7;" " -3;8;"Platform" +3;7;"Platform" 3;9;"iMac27" -3;10;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]" +3;11;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]" 4;5;" 0.17 h" -4;10;"Stratified: False" +4;11;"Stratified: False" +4;13;"Discretized: False" 6;1;"Dataset" 6;2;"Samples" 6;3;"Features" @@ -17,10 +17,11 @@ 6;6;"Leaves" 6;7;"Depth" 6;8;"Score" -6;9;"Score Std." -6;10;"Time" -6;11;"Time Std." -6;12;"Hyperparameters" +6;9;"Stat" +6;10;"Score Std." +6;11;"Time" +6;12;"Time Std." +6;13;"Hyperparameters" 7;1;"balance-scale" 7;2;"625" 7;3;"4" @@ -29,10 +30,11 @@ 7;6;"4" 7;7;"3" 7;8;"0.97056" -7;9;"0.0150468069702512" -7;10;"0.01404867172241211" -7;11;"0.002026269126958884" -7;12;"{'C': 10000, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000, 'multiclass_strategy': 'ovr'}" +7;9;" " +7;10;"0.0150468069702512" +7;11;"0.01404867172241211" +7;12;"0.002026269126958884" +7;13;"{'C': 10000, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000, 'multiclass_strategy': 'ovr'}" 8;1;"balloons" 8;2;"16" 8;3;"4" @@ -41,8 +43,12 @@ 8;6;"2" 8;7;"2" 8;8;"0.86" -8;9;"0.2850146195080759" -8;10;"0.0008541679382324218" -8;11;"3.629469326417878e-05" -8;12;"{'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000, 'multiclass_strategy': 'ovr'}" -10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0454" +8;9;"➶" +8;10;"0.2850146195080759" +8;11;"0.0008541679382324218" +8;12;"3.629469326417878e-05" +8;13;"{'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000, 'multiclass_strategy': 'ovr'}" +11;2;"➶" +11;3;"1" +11;4;"Better than ZeroR + 10.0%" +13;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0454"