Doctorado-ML/benchmark
1
0
Fork 0
mirror of https://github.com/Doctorado-ML/benchmark.git synced 2025-08-16 16:05:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix tests

Browse Source
This commit is contained in:
Ricardo Montañana Gómez
2023-05-22 10:07:28 +02:00
parent 83bd321dd6
commit b6fc9096a1
21 changed files with 500 additions and 468 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
benchmark/Manager.py Normal file
View File

@@ -0,0 +1,120 @@
import os
from types import SimpleNamespace
import xlsxwriter
from benchmark.Results import Report
from benchmark.ResultsFiles import Excel
from benchmark.Utils import Files, Folders, TextColor
def get_input(message="", is_test=False):
return "test" if is_test else input(message)
class Manage:
def __init__(self, summary):
self.summary = summary
def manage_results(self):
"""Manage results showed in the summary
return True if excel file is created False otherwise
"""
def process_file(num, command, path):
num = int(num)
name = self.summary.data_filtered[num]["file"]
file_name_result = os.path.join(path, name)
verb1, verb2 = (
("delete", "Deleting")
if command == cmd.delete
else (
"hide",
"Hiding",
)
)
conf_message = (
TextColor.RED
+ f"Are you sure to {verb1} {file_name_result} (y/n)? "
)
confirm = get_input(message=conf_message)
if confirm == "y":
print(TextColor.YELLOW + f"{verb2} {file_name_result}")
if command == cmd.delete:
os.unlink(file_name_result)
else:
os.rename(
os.path.join(Folders.results, name),
os.path.join(Folders.hidden_results, name),
)
self.summary.data_filtered.pop(num)
get_input(message="Press enter to continue")
self.summary.list_results()
cmd = SimpleNamespace(
quit="q", relist="r", delete="d", hide="h", excel="e"
)
message = (
TextColor.ENDC
+ f"Choose option {str(cmd).replace('namespace', '')}: "
)
path = (
Folders.hidden_results if self.summary.hidden else Folders.results
)
book = None
max_value = len(self.summary.data_filtered)
while True:
match get_input(message=message).split():
case [cmd.relist]:
self.summary.list_results()
case [cmd.quit]:
if book is not None:
book.close()
return True
return False
case [cmd.hide, num] if num.isdigit() and int(num) < max_value:
if self.summary.hidden:
print("Already hidden")
else:
process_file(num, path=path, command=cmd.hide)
case [cmd.delete, num] if num.isdigit() and int(
num
) < max_value:
process_file(num=num, path=path, command=cmd.delete)
case [cmd.excel, num] if num.isdigit() and int(
num
) < max_value:
# Add to excel file result #num
num = int(num)
file_name_result = os.path.join(
path, self.summary.data_filtered[num]["file"]
)
if book is None:
file_name = os.path.join(
Folders.excel, Files.be_list_excel
)
book = xlsxwriter.Workbook(
file_name, {"nan_inf_to_errors": True}
)
excel = Excel(
file_name=file_name_result,
book=book,
compare=self.summary.compare,
)
excel.report()
print(f"Added {file_name_result} to {Files.be_list_excel}")
case [num] if num.isdigit() and int(num) < max_value:
# Report the result #num
num = int(num)
file_name_result = os.path.join(
path, self.summary.data_filtered[num]["file"]
)
try:
rep = Report(
file_name_result, compare=self.summary.compare
)
rep.report()
except ValueError as e:
print(e)
case _:
print("Invalid option. Try again!")

381
benchmark/Results.py
View File

@@ -1,14 +1,6 @@
import math
import os import os
from operator import itemgetter from .ResultsBase import BaseReport, StubReport, Summary
from types import SimpleNamespace from .Utils import Files, Folders, TextColor
import xlsxwriter
from .Datasets import Datasets
from .ResultsBase import BaseReport, StubReport, get_input
from .ResultsFiles import Excel
from .Utils import NO_RESULTS, Files, Folders, TextColor
class Report(BaseReport): class Report(BaseReport):
@@ -189,375 +181,6 @@ class ReportBest(BaseReport):
self.header_line("*") self.header_line("*")
class Summary:
def __init__(self, hidden=False, compare=False) -> None:
self.results = Files().get_all_results(hidden=hidden)
self.data = []
self.data_filtered = []
self.datasets = {}
self.models = set()
self.hidden = hidden
self.compare = compare
def get_models(self):
return sorted(self.models)
def acquire(self, given_score="any") -> None:
"""Get all results"""
for result in self.results:
(
score,
model,
platform,
date,
time,
stratified,
) = Files().split_file_name(result)
if given_score in ("any", score):
self.models.add(model)
report = StubReport(
os.path.join(
Folders.hidden_results
if self.hidden
else Folders.results,
result,
)
)
report.report()
entry = dict(
score=score,
model=model,
title=report.title,
platform=platform,
date=date,
time=time,
stratified=stratified,
file=result,
metric=report.score,
duration=report.duration,
)
self.datasets[result] = report.lines
self.data.append(entry)
def get_results_criteria(
self, score, model, input_data, sort_key, number, nan=False
):
data = self.data.copy() if input_data is None else input_data
if score:
data = [x for x in data if x["score"] == score]
if model:
data = [x for x in data if x["model"] == model]
if nan:
data = [x for x in data if x["metric"] != x["metric"]]
keys = (
itemgetter(sort_key, "time")
if sort_key == "date"
else itemgetter(sort_key, "date", "time")
)
data = sorted(data, key=keys, reverse=True)
if number > 0:
data = data[:number]
return data
def list_results(
self,
score=None,
model=None,
input_data=None,
sort_key="date",
number=0,
nan=False,
) -> None:
"""Print the list of results"""
if self.data_filtered == []:
self.data_filtered = self.get_results_criteria(
score, model, input_data, sort_key, number, nan=nan
)
if self.data_filtered == []:
raise ValueError(NO_RESULTS)
max_file = max(len(x["file"]) for x in self.data_filtered)
max_title = max(len(x["title"]) for x in self.data_filtered)
if self.hidden:
color1 = TextColor.GREEN
color2 = TextColor.YELLOW
else:
color1 = TextColor.LINE1
color2 = TextColor.LINE2
print(color1, end="")
print(
f" # {'Date':10s} {'File':{max_file}s} {'Score':8s} "
f"{'Time(h)':7s} {'Title':s}"
)
print(
"===",
"=" * 10
+ " "
+ "=" * max_file
+ " "
+ "=" * 8
+ " "
+ "=" * 7
+ " "
+ "=" * max_title,
)
print(
"\n".join(
[
(color2 if n % 2 == 0 else color1) + f"{n:3d} "
f"{x['date']} {x['file']:{max_file}s} "
f"{x['metric']:8.5f} "
f"{x['duration']/3600:7.3f} "
f"{x['title']}"
for n, x in enumerate(self.data_filtered)
]
)
)
def manage_results(self):
"""Manage results showed in the summary
return True if excel file is created False otherwise
"""
def process_file(num, command, path):
num = int(num)
name = self.data_filtered[num]["file"]
file_name_result = os.path.join(path, name)
verb1, verb2 = (
("delete", "Deleting")
if command == cmd.delete
else (
"hide",
"Hiding",
)
)
conf_message = (
TextColor.RED
+ f"Are you sure to {verb1} {file_name_result} (y/n)? "
)
confirm = get_input(message=conf_message)
if confirm == "y":
print(TextColor.YELLOW + f"{verb2} {file_name_result}")
if command == cmd.delete:
os.unlink(file_name_result)
else:
os.rename(
os.path.join(Folders.results, name),
os.path.join(Folders.hidden_results, name),
)
self.data_filtered.pop(num)
get_input(message="Press enter to continue")
self.list_results()
cmd = SimpleNamespace(
quit="q", relist="r", delete="d", hide="h", excel="e"
)
message = (
TextColor.ENDC
+ f"Choose option {str(cmd).replace('namespace', '')}: "
)
path = Folders.hidden_results if self.hidden else Folders.results
book = None
max_value = len(self.data_filtered)
while True:
match get_input(message=message).split():
case [cmd.relist]:
self.list_results()
case [cmd.quit]:
if book is not None:
book.close()
return True
return False
case [cmd.hide, num] if num.isdigit() and int(num) < max_value:
if self.hidden:
print("Already hidden")
else:
process_file(num, path=path, command=cmd.hide)
case [cmd.delete, num] if num.isdigit() and int(
num
) < max_value:
process_file(num=num, path=path, command=cmd.delete)
case [cmd.excel, num] if num.isdigit() and int(
num
) < max_value:
# Add to excel file result #num
num = int(num)
file_name_result = os.path.join(
path, self.data_filtered[num]["file"]
)
if book is None:
file_name = os.path.join(
Folders.excel, Files.be_list_excel
)
book = xlsxwriter.Workbook(
file_name, {"nan_inf_to_errors": True}
)
excel = Excel(
file_name=file_name_result,
book=book,
compare=self.compare,
)
excel.report()
print(f"Added {file_name_result} to {Files.be_list_excel}")
case [num] if num.isdigit() and int(num) < max_value:
# Report the result #num
num = int(num)
file_name_result = os.path.join(
path, self.data_filtered[num]["file"]
)
try:
rep = Report(file_name_result, compare=self.compare)
rep.report()
except ValueError as e:
print(e)
case _:
print("Invalid option. Try again!")
def show_result(self, data: dict, title: str = "") -> None:
def whites(n: int) -> str:
return " " * n + color1 + "*"
if data == {}:
print(f"** {title} has No data **")
return
color1 = TextColor.CYAN
color2 = TextColor.YELLOW
file_name = data["file"]
metric = data["metric"]
result = StubReport(os.path.join(Folders.results, file_name))
length = 81
print(color1 + "*" * length)
if title != "":
print(
"*"
+ color2
+ TextColor.BOLD
+ f"{title:^{length - 2}s}"
+ TextColor.ENDC
+ color1
+ "*"
)
print("*" + "-" * (length - 2) + "*")
print("*" + whites(length - 2))
print(
"* "
+ color2
+ f"{result.data['title']:^{length - 4}}"
+ color1
+ " *"
)
print("*" + whites(length - 2))
print(
"* Model: "
+ color2
+ f"{result.data['model']:15s} "
+ color1
+ "Ver. "
+ color2
+ f"{result.data['version']:10s} "
+ color1
+ "Score: "
+ color2
+ f"{result.data['score_name']:10s} "
+ color1
+ "Metric: "
+ color2
+ f"{metric:10.7f}"
+ whites(length - 78)
)
print(color1 + "*" + whites(length - 2))
print(
"* Date : "
+ color2
+ f"{result.data['date']:15s}"
+ color1
+ " Time: "
+ color2
+ f"{result.data['time']:18s} "
+ color1
+ "Time Spent: "
+ color2
+ f"{result.data['duration']:9,.2f}"
+ color1
+ " secs."
+ whites(length - 78)
)
seeds = str(result.data["seeds"])
seeds_len = len(seeds)
print(
"* Seeds: "
+ color2
+ f"{seeds:{seeds_len}s} "
+ color1
+ "Platform: "
+ color2
+ f"{result.data['platform']:17s} "
+ whites(length - 79)
)
print(
"* Stratified: "
+ color2
+ f"{str(result.data['stratified']):15s}"
+ whites(length - 30)
)
print("* " + color2 + f"{file_name:60s}" + whites(length - 63))
print(color1 + "*" + whites(length - 2))
print(color1 + "*" * length)
def best_results(self, criterion=None, value=None, score="accuracy", n=10):
# First filter the same score results (accuracy, f1, ...)
haystack = [x for x in self.data if x["score"] == score]
haystack = (
haystack
if criterion is None or value is None
else [x for x in haystack if x[criterion] == value]
)
if haystack == []:
raise ValueError(NO_RESULTS)
return (
sorted(
haystack,
key=lambda x: -1.0 if math.isnan(x["metric"]) else x["metric"],
reverse=True,
)[:n]
if len(haystack) > 0
else {}
)
def best_result(
self, criterion=None, value=None, score="accuracy"
) -> dict:
return self.best_results(criterion, value, score)[0]
def best_results_datasets(self, score="accuracy") -> dict:
"""Get the best results for each dataset"""
dt = Datasets()
best_results = {}
for dataset in dt:
best_results[dataset] = (1, "", "", "")
haystack = [x for x in self.data if x["score"] == score]
# Search for the best results for each dataset
for entry in haystack:
for dataset in self.datasets[entry["file"]]:
if dataset["score"] < best_results[dataset["dataset"]][0]:
best_results[dataset["dataset"]] = (
dataset["score"],
dataset["hyperparameters"],
entry["file"],
entry["title"],
)
return best_results
def show_top(self, score="accuracy", n=10):
try:
self.list_results(
score=score,
input_data=self.best_results(score=score, n=n),
sort_key="metric",
)
except ValueError as e:
print(e)
class PairCheck: class PairCheck:
def __init__(self, score, model_a, model_b, winners=False, losers=False): def __init__(self, score, model_a, model_b, winners=False, losers=False):
self.score = score self.score = score

278
benchmark/ResultsBase.py
View File

@@ -1,7 +1,11 @@
import abc import abc
import json import json
import math
import os import os
from operator import itemgetter
from benchmark.Datasets import Datasets
from benchmark.Utils import NO_RESULTS, Files, Folders, TextColor
from .Arguments import ALL_METRICS, EnvData from .Arguments import ALL_METRICS, EnvData
from .Datasets import Datasets from .Datasets import Datasets
@@ -9,10 +13,6 @@ from .Experiments import BestResults
from .Utils import Folders, Symbols from .Utils import Folders, Symbols
def get_input(message="", is_test=False):
return "test" if is_test else input(message)
class BestResultsEver: class BestResultsEver:
def __init__(self): def __init__(self):
self.data = {} self.data = {}
@@ -161,3 +161,273 @@ class StubReport(BaseReport):
def footer(self, accuracy: float) -> None: def footer(self, accuracy: float) -> None:
self.accuracy = accuracy self.accuracy = accuracy
self.score = accuracy / self._get_best_accuracy() self.score = accuracy / self._get_best_accuracy()
class Summary:
def __init__(self, hidden=False, compare=False) -> None:
self.results = Files().get_all_results(hidden=hidden)
self.data = []
self.data_filtered = []
self.datasets = {}
self.models = set()
self.hidden = hidden
self.compare = compare
def get_models(self):
return sorted(self.models)
def acquire(self, given_score="any") -> None:
"""Get all results"""
for result in self.results:
(
score,
model,
platform,
date,
time,
stratified,
) = Files().split_file_name(result)
if given_score in ("any", score):
self.models.add(model)
report = StubReport(
os.path.join(
Folders.hidden_results
if self.hidden
else Folders.results,
result,
)
)
report.report()
entry = dict(
score=score,
model=model,
title=report.title,
platform=platform,
date=date,
time=time,
stratified=stratified,
file=result,
metric=report.score,
duration=report.duration,
)
self.datasets[result] = report.lines
self.data.append(entry)
def get_results_criteria(
self, score, model, input_data, sort_key, number, nan=False
):
data = self.data.copy() if input_data is None else input_data
if score:
data = [x for x in data if x["score"] == score]
if model:
data = [x for x in data if x["model"] == model]
if nan:
data = [x for x in data if x["metric"] != x["metric"]]
keys = (
itemgetter(sort_key, "time")
if sort_key == "date"
else itemgetter(sort_key, "date", "time")
)
data = sorted(data, key=keys, reverse=True)
if number > 0:
data = data[:number]
return data
def list_results(
self,
score=None,
model=None,
input_data=None,
sort_key="date",
number=0,
nan=False,
) -> None:
"""Print the list of results"""
if self.data_filtered == []:
self.data_filtered = self.get_results_criteria(
score, model, input_data, sort_key, number, nan=nan
)
if self.data_filtered == []:
raise ValueError(NO_RESULTS)
max_file = max(len(x["file"]) for x in self.data_filtered)
max_title = max(len(x["title"]) for x in self.data_filtered)
if self.hidden:
color1 = TextColor.GREEN
color2 = TextColor.YELLOW
else:
color1 = TextColor.LINE1
color2 = TextColor.LINE2
print(color1, end="")
print(
f" # {'Date':10s} {'File':{max_file}s} {'Score':8s} "
f"{'Time(h)':7s} {'Title':s}"
)
print(
"===",
"=" * 10
+ " "
+ "=" * max_file
+ " "
+ "=" * 8
+ " "
+ "=" * 7
+ " "
+ "=" * max_title,
)
print(
"\n".join(
[
(color2 if n % 2 == 0 else color1) + f"{n:3d} "
f"{x['date']} {x['file']:{max_file}s} "
f"{x['metric']:8.5f} "
f"{x['duration']/3600:7.3f} "
f"{x['title']}"
for n, x in enumerate(self.data_filtered)
]
)
)
def show_result(self, data: dict, title: str = "") -> None:
def whites(n: int) -> str:
return " " * n + color1 + "*"
if data == {}:
print(f"** {title} has No data **")
return
color1 = TextColor.CYAN
color2 = TextColor.YELLOW
file_name = data["file"]
metric = data["metric"]
result = StubReport(os.path.join(Folders.results, file_name))
length = 81
print(color1 + "*" * length)
if title != "":
print(
"*"
+ color2
+ TextColor.BOLD
+ f"{title:^{length - 2}s}"
+ TextColor.ENDC
+ color1
+ "*"
)
print("*" + "-" * (length - 2) + "*")
print("*" + whites(length - 2))
print(
"* "
+ color2
+ f"{result.data['title']:^{length - 4}}"
+ color1
+ " *"
)
print("*" + whites(length - 2))
print(
"* Model: "
+ color2
+ f"{result.data['model']:15s} "
+ color1
+ "Ver. "
+ color2
+ f"{result.data['version']:10s} "
+ color1
+ "Score: "
+ color2
+ f"{result.data['score_name']:10s} "
+ color1
+ "Metric: "
+ color2
+ f"{metric:10.7f}"
+ whites(length - 78)
)
print(color1 + "*" + whites(length - 2))
print(
"* Date : "
+ color2
+ f"{result.data['date']:15s}"
+ color1
+ " Time: "
+ color2
+ f"{result.data['time']:18s} "
+ color1
+ "Time Spent: "
+ color2
+ f"{result.data['duration']:9,.2f}"
+ color1
+ " secs."
+ whites(length - 78)
)
seeds = str(result.data["seeds"])
seeds_len = len(seeds)
print(
"* Seeds: "
+ color2
+ f"{seeds:{seeds_len}s} "
+ color1
+ "Platform: "
+ color2
+ f"{result.data['platform']:17s} "
+ whites(length - 79)
)
print(
"* Stratified: "
+ color2
+ f"{str(result.data['stratified']):15s}"
+ whites(length - 30)
)
print("* " + color2 + f"{file_name:60s}" + whites(length - 63))
print(color1 + "*" + whites(length - 2))
print(color1 + "*" * length)
def best_results(self, criterion=None, value=None, score="accuracy", n=10):
# First filter the same score results (accuracy, f1, ...)
haystack = [x for x in self.data if x["score"] == score]
haystack = (
haystack
if criterion is None or value is None
else [x for x in haystack if x[criterion] == value]
)
if haystack == []:
raise ValueError(NO_RESULTS)
return (
sorted(
haystack,
key=lambda x: -1.0 if math.isnan(x["metric"]) else x["metric"],
reverse=True,
)[:n]
if len(haystack) > 0
else {}
)
def best_result(
self, criterion=None, value=None, score="accuracy"
) -> dict:
return self.best_results(criterion, value, score)[0]
def best_results_datasets(self, score="accuracy") -> dict:
"""Get the best results for each dataset"""
dt = Datasets()
best_results = {}
for dataset in dt:
best_results[dataset] = (1, "", "", "")
haystack = [x for x in self.data if x["score"] == score]
# Search for the best results for each dataset
for entry in haystack:
for dataset in self.datasets[entry["file"]]:
if dataset["score"] < best_results[dataset["dataset"]][0]:
best_results[dataset["dataset"]] = (
dataset["score"],
dataset["hyperparameters"],
entry["file"],
entry["title"],
)
return best_results
def show_top(self, score="accuracy", n=10):
try:
self.list_results(
score=score,
input_data=self.best_results(score=score, n=n),
sort_key="metric",
)
except ValueError as e:
print(e)

2
benchmark/ResultsFiles.py
View File

@@ -12,7 +12,7 @@ from xlsxwriter.exceptions import DuplicateWorksheetName
from ._version import __version__ from ._version import __version__
from .Arguments import EnvData from .Arguments import EnvData
from .Datasets import Datasets from .Datasets import Datasets
from .ResultsBase import BaseReport from .ResultsBase import BaseReport, BestResultsEver, Summary, StubReport
from .Utils import NO_RESULTS, Files, Folders, TextColor from .Utils import NO_RESULTS, Files, Folders, TextColor

3
benchmark/__init__.py
View File

@@ -1,3 +1,4 @@
from .ResultsBase import Summary
from .Datasets import ( from .Datasets import (
Datasets, Datasets,
DatasetsSurcov, DatasetsSurcov,
@@ -5,7 +6,7 @@ from .Datasets import (
DatasetsArff, DatasetsArff,
) )
from .Experiments import Experiment from .Experiments import Experiment
from .Results import Report, Summary from .Results import Report
from ._version import __version__ from ._version import __version__
__author__ = "Ricardo Montañana Gómez" __author__ = "Ricardo Montañana Gómez"

2
benchmark/scripts/be_benchmark.py
View File

@@ -1,5 +1,5 @@
#!/usr/bin/env python #!/usr/bin/env python
from benchmark.Results import Benchmark from benchmark.ResultsFiles import Benchmark
from benchmark.Utils import Files from benchmark.Utils import Files
from benchmark.Arguments import Arguments from benchmark.Arguments import Arguments

2
benchmark/scripts/be_best.py
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env python #!/usr/bin/env python
import json import json
from benchmark.Results import Summary from benchmark.ResultsBase import Summary
from benchmark.Arguments import ALL_METRICS, Arguments from benchmark.Arguments import ALL_METRICS, Arguments

6
benchmark/scripts/be_list.py
View File

@@ -1,8 +1,9 @@
#! /usr/bin/env python #! /usr/bin/env python
import os import os
from benchmark.Results import Summary from benchmark.ResultsBase import Summary
from benchmark.Utils import Files, Folders from benchmark.Utils import Files, Folders
from benchmark.Arguments import Arguments from benchmark.Arguments import Arguments
from benchmark.Manager import Manage
"""List experiments of a model """List experiments of a model
""" """
@@ -27,7 +28,8 @@ def main(args_test=None):
except ValueError as e: except ValueError as e:
print(e) print(e)
return return
excel_generated = data.manage_results() manager = Manage(data)
excel_generated = manager.manage_results()
if excel_generated: if excel_generated:
name = os.path.join(Folders.excel, Files.be_list_excel) name = os.path.join(Folders.excel, Files.be_list_excel)
print(f"Generated file: {name}") print(f"Generated file: {name}")

2
benchmark/scripts/be_summary.py
View File

@@ -1,5 +1,5 @@
#!/usr/bin/env python #!/usr/bin/env python
from benchmark.Results import Summary from benchmark.ResultsBase import Summary
from benchmark.Arguments import ALL_METRICS, Arguments from benchmark.Arguments import ALL_METRICS, Arguments

9
benchmark/tests/Benchmark_test.py
View File

@@ -90,15 +90,6 @@ class BenchmarkTest(TestBase):
self.assertTrue(os.path.exists(benchmark.get_tex_file())) self.assertTrue(os.path.exists(benchmark.get_tex_file()))
self.check_file_file(benchmark.get_tex_file(), "exreport_tex") self.check_file_file(benchmark.get_tex_file(), "exreport_tex")
@staticmethod
def generate_excel_sheet(test, sheet, file_name):
with open(os.path.join("test_files", file_name), "w") as f:
for row in range(1, sheet.max_row + 1):
for col in range(1, sheet.max_column + 1):
value = sheet.cell(row=row, column=col).value
if value is not None:
print(f'{row};{col};"{value}"', file=f)
def test_excel_output(self): def test_excel_output(self):
benchmark = Benchmark("accuracy", visualize=False) benchmark = Benchmark("accuracy", visualize=False)
benchmark.compile_results() benchmark.compile_results()

3
benchmark/tests/Report_test.py
View File

@@ -4,7 +4,8 @@ from unittest.mock import patch
from .TestBase import TestBase from .TestBase import TestBase
from ..Results import Report, ReportBest from ..Results import Report, ReportBest
from ..ResultsFiles import ReportDatasets from ..ResultsFiles import ReportDatasets
from ..ResultsBase import BaseReport, get_input from ..ResultsBase import BaseReport
from ..Manager import get_input
from ..Utils import Symbols from ..Utils import Symbols

2
benchmark/tests/Summary_test.py
View File

@@ -1,7 +1,7 @@
from io import StringIO from io import StringIO
from unittest.mock import patch from unittest.mock import patch
from .TestBase import TestBase from .TestBase import TestBase
from ..Results import Summary from ..ResultsBase import Summary
from ..Utils import NO_RESULTS from ..Utils import NO_RESULTS

26
benchmark/tests/scripts/Be_List_test.py
View File

@@ -10,31 +10,31 @@ class BeListTest(TestBase):
def setUp(self): def setUp(self):
self.prepare_scripts_env() self.prepare_scripts_env()
@patch("benchmark.Results.get_input", return_value="q") @patch("benchmark.Manager.get_input", return_value="q")
def test_be_list(self, input_data): def test_be_list(self, input_data):
stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
self.assertEqual(stderr.getvalue(), "") self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_model") self.check_output_file(stdout, "be_list_model")
@patch("benchmark.Results.get_input", side_effect=iter(["x", "q"])) @patch("benchmark.Manager.get_input", side_effect=iter(["x", "q"]))
def test_be_list_invalid_option(self, input_data): def test_be_list_invalid_option(self, input_data):
stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
self.assertEqual(stderr.getvalue(), "") self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_model_invalid") self.check_output_file(stdout, "be_list_model_invalid")
@patch("benchmark.Results.get_input", side_effect=iter(["0", "q"])) @patch("benchmark.Manager.get_input", side_effect=iter(["0", "q"]))
def test_be_list_report(self, input_data): def test_be_list_report(self, input_data):
stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
self.assertEqual(stderr.getvalue(), "") self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_report") self.check_output_file(stdout, "be_list_report")
@patch("benchmark.Results.get_input", side_effect=iter(["r", "q"])) @patch("benchmark.Manager.get_input", side_effect=iter(["r", "q"]))
def test_be_list_twice(self, input_data): def test_be_list_twice(self, input_data):
stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
self.assertEqual(stderr.getvalue(), "") self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_model_2") self.check_output_file(stdout, "be_list_model_2")
@patch("benchmark.Results.get_input", side_effect=iter(["e 2", "q"])) @patch("benchmark.Manager.get_input", side_effect=iter(["e 2", "q"]))
def test_be_list_report_excel(self, input_data): def test_be_list_report_excel(self, input_data):
stdout, stderr = self.execute_script("be_list", ["-m", "STree"]) stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
self.assertEqual(stderr.getvalue(), "") self.assertEqual(stderr.getvalue(), "")
@@ -45,7 +45,7 @@ class BeListTest(TestBase):
self.check_excel_sheet(sheet, "excel") self.check_excel_sheet(sheet, "excel")
@patch( @patch(
"benchmark.Results.get_input", "benchmark.Manager.get_input",
side_effect=iter(["e 2", "e 1", "q"]), side_effect=iter(["e 2", "e 1", "q"]),
) )
def test_be_list_report_excel_twice(self, input_data): def test_be_list_report_excel_twice(self, input_data):
@@ -58,7 +58,7 @@ class BeListTest(TestBase):
sheet = book["STree2"] sheet = book["STree2"]
self.check_excel_sheet(sheet, "excel2") self.check_excel_sheet(sheet, "excel2")
@patch("benchmark.Results.get_input", return_value="q") @patch("benchmark.Manager.get_input", return_value="q")
def test_be_list_no_data(self, input_data): def test_be_list_no_data(self, input_data):
stdout, stderr = self.execute_script( stdout, stderr = self.execute_script(
"be_list", ["-m", "Wodt", "-s", "f1-macro"] "be_list", ["-m", "Wodt", "-s", "f1-macro"]
@@ -67,7 +67,7 @@ class BeListTest(TestBase):
self.assertEqual(stdout.getvalue(), f"{NO_RESULTS}\n") self.assertEqual(stdout.getvalue(), f"{NO_RESULTS}\n")
@patch( @patch(
"benchmark.Results.get_input", "benchmark.Manager.get_input",
side_effect=iter(["d 0", "y", "", "q"]), side_effect=iter(["d 0", "y", "", "q"]),
) )
# @patch("benchmark.ResultsBase.get_input", side_effect=iter(["q"])) # @patch("benchmark.ResultsBase.get_input", side_effect=iter(["q"]))
@@ -94,7 +94,7 @@ class BeListTest(TestBase):
self.fail("test_be_list_delete() should not raise exception") self.fail("test_be_list_delete() should not raise exception")
@patch( @patch(
"benchmark.Results.get_input", "benchmark.Manager.get_input",
side_effect=iter(["h 0", "y", "", "q"]), side_effect=iter(["h 0", "y", "", "q"]),
) )
def test_be_list_hide(self, input_data): def test_be_list_hide(self, input_data):
@@ -119,25 +119,25 @@ class BeListTest(TestBase):
swap_files(Folders.results, Folders.hidden_results, file_name) swap_files(Folders.results, Folders.hidden_results, file_name)
self.fail("test_be_list_hide() should not raise exception") self.fail("test_be_list_hide() should not raise exception")
@patch("benchmark.Results.get_input", side_effect=iter(["h 0", "q"])) @patch("benchmark.Manager.get_input", side_effect=iter(["h 0", "q"]))
def test_be_list_already_hidden(self, input_data): def test_be_list_already_hidden(self, input_data):
stdout, stderr = self.execute_script("be_list", ["--hidden"]) stdout, stderr = self.execute_script("be_list", ["--hidden"])
self.assertEqual(stderr.getvalue(), "") self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_already_hidden") self.check_output_file(stdout, "be_list_already_hidden")
@patch("benchmark.Results.get_input", side_effect=iter(["h 0", "n", "q"])) @patch("benchmark.Manager.get_input", side_effect=iter(["h 0", "n", "q"]))
def test_be_list_dont_hide(self, input_data): def test_be_list_dont_hide(self, input_data):
stdout, stderr = self.execute_script("be_list", "") stdout, stderr = self.execute_script("be_list", "")
self.assertEqual(stderr.getvalue(), "") self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_default") self.check_output_file(stdout, "be_list_default")
@patch("benchmark.Results.get_input", side_effect=iter(["q"])) @patch("benchmark.Manager.get_input", side_effect=iter(["q"]))
def test_be_list_hidden_nan(self, input_data): def test_be_list_hidden_nan(self, input_data):
stdout, stderr = self.execute_script("be_list", ["--hidden", "--nan"]) stdout, stderr = self.execute_script("be_list", ["--hidden", "--nan"])
self.assertEqual(stderr.getvalue(), "") self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_hidden_nan") self.check_output_file(stdout, "be_list_hidden_nan")
@patch("benchmark.Results.get_input", side_effect=iter(["q"])) @patch("benchmark.Manager.get_input", side_effect=iter(["q"]))
def test_be_list_hidden(self, input_data): def test_be_list_hidden(self, input_data):
stdout, stderr = self.execute_script("be_list", ["--hidden"]) stdout, stderr = self.execute_script("be_list", ["--hidden"])
self.assertEqual(stderr.getvalue(), "") self.assertEqual(stderr.getvalue(), "")

6
benchmark/tests/scripts/Be_Main_test.py
View File

@@ -25,7 +25,7 @@ class BeMainTest(TestBase):
self.check_output_lines( self.check_output_lines(
stdout=stdout, stdout=stdout,
file_name="be_main_dataset", file_name="be_main_dataset",
lines_to_compare=[0, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13], lines_to_compare=[0, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13, 14],
) )
def test_be_main_complete(self): def test_be_main_complete(self):
@@ -37,7 +37,9 @@ class BeMainTest(TestBase):
report_name = stdout.getvalue().splitlines()[-1].split("in ")[1] report_name = stdout.getvalue().splitlines()[-1].split("in ")[1]
self.files.append(report_name) self.files.append(report_name)
self.check_output_lines( self.check_output_lines(
stdout, "be_main_complete", [0, 2, 3, 5, 6, 7, 8, 9, 12, 13, 14] stdout,
"be_main_complete",
[0, 2, 3, 5, 6, 7, 8, 9, 12, 13, 14, 15],
) )
def test_be_main_no_report(self): def test_be_main_no_report(self):

1
benchmark/tests/test_files/be_main_best.test
View File

@@ -11,6 +11,7 @@ Dataset Sampl. Feat. Cls Nodes Leaves Depth Score
balance-scale 625 4 3 23.32 12.16 6.44 0.840160±0.0304 0.013745±0.0019 {'splitter': 'best', 'max_features': 'auto'} balance-scale 625 4 3 23.32 12.16 6.44 0.840160±0.0304 0.013745±0.0019 {'splitter': 'best', 'max_features': 'auto'}
balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000388±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000, 'multiclass_strategy': 'ovr'} balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000388±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000, 'multiclass_strategy': 'ovr'}
************************************************************************************************************************* *************************************************************************************************************************
* ➶ Better than ZeroR + 10.0%.....: 1 *
* accuracy compared to STree_default (liblinear-ovr) .: 0.0422 * * accuracy compared to STree_default (liblinear-ovr) .: 0.0422 *
************************************************************************************************************************* *************************************************************************************************************************
Results in results/results_accuracy_STree_iMac27_2022-05-09_00:15:25_0.json Results in results/results_accuracy_STree_iMac27_2022-05-09_00:15:25_0.json

1
benchmark/tests/test_files/be_main_complete.test
View File

@@ -11,6 +11,7 @@ Dataset Sampl. Feat. Cls Nodes Leaves Depth Score
balance-scale 625 4 3 17.36 9.18 6.18 0.908480±0.0247 0.007388±0.0013 {} balance-scale 625 4 3 17.36 9.18 6.18 0.908480±0.0247 0.007388±0.0013 {}
balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000664±0.0002 {} balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000664±0.0002 {}
************************************************************************************************************************* *************************************************************************************************************************
* ➶ Better than ZeroR + 10.0%.....: 1 *
* accuracy compared to STree_default (liblinear-ovr) .: 0.0390 * * accuracy compared to STree_default (liblinear-ovr) .: 0.0390 *
************************************************************************************************************************* *************************************************************************************************************************
Results in results/results_accuracy_STree_iMac27_2022-05-08_20:14:43_0.json Results in results/results_accuracy_STree_iMac27_2022-05-08_20:14:43_0.json

3
benchmark/tests/test_files/be_main_dataset.test
View File

@@ -8,8 +8,9 @@
Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters
============================== ====== ===== === ======= ======= ======= =============== ================= =============== ============================== ====== ===== === ======= ======= ======= =============== ================= ===============
balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000671±0.0001 {} balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009➶ 0.000671±0.0001 {}
************************************************************************************************************************* *************************************************************************************************************************
* ➶ Better than ZeroR + 10.0%.....: 1 *
* accuracy compared to STree_default (liblinear-ovr) .: 0.0165 * * accuracy compared to STree_default (liblinear-ovr) .: 0.0165 *
************************************************************************************************************************* *************************************************************************************************************************
Partial result file removed: results/results_accuracy_STree_iMac27_2022-05-08_19:38:28_0.json Partial result file removed: results/results_accuracy_STree_iMac27_2022-05-08_19:38:28_0.json

1
benchmark/tests/test_files/be_main_grid.test
View File

@@ -11,6 +11,7 @@ Dataset Sampl. Feat. Cls Nodes Leaves Depth Score
balance-scale 625 4 3 26.12 13.56 7.94 0.910720±0.0249 0.015852±0.0027 {'C': 1.0, 'kernel': 'liblinear', 'multiclass_strategy': 'ovr'} balance-scale 625 4 3 26.12 13.56 7.94 0.910720±0.0249 0.015852±0.0027 {'C': 1.0, 'kernel': 'liblinear', 'multiclass_strategy': 'ovr'}
balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000640±0.0001 {'C': 1.0, 'kernel': 'linear', 'multiclass_strategy': 'ovr'} balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000640±0.0001 {'C': 1.0, 'kernel': 'linear', 'multiclass_strategy': 'ovr'}
************************************************************************************************************************* *************************************************************************************************************************
* ➶ Better than ZeroR + 10.0%.....: 1 *
* accuracy compared to STree_default (liblinear-ovr) .: 0.0391 * * accuracy compared to STree_default (liblinear-ovr) .: 0.0391 *
************************************************************************************************************************* *************************************************************************************************************************
Results in results/results_accuracy_STree_iMac27_2022-05-09_00:21:06_0.json Results in results/results_accuracy_STree_iMac27_2022-05-09_00:21:06_0.json

40
benchmark/tests/test_files/exreport_excel_ODTE.test
View File

@@ -3,12 +3,12 @@
3;1;" Score is accuracy" 3;1;" Score is accuracy"
3;2;" Execution time" 3;2;" Execution time"
3;5;"22,591.47 s" 3;5;"22,591.47 s"
3;7;" " 3;7;"Platform"
3;8;"Platform"
3;9;"Galgo" 3;9;"Galgo"
3;10;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]" 3;11;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]"
4;5;" 6.28 h" 4;5;" 6.28 h"
4;10;"Stratified: False" 4;11;"Stratified: False"
4;13;"Discretized: False"
6;1;"Dataset" 6;1;"Dataset"
6;2;"Samples" 6;2;"Samples"
6;3;"Features" 6;3;"Features"
@@ -17,10 +17,11 @@
6;6;"Leaves" 6;6;"Leaves"
6;7;"Depth" 6;7;"Depth"
6;8;"Score" 6;8;"Score"
6;9;"Score Std." 6;9;"Stat"
6;10;"Time" 6;10;"Score Std."
6;11;"Time Std." 6;11;"Time"
6;12;"Hyperparameters" 6;12;"Time Std."
6;13;"Hyperparameters"
7;1;"balance-scale" 7;1;"balance-scale"
7;2;"625" 7;2;"625"
7;3;"4" 7;3;"4"
@@ -29,10 +30,11 @@
7;6;"4.180599999999999" 7;6;"4.180599999999999"
7;7;"3.536" 7;7;"3.536"
7;8;"0.96352" 7;8;"0.96352"
7;9;"0.02494974148162661" 7;9;" "
7;10;"0.3166321754455567" 7;10;"0.02494974148162661"
7;11;"0.1991881389525559" 7;11;"0.3166321754455567"
7;12;"{'base_estimator__C': 57, 'base_estimator__gamma': 0.1, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1}" 7;12;"0.1991881389525559"
7;13;"{'base_estimator__C': 57, 'base_estimator__gamma': 0.1, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1}"
8;1;"balloons" 8;1;"balloons"
8;2;"16" 8;2;"16"
8;3;"4" 8;3;"4"
@@ -41,8 +43,12 @@
8;6;"1.9976" 8;6;"1.9976"
8;7;"1.9976" 8;7;"1.9976"
8;8;"0.785" 8;8;"0.785"
8;9;"0.2461311755051675" 8;9;"➶"
8;10;"0.1156062078475952" 8;10;"0.2461311755051675"
8;11;"0.0127842418285999" 8;11;"0.1156062078475952"
8;12;"{'base_estimator__C': 5, 'base_estimator__gamma': 0.14, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1}" 8;12;"0.0127842418285999"
10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0434" 8;13;"{'base_estimator__C': 5, 'base_estimator__gamma': 0.14, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1}"
11;2;"➶"
11;3;"1"
11;4;"Better than ZeroR + 10.0%"
13;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0434"

40
benchmark/tests/test_files/exreport_excel_RandomForest.test
View File

@@ -3,12 +3,12 @@
3;1;" Score is accuracy" 3;1;" Score is accuracy"
3;2;" Execution time" 3;2;" Execution time"
3;5;" 272.74 s" 3;5;" 272.74 s"
3;7;" " 3;7;"Platform"
3;8;"Platform"
3;9;"iMac27" 3;9;"iMac27"
3;10;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]" 3;11;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]"
4;5;" 0.08 h" 4;5;" 0.08 h"
4;10;"Stratified: False" 4;11;"Stratified: False"
4;13;"Discretized: False"
6;1;"Dataset" 6;1;"Dataset"
6;2;"Samples" 6;2;"Samples"
6;3;"Features" 6;3;"Features"
@@ -17,10 +17,11 @@
6;6;"Leaves" 6;6;"Leaves"
6;7;"Depth" 6;7;"Depth"
6;8;"Score" 6;8;"Score"
6;9;"Score Std." 6;9;"Stat"
6;10;"Time" 6;10;"Score Std."
6;11;"Time Std." 6;11;"Time"
6;12;"Hyperparameters" 6;12;"Time Std."
6;13;"Hyperparameters"
7;1;"balance-scale" 7;1;"balance-scale"
7;2;"625" 7;2;"625"
7;3;"4" 7;3;"4"
@@ -29,10 +30,11 @@
7;6;"98.42" 7;6;"98.42"
7;7;"10.6814" 7;7;"10.6814"
7;8;"0.83616" 7;8;"0.83616"
7;9;"0.02649630917694009" 7;9;" "
7;10;"0.08222018241882324" 7;10;"0.02649630917694009"
7;11;"0.001302632681512063" 7;11;"0.08222018241882324"
7;12;"{}" 7;12;"0.001302632681512063"
7;13;"{}"
8;1;"balloons" 8;1;"balloons"
8;2;"16" 8;2;"16"
8;3;"4" 8;3;"4"
@@ -41,8 +43,12 @@
8;6;"4.58" 8;6;"4.58"
8;7;"3.0982" 8;7;"3.0982"
8;8;"0.625" 8;8;"0.625"
8;9;"0.249582985531199" 8;9;"➶"
8;10;"0.07016648769378662" 8;10;"0.249582985531199"
8;11;"0.002460508923990468" 8;11;"0.07016648769378662"
8;12;"{}" 8;12;"0.002460508923990468"
10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0363" 8;13;"{}"
11;2;"➶"
11;3;"1"
11;4;"Better than ZeroR + 10.0%"
13;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0363"

40
benchmark/tests/test_files/exreport_excel_STree.test
View File

@@ -3,12 +3,12 @@
3;1;" Score is accuracy" 3;1;" Score is accuracy"
3;2;" Execution time" 3;2;" Execution time"
3;5;" 624.25 s" 3;5;" 624.25 s"
3;7;" " 3;7;"Platform"
3;8;"Platform"
3;9;"iMac27" 3;9;"iMac27"
3;10;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]" 3;11;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]"
4;5;" 0.17 h" 4;5;" 0.17 h"
4;10;"Stratified: False" 4;11;"Stratified: False"
4;13;"Discretized: False"
6;1;"Dataset" 6;1;"Dataset"
6;2;"Samples" 6;2;"Samples"
6;3;"Features" 6;3;"Features"
@@ -17,10 +17,11 @@
6;6;"Leaves" 6;6;"Leaves"
6;7;"Depth" 6;7;"Depth"
6;8;"Score" 6;8;"Score"
6;9;"Score Std." 6;9;"Stat"
6;10;"Time" 6;10;"Score Std."
6;11;"Time Std." 6;11;"Time"
6;12;"Hyperparameters" 6;12;"Time Std."
6;13;"Hyperparameters"
7;1;"balance-scale" 7;1;"balance-scale"
7;2;"625" 7;2;"625"
7;3;"4" 7;3;"4"
@@ -29,10 +30,11 @@
7;6;"4" 7;6;"4"
7;7;"3" 7;7;"3"
7;8;"0.97056" 7;8;"0.97056"
7;9;"0.0150468069702512" 7;9;" "
7;10;"0.01404867172241211" 7;10;"0.0150468069702512"
7;11;"0.002026269126958884" 7;11;"0.01404867172241211"
7;12;"{'C': 10000, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000, 'multiclass_strategy': 'ovr'}" 7;12;"0.002026269126958884"
7;13;"{'C': 10000, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000, 'multiclass_strategy': 'ovr'}"
8;1;"balloons" 8;1;"balloons"
8;2;"16" 8;2;"16"
8;3;"4" 8;3;"4"
@@ -41,8 +43,12 @@
8;6;"2" 8;6;"2"
8;7;"2" 8;7;"2"
8;8;"0.86" 8;8;"0.86"
8;9;"0.2850146195080759" 8;9;"➶"
8;10;"0.0008541679382324218" 8;10;"0.2850146195080759"
8;11;"3.629469326417878e-05" 8;11;"0.0008541679382324218"
8;12;"{'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000, 'multiclass_strategy': 'ovr'}" 8;12;"3.629469326417878e-05"
10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0454" 8;13;"{'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000, 'multiclass_strategy': 'ovr'}"
11;2;"➶"
11;3;"1"
11;4;"Better than ZeroR + 10.0%"
13;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0454"