Doctorado-ML/benchmark
1
0
Fork 0
mirror of https://github.com/Doctorado-ML/benchmark.git synced 2025-08-16 07:55:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix tests

Browse Source
This commit is contained in:
Ricardo Montañana Gómez
2023-05-22 10:07:28 +02:00
parent 83bd321dd6
commit b6fc9096a1
21 changed files with 500 additions and 468 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
benchmark/Manager.py Normal file
View File

@@ -0,0 +1,120 @@
import os
from types import SimpleNamespace
import xlsxwriter
from benchmark.Results import Report
from benchmark.ResultsFiles import Excel
from benchmark.Utils import Files, Folders, TextColor
def get_input(message="", is_test=False):
return "test" if is_test else input(message)
class Manage:
def __init__(self, summary):
self.summary = summary
def manage_results(self):
"""Manage results showed in the summary
return True if excel file is created False otherwise
"""
def process_file(num, command, path):
num = int(num)
name = self.summary.data_filtered[num]["file"]
file_name_result = os.path.join(path, name)
verb1, verb2 = (
("delete", "Deleting")
if command == cmd.delete
else (
"hide",
"Hiding",
)
)
conf_message = (
TextColor.RED
+ f"Are you sure to {verb1} {file_name_result} (y/n)? "
)
confirm = get_input(message=conf_message)
if confirm == "y":
print(TextColor.YELLOW + f"{verb2} {file_name_result}")
if command == cmd.delete:
os.unlink(file_name_result)
else:
os.rename(
os.path.join(Folders.results, name),
os.path.join(Folders.hidden_results, name),
)
self.summary.data_filtered.pop(num)
get_input(message="Press enter to continue")
self.summary.list_results()
cmd = SimpleNamespace(
quit="q", relist="r", delete="d", hide="h", excel="e"
)
message = (
TextColor.ENDC
+ f"Choose option {str(cmd).replace('namespace', '')}: "
)
path = (
Folders.hidden_results if self.summary.hidden else Folders.results
)
book = None
max_value = len(self.summary.data_filtered)
while True:
match get_input(message=message).split():
case [cmd.relist]:
self.summary.list_results()
case [cmd.quit]:
if book is not None:
book.close()
return True
return False
case [cmd.hide, num] if num.isdigit() and int(num) < max_value:
if self.summary.hidden:
print("Already hidden")
else:
process_file(num, path=path, command=cmd.hide)
case [cmd.delete, num] if num.isdigit() and int(
num
) < max_value:
process_file(num=num, path=path, command=cmd.delete)
case [cmd.excel, num] if num.isdigit() and int(
num
) < max_value:
# Add to excel file result #num
num = int(num)
file_name_result = os.path.join(
path, self.summary.data_filtered[num]["file"]
)
if book is None:
file_name = os.path.join(
Folders.excel, Files.be_list_excel
)
book = xlsxwriter.Workbook(
file_name, {"nan_inf_to_errors": True}
)
excel = Excel(
file_name=file_name_result,
book=book,
compare=self.summary.compare,
)
excel.report()
print(f"Added {file_name_result} to {Files.be_list_excel}")
case [num] if num.isdigit() and int(num) < max_value:
# Report the result #num
num = int(num)
file_name_result = os.path.join(
path, self.summary.data_filtered[num]["file"]
)
try:
rep = Report(
file_name_result, compare=self.summary.compare
)
rep.report()
except ValueError as e:
print(e)
case _:
print("Invalid option. Try again!")

381
benchmark/Results.py
View File

@@ -1,14 +1,6 @@
import math
import os
from operator import itemgetter
from types import SimpleNamespace
import xlsxwriter
from .Datasets import Datasets
from .ResultsBase import BaseReport, StubReport, get_input
from .ResultsFiles import Excel
from .Utils import NO_RESULTS, Files, Folders, TextColor
from .ResultsBase import BaseReport, StubReport, Summary
from .Utils import Files, Folders, TextColor
class Report(BaseReport):
@@ -189,375 +181,6 @@ class ReportBest(BaseReport):
self.header_line("*")
class Summary:
def __init__(self, hidden=False, compare=False) -> None:
self.results = Files().get_all_results(hidden=hidden)
self.data = []
self.data_filtered = []
self.datasets = {}
self.models = set()
self.hidden = hidden
self.compare = compare
def get_models(self):
return sorted(self.models)
def acquire(self, given_score="any") -> None:
"""Get all results"""
for result in self.results:
(
score,
model,
platform,
date,
time,
stratified,
) = Files().split_file_name(result)
if given_score in ("any", score):
self.models.add(model)
report = StubReport(
os.path.join(
Folders.hidden_results
if self.hidden
else Folders.results,
result,
)
)
report.report()
entry = dict(
score=score,
model=model,
title=report.title,
platform=platform,
date=date,
time=time,
stratified=stratified,
file=result,
metric=report.score,
duration=report.duration,
)
self.datasets[result] = report.lines
self.data.append(entry)
def get_results_criteria(
self, score, model, input_data, sort_key, number, nan=False
):
data = self.data.copy() if input_data is None else input_data
if score:
data = [x for x in data if x["score"] == score]
if model:
data = [x for x in data if x["model"] == model]
if nan:
data = [x for x in data if x["metric"] != x["metric"]]
keys = (
itemgetter(sort_key, "time")
if sort_key == "date"
else itemgetter(sort_key, "date", "time")
)
data = sorted(data, key=keys, reverse=True)
if number > 0:
data = data[:number]
return data
def list_results(
self,
score=None,
model=None,
input_data=None,
sort_key="date",
number=0,
nan=False,
) -> None:
"""Print the list of results"""
if self.data_filtered == []:
self.data_filtered = self.get_results_criteria(
score, model, input_data, sort_key, number, nan=nan
)
if self.data_filtered == []:
raise ValueError(NO_RESULTS)
max_file = max(len(x["file"]) for x in self.data_filtered)
max_title = max(len(x["title"]) for x in self.data_filtered)
if self.hidden:
color1 = TextColor.GREEN
color2 = TextColor.YELLOW
else:
color1 = TextColor.LINE1
color2 = TextColor.LINE2
print(color1, end="")
print(
f" # {'Date':10s} {'File':{max_file}s} {'Score':8s} "
f"{'Time(h)':7s} {'Title':s}"
)
print(
"===",
"=" * 10
+ " "
+ "=" * max_file
+ " "
+ "=" * 8
+ " "
+ "=" * 7
+ " "
+ "=" * max_title,
)
print(
"\n".join(
[
(color2 if n % 2 == 0 else color1) + f"{n:3d} "
f"{x['date']} {x['file']:{max_file}s} "
f"{x['metric']:8.5f} "
f"{x['duration']/3600:7.3f} "
f"{x['title']}"
for n, x in enumerate(self.data_filtered)
]
)
)
def manage_results(self):
"""Manage results showed in the summary
return True if excel file is created False otherwise
"""
def process_file(num, command, path):
num = int(num)
name = self.data_filtered[num]["file"]
file_name_result = os.path.join(path, name)
verb1, verb2 = (
("delete", "Deleting")
if command == cmd.delete
else (
"hide",
"Hiding",
)
)
conf_message = (
TextColor.RED
+ f"Are you sure to {verb1} {file_name_result} (y/n)? "
)
confirm = get_input(message=conf_message)
if confirm == "y":
print(TextColor.YELLOW + f"{verb2} {file_name_result}")
if command == cmd.delete:
os.unlink(file_name_result)
else:
os.rename(
os.path.join(Folders.results, name),
os.path.join(Folders.hidden_results, name),
)
self.data_filtered.pop(num)
get_input(message="Press enter to continue")
self.list_results()
cmd = SimpleNamespace(
quit="q", relist="r", delete="d", hide="h", excel="e"
)
message = (
TextColor.ENDC
+ f"Choose option {str(cmd).replace('namespace', '')}: "
)
path = Folders.hidden_results if self.hidden else Folders.results
book = None
max_value = len(self.data_filtered)
while True:
match get_input(message=message).split():
case [cmd.relist]:
self.list_results()
case [cmd.quit]:
if book is not None:
book.close()
return True
return False
case [cmd.hide, num] if num.isdigit() and int(num) < max_value:
if self.hidden:
print("Already hidden")
else:
process_file(num, path=path, command=cmd.hide)
case [cmd.delete, num] if num.isdigit() and int(
num
) < max_value:
process_file(num=num, path=path, command=cmd.delete)
case [cmd.excel, num] if num.isdigit() and int(
num
) < max_value:
# Add to excel file result #num
num = int(num)
file_name_result = os.path.join(
path, self.data_filtered[num]["file"]
)
if book is None:
file_name = os.path.join(
Folders.excel, Files.be_list_excel
)
book = xlsxwriter.Workbook(
file_name, {"nan_inf_to_errors": True}
)
excel = Excel(
file_name=file_name_result,
book=book,
compare=self.compare,
)
excel.report()
print(f"Added {file_name_result} to {Files.be_list_excel}")
case [num] if num.isdigit() and int(num) < max_value:
# Report the result #num
num = int(num)
file_name_result = os.path.join(
path, self.data_filtered[num]["file"]
)
try:
rep = Report(file_name_result, compare=self.compare)
rep.report()
except ValueError as e:
print(e)
case _:
print("Invalid option. Try again!")
def show_result(self, data: dict, title: str = "") -> None:
def whites(n: int) -> str:
return " " * n + color1 + "*"
if data == {}:
print(f"** {title} has No data **")
return
color1 = TextColor.CYAN
color2 = TextColor.YELLOW
file_name = data["file"]
metric = data["metric"]
result = StubReport(os.path.join(Folders.results, file_name))
length = 81
print(color1 + "*" * length)
if title != "":
print(
"*"
+ color2
+ TextColor.BOLD
+ f"{title:^{length - 2}s}"
+ TextColor.ENDC
+ color1
+ "*"
)
print("*" + "-" * (length - 2) + "*")
print("*" + whites(length - 2))
print(
"* "
+ color2
+ f"{result.data['title']:^{length - 4}}"
+ color1
+ " *"
)
print("*" + whites(length - 2))
print(
"* Model: "
+ color2
+ f"{result.data['model']:15s} "
+ color1
+ "Ver. "
+ color2
+ f"{result.data['version']:10s} "
+ color1
+ "Score: "
+ color2
+ f"{result.data['score_name']:10s} "
+ color1
+ "Metric: "
+ color2
+ f"{metric:10.7f}"
+ whites(length - 78)
)
print(color1 + "*" + whites(length - 2))
print(
"* Date : "
+ color2
+ f"{result.data['date']:15s}"
+ color1
+ " Time: "
+ color2
+ f"{result.data['time']:18s} "
+ color1
+ "Time Spent: "
+ color2
+ f"{result.data['duration']:9,.2f}"
+ color1
+ " secs."
+ whites(length - 78)
)
seeds = str(result.data["seeds"])
seeds_len = len(seeds)
print(
"* Seeds: "
+ color2
+ f"{seeds:{seeds_len}s} "
+ color1
+ "Platform: "
+ color2
+ f"{result.data['platform']:17s} "
+ whites(length - 79)
)
print(
"* Stratified: "
+ color2
+ f"{str(result.data['stratified']):15s}"
+ whites(length - 30)
)
print("* " + color2 + f"{file_name:60s}" + whites(length - 63))
print(color1 + "*" + whites(length - 2))
print(color1 + "*" * length)
def best_results(self, criterion=None, value=None, score="accuracy", n=10):
# First filter the same score results (accuracy, f1, ...)
haystack = [x for x in self.data if x["score"] == score]
haystack = (
haystack
if criterion is None or value is None
else [x for x in haystack if x[criterion] == value]
)
if haystack == []:
raise ValueError(NO_RESULTS)
return (
sorted(
haystack,
key=lambda x: -1.0 if math.isnan(x["metric"]) else x["metric"],
reverse=True,
)[:n]
if len(haystack) > 0
else {}
)
def best_result(
self, criterion=None, value=None, score="accuracy"
) -> dict:
return self.best_results(criterion, value, score)[0]
def best_results_datasets(self, score="accuracy") -> dict:
"""Get the best results for each dataset"""
dt = Datasets()
best_results = {}
for dataset in dt:
best_results[dataset] = (1, "", "", "")
haystack = [x for x in self.data if x["score"] == score]
# Search for the best results for each dataset
for entry in haystack:
for dataset in self.datasets[entry["file"]]:
if dataset["score"] < best_results[dataset["dataset"]][0]:
best_results[dataset["dataset"]] = (
dataset["score"],
dataset["hyperparameters"],
entry["file"],
entry["title"],
)
return best_results
def show_top(self, score="accuracy", n=10):
try:
self.list_results(
score=score,
input_data=self.best_results(score=score, n=n),
sort_key="metric",
)
except ValueError as e:
print(e)
class PairCheck:
def __init__(self, score, model_a, model_b, winners=False, losers=False):
self.score = score

278
benchmark/ResultsBase.py
View File

@@ -1,7 +1,11 @@
import abc
import json
import math
import os
from operator import itemgetter
from benchmark.Datasets import Datasets
from benchmark.Utils import NO_RESULTS, Files, Folders, TextColor
from .Arguments import ALL_METRICS, EnvData
from .Datasets import Datasets
@@ -9,10 +13,6 @@ from .Experiments import BestResults
from .Utils import Folders, Symbols
def get_input(message="", is_test=False):
return "test" if is_test else input(message)
class BestResultsEver:
def __init__(self):
self.data = {}
@@ -161,3 +161,273 @@ class StubReport(BaseReport):
def footer(self, accuracy: float) -> None:
self.accuracy = accuracy
self.score = accuracy / self._get_best_accuracy()
class Summary:
def __init__(self, hidden=False, compare=False) -> None:
self.results = Files().get_all_results(hidden=hidden)
self.data = []
self.data_filtered = []
self.datasets = {}
self.models = set()
self.hidden = hidden
self.compare = compare
def get_models(self):
return sorted(self.models)
def acquire(self, given_score="any") -> None:
"""Get all results"""
for result in self.results:
(
score,
model,
platform,
date,
time,
stratified,
) = Files().split_file_name(result)
if given_score in ("any", score):
self.models.add(model)
report = StubReport(
os.path.join(
Folders.hidden_results
if self.hidden
else Folders.results,
result,
)
)
report.report()
entry = dict(
score=score,
model=model,
title=report.title,
platform=platform,
date=date,
time=time,
stratified=stratified,
file=result,
metric=report.score,
duration=report.duration,
)
self.datasets[result] = report.lines
self.data.append(entry)
def get_results_criteria(
self, score, model, input_data, sort_key, number, nan=False
):
data = self.data.copy() if input_data is None else input_data
if score:
data = [x for x in data if x["score"] == score]
if model:
data = [x for x in data if x["model"] == model]
if nan:
data = [x for x in data if x["metric"] != x["metric"]]
keys = (
itemgetter(sort_key, "time")
if sort_key == "date"
else itemgetter(sort_key, "date", "time")
)
data = sorted(data, key=keys, reverse=True)
if number > 0:
data = data[:number]
return data
def list_results(
self,
score=None,
model=None,
input_data=None,
sort_key="date",
number=0,
nan=False,
) -> None:
"""Print the list of results"""
if self.data_filtered == []:
self.data_filtered = self.get_results_criteria(
score, model, input_data, sort_key, number, nan=nan
)
if self.data_filtered == []:
raise ValueError(NO_RESULTS)
max_file = max(len(x["file"]) for x in self.data_filtered)
max_title = max(len(x["title"]) for x in self.data_filtered)
if self.hidden:
color1 = TextColor.GREEN
color2 = TextColor.YELLOW
else:
color1 = TextColor.LINE1
color2 = TextColor.LINE2
print(color1, end="")
print(
f" # {'Date':10s} {'File':{max_file}s} {'Score':8s} "
f"{'Time(h)':7s} {'Title':s}"
)
print(
"===",
"=" * 10
+ " "
+ "=" * max_file
+ " "
+ "=" * 8
+ " "
+ "=" * 7
+ " "
+ "=" * max_title,
)
print(
"\n".join(
[
(color2 if n % 2 == 0 else color1) + f"{n:3d} "
f"{x['date']} {x['file']:{max_file}s} "
f"{x['metric']:8.5f} "
f"{x['duration']/3600:7.3f} "
f"{x['title']}"
for n, x in enumerate(self.data_filtered)
]
)
)
def show_result(self, data: dict, title: str = "") -> None:
def whites(n: int) -> str:
return " " * n + color1 + "*"
if data == {}:
print(f"** {title} has No data **")
return
color1 = TextColor.CYAN
color2 = TextColor.YELLOW
file_name = data["file"]
metric = data["metric"]
result = StubReport(os.path.join(Folders.results, file_name))
length = 81
print(color1 + "*" * length)
if title != "":
print(
"*"
+ color2
+ TextColor.BOLD
+ f"{title:^{length - 2}s}"
+ TextColor.ENDC
+ color1
+ "*"
)
print("*" + "-" * (length - 2) + "*")
print("*" + whites(length - 2))
print(
"* "
+ color2
+ f"{result.data['title']:^{length - 4}}"
+ color1
+ " *"
)
print("*" + whites(length - 2))
print(
"* Model: "
+ color2
+ f"{result.data['model']:15s} "
+ color1
+ "Ver. "
+ color2
+ f"{result.data['version']:10s} "
+ color1
+ "Score: "
+ color2
+ f"{result.data['score_name']:10s} "
+ color1
+ "Metric: "
+ color2
+ f"{metric:10.7f}"
+ whites(length - 78)
)
print(color1 + "*" + whites(length - 2))
print(
"* Date : "
+ color2
+ f"{result.data['date']:15s}"
+ color1
+ " Time: "
+ color2
+ f"{result.data['time']:18s} "
+ color1
+ "Time Spent: "
+ color2
+ f"{result.data['duration']:9,.2f}"
+ color1
+ " secs."
+ whites(length - 78)
)
seeds = str(result.data["seeds"])
seeds_len = len(seeds)
print(
"* Seeds: "
+ color2
+ f"{seeds:{seeds_len}s} "
+ color1
+ "Platform: "
+ color2
+ f"{result.data['platform']:17s} "
+ whites(length - 79)
)
print(
"* Stratified: "
+ color2
+ f"{str(result.data['stratified']):15s}"
+ whites(length - 30)
)
print("* " + color2 + f"{file_name:60s}" + whites(length - 63))
print(color1 + "*" + whites(length - 2))
print(color1 + "*" * length)
def best_results(self, criterion=None, value=None, score="accuracy", n=10):
# First filter the same score results (accuracy, f1, ...)
haystack = [x for x in self.data if x["score"] == score]
haystack = (
haystack
if criterion is None or value is None
else [x for x in haystack if x[criterion] == value]
)
if haystack == []:
raise ValueError(NO_RESULTS)
return (
sorted(
haystack,
key=lambda x: -1.0 if math.isnan(x["metric"]) else x["metric"],
reverse=True,
)[:n]
if len(haystack) > 0
else {}
)
def best_result(
self, criterion=None, value=None, score="accuracy"
) -> dict:
return self.best_results(criterion, value, score)[0]
def best_results_datasets(self, score="accuracy") -> dict:
"""Get the best results for each dataset"""
dt = Datasets()
best_results = {}
for dataset in dt:
best_results[dataset] = (1, "", "", "")
haystack = [x for x in self.data if x["score"] == score]
# Search for the best results for each dataset
for entry in haystack:
for dataset in self.datasets[entry["file"]]:
if dataset["score"] < best_results[dataset["dataset"]][0]:
best_results[dataset["dataset"]] = (
dataset["score"],
dataset["hyperparameters"],
entry["file"],
entry["title"],
)
return best_results
def show_top(self, score="accuracy", n=10):
try:
self.list_results(
score=score,
input_data=self.best_results(score=score, n=n),
sort_key="metric",
)
except ValueError as e:
print(e)

2
benchmark/ResultsFiles.py
View File

@@ -12,7 +12,7 @@ from xlsxwriter.exceptions import DuplicateWorksheetName
from ._version import __version__
from .Arguments import EnvData
from .Datasets import Datasets
from .ResultsBase import BaseReport
from .ResultsBase import BaseReport, BestResultsEver, Summary, StubReport
from .Utils import NO_RESULTS, Files, Folders, TextColor

3
benchmark/__init__.py
View File

@@ -1,3 +1,4 @@
from .ResultsBase import Summary
from .Datasets import (
Datasets,
DatasetsSurcov,
@@ -5,7 +6,7 @@ from .Datasets import (
DatasetsArff,
)
from .Experiments import Experiment
from .Results import Report, Summary
from .Results import Report
from ._version import __version__
__author__ = "Ricardo Montañana Gómez"

2
benchmark/scripts/be_benchmark.py
View File

@@ -1,5 +1,5 @@
#!/usr/bin/env python
from benchmark.Results import Benchmark
from benchmark.ResultsFiles import Benchmark
from benchmark.Utils import Files
from benchmark.Arguments import Arguments

2
benchmark/scripts/be_best.py
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env python
import json
from benchmark.Results import Summary
from benchmark.ResultsBase import Summary
from benchmark.Arguments import ALL_METRICS, Arguments

6
benchmark/scripts/be_list.py
View File

@@ -1,8 +1,9 @@
#! /usr/bin/env python
import os
from benchmark.Results import Summary
from benchmark.ResultsBase import Summary
from benchmark.Utils import Files, Folders
from benchmark.Arguments import Arguments
from benchmark.Manager import Manage
"""List experiments of a model
"""
@@ -27,7 +28,8 @@ def main(args_test=None):
except ValueError as e:
print(e)
return
excel_generated = data.manage_results()
manager = Manage(data)
excel_generated = manager.manage_results()
if excel_generated:
name = os.path.join(Folders.excel, Files.be_list_excel)
print(f"Generated file: {name}")

2
benchmark/scripts/be_summary.py
View File

@@ -1,5 +1,5 @@
#!/usr/bin/env python
from benchmark.Results import Summary
from benchmark.ResultsBase import Summary
from benchmark.Arguments import ALL_METRICS, Arguments

9
benchmark/tests/Benchmark_test.py
View File

@@ -90,15 +90,6 @@ class BenchmarkTest(TestBase):
self.assertTrue(os.path.exists(benchmark.get_tex_file()))
self.check_file_file(benchmark.get_tex_file(), "exreport_tex")
@staticmethod
def generate_excel_sheet(test, sheet, file_name):
with open(os.path.join("test_files", file_name), "w") as f:
for row in range(1, sheet.max_row + 1):
for col in range(1, sheet.max_column + 1):
value = sheet.cell(row=row, column=col).value
if value is not None:
print(f'{row};{col};"{value}"', file=f)
def test_excel_output(self):
benchmark = Benchmark("accuracy", visualize=False)
benchmark.compile_results()

3
benchmark/tests/Report_test.py
View File

@@ -4,7 +4,8 @@ from unittest.mock import patch
from .TestBase import TestBase
from ..Results import Report, ReportBest
from ..ResultsFiles import ReportDatasets
from ..ResultsBase import BaseReport, get_input
from ..ResultsBase import BaseReport
from ..Manager import get_input
from ..Utils import Symbols

2
benchmark/tests/Summary_test.py
View File

@@ -1,7 +1,7 @@
from io import StringIO
from unittest.mock import patch
from .TestBase import TestBase
from ..Results import Summary
from ..ResultsBase import Summary
from ..Utils import NO_RESULTS

26
benchmark/tests/scripts/Be_List_test.py
View File

@@ -10,31 +10,31 @@ class BeListTest(TestBase):
def setUp(self):
self.prepare_scripts_env()
@patch("benchmark.Results.get_input", return_value="q")
@patch("benchmark.Manager.get_input", return_value="q")
def test_be_list(self, input_data):
stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_model")
@patch("benchmark.Results.get_input", side_effect=iter(["x", "q"]))
@patch("benchmark.Manager.get_input", side_effect=iter(["x", "q"]))
def test_be_list_invalid_option(self, input_data):
stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_model_invalid")
@patch("benchmark.Results.get_input", side_effect=iter(["0", "q"]))
@patch("benchmark.Manager.get_input", side_effect=iter(["0", "q"]))
def test_be_list_report(self, input_data):
stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_report")
@patch("benchmark.Results.get_input", side_effect=iter(["r", "q"]))
@patch("benchmark.Manager.get_input", side_effect=iter(["r", "q"]))
def test_be_list_twice(self, input_data):
stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_model_2")
@patch("benchmark.Results.get_input", side_effect=iter(["e 2", "q"]))
@patch("benchmark.Manager.get_input", side_effect=iter(["e 2", "q"]))
def test_be_list_report_excel(self, input_data):
stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
self.assertEqual(stderr.getvalue(), "")
@@ -45,7 +45,7 @@ class BeListTest(TestBase):
self.check_excel_sheet(sheet, "excel")
@patch(
"benchmark.Results.get_input",
"benchmark.Manager.get_input",
side_effect=iter(["e 2", "e 1", "q"]),
)
def test_be_list_report_excel_twice(self, input_data):
@@ -58,7 +58,7 @@ class BeListTest(TestBase):
sheet = book["STree2"]
self.check_excel_sheet(sheet, "excel2")
@patch("benchmark.Results.get_input", return_value="q")
@patch("benchmark.Manager.get_input", return_value="q")
def test_be_list_no_data(self, input_data):
stdout, stderr = self.execute_script(
"be_list", ["-m", "Wodt", "-s", "f1-macro"]
@@ -67,7 +67,7 @@ class BeListTest(TestBase):
self.assertEqual(stdout.getvalue(), f"{NO_RESULTS}\n")
@patch(
"benchmark.Results.get_input",
"benchmark.Manager.get_input",
side_effect=iter(["d 0", "y", "", "q"]),
)
# @patch("benchmark.ResultsBase.get_input", side_effect=iter(["q"]))
@@ -94,7 +94,7 @@ class BeListTest(TestBase):
self.fail("test_be_list_delete() should not raise exception")
@patch(
"benchmark.Results.get_input",
"benchmark.Manager.get_input",
side_effect=iter(["h 0", "y", "", "q"]),
)
def test_be_list_hide(self, input_data):
@@ -119,25 +119,25 @@ class BeListTest(TestBase):
swap_files(Folders.results, Folders.hidden_results, file_name)
self.fail("test_be_list_hide() should not raise exception")
@patch("benchmark.Results.get_input", side_effect=iter(["h 0", "q"]))
@patch("benchmark.Manager.get_input", side_effect=iter(["h 0", "q"]))
def test_be_list_already_hidden(self, input_data):
stdout, stderr = self.execute_script("be_list", ["--hidden"])
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_already_hidden")
@patch("benchmark.Results.get_input", side_effect=iter(["h 0", "n", "q"]))
@patch("benchmark.Manager.get_input", side_effect=iter(["h 0", "n", "q"]))
def test_be_list_dont_hide(self, input_data):
stdout, stderr = self.execute_script("be_list", "")
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_default")
@patch("benchmark.Results.get_input", side_effect=iter(["q"]))
@patch("benchmark.Manager.get_input", side_effect=iter(["q"]))
def test_be_list_hidden_nan(self, input_data):
stdout, stderr = self.execute_script("be_list", ["--hidden", "--nan"])
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_hidden_nan")
@patch("benchmark.Results.get_input", side_effect=iter(["q"]))
@patch("benchmark.Manager.get_input", side_effect=iter(["q"]))
def test_be_list_hidden(self, input_data):
stdout, stderr = self.execute_script("be_list", ["--hidden"])
self.assertEqual(stderr.getvalue(), "")

6
benchmark/tests/scripts/Be_Main_test.py
View File

@@ -25,7 +25,7 @@ class BeMainTest(TestBase):
self.check_output_lines(
stdout=stdout,
file_name="be_main_dataset",
lines_to_compare=[0, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13],
lines_to_compare=[0, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13, 14],
)
def test_be_main_complete(self):
@@ -37,7 +37,9 @@ class BeMainTest(TestBase):
report_name = stdout.getvalue().splitlines()[-1].split("in ")[1]
self.files.append(report_name)
self.check_output_lines(
stdout, "be_main_complete", [0, 2, 3, 5, 6, 7, 8, 9, 12, 13, 14]
stdout,
"be_main_complete",
[0, 2, 3, 5, 6, 7, 8, 9, 12, 13, 14, 15],
)
def test_be_main_no_report(self):

1
benchmark/tests/test_files/be_main_best.test
View File

@@ -11,6 +11,7 @@ Dataset Sampl. Feat. Cls Nodes Leaves Depth Score
balance-scale 625 4 3 23.32 12.16 6.44 0.840160±0.0304 0.013745±0.0019 {'splitter': 'best', 'max_features': 'auto'}
balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000388±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000, 'multiclass_strategy': 'ovr'}
*************************************************************************************************************************
* ➶ Better than ZeroR + 10.0%.....: 1 *
* accuracy compared to STree_default (liblinear-ovr) .: 0.0422 *
*************************************************************************************************************************
Results in results/results_accuracy_STree_iMac27_2022-05-09_00:15:25_0.json

1
benchmark/tests/test_files/be_main_complete.test
View File

@@ -11,6 +11,7 @@ Dataset Sampl. Feat. Cls Nodes Leaves Depth Score
balance-scale 625 4 3 17.36 9.18 6.18 0.908480±0.0247 0.007388±0.0013 {}
balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000664±0.0002 {}
*************************************************************************************************************************
* ➶ Better than ZeroR + 10.0%.....: 1 *
* accuracy compared to STree_default (liblinear-ovr) .: 0.0390 *
*************************************************************************************************************************
Results in results/results_accuracy_STree_iMac27_2022-05-08_20:14:43_0.json

3
benchmark/tests/test_files/be_main_dataset.test
View File

@@ -8,8 +8,9 @@
Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters
============================== ====== ===== === ======= ======= ======= =============== ================= ===============
balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000671±0.0001 {}
balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009➶ 0.000671±0.0001 {}
*************************************************************************************************************************
* ➶ Better than ZeroR + 10.0%.....: 1 *
* accuracy compared to STree_default (liblinear-ovr) .: 0.0165 *
*************************************************************************************************************************
Partial result file removed: results/results_accuracy_STree_iMac27_2022-05-08_19:38:28_0.json

1
benchmark/tests/test_files/be_main_grid.test
View File

@@ -11,6 +11,7 @@ Dataset Sampl. Feat. Cls Nodes Leaves Depth Score
balance-scale 625 4 3 26.12 13.56 7.94 0.910720±0.0249 0.015852±0.0027 {'C': 1.0, 'kernel': 'liblinear', 'multiclass_strategy': 'ovr'}
balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000640±0.0001 {'C': 1.0, 'kernel': 'linear', 'multiclass_strategy': 'ovr'}
*************************************************************************************************************************
* ➶ Better than ZeroR + 10.0%.....: 1 *
* accuracy compared to STree_default (liblinear-ovr) .: 0.0391 *
*************************************************************************************************************************
Results in results/results_accuracy_STree_iMac27_2022-05-09_00:21:06_0.json

40
benchmark/tests/test_files/exreport_excel_ODTE.test
View File

@@ -3,12 +3,12 @@
3;1;" Score is accuracy"
3;2;" Execution time"
3;5;"22,591.47 s"
3;7;" "
3;8;"Platform"
3;7;"Platform"
3;9;"Galgo"
3;10;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]"
3;11;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]"
4;5;" 6.28 h"
4;10;"Stratified: False"
4;11;"Stratified: False"
4;13;"Discretized: False"
6;1;"Dataset"
6;2;"Samples"
6;3;"Features"
@@ -17,10 +17,11 @@
6;6;"Leaves"
6;7;"Depth"
6;8;"Score"
6;9;"Score Std."
6;10;"Time"
6;11;"Time Std."
6;12;"Hyperparameters"
6;9;"Stat"
6;10;"Score Std."
6;11;"Time"
6;12;"Time Std."
6;13;"Hyperparameters"
7;1;"balance-scale"
7;2;"625"
7;3;"4"
@@ -29,10 +30,11 @@
7;6;"4.180599999999999"
7;7;"3.536"
7;8;"0.96352"
7;9;"0.02494974148162661"
7;10;"0.3166321754455567"
7;11;"0.1991881389525559"
7;12;"{'base_estimator__C': 57, 'base_estimator__gamma': 0.1, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1}"
7;9;" "
7;10;"0.02494974148162661"
7;11;"0.3166321754455567"
7;12;"0.1991881389525559"
7;13;"{'base_estimator__C': 57, 'base_estimator__gamma': 0.1, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1}"
8;1;"balloons"
8;2;"16"
8;3;"4"
@@ -41,8 +43,12 @@
8;6;"1.9976"
8;7;"1.9976"
8;8;"0.785"
8;9;"0.2461311755051675"
8;10;"0.1156062078475952"
8;11;"0.0127842418285999"
8;12;"{'base_estimator__C': 5, 'base_estimator__gamma': 0.14, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1}"
10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0434"
8;9;"➶"
8;10;"0.2461311755051675"
8;11;"0.1156062078475952"
8;12;"0.0127842418285999"
8;13;"{'base_estimator__C': 5, 'base_estimator__gamma': 0.14, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1}"
11;2;"➶"
11;3;"1"
11;4;"Better than ZeroR + 10.0%"
13;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0434"

40
benchmark/tests/test_files/exreport_excel_RandomForest.test
View File

@@ -3,12 +3,12 @@
3;1;" Score is accuracy"
3;2;" Execution time"
3;5;" 272.74 s"
3;7;" "
3;8;"Platform"
3;7;"Platform"
3;9;"iMac27"
3;10;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]"
3;11;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]"
4;5;" 0.08 h"
4;10;"Stratified: False"
4;11;"Stratified: False"
4;13;"Discretized: False"
6;1;"Dataset"
6;2;"Samples"
6;3;"Features"
@@ -17,10 +17,11 @@
6;6;"Leaves"
6;7;"Depth"
6;8;"Score"
6;9;"Score Std."
6;10;"Time"
6;11;"Time Std."
6;12;"Hyperparameters"
6;9;"Stat"
6;10;"Score Std."
6;11;"Time"
6;12;"Time Std."
6;13;"Hyperparameters"
7;1;"balance-scale"
7;2;"625"
7;3;"4"
@@ -29,10 +30,11 @@
7;6;"98.42"
7;7;"10.6814"
7;8;"0.83616"
7;9;"0.02649630917694009"
7;10;"0.08222018241882324"
7;11;"0.001302632681512063"
7;12;"{}"
7;9;" "
7;10;"0.02649630917694009"
7;11;"0.08222018241882324"
7;12;"0.001302632681512063"
7;13;"{}"
8;1;"balloons"
8;2;"16"
8;3;"4"
@@ -41,8 +43,12 @@
8;6;"4.58"
8;7;"3.0982"
8;8;"0.625"
8;9;"0.249582985531199"
8;10;"0.07016648769378662"
8;11;"0.002460508923990468"
8;12;"{}"
10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0363"
8;9;"➶"
8;10;"0.249582985531199"
8;11;"0.07016648769378662"
8;12;"0.002460508923990468"
8;13;"{}"
11;2;"➶"
11;3;"1"
11;4;"Better than ZeroR + 10.0%"
13;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0363"

40
benchmark/tests/test_files/exreport_excel_STree.test
View File

@@ -3,12 +3,12 @@
3;1;" Score is accuracy"
3;2;" Execution time"
3;5;" 624.25 s"
3;7;" "
3;8;"Platform"
3;7;"Platform"
3;9;"iMac27"
3;10;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]"
3;11;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]"
4;5;" 0.17 h"
4;10;"Stratified: False"
4;11;"Stratified: False"
4;13;"Discretized: False"
6;1;"Dataset"
6;2;"Samples"
6;3;"Features"
@@ -17,10 +17,11 @@
6;6;"Leaves"
6;7;"Depth"
6;8;"Score"
6;9;"Score Std."
6;10;"Time"
6;11;"Time Std."
6;12;"Hyperparameters"
6;9;"Stat"
6;10;"Score Std."
6;11;"Time"
6;12;"Time Std."
6;13;"Hyperparameters"
7;1;"balance-scale"
7;2;"625"
7;3;"4"
@@ -29,10 +30,11 @@
7;6;"4"
7;7;"3"
7;8;"0.97056"
7;9;"0.0150468069702512"
7;10;"0.01404867172241211"
7;11;"0.002026269126958884"
7;12;"{'C': 10000, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000, 'multiclass_strategy': 'ovr'}"
7;9;" "
7;10;"0.0150468069702512"
7;11;"0.01404867172241211"
7;12;"0.002026269126958884"
7;13;"{'C': 10000, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000, 'multiclass_strategy': 'ovr'}"
8;1;"balloons"
8;2;"16"
8;3;"4"
@@ -41,8 +43,12 @@
8;6;"2"
8;7;"2"
8;8;"0.86"
8;9;"0.2850146195080759"
8;10;"0.0008541679382324218"
8;11;"3.629469326417878e-05"
8;12;"{'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000, 'multiclass_strategy': 'ovr'}"
10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0454"
8;9;"➶"
8;10;"0.2850146195080759"
8;11;"0.0008541679382324218"
8;12;"3.629469326417878e-05"
8;13;"{'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000, 'multiclass_strategy': 'ovr'}"
11;2;"➶"
11;3;"1"
11;4;"Better than ZeroR + 10.0%"
13;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0454"