Doctorado-ML/benchmark
1
0
Fork 0
mirror of https://github.com/Doctorado-ML/benchmark.git synced 2025-08-16 16:05:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix test issues

Browse Source
This commit is contained in:
Ricardo Montañana Gómez
2022-05-06 11:15:29 +02:00
parent 3056bb649a
commit d87c7064a9
17 changed files with 111 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
benchmark/Experiments.py
View File

@@ -156,7 +156,7 @@ class BestResults:
self._process_datafile(results, data, name)
found = True
if not found:
raise ValueError(f"No results found")
raise ValueError("** No results found **")
# Build best results json file
output = {}
datasets = Datasets()

34
benchmark/Results.py
View File

@@ -521,6 +521,34 @@ class Excel(BaseReport):
self.book.close()
class ReportDatasets:
@staticmethod
def report():
data_sets = Datasets()
color_line = TextColor.LINE1
print(color_line, end="")
print(f"{'Dataset':30s} Samp. Feat. Cls Balance")
print("=" * 30 + " ===== ===== === " + "=" * 40)
for dataset in data_sets:
X, y = data_sets.load(dataset)
color_line = (
TextColor.LINE2
if color_line == TextColor.LINE1
else TextColor.LINE1
)
values, counts = np.unique(y, return_counts=True)
comp = ""
sep = ""
for count in counts:
comp += f"{sep}{count/sum(counts)*100:5.2f}%"
sep = "/ "
print(color_line, end="")
print(
f"{dataset:30s} {X.shape[0]:5,d} {X.shape[1]:5,d} "
f"{len(np.unique(y)):3d} {comp:40s}"
)
class SQL(BaseReport):
table_name = "results"
@@ -1112,8 +1140,7 @@ class Summary:
score, model, input_data, sort_key, number
)
if data == []:
print("*No results found*")
exit(1)
raise ValueError("** No results found **")
max_file = max(len(x["file"]) for x in data)
max_title = max(len(x["title"]) for x in data)
if self.hidden:
@@ -1285,11 +1312,14 @@ class Summary:
return best_results
def show_top(self, score="accuracy", n=10):
try:
self.list_results(
score=score,
input_data=self.best_results(score=score, n=n),
sort_key="metric",
)
except ValueError as e:
print(e)
class PairCheck:

1
benchmark/Utils.py
View File

@@ -9,6 +9,7 @@ class Folders:
hidden_results = "hidden_results"
exreport = "exreport"
report = os.path.join(exreport, "exreport_output")
img = "img"
@staticmethod
def src():

7
benchmark/scripts/be_build_best.py
View File

@@ -9,14 +9,15 @@ from benchmark.Arguments import Arguments
def main():
arguments = Arguments()
arguments.xset("score").xset("report").xset("model")
arguments.xset("score", mandatory=True).xset("report")
arguments.xset("model", mandatory=True)
args = arguments.parse()
datasets = Datasets()
best = BestResults(args.score, args.model, datasets)
try:
best.build()
except ValueError:
print("** No results found **")
except ValueError as e:
print(e)
else:
if args.report:
report = ReportBest(args.score, args.model, best=True, grid=False)

2
benchmark/scripts/be_grid.py
View File

@@ -8,7 +8,7 @@ from benchmark.Arguments import Arguments
def main():
arguments = Arguments()
arguments.xset("score").xset("platform").xset("model")
arguments.xset("score").xset("platform").xset("model", mandatory=True)
arguments.xset("quiet").xset("stratified").xset("dataset").xset("n_folds")
args = arguments.parse()
if not args.quiet:

4
benchmark/scripts/be_list.py
View File

@@ -15,12 +15,16 @@ def main():
args = arguments.parse()
data = Summary(hidden=args.hidden)
data.acquire()
try:
data.list_results(
score=args.score,
model=args.model,
sort_key=args.key,
number=args.number,
)
except ValueError as e:
print(e)
else:
if args.nan:
results_nan = []
results = data.get_results_criteria(

4
benchmark/scripts/be_main.py
View File

@@ -10,10 +10,10 @@ from benchmark.Arguments import Arguments
def main():
arguments = Arguments()
arguments.xset("stratified").xset("score").xset("model").xset("dataset")
arguments.xset("stratified").xset("score").xset("model", mandatory=True)
arguments.xset("n_folds").xset("platform").xset("quiet").xset("title")
arguments.xset("hyperparameters").xset("paramfile").xset("report")
arguments.xset("grid_paramfile")
arguments.xset("grid_paramfile").xset("dataset")
args = arguments.parse()
report = args.report or args.dataset is not None
if args.grid_paramfile:

3
benchmark/scripts/be_print_strees.py
View File

@@ -62,7 +62,6 @@ def add_color(source):
def print_stree(clf, dataset, X, y, color, quiet):
output_folder = "img"
samples, features = X.shape
classes = max(y) + 1
accuracy = clf.score(X, y)
@@ -72,7 +71,7 @@ def print_stree(clf, dataset, X, y, color, quiet):
if color:
dot_source = add_color(dot_source)
grp = Source(dot_source)
file_name = os.path.join(output_folder, f"stree_{dataset}")
file_name = os.path.join(Folders.img, f"stree_{dataset}")
grp.render(format="png", filename=f"{file_name}")
os.remove(f"{file_name}")
print(f"File {file_name}.png generated")

38
benchmark/scripts/be_report.py
View File

@@ -1,11 +1,6 @@
#!/usr/bin/env python
import numpy as np
from benchmark.Experiments import Datasets
from benchmark.Results import Report, Excel, SQL, ReportBest
from benchmark.Utils import (
Files,
TextColor,
)
from benchmark.Results import Report, Excel, SQL, ReportBest, ReportDatasets
from benchmark.Utils import Files
from benchmark.Arguments import Arguments
@@ -15,32 +10,6 @@ If no argument is set, displays the datasets and its characteristics
"""
def default_report():
sets = Datasets()
color_line = TextColor.LINE1
print(color_line, end="")
print(f"{'Dataset':30s} Samp. Feat Cls Balance")
print("=" * 30 + " ===== ==== === " + "=" * 40)
for line in sets:
X, y = sets.load(line)
color_line = (
TextColor.LINE2
if color_line == TextColor.LINE1
else TextColor.LINE1
)
values, counts = np.unique(y, return_counts=True)
comp = ""
sep = ""
for value, count in zip(values, counts):
comp += f"{sep}{count/sum(counts)*100:5.2f}%"
sep = "/ "
print(color_line, end="")
print(
f"{line:30s} {X.shape[0]:5,d} {X.shape[1]:4d} "
f"{len(np.unique(y)):3d} {comp:40s}"
)
def main():
arguments = Arguments()
arguments.xset("file").xset("excel").xset("sql").xset("compare")
@@ -48,11 +17,10 @@ def main():
"score"
)
args = arguments.parse()
if args.grid:
args.best = False
if args.file is None and args.best is None:
default_report()
ReportDatasets.report()
else:
if args.best is not None or args.grid is not None:
report = ReportBest(args.score, args.model, args.best, args.grid)

3
benchmark/tests/GridSearch_test.py
View File

@@ -37,7 +37,8 @@ class GridSearchTest(TestBase):
],
".",
)
_ = self.build_exp()
grid = self.build_exp()
grid._init_data()
# check the output file is initialized
with open(file_name) as f:
data = json.load(f)

24
benchmark/tests/test_files/summary_show_results.test
View File

@@ -1,12 +1,12 @@
*********************************************************************************
* *
* With gridsearched hyperparameters *
* *
* Model: STree Ver. 1.2.3 Score: accuracy Metric: 0.0454434 *
* *
* Date : 2021-09-30 Time: 11:42:07 Time Spent: 624.25 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 *
* Stratified: False *
* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json *
* *
*********************************************************************************
*********************************************************************************
* *
*  With gridsearched hyperparameters  *
* *
* Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 *
* *
* Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 *
* Stratified: False *
* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json *
* *
*********************************************************************************

26
benchmark/tests/test_files/summary_show_results_title.test
View File

@@ -1,14 +1,14 @@
*********************************************************************************
* **Title** *
*********************************************************************************
* **Title** *
*-------------------------------------------------------------------------------*
* *
* With gridsearched hyperparameters *
* *
* Model: STree Ver. 1.2.3 Score: accuracy Metric: 0.0454434 *
* *
* Date : 2021-09-30 Time: 11:42:07 Time Spent: 624.25 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 *
* Stratified: False *
* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json *
* *
*********************************************************************************
* *
*  With gridsearched hyperparameters  *
* *
* Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 *
* *
* Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 *
* Stratified: False *
* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json *
* *
*********************************************************************************