Doctorado-ML/benchmark
1
0
Fork 0
mirror of https://github.com/Doctorado-ML/benchmark.git synced 2025-08-15 15:35:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix test issues

Browse Source
This commit is contained in:
Ricardo Montañana Gómez
2022-05-06 11:15:29 +02:00
parent 3056bb649a
commit d87c7064a9
17 changed files with 111 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
benchmark/Experiments.py
View File

@@ -156,7 +156,7 @@ class BestResults:
self._process_datafile(results, data, name)
found = True
if not found:
raise ValueError(f"No results found")
raise ValueError("** No results found **")
# Build best results json file
output = {}
datasets = Datasets()

44
benchmark/Results.py
View File

@@ -521,6 +521,34 @@ class Excel(BaseReport):
self.book.close()
class ReportDatasets:
@staticmethod
def report():
data_sets = Datasets()
color_line = TextColor.LINE1
print(color_line, end="")
print(f"{'Dataset':30s} Samp. Feat. Cls Balance")
print("=" * 30 + " ===== ===== === " + "=" * 40)
for dataset in data_sets:
X, y = data_sets.load(dataset)
color_line = (
TextColor.LINE2
if color_line == TextColor.LINE1
else TextColor.LINE1
)
values, counts = np.unique(y, return_counts=True)
comp = ""
sep = ""
for count in counts:
comp += f"{sep}{count/sum(counts)*100:5.2f}%"
sep = "/ "
print(color_line, end="")
print(
f"{dataset:30s} {X.shape[0]:5,d} {X.shape[1]:5,d} "
f"{len(np.unique(y)):3d} {comp:40s}"
)
class SQL(BaseReport):
table_name = "results"
@@ -1112,8 +1140,7 @@ class Summary:
score, model, input_data, sort_key, number
)
if data == []:
print("*No results found*")
exit(1)
raise ValueError("** No results found **")
max_file = max(len(x["file"]) for x in data)
max_title = max(len(x["title"]) for x in data)
if self.hidden:
@@ -1285,11 +1312,14 @@ class Summary:
return best_results
def show_top(self, score="accuracy", n=10):
self.list_results(
score=score,
input_data=self.best_results(score=score, n=n),
sort_key="metric",
)
try:
self.list_results(
score=score,
input_data=self.best_results(score=score, n=n),
sort_key="metric",
)
except ValueError as e:
print(e)
class PairCheck:

1
benchmark/Utils.py
View File

@@ -9,6 +9,7 @@ class Folders:
hidden_results = "hidden_results"
exreport = "exreport"
report = os.path.join(exreport, "exreport_output")
img = "img"
@staticmethod
def src():

7
benchmark/scripts/be_build_best.py
View File

@@ -9,14 +9,15 @@ from benchmark.Arguments import Arguments
def main():
arguments = Arguments()
arguments.xset("score").xset("report").xset("model")
arguments.xset("score", mandatory=True).xset("report")
arguments.xset("model", mandatory=True)
args = arguments.parse()
datasets = Datasets()
best = BestResults(args.score, args.model, datasets)
try:
best.build()
except ValueError:
print("** No results found **")
except ValueError as e:
print(e)
else:
if args.report:
report = ReportBest(args.score, args.model, best=True, grid=False)

2
benchmark/scripts/be_grid.py
View File

@@ -8,7 +8,7 @@ from benchmark.Arguments import Arguments
def main():
arguments = Arguments()
arguments.xset("score").xset("platform").xset("model")
arguments.xset("score").xset("platform").xset("model", mandatory=True)
arguments.xset("quiet").xset("stratified").xset("dataset").xset("n_folds")
args = arguments.parse()
if not args.quiet:

54
benchmark/scripts/be_list.py
View File

@@ -15,35 +15,39 @@ def main():
args = arguments.parse()
data = Summary(hidden=args.hidden)
data.acquire()
data.list_results(
score=args.score,
model=args.model,
sort_key=args.key,
number=args.number,
)
if args.nan:
results_nan = []
results = data.get_results_criteria(
try:
data.list_results(
score=args.score,
model=args.model,
input_data=None,
sort_key=args.key,
number=args.number,
)
for result in results:
if result["metric"] != result["metric"]:
results_nan.append(result)
if results_nan != []:
print(
"\n"
+ "*" * 30
+ " Results with nan moved to hidden "
+ "*" * 30
except ValueError as e:
print(e)
else:
if args.nan:
results_nan = []
results = data.get_results_criteria(
score=args.score,
model=args.model,
input_data=None,
sort_key=args.key,
number=args.number,
)
data.list_results(input_data=results_nan)
for result in results_nan:
name = result["file"]
os.rename(
os.path.join(Folders.results, name),
os.path.join(Folders.hidden_results, name),
for result in results:
if result["metric"] != result["metric"]:
results_nan.append(result)
if results_nan != []:
print(
"\n"
+ "*" * 30
+ " Results with nan moved to hidden "
+ "*" * 30
)
data.list_results(input_data=results_nan)
for result in results_nan:
name = result["file"]
os.rename(
os.path.join(Folders.results, name),
os.path.join(Folders.hidden_results, name),
)

4
benchmark/scripts/be_main.py
View File

@@ -10,10 +10,10 @@ from benchmark.Arguments import Arguments
def main():
arguments = Arguments()
arguments.xset("stratified").xset("score").xset("model").xset("dataset")
arguments.xset("stratified").xset("score").xset("model", mandatory=True)
arguments.xset("n_folds").xset("platform").xset("quiet").xset("title")
arguments.xset("hyperparameters").xset("paramfile").xset("report")
arguments.xset("grid_paramfile")
arguments.xset("grid_paramfile").xset("dataset")
args = arguments.parse()
report = args.report or args.dataset is not None
if args.grid_paramfile:

3
benchmark/scripts/be_print_strees.py
View File

@@ -62,7 +62,6 @@ def add_color(source):
def print_stree(clf, dataset, X, y, color, quiet):
output_folder = "img"
samples, features = X.shape
classes = max(y) + 1
accuracy = clf.score(X, y)
@@ -72,7 +71,7 @@ def print_stree(clf, dataset, X, y, color, quiet):
if color:
dot_source = add_color(dot_source)
grp = Source(dot_source)
file_name = os.path.join(output_folder, f"stree_{dataset}")
file_name = os.path.join(Folders.img, f"stree_{dataset}")
grp.render(format="png", filename=f"{file_name}")
os.remove(f"{file_name}")
print(f"File {file_name}.png generated")

38
benchmark/scripts/be_report.py
View File

@@ -1,11 +1,6 @@
#!/usr/bin/env python
import numpy as np
from benchmark.Experiments import Datasets
from benchmark.Results import Report, Excel, SQL, ReportBest
from benchmark.Utils import (
Files,
TextColor,
)
from benchmark.Results import Report, Excel, SQL, ReportBest, ReportDatasets
from benchmark.Utils import Files
from benchmark.Arguments import Arguments
@@ -15,32 +10,6 @@ If no argument is set, displays the datasets and its characteristics
"""
def default_report():
sets = Datasets()
color_line = TextColor.LINE1
print(color_line, end="")
print(f"{'Dataset':30s} Samp. Feat Cls Balance")
print("=" * 30 + " ===== ==== === " + "=" * 40)
for line in sets:
X, y = sets.load(line)
color_line = (
TextColor.LINE2
if color_line == TextColor.LINE1
else TextColor.LINE1
)
values, counts = np.unique(y, return_counts=True)
comp = ""
sep = ""
for value, count in zip(values, counts):
comp += f"{sep}{count/sum(counts)*100:5.2f}%"
sep = "/ "
print(color_line, end="")
print(
f"{line:30s} {X.shape[0]:5,d} {X.shape[1]:4d} "
f"{len(np.unique(y)):3d} {comp:40s}"
)
def main():
arguments = Arguments()
arguments.xset("file").xset("excel").xset("sql").xset("compare")
@@ -48,11 +17,10 @@ def main():
"score"
)
args = arguments.parse()
if args.grid:
args.best = False
if args.file is None and args.best is None:
default_report()
ReportDatasets.report()
else:
if args.best is not None or args.grid is not None:
report = ReportBest(args.score, args.model, args.best, args.grid)

3
benchmark/tests/GridSearch_test.py
View File

@@ -37,7 +37,8 @@ class GridSearchTest(TestBase):
],
".",
)
_ = self.build_exp()
grid = self.build_exp()
grid._init_data()
# check the output file is initialized
with open(file_name) as f:
data = json.load(f)

2
benchmark/tests/test_files/summary_list_hidden.test
View File

@@ -1,3 +1,3 @@
Date File Score Time(h) Title
Date File Score Time(h) Title
========== ======================================================== ======== ======= =======
2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json 0.97446 0.098 default

2
benchmark/tests/test_files/summary_list_model.test
View File

@@ -1,4 +1,4 @@
Date File Score Time(h) Title
Date File Score Time(h) Title
========== ============================================================= ======== ======= =================================
2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B
2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A

2
benchmark/tests/test_files/summary_list_n.test
View File

@@ -1,4 +1,4 @@
Date File Score Time(h) Title
Date File Score Time(h) Title
========== =============================================================== ======== ======= ============================================
2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init
2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest

2
benchmark/tests/test_files/summary_list_score.test
View File

@@ -1,4 +1,4 @@
Date File Score Time(h) Title
Date File Score Time(h) Title
========== =============================================================== ======== ======= ============================================
2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init
2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest

24
benchmark/tests/test_files/summary_show_results.test
View File

@@ -1,12 +1,12 @@
*********************************************************************************
* *
* With gridsearched hyperparameters *
* *
* Model: STree Ver. 1.2.3 Score: accuracy Metric: 0.0454434 *
* *
* Date : 2021-09-30 Time: 11:42:07 Time Spent: 624.25 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 *
* Stratified: False *
* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json *
* *
*********************************************************************************
*********************************************************************************
* *
*  With gridsearched hyperparameters  *
* *
* Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 *
* *
* Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 *
* Stratified: False *
* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json *
* *
*********************************************************************************

26
benchmark/tests/test_files/summary_show_results_title.test
View File

@@ -1,14 +1,14 @@
*********************************************************************************
* **Title** *
*********************************************************************************
* **Title** *
*-------------------------------------------------------------------------------*
* *
* With gridsearched hyperparameters *
* *
* Model: STree Ver. 1.2.3 Score: accuracy Metric: 0.0454434 *
* *
* Date : 2021-09-30 Time: 11:42:07 Time Spent: 624.25 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 *
* Stratified: False *
* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json *
* *
*********************************************************************************
* *
*  With gridsearched hyperparameters  *
* *
* Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 *
* *
* Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 *
* Stratified: False *
* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json *
* *
*********************************************************************************

2
benchmark/tests/test_files/summary_show_top.test
View File

@@ -1,4 +1,4 @@
Date File Score Time(h) Title
Date File Score Time(h) Title
========== =============================================================== ======== ======= ============================================
2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters
2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init