Doctorado-ML/benchmark
1
0
Fork 0
mirror of https://github.com/Doctorado-ML/benchmark.git synced 2025-08-16 16:05:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix test issues

Browse Source
This commit is contained in:
Ricardo Montañana Gómez
2022-05-06 11:15:29 +02:00
parent 3056bb649a
commit d87c7064a9
17 changed files with 111 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
benchmark/Experiments.py
View File

@@ -156,7 +156,7 @@ class BestResults:
self._process_datafile(results, data, name) self._process_datafile(results, data, name)
found = True found = True
if not found: if not found:
raise ValueError(f"No results found") raise ValueError("** No results found **")
# Build best results json file # Build best results json file
output = {} output = {}
datasets = Datasets() datasets = Datasets()

44
benchmark/Results.py
View File

@@ -521,6 +521,34 @@ class Excel(BaseReport):
self.book.close() self.book.close()
class ReportDatasets:
@staticmethod
def report():
data_sets = Datasets()
color_line = TextColor.LINE1
print(color_line, end="")
print(f"{'Dataset':30s} Samp. Feat. Cls Balance")
print("=" * 30 + " ===== ===== === " + "=" * 40)
for dataset in data_sets:
X, y = data_sets.load(dataset)
color_line = (
TextColor.LINE2
if color_line == TextColor.LINE1
else TextColor.LINE1
)
values, counts = np.unique(y, return_counts=True)
comp = ""
sep = ""
for count in counts:
comp += f"{sep}{count/sum(counts)*100:5.2f}%"
sep = "/ "
print(color_line, end="")
print(
f"{dataset:30s} {X.shape[0]:5,d} {X.shape[1]:5,d} "
f"{len(np.unique(y)):3d} {comp:40s}"
)
class SQL(BaseReport): class SQL(BaseReport):
table_name = "results" table_name = "results"
@@ -1112,8 +1140,7 @@ class Summary:
score, model, input_data, sort_key, number score, model, input_data, sort_key, number
) )
if data == []: if data == []:
print("*No results found*") raise ValueError("** No results found **")
exit(1)
max_file = max(len(x["file"]) for x in data) max_file = max(len(x["file"]) for x in data)
max_title = max(len(x["title"]) for x in data) max_title = max(len(x["title"]) for x in data)
if self.hidden: if self.hidden:
@@ -1285,11 +1312,14 @@ class Summary:
return best_results return best_results
def show_top(self, score="accuracy", n=10): def show_top(self, score="accuracy", n=10):
self.list_results( try:
score=score, self.list_results(
input_data=self.best_results(score=score, n=n), score=score,
sort_key="metric", input_data=self.best_results(score=score, n=n),
) sort_key="metric",
)
except ValueError as e:
print(e)
class PairCheck: class PairCheck:

1
benchmark/Utils.py
View File

@@ -9,6 +9,7 @@ class Folders:
hidden_results = "hidden_results" hidden_results = "hidden_results"
exreport = "exreport" exreport = "exreport"
report = os.path.join(exreport, "exreport_output") report = os.path.join(exreport, "exreport_output")
img = "img"
@staticmethod @staticmethod
def src(): def src():

7
benchmark/scripts/be_build_best.py
View File

@@ -9,14 +9,15 @@ from benchmark.Arguments import Arguments
def main(): def main():
arguments = Arguments() arguments = Arguments()
arguments.xset("score").xset("report").xset("model") arguments.xset("score", mandatory=True).xset("report")
arguments.xset("model", mandatory=True)
args = arguments.parse() args = arguments.parse()
datasets = Datasets() datasets = Datasets()
best = BestResults(args.score, args.model, datasets) best = BestResults(args.score, args.model, datasets)
try: try:
best.build() best.build()
except ValueError: except ValueError as e:
print("** No results found **") print(e)
else: else:
if args.report: if args.report:
report = ReportBest(args.score, args.model, best=True, grid=False) report = ReportBest(args.score, args.model, best=True, grid=False)

2
benchmark/scripts/be_grid.py
View File

@@ -8,7 +8,7 @@ from benchmark.Arguments import Arguments
def main(): def main():
arguments = Arguments() arguments = Arguments()
arguments.xset("score").xset("platform").xset("model") arguments.xset("score").xset("platform").xset("model", mandatory=True)
arguments.xset("quiet").xset("stratified").xset("dataset").xset("n_folds") arguments.xset("quiet").xset("stratified").xset("dataset").xset("n_folds")
args = arguments.parse() args = arguments.parse()
if not args.quiet: if not args.quiet:

54
benchmark/scripts/be_list.py
View File

@@ -15,35 +15,39 @@ def main():
args = arguments.parse() args = arguments.parse()
data = Summary(hidden=args.hidden) data = Summary(hidden=args.hidden)
data.acquire() data.acquire()
data.list_results( try:
score=args.score, data.list_results(
model=args.model,
sort_key=args.key,
number=args.number,
)
if args.nan:
results_nan = []
results = data.get_results_criteria(
score=args.score, score=args.score,
model=args.model, model=args.model,
input_data=None,
sort_key=args.key, sort_key=args.key,
number=args.number, number=args.number,
) )
for result in results: except ValueError as e:
if result["metric"] != result["metric"]: print(e)
results_nan.append(result) else:
if results_nan != []: if args.nan:
print( results_nan = []
"\n" results = data.get_results_criteria(
+ "*" * 30 score=args.score,
+ " Results with nan moved to hidden " model=args.model,
+ "*" * 30 input_data=None,
sort_key=args.key,
number=args.number,
) )
data.list_results(input_data=results_nan) for result in results:
for result in results_nan: if result["metric"] != result["metric"]:
name = result["file"] results_nan.append(result)
os.rename( if results_nan != []:
os.path.join(Folders.results, name), print(
os.path.join(Folders.hidden_results, name), "\n"
+ "*" * 30
+ " Results with nan moved to hidden "
+ "*" * 30
) )
data.list_results(input_data=results_nan)
for result in results_nan:
name = result["file"]
os.rename(
os.path.join(Folders.results, name),
os.path.join(Folders.hidden_results, name),
)

4
benchmark/scripts/be_main.py
View File

@@ -10,10 +10,10 @@ from benchmark.Arguments import Arguments
def main(): def main():
arguments = Arguments() arguments = Arguments()
arguments.xset("stratified").xset("score").xset("model").xset("dataset") arguments.xset("stratified").xset("score").xset("model", mandatory=True)
arguments.xset("n_folds").xset("platform").xset("quiet").xset("title") arguments.xset("n_folds").xset("platform").xset("quiet").xset("title")
arguments.xset("hyperparameters").xset("paramfile").xset("report") arguments.xset("hyperparameters").xset("paramfile").xset("report")
arguments.xset("grid_paramfile") arguments.xset("grid_paramfile").xset("dataset")
args = arguments.parse() args = arguments.parse()
report = args.report or args.dataset is not None report = args.report or args.dataset is not None
if args.grid_paramfile: if args.grid_paramfile:

3
benchmark/scripts/be_print_strees.py
View File

@@ -62,7 +62,6 @@ def add_color(source):
def print_stree(clf, dataset, X, y, color, quiet): def print_stree(clf, dataset, X, y, color, quiet):
output_folder = "img"
samples, features = X.shape samples, features = X.shape
classes = max(y) + 1 classes = max(y) + 1
accuracy = clf.score(X, y) accuracy = clf.score(X, y)
@@ -72,7 +71,7 @@ def print_stree(clf, dataset, X, y, color, quiet):
if color: if color:
dot_source = add_color(dot_source) dot_source = add_color(dot_source)
grp = Source(dot_source) grp = Source(dot_source)
file_name = os.path.join(output_folder, f"stree_{dataset}") file_name = os.path.join(Folders.img, f"stree_{dataset}")
grp.render(format="png", filename=f"{file_name}") grp.render(format="png", filename=f"{file_name}")
os.remove(f"{file_name}") os.remove(f"{file_name}")
print(f"File {file_name}.png generated") print(f"File {file_name}.png generated")

38
benchmark/scripts/be_report.py
View File

@@ -1,11 +1,6 @@
#!/usr/bin/env python #!/usr/bin/env python
import numpy as np from benchmark.Results import Report, Excel, SQL, ReportBest, ReportDatasets
from benchmark.Experiments import Datasets from benchmark.Utils import Files
from benchmark.Results import Report, Excel, SQL, ReportBest
from benchmark.Utils import (
Files,
TextColor,
)
from benchmark.Arguments import Arguments from benchmark.Arguments import Arguments
@@ -15,32 +10,6 @@ If no argument is set, displays the datasets and its characteristics
""" """
def default_report():
sets = Datasets()
color_line = TextColor.LINE1
print(color_line, end="")
print(f"{'Dataset':30s} Samp. Feat Cls Balance")
print("=" * 30 + " ===== ==== === " + "=" * 40)
for line in sets:
X, y = sets.load(line)
color_line = (
TextColor.LINE2
if color_line == TextColor.LINE1
else TextColor.LINE1
)
values, counts = np.unique(y, return_counts=True)
comp = ""
sep = ""
for value, count in zip(values, counts):
comp += f"{sep}{count/sum(counts)*100:5.2f}%"
sep = "/ "
print(color_line, end="")
print(
f"{line:30s} {X.shape[0]:5,d} {X.shape[1]:4d} "
f"{len(np.unique(y)):3d} {comp:40s}"
)
def main(): def main():
arguments = Arguments() arguments = Arguments()
arguments.xset("file").xset("excel").xset("sql").xset("compare") arguments.xset("file").xset("excel").xset("sql").xset("compare")
@@ -48,11 +17,10 @@ def main():
"score" "score"
) )
args = arguments.parse() args = arguments.parse()
if args.grid: if args.grid:
args.best = False args.best = False
if args.file is None and args.best is None: if args.file is None and args.best is None:
default_report() ReportDatasets.report()
else: else:
if args.best is not None or args.grid is not None: if args.best is not None or args.grid is not None:
report = ReportBest(args.score, args.model, args.best, args.grid) report = ReportBest(args.score, args.model, args.best, args.grid)

3
benchmark/tests/GridSearch_test.py
View File

@@ -37,7 +37,8 @@ class GridSearchTest(TestBase):
], ],
".", ".",
) )
_ = self.build_exp() grid = self.build_exp()
grid._init_data()
# check the output file is initialized # check the output file is initialized
with open(file_name) as f: with open(file_name) as f:
data = json.load(f) data = json.load(f)

2
benchmark/tests/test_files/summary_list_hidden.test
View File

@@ -1,3 +1,3 @@
Date File Score Time(h) Title Date File Score Time(h) Title
========== ======================================================== ======== ======= ======= ========== ======================================================== ======== ======= =======
2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json 0.97446 0.098 default 2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json 0.97446 0.098 default

2
benchmark/tests/test_files/summary_list_model.test
View File

@@ -1,4 +1,4 @@
Date File Score Time(h) Title Date File Score Time(h) Title
========== ============================================================= ======== ======= ================================= ========== ============================================================= ======== ======= =================================
2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B
2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A

2
benchmark/tests/test_files/summary_list_n.test
View File

@@ -1,4 +1,4 @@
Date File Score Time(h) Title Date File Score Time(h) Title
========== =============================================================== ======== ======= ============================================ ========== =============================================================== ======== ======= ============================================
2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init
2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest

2
benchmark/tests/test_files/summary_list_score.test
View File

@@ -1,4 +1,4 @@
Date File Score Time(h) Title Date File Score Time(h) Title
========== =============================================================== ======== ======= ============================================ ========== =============================================================== ======== ======= ============================================
2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init
2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest

24
benchmark/tests/test_files/summary_show_results.test
View File

@@ -1,12 +1,12 @@
********************************************************************************* *********************************************************************************
* * * *
* With gridsearched hyperparameters * *  With gridsearched hyperparameters  *
* * * *
* Model: STree Ver. 1.2.3 Score: accuracy Metric: 0.0454434 * * Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 *
* * * *
* Date : 2021-09-30 Time: 11:42:07 Time Spent: 624.25 secs. * * Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 * * Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 *
* Stratified: False * * Stratified: False *
* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json *
* * * *
********************************************************************************* *********************************************************************************

26
benchmark/tests/test_files/summary_show_results_title.test
View File

@@ -1,14 +1,14 @@
********************************************************************************* *********************************************************************************
* **Title** * * **Title** *
*-------------------------------------------------------------------------------* *-------------------------------------------------------------------------------*
* * * *
* With gridsearched hyperparameters * *  With gridsearched hyperparameters  *
* * * *
* Model: STree Ver. 1.2.3 Score: accuracy Metric: 0.0454434 * * Model: STree Ver. 1.2.3 Score: accuracy Metric:  0.0454434 *
* * * *
* Date : 2021-09-30 Time: 11:42:07 Time Spent: 624.25 secs. * * Date : 2021-09-30  Time: 11:42:07 Time Spent:  624.25 secs. *
* Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 * * Seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Platform: iMac27 *
* Stratified: False * * Stratified: False *
* results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json * * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json *
* * * *
********************************************************************************* *********************************************************************************

2
benchmark/tests/test_files/summary_show_top.test
View File

@@ -1,4 +1,4 @@
Date File Score Time(h) Title Date File Score Time(h) Title
========== =============================================================== ======== ======= ============================================ ========== =============================================================== ======== ======= ============================================
2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters
2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init