Doctorado-ML/benchmark
1
0
Fork 0
mirror of https://github.com/Doctorado-ML/benchmark.git synced 2025-08-16 07:55:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add consistent comparative results to reports

Browse Source
This commit is contained in:
Ricardo Montañana Gómez
2022-10-25 15:00:37 +02:00
parent 29c4b4ceef
commit b24a508d1c
18 changed files with 75 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
benchmark/Results.py
View File

@@ -9,16 +9,37 @@ import xlsxwriter
import numpy as np
from .Experiments import BestResults
from .Datasets import Datasets
from .Arguments import EnvData, ALL_METRICS
from .Utils import (
Folders,
Files,
Symbols,
BEST_ACCURACY_STREE,
TextColor,
NO_RESULTS,
)
class BestResultsEver:
def __init__(self):
self.data = {}
for i in ["Tanveer", "Surcov", "Arff"]:
self.data[i] = {}
for metric in ALL_METRICS:
self.data[i][metric.replace("-", "_")] = ["self", 1.0]
self.data[i][metric] = ["self", 1.0]
self.data["Tanveer"]["accuracy"] = [
"STree_default (liblinear-ovr)",
40.282203,
]
self.data["Arff"]["accuracy"] = [
"STree_default (linear-ovo)",
21.9765,
]
def get_name_value(self, key, score):
return self.data[key][score]
class BaseReport(abc.ABC):
def __init__(self, file_name, best_file=False):
self.file_name = file_name
@@ -30,7 +51,20 @@ class BaseReport(abc.ABC):
with open(self.file_name) as f:
self.data = json.load(f)
self.best_acc_file = best_file
self.lines = self.data if best_file else self.data["results"]
if best_file:
self.lines = self.data
else:
self.lines = self.data["results"]
self.score_name = self.data["score_name"]
self.__compute_best_results_ever()
def __compute_best_results_ever(self):
args = EnvData.load()
key = args["source_data"]
best = BestResultsEver()
self.best_score_name, self.best_score_value = best.get_name_value(
key, self.score_name
)
def _get_accuracy(self, item):
return self.data[item][0] if self.best_acc_file else item["score"]
@@ -69,6 +103,12 @@ class BaseReport(abc.ABC):
}
return meaning[status]
def _get_best_accuracy(self):
return self.best_score_value
def _get_message_best_accuracy(self):
return f"{self.score_name} compared to {self.best_score_name} .:"
@abc.abstractmethod
def header(self) -> None:
pass
@@ -188,8 +228,8 @@ class Report(BaseReport):
f" {key} {self._status_meaning(key)} .....: {value:2d}"
)
self.header_line(
f" Accuracy compared to stree_default (liblinear-ovr) .: "
f"{accuracy/BEST_ACCURACY_STREE:7.4f}"
f" {self._get_message_best_accuracy()} "
f"{accuracy/self._get_best_accuracy():7.4f}"
)
self.header_line("*")
@@ -209,12 +249,12 @@ class ReportBest(BaseReport):
if best
else Files.grid_output(score, model)
)
file_name = os.path.join(Folders.results, name)
self.best = best
self.grid = grid
file_name = os.path.join(Folders.results, name)
super().__init__(file_name, best_file=True)
self.score_name = score
self.model = model
super().__init__(file_name, best_file=True)
def header_line(self, text: str) -> None:
length = sum(self.header_lengths) + len(self.header_lengths) - 3
@@ -254,8 +294,8 @@ class ReportBest(BaseReport):
def footer(self, accuracy):
self.header_line("*")
self.header_line(
f" Scores compared to stree_default accuracy (liblinear-ovr) .: "
f"{accuracy/BEST_ACCURACY_STREE:7.4f}"
f" {self._get_message_best_accuracy()} "
f"{accuracy/self._get_best_accuracy():7.4f}"
)
self.header_line("*")
@@ -509,8 +549,8 @@ class Excel(BaseReport):
self.sheet.write(self.row, 3, self._status_meaning(key), bold)
self.row += 1
message = (
f"** Accuracy compared to stree_default (liblinear-ovr) .: "
f"{accuracy/BEST_ACCURACY_STREE:7.4f}"
f"** {self._get_message_best_accuracy()} "
f"{accuracy/self._get_best_accuracy():7.4f}"
)
bold = self.book.add_format({"bold": True, "font_size": 14})
# set width of the hyperparams column with the maximum width
@@ -634,6 +674,13 @@ class Benchmark:
self._report = {}
self._datasets = set()
self.visualize = visualize
self.__compute_best_results_ever()
def __compute_best_results_ever(self):
args = EnvData.load()
key = args["source_data"]
best = BestResultsEver()
_, self.best_score_value = best.get_name_value(key, self._score)
def get_result_file_name(self):
return os.path.join(Folders.exreport, Files.exreport(self._score))
@@ -971,7 +1018,7 @@ class Benchmark:
sheet.write_formula(
row,
col + 1,
f"=sum({range_metric})/{BEST_ACCURACY_STREE}",
f"=sum({range_metric})/{self.best_score_value}",
decimal_total,
)
range_rank = (
@@ -1063,7 +1110,7 @@ class StubReport(BaseReport):
def footer(self, accuracy: float) -> None:
self.accuracy = accuracy
self.score = accuracy / BEST_ACCURACY_STREE
self.score = accuracy / self._get_best_accuracy()
class Summary:

1
benchmark/Utils.py
View File

@@ -1,7 +1,6 @@
import os
import subprocess
BEST_ACCURACY_STREE = 40.282203
NO_RESULTS = "** No results found **"
NO_ENV = "File .env not found"

2
benchmark/tests/test_files/be_build_best_report.test
View File

@@ -7,5 +7,5 @@ Dataset Score File/Message
balance-scale 0.963520 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json {'base_estimator__C': 57, 'base_estimator__gamma': 0.1, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1}
balloons 0.785000 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json {'base_estimator__C': 5, 'base_estimator__gamma': 0.14, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1}
******************************************************************************************************************************************************************
* Scores compared to stree_default accuracy (liblinear-ovr) .: 0.0434 *
* accuracy compared to STree_default (liblinear-ovr) .: 0.0434 *
******************************************************************************************************************************************************************

2
benchmark/tests/test_files/be_main_best.test
View File

@@ -11,6 +11,6 @@ Dataset Sampl. Feat. Cls Nodes Leaves Depth Score
balance-scale 625 4 3 23.32 12.16 6.44 0.840160±0.0304 0.013745±0.0019 {'splitter': 'best', 'max_features': 'auto'}
balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000388±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'}
************************************************************************************************************************
* Accuracy compared to stree_default (liblinear-ovr) .: 0.0422 *
* accuracy compared to STree_default (liblinear-ovr) .: 0.0422 *
************************************************************************************************************************
Results in results/results_accuracy_STree_iMac27_2022-05-09_00:15:25_0.json

2
benchmark/tests/test_files/be_main_complete.test
View File

@@ -11,6 +11,6 @@ Dataset Sampl. Feat. Cls Nodes Leaves Depth Score
balance-scale 625 4 3 17.36 9.18 6.18 0.908480±0.0247 0.007388±0.0013 {}
balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000664±0.0002 {}
************************************************************************************************************************
* Accuracy compared to stree_default (liblinear-ovr) .: 0.0390 *
* accuracy compared to STree_default (liblinear-ovr) .: 0.0390 *
************************************************************************************************************************
Results in results/results_accuracy_STree_iMac27_2022-05-08_20:14:43_0.json

2
benchmark/tests/test_files/be_main_dataset.test
View File

@@ -10,6 +10,6 @@ Dataset Sampl. Feat. Cls Nodes Leaves Depth Score
============================== ====== ===== === ======= ======= ======= =============== ================ ===============
balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000671±0.0001 {}
************************************************************************************************************************
* Accuracy compared to stree_default (liblinear-ovr) .: 0.0165 *
* accuracy compared to STree_default (liblinear-ovr) .: 0.0165 *
************************************************************************************************************************
Partial result file removed: results/results_accuracy_STree_iMac27_2022-05-08_19:38:28_0.json

2
benchmark/tests/test_files/be_main_grid.test
View File

@@ -11,6 +11,6 @@ Dataset Sampl. Feat. Cls Nodes Leaves Depth Score
balance-scale 625 4 3 26.12 13.56 7.94 0.910720±0.0249 0.015852±0.0027 {'C': 1.0, 'kernel': 'liblinear', 'multiclass_strategy': 'ovr'}
balloons 16 4 2 4.64 2.82 2.66 0.663333±0.3009 0.000640±0.0001 {'C': 1.0, 'kernel': 'linear', 'multiclass_strategy': 'ovr'}
************************************************************************************************************************
* Accuracy compared to stree_default (liblinear-ovr) .: 0.0391 *
* accuracy compared to STree_default (liblinear-ovr) .: 0.0391 *
************************************************************************************************************************
Results in results/results_accuracy_STree_iMac27_2022-05-09_00:21:06_0.json

2
benchmark/tests/test_files/excel.test
View File

@@ -45,4 +45,4 @@
8;10;"0.0008541679382324218"
8;11;"3.629469326417878e-05"
8;12;"{'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'}"
10;1;"** Accuracy compared to stree_default (liblinear-ovr) .: 0.0454"
10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0454"

2
benchmark/tests/test_files/excel_add_ODTE.test
View File

@@ -45,4 +45,4 @@
8;10;"0.1156062078475952"
8;11;"0.0127842418285999"
8;12;"{'base_estimator__C': 5, 'base_estimator__gamma': 0.14, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1}"
10;1;"** Accuracy compared to stree_default (liblinear-ovr) .: 0.0434"
10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0434"

2
benchmark/tests/test_files/excel_add_STree.test
View File

@@ -43,4 +43,4 @@
8;10;"0.02120100021362305"
8;11;"0.003526023309468471"
8;12;"{'splitter': 'best', 'max_features': 'auto'}"
10;1;"** Accuracy compared to stree_default (liblinear-ovr) .: 0.0416"
10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0416"

2
benchmark/tests/test_files/excel_compared.test
View File

@@ -49,4 +49,4 @@
11;2;"✔"
11;3;1
11;4;"Equal to best"
13;1;"** Accuracy compared to stree_default (liblinear-ovr) .: 0.0454"
13;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0454"

2
benchmark/tests/test_files/exreport_excel_ODTE.test
View File

@@ -45,4 +45,4 @@
8;10;"0.1156062078475952"
8;11;"0.0127842418285999"
8;12;"{'base_estimator__C': 5, 'base_estimator__gamma': 0.14, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1}"
10;1;"** Accuracy compared to stree_default (liblinear-ovr) .: 0.0434"
10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0434"

2
benchmark/tests/test_files/exreport_excel_RandomForest.test
View File

@@ -45,4 +45,4 @@
8;10;"0.07016648769378662"
8;11;"0.002460508923990468"
8;12;"{}"
10;1;"** Accuracy compared to stree_default (liblinear-ovr) .: 0.0363"
10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0363"

2
benchmark/tests/test_files/exreport_excel_STree.test
View File

@@ -45,4 +45,4 @@
8;10;"0.0008541679382324218"
8;11;"3.629469326417878e-05"
8;12;"{'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'}"
10;1;"** Accuracy compared to stree_default (liblinear-ovr) .: 0.0454"
10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0454"

2
benchmark/tests/test_files/report.test
View File

@@ -11,5 +11,5 @@ Dataset Sampl. Feat. Cls Nodes Leaves Depth Score
balance-scale 625 4 3 7.00 4.00 3.00 0.970560±0.0150 0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'}
balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'}
************************************************************************************************************************
* Accuracy compared to stree_default (liblinear-ovr) .: 0.0454 *
* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 *
************************************************************************************************************************

2
benchmark/tests/test_files/report_best.test
View File

@@ -7,5 +7,5 @@ Dataset Score File/Message
balance-scale 0.980000 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json {'splitter': 'best', 'max_features': 'auto'}
balloons 0.860000 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'}
******************************************************************************************************************************************************************
* Scores compared to stree_default accuracy (liblinear-ovr) .: 0.0457 *
* accuracy compared to STree_default (liblinear-ovr) .: 0.0457 *
******************************************************************************************************************************************************************

2
benchmark/tests/test_files/report_compared.test
View File

@@ -12,5 +12,5 @@ Dataset Sampl. Feat. Cls Nodes Leaves Depth Score
balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850✔ 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'}
************************************************************************************************************************
* ✔ Equal to best .....: 1 *
* Accuracy compared to stree_default (liblinear-ovr) .: 0.0454 *
* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 *
************************************************************************************************************************

2
benchmark/tests/test_files/report_grid.test
View File

@@ -7,5 +7,5 @@ Dataset Score File/Message
balance-scale 0.919995 v. 1.2.4, Computed on Test on 2022-02-22 at 12:00:00 took 1s {'C': 1.0, 'kernel': 'liblinear', 'multiclass_strategy': 'ovr'}
balloons 0.625000 v. 1.2.4, Computed on Test on 2022-02-22 at 12:00:00 took 1s {'C': 1.0, 'kernel': 'linear', 'multiclass_strategy': 'ovr'}
******************************************************************************************************************************************************************
* Scores compared to stree_default accuracy (liblinear-ovr) .: 0.0384 *
* accuracy compared to STree_default (liblinear-ovr) .: 0.0384 *
******************************************************************************************************************************************************************