Doctorado-ML/benchmark
1
0
Fork 0
mirror of https://github.com/Doctorado-ML/benchmark.git synced 2025-08-17 00:15:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Language and language version to reports

Browse Source
This commit is contained in:
Ricardo Montañana Gómez
2022-11-01 02:07:24 +01:00
parent 5532beb88a
commit 97718e6e82
25 changed files with 613 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
benchmark/Experiments.py
View File

@@ -1,4 +1,5 @@
import os
import sys
import json
import random
import warnings
@@ -162,6 +163,10 @@ class Experiment:
def get_output_file(self):
return self.output_file
@staticmethod
def get_python_version():
return "{}.{}".format(sys.version_info.major, sys.version_info.minor)
def _build_classifier(self, random_state, hyperparameters):
self.model = Models.get_model(self.model_name, random_state)
clf = self.model
@@ -193,7 +198,7 @@ class Experiment:
shuffle=True, random_state=random_state, n_splits=self.folds
)
clf = self._build_classifier(random_state, hyperparameters)
self.version = clf.version() if hasattr(clf, "version") else "-"
self.version = Models.get_version(self.model_name, clf)
with warnings.catch_warnings():
warnings.filterwarnings("ignore")
res = cross_validate(
@@ -243,6 +248,8 @@ class Experiment:
output["duration"] = self.duration
output["seeds"] = self.random_seeds
output["platform"] = self.platform
output["language_version"] = self.get_python_version()
output["language"] = "Python"
output["results"] = self.results
with open(self.output_file, "w") as f:
json.dump(output, f)

14
benchmark/Models.py
View File

@@ -11,6 +11,8 @@ from stree import Stree
from wodt import Wodt
from odte import Odte
from xgboost import XGBClassifier
import sklearn
import xgboost
class Models:
@@ -89,3 +91,15 @@ class Models:
nodes, leaves = result.nodes_leaves()
depth = result.depth_ if hasattr(result, "depth_") else 0
return nodes, leaves, depth
@staticmethod
def get_version(name, clf):
if hasattr(clf, "version"):
return clf.version()
if name in ["Cart", "ExtraTree", "RandomForest", "GBC", "SVC"]:
return sklearn.__version__
elif name.startswith("Bagging") or name.startswith("AdaBoost"):
return sklearn.__version__
elif name == "XGBoost":
return xgboost.__version__
return "Error"

7
benchmark/Results.py
View File

@@ -16,6 +16,7 @@ from .Utils import (
Symbols,
TextColor,
NO_RESULTS,
PYTHON_VERSION,
)
@@ -196,7 +197,8 @@ class Report(BaseReport):
self._compare_totals = {}
self.header_line("*")
self.header_line(
f" Report {self.data['model']} ver. {self.data['version']}"
f" {self.data['model']} ver. {self.data['version']}"
f" {self.data['language']} ver. {self.data['language_version']}"
f" with {self.data['folds']} Folds "
f"cross validation and {len(self.data['seeds'])} random seeds. "
f"{self.data['date']} {self.data['time']}"
@@ -347,7 +349,8 @@ class Excel(BaseReport):
def get_title(self):
return (
f" Report {self.data['model']} ver. {self.data['version']}"
f" {self.data['model']} ver. {self.data['version']}"
f" {self.data['language']} ver. {self.data['language_version']}"
f" with {self.data['folds']} Folds "
f"cross validation and {len(self.data['seeds'])} random seeds. "
f"{self.data['date']} {self.data['time']}"

2
benchmark/Utils.py
View File

@@ -1,6 +1,8 @@
import os
import sys
import subprocess
PYTHON_VERSION = "{}.{}".format(sys.version_info.major, sys.version_info.minor)
NO_RESULTS = "** No results found **"
NO_ENV = "File .env not found"

1
benchmark/tests/Excel_test.py
View File

@@ -4,6 +4,7 @@ from xlsxwriter import Workbook
from .TestBase import TestBase
from ..Results import Excel
from ..Utils import Folders
import benchmark.Utils
class ExcelTest(TestBase):

34
benchmark/tests/Models_test.py
View File

@@ -15,6 +15,8 @@ from odte import Odte
from xgboost import XGBClassifier
from .TestBase import TestBase
from ..Models import Models
import xgboost
import sklearn
class ModelTest(TestBase):
@@ -33,6 +35,38 @@ class ModelTest(TestBase):
for key, value in test.items():
self.assertIsInstance(Models.get_model(key), value)
def test_Models_version(self):
def ver_stree():
return "1.2.3"
def ver_wodt():
return "h.j.k"
def ver_odte():
return "4.5.6"
test = {
"STree": [ver_stree, "1.2.3"],
"Wodt": [ver_wodt, "h.j.k"],
"ODTE": [ver_odte, "4.5.6"],
"RandomForest": [None, "7.8.9"],
"BaggingStree": [None, "x.y.z"],
"AdaBoostStree": [None, "w.x.z"],
"XGBoost": [None, "10.11.12"],
}
for key, value in test.items():
clf = Models.get_model(key)
if key in ["STree", "Wodt", "ODTE"]:
clf.version = value[0]
elif key == "XGBoost":
xgboost.__version__ = value[1]
else:
sklearn.__version__ = value[1]
self.assertEqual(Models.get_version(key, clf), value[1])
def test_bogus_Model_Version(self):
self.assertEqual(Models.get_version("unknown", None), "Error")
def test_BaggingStree(self):
clf = Models.get_model("BaggingStree")
self.assertIsInstance(clf, BaggingClassifier)

2
benchmark/tests/results/results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json
View File

@@ -3,6 +3,8 @@
"title": "Gridsearched hyperparams v022.1b random_init",
"model": "ODTE",
"version": "0.3.2",
"language_version": "3.11x",
"language": "Python",
"stratified": false,
"folds": 5,
"date": "2022-04-20",

2
benchmark/tests/results/results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json
View File

@@ -3,6 +3,8 @@
"title": "Test default paramters with RandomForest",
"model": "RandomForest",
"version": "-",
"language_version": "3.11x",
"language": "Python",
"stratified": false,
"folds": 5,
"date": "2022-01-14",

2
benchmark/tests/results/results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json
View File

@@ -3,6 +3,8 @@
"model": "STree",
"stratified": false,
"folds": 5,
"language_version": "3.11x",
"language": "Python",
"date": "2021-09-30",
"time": "11:42:07",
"duration": 624.2505249977112,

2
benchmark/tests/results/results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json
View File

@@ -1,6 +1,8 @@
{
"score_name": "accuracy",
"model": "STree",
"language": "Python",
"language_version": "3.11x",
"stratified": false,
"folds": 5,
"date": "2021-10-27",

2
benchmark/tests/results/results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json
View File

@@ -1,6 +1,8 @@
{
"score_name": "accuracy",
"model": "STree",
"language_version": "3.11x",
"language": "Python",
"stratified": false,
"folds": 5,
"date": "2021-11-01",

2
benchmark/tests/test_files/be_main_best.test
View File

@@ -1,5 +1,5 @@
************************************************************************************************************************
* Report STree ver. 1.2.4 with 5 Folds cross validation and 10 random seeds. 2022-05-09 00:15:25 *
* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-09 00:15:25 *
* test *
* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False *
* Execution took 0.80 seconds, 0.00 hours, on iMac27 *

2
benchmark/tests/test_files/be_main_complete.test
View File

@@ -1,5 +1,5 @@
************************************************************************************************************************
* Report STree ver. 1.2.4 with 5 Folds cross validation and 10 random seeds. 2022-05-08 20:14:43 *
* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-08 20:14:43 *
* test *
* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False *
* Execution took 0.48 seconds, 0.00 hours, on iMac27 *

2
benchmark/tests/test_files/be_main_dataset.test
View File

@@ -1,5 +1,5 @@
************************************************************************************************************************
* Report STree ver. 1.2.4 with 5 Folds cross validation and 10 random seeds. 2022-05-08 19:38:28 *
* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-08 19:38:28 *
* test *
* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False *
* Execution took 0.06 seconds, 0.00 hours, on iMac27 *

2
benchmark/tests/test_files/be_main_grid.test
View File

@@ -1,5 +1,5 @@
************************************************************************************************************************
* Report STree ver. 1.2.4 with 5 Folds cross validation and 10 random seeds. 2022-05-09 00:21:06 *
* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-09 00:21:06 *
* test *
* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False *
* Execution took 0.89 seconds, 0.00 hours, on iMac27 *

2
benchmark/tests/test_files/excel.test
View File

@@ -1,4 +1,4 @@
1;1;" Report STree ver. 1.2.3 with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07"
1;1;" STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07"
2;1;" With gridsearched hyperparameters"
3;1;" Score is accuracy"
3;2;" Execution time"

2
benchmark/tests/test_files/excel_add_ODTE.test
View File

@@ -1,4 +1,4 @@
1;1;" Report ODTE ver. 0.3.2 with 5 Folds cross validation and 10 random seeds. 2022-04-20 10:52:20"
1;1;" ODTE ver. 0.3.2 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-04-20 10:52:20"
2;1;" Gridsearched hyperparams v022.1b random_init"
3;1;" Score is accuracy"
3;2;" Execution time"

2
benchmark/tests/test_files/excel_add_STree.test
View File

@@ -1,4 +1,4 @@
1;1;" Report STree ver. 1.2.3 with 5 Folds cross validation and 10 random seeds. 2021-10-27 09:40:40"
1;1;" STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-10-27 09:40:40"
2;1;" default A"
3;1;" Score is accuracy"
3;2;" Execution time"

2
benchmark/tests/test_files/excel_compared.test
View File

@@ -1,4 +1,4 @@
1;1;" Report STree ver. 1.2.3 with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07"
1;1;" STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07"
2;1;" With gridsearched hyperparameters"
3;1;" Score is accuracy"
3;2;" Execution time"

2
benchmark/tests/test_files/exreport_excel_ODTE.test
View File

@@ -1,4 +1,4 @@
1;1;" Report ODTE ver. 0.3.2 with 5 Folds cross validation and 10 random seeds. 2022-04-20 10:52:20"
1;1;" ODTE ver. 0.3.2 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-04-20 10:52:20"
2;1;" Gridsearched hyperparams v022.1b random_init"
3;1;" Score is accuracy"
3;2;" Execution time"

2
benchmark/tests/test_files/exreport_excel_RandomForest.test
View File

@@ -1,4 +1,4 @@
1;1;" Report RandomForest ver. - with 5 Folds cross validation and 10 random seeds. 2022-01-14 12:39:30"
1;1;" RandomForest ver. - Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-01-14 12:39:30"
2;1;" Test default paramters with RandomForest"
3;1;" Score is accuracy"
3;2;" Execution time"

2
benchmark/tests/test_files/exreport_excel_STree.test
View File

@@ -1,4 +1,4 @@
1;1;" Report STree ver. 1.2.3 with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07"
1;1;" STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07"
2;1;" With gridsearched hyperparameters"
3;1;" Score is accuracy"
3;2;" Execution time"

2
benchmark/tests/test_files/report.test
View File

@@ -1,5 +1,5 @@
************************************************************************************************************************
* Report STree ver. 1.2.3 with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 *
* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 *
* With gridsearched hyperparameters *
* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False *
* Execution took 624.25 seconds, 0.17 hours, on iMac27 *

2
benchmark/tests/test_files/report_compared.test
View File

@@ -1,5 +1,5 @@
************************************************************************************************************************
* Report STree ver. 1.2.3 with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 *
* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 *
* With gridsearched hyperparameters *
* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False *
* Execution took 624.25 seconds, 0.17 hours, on iMac27 *