Doctorado-ML/benchmark
1
0
Fork 0
mirror of https://github.com/Doctorado-ML/benchmark.git synced 2025-08-16 07:55:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Complete tests adding excel to be_list

Browse Source
This commit is contained in:
Ricardo Montañana Gómez
2022-11-17 12:00:30 +01:00
parent 8a9342c97b
commit dfd4f8179b
13 changed files with 238 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
benchmark/Results.py
View File

@@ -22,8 +22,8 @@ from .Utils import (
from ._version import __version__
def get_input():
return input()
def get_input(is_test):
return "test" if is_test else input()
class BestResultsEver:
@@ -1477,31 +1477,29 @@ class Summary:
)
)
def manage_results(self, excel, isTest):
def manage_results(self, excel, is_test):
"""Manage results showed in the summary
return True if excel file is created False otherwise
"""
num = ""
first = True
book = None
while True and not isTest:
if not first:
print(f"Invalid option {num}. Try again!")
first = False
while True:
print(
"Which result do you want to report? (q to quit, r to list "
"again, number to report): ",
end="",
)
num = get_input()
if num == "n":
return
num = get_input(is_test)
if num == "r":
self.list_results()
if num == "q":
if excel:
if book is not None:
book.close()
return
return True
return False
if num.isdigit() and int(num) < len(self.data) and int(num) >= 0:
rep = Report(self.data[int(num)]["file"], self.hidden)
rep = Report(self.data_filtered[int(num)]["file"], self.hidden)
rep.report()
if excel and not self.hidden:
if book is None:
@@ -1510,10 +1508,13 @@ class Summary:
file_name, {"nan_inf_to_errors": True}
)
excel = Excel(
file_name=self.data[int(num)]["file"],
file_name=self.data_filtered[int(num)]["file"],
book=book,
)
excel.report()
else:
if num not in ("r", "q"):
print(f"Invalid option {num}. Try again!")
def show_result(self, data: dict, title: str = "") -> None:
def whites(n: int) -> str:

8
benchmark/scripts/be_list.py
View File

@@ -24,9 +24,11 @@ def main(args_test=None):
number=args.number,
)
is_test = args_test is not None
data.manage_results(args.excel, is_test)
if args.excel:
Files.open(Files.be_list_excel, is_test)
if not args.nan:
excel_generated = data.manage_results(args.excel, is_test)
if args.excel and excel_generated:
print(f"Generated file: {Files.be_list_excel}")
Files.open(Files.be_list_excel, is_test)
except ValueError as e:
print(e)
else:

1
benchmark/tests/.gitignore vendored
View File

@@ -1 +1,2 @@
ReportDatasets.xlsx
some_results.xlsx

5
benchmark/tests/Report_test.py
View File

@@ -2,11 +2,14 @@ import os
from io import StringIO
from unittest.mock import patch
from .TestBase import TestBase
from ..Results import Report, BaseReport, ReportBest, ReportDatasets
from ..Results import Report, BaseReport, ReportBest, ReportDatasets, get_input
from ..Utils import Symbols
class ReportTest(TestBase):
def test_get_input(self):
self.assertEqual(get_input(is_test=True), "test")
def test_BaseReport(self):
with patch.multiple(BaseReport, __abstractmethods__=set()):
file_name = os.path.join(

4
benchmark/tests/scripts/Be_Grid_test.py
View File

@@ -4,6 +4,10 @@ from ...Utils import Folders, Files
from ..TestBase import TestBase
def get_test():
return "hola"
class BeGridTest(TestBase):
def setUp(self):
self.prepare_scripts_env()

66
benchmark/tests/scripts/Be_List_test.py
View File

@@ -1,7 +1,7 @@
import os
from unittest.mock import patch
from ...Utils import Folders, NO_RESULTS
from ...Results import Summary
from openpyxl import load_workbook
from ...Utils import Folders, Files, NO_RESULTS
from ..TestBase import TestBase
@@ -9,13 +9,64 @@ class BeListTest(TestBase):
def setUp(self):
self.prepare_scripts_env()
# @patch("...Results.get_input()", return_value="n")
def test_be_list(self):
@patch("benchmark.Results.get_input", return_value="q")
def test_be_list(self, input_data):
stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "summary_list_model")
self.check_output_file(stdout, "be_list_model")
def test_be_list_no_data(self):
@patch("benchmark.Results.get_input", side_effect=iter(["x", "q"]))
def test_be_list_invalid_option(self, input_data):
stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_model_invalid")
@patch("benchmark.Results.get_input", side_effect=iter(["0", "q"]))
def test_be_list_report(self, input_data):
stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_report")
@patch("benchmark.Results.get_input", side_effect=iter(["q"]))
def test_be_list_report_excel_none(self, input_data):
stdout, stderr = self.execute_script(
"be_list", ["-m", "STree", "-x", "1"]
)
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_model")
@patch("benchmark.Results.get_input", side_effect=iter(["r", "q"]))
def test_be_list_twice(self, input_data):
stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_model_2")
@patch("benchmark.Results.get_input", side_effect=iter(["2", "q"]))
def test_be_list_report_excel(self, input_data):
stdout, stderr = self.execute_script(
"be_list", ["-m", "STree", "-x", "1"]
)
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_report_excel")
book = load_workbook(Files.be_list_excel)
sheet = book["STree"]
self.check_excel_sheet(sheet, "excel")
@patch("benchmark.Results.get_input", side_effect=iter(["2", "1", "q"]))
def test_be_list_report_excel_twice(self, input_data):
stdout, stderr = self.execute_script(
"be_list", ["-m", "STree", "-x", "1"]
)
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_report_excel_2")
book = load_workbook(Files.be_list_excel)
sheet = book["STree"]
self.check_excel_sheet(sheet, "excel")
sheet = book["STree2"]
self.check_excel_sheet(sheet, "excel2")
@patch("benchmark.Results.get_input", return_value="q")
def test_be_list_no_data(self, input_data):
stdout, stderr = self.execute_script(
"be_list", ["-m", "Wodt", "-s", "f1-macro"]
)
@@ -44,7 +95,8 @@ class BeListTest(TestBase):
swap_files(Folders.results, Folders.hidden_results, file_name)
self.fail("test_be_list_nan() should not raise exception")
def test_be_list_nan_no_nan(self):
@patch("benchmark.Results.get_input", return_value="q")
def test_be_list_nan_no_nan(self, input_data):
stdout, stderr = self.execute_script("be_list", ["--nan", "1"])
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_no_nan")

6
benchmark/tests/test_files/be_list_model.test Normal file
View File

@@ -0,0 +1,6 @@
 # Date File Score Time(h) Title
=== ========== ============================================================= ======== ======= =================================
 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B
 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A
 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters
Which result do you want to report? (q to quit, r to list again, number to report):

11
benchmark/tests/test_files/be_list_model_2.test Normal file
View File

@@ -0,0 +1,11 @@
 # Date File Score Time(h) Title
=== ========== ============================================================= ======== ======= =================================
 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B
 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A
 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters
Which result do you want to report? (q to quit, r to list again, number to report):  # Date File Score Time(h) Title
=== ========== ============================================================= ======== ======= =================================
 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B
 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A
 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters
Which result do you want to report? (q to quit, r to list again, number to report):

7
benchmark/tests/test_files/be_list_model_invalid.test Normal file
View File

@@ -0,0 +1,7 @@
 # Date File Score Time(h) Title
=== ========== ============================================================= ======== ======= =================================
 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B
 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A
 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters
Which result do you want to report? (q to quit, r to list again, number to report): Invalid option x. Try again!
Which result do you want to report? (q to quit, r to list again, number to report):

21
benchmark/tests/test_files/be_list_report.test Normal file
View File

@@ -0,0 +1,21 @@
 # Date File Score Time(h) Title
=== ========== ============================================================= ======== ======= =================================
 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B
 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A
 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters
Which result do you want to report? (q to quit, r to list again, number to report): *************************************************************************************************************************
* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-11-01 19:17:07 *
* default B *
* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False *
* Execution took 4115.04 seconds, 1.14 hours, on macbook-pro *
* Score is accuracy *
*************************************************************************************************************************
Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters
============================== ====== ===== === ======= ======= ======= =============== ================= ===============
balance-scale 625 4 3 18.78 9.88 5.90 0.970000±0.0020 0.233304±0.0481 {'max_features': 'auto', 'splitter': 'mutual'}
balloons 16 4 2 4.72 2.86 2.78 0.556667±0.2941 0.021352±0.0058 {'max_features': 'auto', 'splitter': 'mutual'}
*************************************************************************************************************************
* accuracy compared to STree_default (liblinear-ovr) .: 0.0379 *
*************************************************************************************************************************
Which result do you want to report? (q to quit, r to list again, number to report):

21
benchmark/tests/test_files/be_list_report_excel.test Normal file
View File

@@ -0,0 +1,21 @@
 # Date File Score Time(h) Title
=== ========== ============================================================= ======== ======= =================================
 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B
 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A
 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters
Which result do you want to report? (q to quit, r to list again, number to report): *************************************************************************************************************************
* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 *
* With gridsearched hyperparameters *
* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False *
* Execution took 624.25 seconds, 0.17 hours, on iMac27 *
* Score is accuracy *
*************************************************************************************************************************
Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters
============================== ====== ===== === ======= ======= ======= =============== ================= ===============
balance-scale 625 4 3 7.00 4.00 3.00 0.970560±0.0150 0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'}
balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'}
*************************************************************************************************************************
* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 *
*************************************************************************************************************************
Which result do you want to report? (q to quit, r to list again, number to report): Generated file: some_results.xlsx

36
benchmark/tests/test_files/be_list_report_excel_2.test Normal file
View File

@@ -0,0 +1,36 @@
 # Date File Score Time(h) Title
=== ========== ============================================================= ======== ======= =================================
 0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B
 1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A
 2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters
Which result do you want to report? (q to quit, r to list again, number to report): *************************************************************************************************************************
* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07 *
* With gridsearched hyperparameters *
* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False *
* Execution took 624.25 seconds, 0.17 hours, on iMac27 *
* Score is accuracy *
*************************************************************************************************************************
Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters
============================== ====== ===== === ======= ======= ======= =============== ================= ===============
balance-scale 625 4 3 7.00 4.00 3.00 0.970560±0.0150 0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'}
balloons 16 4 2 3.00 2.00 2.00 0.860000±0.2850 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'}
*************************************************************************************************************************
* accuracy compared to STree_default (liblinear-ovr) .: 0.0454 *
*************************************************************************************************************************
Which result do you want to report? (q to quit, r to list again, number to report): *************************************************************************************************************************
* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-10-27 09:40:40 *
* default A *
* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False *
* Execution took 3395.01 seconds, 0.94 hours, on iMac27 *
* Score is accuracy *
*************************************************************************************************************************
Dataset Sampl. Feat. Cls Nodes Leaves Depth Score Time Hyperparameters
============================== ====== ===== === ======= ======= ======= =============== ================= ===============
balance-scale 625 4 3 11.08 5.90 5.90 0.980000±0.0010 0.285207±0.0603 {'splitter': 'best', 'max_features': 'auto'}
balloons 16 4 2 4.12 2.56 2.56 0.695000±0.2757 0.021201±0.0035 {'splitter': 'best', 'max_features': 'auto'}
*************************************************************************************************************************
* accuracy compared to STree_default (liblinear-ovr) .: 0.0416 *
*************************************************************************************************************************
Which result do you want to report? (q to quit, r to list again, number to report): Generated file: some_results.xlsx

48
benchmark/tests/test_files/excel2.test Normal file
View File

@@ -0,0 +1,48 @@
1;1;" STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-10-27 09:40:40"
2;1;" default A"
3;1;" Score is accuracy"
3;2;" Execution time"
3;5;"3,395.01 s"
3;7;" "
3;8;"Platform"
3;9;"iMac27"
3;10;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]"
4;5;" 0.94 h"
4;10;"Stratified: False"
6;1;"Dataset"
6;2;"Samples"
6;3;"Features"
6;4;"Classes"
6;5;"Nodes"
6;6;"Leaves"
6;7;"Depth"
6;8;"Score"
6;9;"Score Std."
6;10;"Time"
6;11;"Time Std."
6;12;"Hyperparameters"
7;1;"balance-scale"
7;2;"625"
7;3;"4"
7;4;"3"
7;5;"11.08"
7;6;"5.9"
7;7;"5.9"
7;8;"0.98"
7;9;"0.001"
7;10;"0.2852065515518188"
7;11;"0.06031593282605064"
7;12;"{'splitter': 'best', 'max_features': 'auto'}"
8;1;"balloons"
8;2;"16"
8;3;"4"
8;4;"2"
8;5;"4.12"
8;6;"2.56"
8;7;"2.56"
8;8;"0.695"
8;9;"0.2756860130252853"
8;10;"0.02120100021362305"
8;11;"0.003526023309468471"
8;12;"{'splitter': 'best', 'max_features': 'auto'}"
10;1;"** accuracy compared to STree_default (liblinear-ovr) .: 0.0416"