Doctorado-ML/benchmark
1
0
Fork 0
mirror of https://github.com/Doctorado-ML/benchmark.git synced 2025-08-15 15:35:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update tests

Browse Source
This commit is contained in:
Ricardo Montañana Gómez
2022-11-22 23:32:28 +01:00
parent 3a2ec38671
commit 71a11110bd
14 changed files with 124 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
benchmark/Arguments.py
View File

@@ -176,7 +176,7 @@ class Arguments(argparse.ArgumentParser):
"action": "store_true",
"required": False,
"default": False,
"help": "Move nan results to hidden folder",
"help": "List nan results to hidden folder",
},
],
"number": [

8
benchmark/Results.py
View File

@@ -23,7 +23,7 @@ from .Utils import (
from ._version import __version__
def get_input(is_test=False, message=""):
def get_input(message="", is_test=False):
return "test" if is_test else input(message)
@@ -1476,7 +1476,7 @@ class Summary:
)
)
def manage_results(self, is_test):
def manage_results(self):
"""Manage results showed in the summary
return True if excel file is created False otherwise
"""
@@ -1497,7 +1497,7 @@ class Summary:
TextColor.RED
+ f"Are you sure to {verb1} {file_name_result} (y/n)? "
)
confirm = get_input(message=conf_message, is_test=is_test)
confirm = get_input(message=conf_message)
if confirm == "y":
print(TextColor.YELLOW + f"{verb2} {file_name_result}")
if command == cmd.delete:
@@ -1522,7 +1522,7 @@ class Summary:
book = None
max_value = len(self.data)
while True:
match get_input(message=message, is_test=is_test).split():
match get_input(message=message).split():
case [cmd.relist]:
self.list_results()
case [cmd.quit]:

7
benchmark/scripts/be_list.py
View File

@@ -8,10 +8,10 @@ from benchmark.Arguments import Arguments
def main(args_test=None):
is_test = args_test is not None
arguments = Arguments(prog="be_list")
arguments.xset("number").xset("model", required=False).xset("key")
arguments.xset("score", required=False).xset("compare").xset("hidden")
arguments.xset("nan")
args = arguments.parse(args_test)
data = Summary(hidden=args.hidden, compare=args.compare)
data.acquire()
@@ -21,11 +21,12 @@ def main(args_test=None):
model=args.model,
sort_key=args.key,
number=args.number,
nan=args.nan,
)
except ValueError as e:
print(e)
return
excel_generated = data.manage_results(is_test=is_test)
excel_generated = data.manage_results()
if excel_generated:
print(f"Generated file: {Files.be_list_excel}")
Files.open(Files.be_list_excel, is_test)
Files.open(Files.be_list_excel, test=args_test is not None)

7
benchmark/scripts/be_report.py
View File

@@ -5,12 +5,13 @@ from benchmark.Arguments import Arguments
"""Build report on screen of a result file, optionally generate excel and sql
file, and can compare results of report with best results obtained by model
file, and can compare results of report wibth best results obtained by model
If no argument is set, displays the datasets and its characteristics
"""
def main(args_test=None):
is_test = args_test is not None
arguments = Arguments(prog="be_report")
arguments.add_subparser()
arguments.add_subparsers_options(
@@ -72,13 +73,11 @@ def main(args_test=None):
compare=args.compare,
)
excel.report()
is_test = args_test is not None
Files.open(excel.get_file_name(), is_test)
case "datasets":
report = ReportDatasets(args.excel)
report.report()
if args.excel:
is_test = args_test is not None
Files.open(report.get_file_name(), is_test)
case _:
arguments.parse(["-h"])
arguments.print_help()

64
benchmark/tests/scripts/Be_List_test.py
View File

@@ -1,4 +1,5 @@
import os
import shutil
from unittest.mock import patch
from openpyxl import load_workbook
from ...Utils import Folders, Files, NO_RESULTS
@@ -63,7 +64,35 @@ class BeListTest(TestBase):
self.assertEqual(stderr.getvalue(), "")
self.assertEqual(stdout.getvalue(), f"{NO_RESULTS}\n")
@patch("benchmark.Results.get_input", side_effect=iter(["h 0", "y", "q"]))
@patch(
"benchmark.Results.get_input", side_effect=iter(["d 0", "y", "", "q"])
)
# @patch("benchmark.Results.get_input", side_effect=iter(["q"]))
def test_be_list_delete(self, input_data):
def copy_files(source_folder, target_folder, file_name):
source = os.path.join(source_folder, file_name)
target = os.path.join(target_folder, file_name)
shutil.copyfile(source, target)
file_name = (
"results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:"
"35_0.json"
)
# move nan result from hidden to results
copy_files(Folders.hidden_results, Folders.results, file_name)
try:
# list and delete result
stdout, stderr = self.execute_script("be_list", "")
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_delete")
except Exception:
# delete the result copied if be_list couldn't
os.unlink(os.path.join(Folders.results, file_name))
self.fail("test_be_list_delete() should not raise exception")
@patch(
"benchmark.Results.get_input", side_effect=iter(["h 0", "y", "", "q"])
)
def test_be_list_hide(self, input_data):
def swap_files(source_folder, target_folder, file_name):
source = os.path.join(source_folder, file_name)
@@ -78,20 +107,37 @@ class BeListTest(TestBase):
swap_files(Folders.hidden_results, Folders.results, file_name)
try:
# list and move nan result to hidden again
stdout, stderr = self.execute_script("be_list", [])
stdout, stderr = self.execute_script("be_list", "")
self.assertEqual(stderr.getvalue(), "")
print(stdout.getvalue())
# self.check_output_file(stdout, "be_list_nan")
self.check_output_file(stdout, "be_list_hide")
except Exception:
# move back nan result file if be_list couldn't
# delete the result copied if be_list couldn't
swap_files(Folders.results, Folders.hidden_results, file_name)
self.fail("test_be_list_hide() should not raise exception")
@patch("benchmark.Results.get_input", return_value="q")
def test_be_list_nan_no_nan(self, input_data):
stdout, stderr = self.execute_script("be_list", ["--nan"])
@patch("benchmark.Results.get_input", side_effect=iter(["h 0", "q"]))
def test_be_list_already_hidden(self, input_data):
stdout, stderr = self.execute_script("be_list", ["--hidden"])
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_no_nan")
self.check_output_file(stdout, "be_list_already_hidden")
@patch("benchmark.Results.get_input", side_effect=iter(["h 0", "n", "q"]))
def test_be_list_dont_hide(self, input_data):
stdout, stderr = self.execute_script("be_list", "")
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_default")
@patch("benchmark.Results.get_input", side_effect=iter(["q"]))
def test_be_list_hidden_nan(self, input_data):
stdout, stderr = self.execute_script("be_list", ["--hidden", "--nan"])
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_hidden_nan")
@patch("benchmark.Results.get_input", side_effect=iter(["q"]))
def test_be_list_hidden(self, input_data):
stdout, stderr = self.execute_script("be_list", ["--hidden"])
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "be_list_hidden")
def test_be_no_env(self):
path = os.getcwd()

14
benchmark/tests/scripts/Be_Report_test.py
View File

@@ -104,14 +104,14 @@ class BeReportTest(TestBase):
def test_be_report_unknown_subcommand(self, stderr):
with self.assertRaises(SystemExit) as msg:
module = self.search_script("be_report")
module.main(["unknown", "accuracy", "-m", "STree"])
module.main(["unknown"])
self.assertEqual(msg.exception.code, 2)
self.assertEqual(
stderr.getvalue(),
"usage: be_report [-h] {best,grid,file,datasets} ...\n"
"be_report: error: argument subcommand: invalid choice: "
"'unknown' (choose from 'best', 'grid', 'file', 'datasets')\n",
)
self.check_output_file(stderr, "report_unknown_subcommand")
def test_be_report_without_subcommand(self):
stdout, stderr = self.execute_script("be_report", "")
self.assertEqual(stderr.getvalue(), "")
self.check_output_file(stdout, "report_without_subcommand")
def test_be_report_excel_compared(self):
file_name = "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json"

5
benchmark/tests/test_files/be_list_already_hidden.test Normal file
View File

@@ -0,0 +1,5 @@
 # Date File Score Time(h) Title
=== ========== ================================================================ ======== ======= =======================
 0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters
 1 2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json 0.97446 0.098 default
Already hidden

0
benchmark/tests/test_files/be_list_no_nan.test → benchmark/tests/test_files/be_list_default.test
View File

16
benchmark/tests/test_files/be_list_delete.test Normal file
View File

@@ -0,0 +1,16 @@
 # Date File Score Time(h) Title
=== ========== ================================================================ ======== ======= ============================================
 0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters
 1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init
 2 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest
 3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B
 4 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A
 5 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters
Deleting results/results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json
 # Date File Score Time(h) Title
=== ========== =============================================================== ======== ======= ============================================
 0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init
 1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest
 2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B
 3 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A
 4 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters

4
benchmark/tests/test_files/be_list_hidden.test Normal file
View File

@@ -0,0 +1,4 @@
 # Date File Score Time(h) Title
=== ========== ================================================================ ======== ======= =======================
 0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters
 1 2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json 0.97446 0.098 default

3
benchmark/tests/test_files/be_list_hidden_nan.test Normal file
View File

@@ -0,0 +1,3 @@
 # Date File Score Time(h) Title
=== ========== ================================================================ ======== ======= =======================
 0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters

13
benchmark/tests/test_files/be_list_nan.test → benchmark/tests/test_files/be_list_hide.test
View File

@@ -6,8 +6,11 @@
 3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B
 4 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A
 5 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters
****************************** Results with nan moved to hidden ******************************
 # Date File Score Time(h) Title
=== ========== ================================================================ ======== ======= =======================
 0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json nan 3.091 Default hyperparameters
Hiding results/results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json
 # Date File Score Time(h) Title
=== ========== =============================================================== ======== ======= ============================================
 0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json 0.04341 6.275 Gridsearched hyperparams v022.1b random_init
 1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json 0.03627 0.076 Test default paramters with RandomForest
 2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json 0.03790 1.143 default B
 3 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json 0.04158 0.943 default A
 4 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json 0.04544 0.173 With gridsearched hyperparameters

2
benchmark/tests/test_files/report_unknown_subcommand.test Normal file
View File

@@ -0,0 +1,2 @@
usage: be_report [-h] {best,grid,file,datasets} ...
be_report: error: argument subcommand: invalid choice: 'unknown' (choose from 'best', 'grid', 'file', 'datasets')

12
benchmark/tests/test_files/report_without_subcommand.test Normal file
View File

@@ -0,0 +1,12 @@
usage: be_report [-h] {best,grid,file,datasets} ...
positional arguments:
{best,grid,file,datasets}
help for subcommand
best Report best results obtained by any model/score. See be_build_best
grid Report grid results obtained by any model/score. See be_build_grid
file Report file results
datasets Report datasets information
options:
-h, --help show this help message and exit