From a17166ed312c494f9e0fcef7c346ef42187eae87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Mon, 25 Apr 2022 02:35:23 +0200 Subject: [PATCH] Add benchmark tests --- benchmark/Results.py | 14 ++-- benchmark/benchmark.r | 6 +- benchmark/tests/Benchmark_test.py | 53 ++++++++++++++ benchmark/tests/Excel_test.py | 33 +++++---- benchmark/tests/SQL_test.py | 71 ++++--------------- benchmark/tests/Util_test.py | 2 + benchmark/tests/__init__.py | 2 + benchmark/tests/exreport/.gitignore | 2 + benchmark/tests/results/exreport_accuracy.txt | 0 .../tests/results/exreport_err_accuracy.txt | 2 + ...omForest_iMac27_2022-01-14_12:39:30_0.json | 43 +++++++++++ benchmark/tests/test_files/exreport.test | 22 ++++++ benchmark/tests/test_files/exreport_csv.test | 7 ++ benchmark/tests/test_files/sql.test | 2 + 14 files changed, 176 insertions(+), 83 deletions(-) create mode 100644 benchmark/tests/Benchmark_test.py create mode 100644 benchmark/tests/exreport/.gitignore create mode 100644 benchmark/tests/results/exreport_accuracy.txt create mode 100644 benchmark/tests/results/exreport_err_accuracy.txt create mode 100644 benchmark/tests/results/results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json create mode 100644 benchmark/tests/test_files/exreport.test create mode 100644 benchmark/tests/test_files/exreport_csv.test create mode 100644 benchmark/tests/test_files/sql.test diff --git a/benchmark/Results.py b/benchmark/Results.py index b04513c..d070946 100644 --- a/benchmark/Results.py +++ b/benchmark/Results.py @@ -597,7 +597,7 @@ class Benchmark: self._datasets = set() def get_result_file_name(self): - return os.path.join(Folders.results, Files.exreport(self._score)) + return os.path.join(Folders.exreport, Files.exreport(self._score)) def compile_results(self): summary = Summary() @@ -644,7 +644,7 @@ class Benchmark: print("*" * length) print(message) print("*" * length) - with open(os.path.join(Folders.results, file)) as f: + with open(os.path.join(Folders.exreport, file)) as f: data = f.read().splitlines() for line in data: print(line) @@ -659,17 +659,23 @@ class Benchmark: print("Error: %s : %s" % (Folders.report, os_error.strerror)) # Compute Friedman & Holm Tests fout = open( - os.path.join(Folders.results, Files.exreport_output(self._score)), + os.path.join(Folders.exreport, Files.exreport_output(self._score)), "w", ) ferr = open( - os.path.join(Folders.results, Files.exreport_err(self._score)), "w" + os.path.join(Folders.exreport, Files.exreport_err(self._score)), + "w", + ) + print( + "*********************", + os.path.join(Folders.src(), Files.benchmark_r), ) result = subprocess.run( [ "Rscript", os.path.join(Folders.src(), Files.benchmark_r), self._score, + os.path.join(Folders.exreport, f"exreport_{self._score}"), ], stdout=fout, stderr=ferr, diff --git a/benchmark/benchmark.r b/benchmark/benchmark.r index 588b3dd..80fa0d6 100644 --- a/benchmark/benchmark.r +++ b/benchmark/benchmark.r @@ -1,9 +1,9 @@ library(glue) args = commandArgs(trailingOnly=TRUE) -if (length(args)!=1) { - stop("Only one argument must be supplied (score).n", call.=FALSE) +if (length(args)!=2) { + stop("Only two arguments must be supplied (score & input_file).n", call.=FALSE) } -csv_file <- glue("results/exreport_{args[1]}.csv") +csv_file <- glue("{args[2]}.csv") destination <- "exreport/" results <- read.csv(csv_file) library(exreport) diff --git a/benchmark/tests/Benchmark_test.py b/benchmark/tests/Benchmark_test.py new file mode 100644 index 0000000..dd71b22 --- /dev/null +++ b/benchmark/tests/Benchmark_test.py @@ -0,0 +1,53 @@ +import os +import unittest +import shutil +from io import StringIO +from unittest.mock import patch +from ..Utils import Folders +from ..Results import Benchmark + + +class BenchmarkTest(unittest.TestCase): + def __init__(self, *args, **kwargs): + os.chdir(os.path.dirname(os.path.abspath(__file__))) + super().__init__(*args, **kwargs) + + def tearDown(self) -> None: + files = [ + # "exreport_accuracy.csv", + "exreport_accuracy.txt", + "exreport_err_accuracy.txt", + ] + for file_name in files: + file_name = os.path.join(Folders.exreport, file_name) + if os.path.exists(file_name): + os.remove(file_name) + if os.path.exists(Folders.report): + shutil.rmtree(Folders.report) + if os.path.exists("Rplots.pdf"): + os.remove("Rplots.pdf") + return super().tearDown() + + def test_csv(self): + benchmark = Benchmark("accuracy") + benchmark.compile_results() + benchmark.save_results() + with open(benchmark.get_result_file_name()) as f: + computed = f.readlines() + with open(os.path.join("test_files", "exreport_csv.test")) as f_exp: + expected = f_exp.readlines() + self.assertEqual(computed, expected) + + def test_exreport(self): + benchmark = Benchmark("accuracy") + benchmark.compile_results() + benchmark.save_results() + with patch("sys.stdout", new=StringIO()) as fake_out: + benchmark.exreport() + with open(os.path.join("test_files", "exreport.test")) as f: + expected_t = f.read() + computed_t = fake_out.getvalue() + computed_t = computed_t.split("\n") + computed_t.pop(0) + for computed, expected in zip(computed_t, expected_t.split("\n")): + self.assertEqual(computed, expected) diff --git a/benchmark/tests/Excel_test.py b/benchmark/tests/Excel_test.py index 447a47b..dfe2227 100644 --- a/benchmark/tests/Excel_test.py +++ b/benchmark/tests/Excel_test.py @@ -38,44 +38,43 @@ class ExcelTest(unittest.TestCase): self.assertEqual(sheet.cell(int(row), int(col)).value, value) def test_report_excel_compared(self): - file_name = ( - "results/results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json" - ) + file_name = "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json" report = Excel(file_name, compare=True) report.report() file_output = report.get_file_name() book = load_workbook(file_output) sheet = book["STree"] - self.check_excel_sheet(sheet, "test_files/excel_compared.test") + self.check_excel_sheet( + sheet, os.path.join("test_files", "excel_compared.test") + ) def test_report_excel(self): - file_name = ( - "results/results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json" - ) + file_name = "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json" report = Excel(file_name, compare=False) report.report() file_output = report.get_file_name() book = load_workbook(file_output) sheet = book["STree"] - self.check_excel_sheet(sheet, "test_files/excel.test") + self.check_excel_sheet(sheet, os.path.join("test_files", "excel.test")) def test_Excel_Add_sheet(self): - file_name = ( - "results/results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json" - ) + file_name = "results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json" excel_file_name = file_name.replace(".json", ".xlsx") - book = Workbook(excel_file_name) + book = Workbook(os.path.join(Folders.results, excel_file_name)) excel = Excel(file_name=file_name, book=book) excel.report() report = Excel( - file_name="results/results_accuracy_ODTE_Galgo_2022-04-20_10:52:20" - "_0.json", + file_name="results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json", book=book, ) report.report() book.close() - book = load_workbook(excel_file_name) + book = load_workbook(os.path.join(Folders.results, excel_file_name)) sheet = book["STree"] - self.check_excel_sheet(sheet, "test_files/excel_add_STree.test") + self.check_excel_sheet( + sheet, os.path.join("test_files", "excel_add_STree.test") + ) sheet = book["ODTE"] - self.check_excel_sheet(sheet, "test_files/excel_add_ODTE.test") + self.check_excel_sheet( + sheet, os.path.join("test_files", "excel_add_ODTE.test") + ) diff --git a/benchmark/tests/SQL_test.py b/benchmark/tests/SQL_test.py index 1f34fe3..f0e89c8 100644 --- a/benchmark/tests/SQL_test.py +++ b/benchmark/tests/SQL_test.py @@ -1,9 +1,6 @@ import os -import csv import unittest -from openpyxl import load_workbook -from xlsxwriter import Workbook -from ..Results import Excel +from ..Results import SQL from ..Utils import Folders @@ -14,8 +11,6 @@ class SQLTest(unittest.TestCase): def tearDown(self) -> None: files = [ - "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.xlsx", - "results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.xlsx", "results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.sql", ] for file_name in files: @@ -24,58 +19,16 @@ class SQLTest(unittest.TestCase): os.remove(file_name) return super().tearDown() - def check_excel_sheet(self, sheet, file_name): - with open(file_name, "r") as f: - expected = csv.reader(f, delimiter=";") - for row, col, value in expected: - if value.isdigit(): - value = int(value) - else: - try: - value = float(value) - except ValueError: - pass - self.assertEqual(sheet.cell(int(row), int(col)).value, value) - - def test_report_excel_compared(self): - file_name = ( - "results/results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json" - ) - report = Excel(file_name, compare=True) + def test_report_SQL(self): + file_name = "results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json" + report = SQL(file_name) report.report() - file_output = report.get_file_name() - book = load_workbook(file_output) - sheet = book["STree"] - self.check_excel_sheet(sheet, "test_files/excel_compared.test") + file_name = os.path.join( + Folders.results, file_name.replace(".json", ".sql") + ) - def test_report_excel(self): - file_name = ( - "results/results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json" - ) - report = Excel(file_name, compare=False) - report.report() - file_output = report.get_file_name() - book = load_workbook(file_output) - sheet = book["STree"] - self.check_excel_sheet(sheet, "test_files/excel.test") - - def test_Excel_Add_sheet(self): - file_name = ( - "results/results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json" - ) - excel_file_name = file_name.replace(".json", ".xlsx") - book = Workbook(excel_file_name) - excel = Excel(file_name=file_name, book=book) - excel.report() - report = Excel( - file_name="results/results_accuracy_ODTE_Galgo_2022-04-20_10:52:20" - "_0.json", - book=book, - ) - report.report() - book.close() - book = load_workbook(excel_file_name) - sheet = book["STree"] - self.check_excel_sheet(sheet, "test_files/excel_add_STree.test") - sheet = book["ODTE"] - self.check_excel_sheet(sheet, "test_files/excel_add_ODTE.test") + with open(file_name, "r") as file: + computed = file.read() + with open(os.path.join("test_files", "sql.test")) as f_exp: + expected = f_exp.read() + self.assertEqual(computed, expected) diff --git a/benchmark/tests/Util_test.py b/benchmark/tests/Util_test.py index 5c36935..d142d77 100644 --- a/benchmark/tests/Util_test.py +++ b/benchmark/tests/Util_test.py @@ -122,6 +122,8 @@ class UtilTest(unittest.TestCase): "results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0." "json", "results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json", + "results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0." + "json", ], ) self.assertCountEqual( diff --git a/benchmark/tests/__init__.py b/benchmark/tests/__init__.py index 2d9851b..f4f5dbf 100644 --- a/benchmark/tests/__init__.py +++ b/benchmark/tests/__init__.py @@ -7,6 +7,7 @@ from .GridSearch_test import GridSearchTest from .Report_test import ReportTest from .Excel_test import ExcelTest from .SQL_test import SQLTest +from .Benchmark_test import BenchmarkTest all = [ "UtilTest", @@ -18,4 +19,5 @@ all = [ "ReportTest", "ExcelTest", "SQLTest", + "BenchmarkTest", ] diff --git a/benchmark/tests/exreport/.gitignore b/benchmark/tests/exreport/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/benchmark/tests/exreport/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/benchmark/tests/results/exreport_accuracy.txt b/benchmark/tests/results/exreport_accuracy.txt new file mode 100644 index 0000000..e69de29 diff --git a/benchmark/tests/results/exreport_err_accuracy.txt b/benchmark/tests/results/exreport_err_accuracy.txt new file mode 100644 index 0000000..84d409d --- /dev/null +++ b/benchmark/tests/results/exreport_err_accuracy.txt @@ -0,0 +1,2 @@ +Error: Only one argument must be supplied (score).n +Ejecución interrumpida diff --git a/benchmark/tests/results/results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json b/benchmark/tests/results/results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json new file mode 100644 index 0000000..ef522ed --- /dev/null +++ b/benchmark/tests/results/results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json @@ -0,0 +1,43 @@ +{ + "score_name": "accuracy", + "title": "Test default paramters with RandomForest", + "model": "RandomForest", + "version": "-", + "stratified": false, + "folds": 5, + "date": "2022-01-14", + "time": "12:39:30", + "duration": 272.7363500595093, + "seeds": [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1], + "platform": "iMac27", + "results": [ + { + "dataset": "balance-scale", + "samples": 625, + "features": 4, + "classes": 3, + "hyperparameters": {}, + "nodes": 196.91440000000003, + "leaves": 98.42, + "depth": 10.681399999999998, + "score": 0.83616, + "score_std": 0.02649630917694009, + "time": 0.08222018241882324, + "time_std": 0.0013026326815120633 + }, + { + "dataset": "balloons", + "samples": 16, + "features": 4, + "classes": 2, + "hyperparameters": {}, + "nodes": 9.110800000000001, + "leaves": 4.58, + "depth": 3.0982, + "score": 0.625, + "score_std": 0.24958298553119898, + "time": 0.07016648769378662, + "time_std": 0.002460508923990468 + } + ] +} diff --git a/benchmark/tests/test_files/exreport.test b/benchmark/tests/test_files/exreport.test new file mode 100644 index 0000000..77de7c9 --- /dev/null +++ b/benchmark/tests/test_files/exreport.test @@ -0,0 +1,22 @@ +**************************************************************************************************** +Benchmark Ok +**************************************************************************************************** +--------------------------------------------------------------------- +Friedman test, objetive maximize output variable accuracy. Obtained p-value: 1.3534e-01 +Chi squared with 2 degrees of freedom statistic: 4.0000 +Test accepted: p-value: 1.3534e-01 >= 0.0500 +--------------------------------------------------------------------- +Control post hoc test for output accuracy +Adjust method: Holm + +Control method: STree +p-values: + ODTE 0.3173 + RandomForest 0.0910 +--------------------------------------------------------------------- +$testMultiple + classifier pvalue rank win tie loss +STree STree NA 1 NA NA NA +ODTE ODTE 0.31731051 2 2 0 0 +RandomForest RandomForest 0.09100053 3 2 0 0 + diff --git a/benchmark/tests/test_files/exreport_csv.test b/benchmark/tests/test_files/exreport_csv.test new file mode 100644 index 0000000..1d91b1d --- /dev/null +++ b/benchmark/tests/test_files/exreport_csv.test @@ -0,0 +1,7 @@ +classifier, dataset, accuracy, stdev, file_name +ODTE, balance-scale, 0.96352, 0.024949741481626608, results/results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json +ODTE, balloons, 0.785, 0.2461311755051675, results/results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json +RandomForest, balance-scale, 0.83616, 0.02649630917694009, results/results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json +RandomForest, balloons, 0.625, 0.24958298553119898, results/results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json +STree, balance-scale, 0.97056, 0.015046806970251203, results/results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json +STree, balloons, 0.86, 0.28501461950807594, results/results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json diff --git a/benchmark/tests/test_files/sql.test b/benchmark/tests/test_files/sql.test new file mode 100644 index 0000000..9130d6c --- /dev/null +++ b/benchmark/tests/test_files/sql.test @@ -0,0 +1,2 @@ +replace into results (date,time,type,title,stratified,score_name,score,score_std,dataset,classifier,version,norm,stand,time_spent,time_spent_std,parameters,nodes,leaves,depth,platform,nfolds,seeds) values('2022-04-20','10:52:20','crossval','Gridsearched hyperparams v022.1b random_init','0','accuracy','0.96352','0.024949741481626608','balance-scale','ODTE','0.3.2','0','1','0.31663217544555666','0.19918813895255585','{"base_estimator__C": 57, "base_estimator__gamma": 0.1, "base_estimator__kernel": "rbf", "base_estimator__multiclass_strategy": "ovr", "n_estimators": 100, "n_jobs": -1}','7.361199999999999','4.180599999999999','3.536','Galgo','5','[57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]'); +replace into results (date,time,type,title,stratified,score_name,score,score_std,dataset,classifier,version,norm,stand,time_spent,time_spent_std,parameters,nodes,leaves,depth,platform,nfolds,seeds) values('2022-04-20','10:52:20','crossval','Gridsearched hyperparams v022.1b random_init','0','accuracy','0.785','0.2461311755051675','balloons','ODTE','0.3.2','0','1','0.11560620784759522','0.012784241828599895','{"base_estimator__C": 5, "base_estimator__gamma": 0.14, "base_estimator__kernel": "rbf", "base_estimator__multiclass_strategy": "ovr", "n_estimators": 100, "n_jobs": -1}','2.9951999999999996','1.9975999999999998','1.9975999999999998','Galgo','5','[57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]');