diff --git a/benchmark/Arguments.py b/benchmark/Arguments.py index 80fae6d..cb43ef7 100644 --- a/benchmark/Arguments.py +++ b/benchmark/Arguments.py @@ -13,21 +13,27 @@ ALL_METRICS = ( class EnvData: - @staticmethod - def load(): - args = {} + def __init__(self): + self.args = {} + + def load(self): try: with open(Files.dot_env) as f: for line in f.read().splitlines(): if line == "" or line.startswith("#"): continue key, value = line.split("=") - args[key] = value + self.args[key] = value except FileNotFoundError: print(NO_ENV, file=sys.stderr) exit(1) else: - return args + return self.args + + def save(self): + with open(Files.dot_env, "w") as f: + for key, value in self.args.items(): + f.write(f"{key}={value}\n") class EnvDefault(argparse.Action): @@ -35,7 +41,7 @@ class EnvDefault(argparse.Action): def __init__( self, envvar, required=True, default=None, mandatory=False, **kwargs ): - self._args = EnvData.load() + self._args = EnvData().load() self._overrides = {} if required and not mandatory: default = self._args[envvar] diff --git a/benchmark/Datasets.py b/benchmark/Datasets.py index 98e2709..ebb5e52 100644 --- a/benchmark/Datasets.py +++ b/benchmark/Datasets.py @@ -109,7 +109,7 @@ class DatasetsSurcov: class Datasets: def __init__(self, dataset_name=None, discretize=None): - env_data = EnvData.load() + env_data = EnvData().load() # DatasetsSurcov, DatasetsTanveer, DatasetsArff,... source_name = getattr( __import__(__name__), diff --git a/benchmark/Experiments.py b/benchmark/Experiments.py index fd2b739..e4dbeac 100644 --- a/benchmark/Experiments.py +++ b/benchmark/Experiments.py @@ -22,7 +22,7 @@ from .Arguments import EnvData class Randomized: @staticmethod def seeds(): - return json.loads(EnvData.load()["seeds"]) + return json.loads(EnvData().load()["seeds"]) class BestResults: @@ -117,7 +117,7 @@ class Experiment: discretize=None, folds=5, ): - env_data = EnvData.load() + env_data = EnvData().load() today = datetime.now() self.time = today.strftime("%H:%M:%S") self.date = today.strftime("%Y-%m-%d") diff --git a/benchmark/Results.py b/benchmark/Results.py index 5669268..7a4317d 100644 --- a/benchmark/Results.py +++ b/benchmark/Results.py @@ -71,7 +71,6 @@ class Report(BaseReport): self._load_best_results( self.data["score_name"], self.data["model"] ) - self._compare_totals = {} self.header_line("*") self.header_line( f" {self.data['model']} ver. {self.data['version']}" diff --git a/benchmark/ResultsBase.py b/benchmark/ResultsBase.py index 0c3fde3..94add18 100644 --- a/benchmark/ResultsBase.py +++ b/benchmark/ResultsBase.py @@ -52,10 +52,11 @@ class BaseReport(abc.ABC): self.score_name = self.data["score_name"] self.__load_env_data() self.__compute_best_results_ever() + self._compare_totals = {} def __load_env_data(self): # Set the labels for nodes, leaves, depth - env_data = EnvData.load() + env_data = EnvData().load() self.nodes_label = env_data["nodes"] self.leaves_label = env_data["leaves"] self.depth_label = env_data["depth"] @@ -149,6 +150,7 @@ class BaseReport(abc.ABC): class StubReport(BaseReport): def __init__(self, file_name): + self.compare = False super().__init__(file_name=file_name, best_file=False) def print_line(self, line) -> None: @@ -165,7 +167,7 @@ class StubReport(BaseReport): class Summary: def __init__(self, hidden=False, compare=False) -> None: - self.results = Files().get_all_results(hidden=hidden) + self.results = Files.get_all_results(hidden=hidden) self.data = [] self.data_filtered = [] self.datasets = {} diff --git a/benchmark/ResultsFiles.py b/benchmark/ResultsFiles.py index 7e87d83..cfa8fde 100644 --- a/benchmark/ResultsFiles.py +++ b/benchmark/ResultsFiles.py @@ -620,7 +620,7 @@ class Benchmark: self.__compute_best_results_ever() def __compute_best_results_ever(self): - args = EnvData.load() + args = EnvData().load() key = args["source_data"] best = BestResultsEver() _, self.best_score_value = best.get_name_value(key, self._score) diff --git a/benchmark/Utils.py b/benchmark/Utils.py index 177b49f..229b187 100644 --- a/benchmark/Utils.py +++ b/benchmark/Utils.py @@ -108,7 +108,8 @@ class Files: ) return None - def get_all_results(self, hidden) -> list[str]: + @staticmethod + def get_all_results(hidden) -> list[str]: result_path = os.path.join( ".", Folders.hidden_results if hidden else Folders.results ) @@ -117,7 +118,7 @@ class Files: else: raise ValueError(f"{result_path} does not exist") result = [] - prefix, suffix = self.results_suffixes() + prefix, suffix = Files.results_suffixes() for result_file in files_list: if result_file.startswith(prefix) and result_file.endswith(suffix): result.append(result_file) diff --git a/benchmark/scripts/be_flask.py b/benchmark/scripts/be_flask.py new file mode 100755 index 0000000..0581b19 --- /dev/null +++ b/benchmark/scripts/be_flask.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +import os +import json +import webbrowser +from benchmark.Utils import Files, Folders, Symbols +from benchmark.Arguments import Arguments, EnvData +from benchmark.ResultsBase import StubReport +from flask import Flask +from flask import render_template, request, redirect, url_for + + +# Launch a flask server to serve the results +app = Flask(__name__) +FRAMEWORK = "framework" +FRAMEWORKS = "frameworks" +HIDDEN = "hidden" + + +def process_data(file_name, data): + report = StubReport(os.path.join(Folders.results, file_name)) + new_list = [] + for result in data["results"]: + symbol = report._compute_status(result["dataset"], result["score"]) + result["symbol"] = symbol if symbol != " " else " " + new_list.append(result) + data["results"] = new_list + # Compute summary with explanation of symbols + summary = {} + for key, value in report._compare_totals.items(): + summary[key] = (report._status_meaning(key), value) + return summary + + +@app.route("/index") +@app.route("/") +def index(): + # Get a list of files in a directory + files = Files.get_all_results(hidden=app.config[HIDDEN]) + candidate = app.config[FRAMEWORKS].copy() + candidate.remove(app.config[FRAMEWORK]) + return render_template( + f"select_{app.config[FRAMEWORK]}.html", + files=files, + framework=candidate[0], + ) + + +@app.route("/show", methods=["post"]) +def show(): + selected_file = request.form["selected-file"] + with open(os.path.join(Folders.results, selected_file)) as f: + data = json.load(f) + summary = process_data(selected_file, data) + return render_template( + f"report_{app.config[FRAMEWORK]}.html", data=data, summary=summary + ) + + +@app.route("/config/") +def config(framework): + if not framework in app.config[FRAMEWORKS]: + message = f"framework {framework} not supported" + return render_template("error.html", message=message) + env = EnvData() + env.load() + env.args[FRAMEWORK] = framework + env.save() + app.config[FRAMEWORK] = framework + return redirect(url_for("index")) + + +def main(args_test=None): + arguments = Arguments(prog="be_flask") + arguments.xset("model", required=False) + arguments.xset("score", required=False).xset("compare").xset("hidden") + arguments.xset("nan") + args = arguments.parse(args_test) + app.config[FRAMEWORK] = EnvData().load()[FRAMEWORK] + app.config[HIDDEN] = args.hidden + app.config[FRAMEWORKS] = ["bootstrap", "bulma"] + webbrowser.open_new("http://127.0.0.1:1234/") + app.run(port=1234) + + # Poner checkboxes para seleccionar resultados y poner un botón abajo para hacer un excel con los seleccionados + # Calcular símbolo igual que en list, o bien si ha puesto el parámetro de compare, con best o con zeror diff --git a/benchmark/scripts/templates/error.html b/benchmark/scripts/templates/error.html new file mode 100644 index 0000000..4156b77 --- /dev/null +++ b/benchmark/scripts/templates/error.html @@ -0,0 +1,23 @@ + + + + + Error + + + + +
+ +
+ + + \ No newline at end of file diff --git a/benchmark/scripts/templates/partials/table_report.html b/benchmark/scripts/templates/partials/table_report.html new file mode 100644 index 0000000..fa03681 --- /dev/null +++ b/benchmark/scripts/templates/partials/table_report.html @@ -0,0 +1,28 @@ +{% for item in data.results %} + + + {{item.dataset}} + + + {{'{:,}'.format(item.samples)}} + + + {{"%d" % item.features}} + + + {{"%d" % item.classes}} + + + {{'{:,.2f}'.format(item.nodes)}} + + + {{"%.6f±%.4f" % (item.score, item.score_std)}} {{ item.symbol|safe }} + + + {{"%.6f±%.4f" % (item.time, item.time_std)}} + + + {{item.hyperparameters}} + + +{% endfor %} \ No newline at end of file diff --git a/benchmark/scripts/templates/partials/table_select.html b/benchmark/scripts/templates/partials/table_select.html new file mode 100644 index 0000000..d217c0f --- /dev/null +++ b/benchmark/scripts/templates/partials/table_select.html @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + {% for file in files %} + {% set parts = file.split('_') %} + {% set stratified = parts[6].split('.')[0] %} + + + + + + + + + + {% endfor %} + +
ModelMetricPlatformDateTimeStratified
{{ parts[2] }}{{ parts[1] }}{{ parts[3] }}{{ parts[4] }}{{ parts[5] }}{{ 'True' if stratified =='1' else 'False' }} + {{ button_pre | safe }}{{ file }}{{ button_post | safe }} +
\ No newline at end of file diff --git a/benchmark/scripts/templates/partials/table_summary.html b/benchmark/scripts/templates/partials/table_summary.html new file mode 100644 index 0000000..dda3edf --- /dev/null +++ b/benchmark/scripts/templates/partials/table_summary.html @@ -0,0 +1,13 @@ +{% for key, value in summary.items() %} + + + {{key}} + + + {{value[0]}} + + + {{'{:,}'.format(value[1])}} + + +{% endfor %} \ No newline at end of file diff --git a/benchmark/scripts/templates/report_bootstrap.html b/benchmark/scripts/templates/report_bootstrap.html new file mode 100644 index 0000000..dcbedc9 --- /dev/null +++ b/benchmark/scripts/templates/report_bootstrap.html @@ -0,0 +1,105 @@ + + + + + + + + Report Viewer + + + + + {% set center = "text-center" %} + {% set right = "text-end" %} +
+
+
+
+ +

{{ data.title }}

+
+
+ + + + + + + + {% if data.duration > 7200 %} + {% set unit = "h" %} + {% set divider = 3600 %} + {% else %} + {% set unit = "min" %} + {% set divider = 60 %} + {% endif %} + + + + + + + + + + + + + + + + + + + + + + +
PlatformModelDateTimeDuration ({{ unit }})StratifiedDiscretized# Folds
{{ data.platform }}{{ data.model }} {{ data.version }}{{ data.date}}{{ data.time}}{{ "%.2f" % (data.duration/divider) }}{{ data.stratified }}{{ data.discretized }}{{ data.folds }}
Language{{ data.language }} {{ data.language_version }}Seeds{{ data.seeds }}
+ + + + + + + + + + + + + + + {% include "partials/table_report.html" %} + +
DatasetSamplesFeaturesClassesNodes{{data.score_name|capitalize}}Timehyperparameters
+ {% if summary|length > 0 %} +
+ + + + + + + + + {% include "partials/table_summary.html" %} +
SymbolMeaningCount
+
+ {% endif %} + + + Total score: {{ "%.6f" % (data.results | sum(attribute="score")) }} + +
+
+
+ + + \ No newline at end of file diff --git a/benchmark/scripts/templates/report_bulma.html b/benchmark/scripts/templates/report_bulma.html new file mode 100644 index 0000000..a2da25b --- /dev/null +++ b/benchmark/scripts/templates/report_bulma.html @@ -0,0 +1,110 @@ + + + + + + + + Report Viewer + + + + + {% set center = "has-text-centered" %} + {% set right = "has-text-right" %} +
+
+
+
+
+ +

{{ data.title }}

+
+
+
+
+
+
+
+ + + + + + + + {% if data.duration > 7200 %} + {% set unit = "h" %} + {% set divider = 3600 %} + {% else %} + {% set unit = "min" %} + {% set divider = 60 %} + {% endif %} + + + + + + + + + + + + + + + + + + + + + + +
PlatformModelDateTimeDuration ({{ unit }})StratifiedDiscretized# Folds
{{ data.platform }}{{ data.model }} {{ data.version }}{{ data.date}}{{ data.time}}{{ "%.2f" % (data.duration/divider) }}{{ data.stratified }}{{ data.discretized }}{{ data.folds }}
Language{{ data.language }} {{ data.language_version }}Seeds{{ data.seeds }}
+ + + + + + + + + + + + + + + {% include "partials/table_report.html" %} + +
DatasetSamplesFeaturesClassesNodes{{data.score_name|capitalize}}Timehyperparameters
+ {% if summary|length > 0 %} +
+ + + + + + + + + {% include "partials/table_summary.html" %} +
SymbolMeaningCount
+
+ {% endif %} +

+ + Total score: {{ "%.6f" % (data.results | sum(attribute="score")) }} +

+
+
+
+ + + \ No newline at end of file diff --git a/benchmark/scripts/templates/select_bootstrap.html b/benchmark/scripts/templates/select_bootstrap.html new file mode 100644 index 0000000..1e2aba2 --- /dev/null +++ b/benchmark/scripts/templates/select_bootstrap.html @@ -0,0 +1,69 @@ + + + + + Benchmark + + + + + + + +
+

Benchmark Results

+ + {% set table_class = "table table-striped table-hover" %} + {% set button_pre = '' %} + {% include "partials/table_select.html" %} +
+ + + + + + + \ No newline at end of file diff --git a/benchmark/scripts/templates/select_bulma.html b/benchmark/scripts/templates/select_bulma.html new file mode 100644 index 0000000..c306978 --- /dev/null +++ b/benchmark/scripts/templates/select_bulma.html @@ -0,0 +1,63 @@ + + + + + Benchmark Results + + + + + + +
+

Benchmark Results

+ + {% set table_class = "table is-striped is-hoverable cell-border" %} + {% set button_pre = 'View' %} + {% include "partials/table_select.html" %} +
+ + + + + + \ No newline at end of file diff --git a/benchmark/tests/Util_test.py b/benchmark/tests/Util_test.py index 3cca34d..81b4919 100644 --- a/benchmark/tests/Util_test.py +++ b/benchmark/tests/Util_test.py @@ -118,7 +118,7 @@ class UtilTest(TestBase): def test_Files_get_results(self): os.chdir(os.path.dirname(os.path.abspath(__file__))) self.assertCountEqual( - Files().get_all_results(hidden=False), + Files.get_all_results(hidden=False), [ "results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json", "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json", @@ -130,7 +130,7 @@ class UtilTest(TestBase): ], ) self.assertCountEqual( - Files().get_all_results(hidden=True), + Files.get_all_results(hidden=True), [ "results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json", "results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_" @@ -143,7 +143,7 @@ class UtilTest(TestBase): # check with results os.rename(Folders.results, f"{Folders.results}.test") try: - Files().get_all_results(hidden=False) + Files.get_all_results(hidden=False) except ValueError: pass else: @@ -153,7 +153,7 @@ class UtilTest(TestBase): # check with hidden_results os.rename(Folders.hidden_results, f"{Folders.hidden_results}.test") try: - Files().get_all_results(hidden=True) + Files.get_all_results(hidden=True) except ValueError: pass else: diff --git a/setup.py b/setup.py index ea9875c..e5693ac 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ def script_names(): "report", "summary", "init_project", + "flask", ] result = [] for script in scripts: