diff --git a/benchmark/Arguments.py b/benchmark/Arguments.py index 80fae6d..26d3d97 100644 --- a/benchmark/Arguments.py +++ b/benchmark/Arguments.py @@ -13,21 +13,27 @@ ALL_METRICS = ( class EnvData: - @staticmethod - def load(): - args = {} + def __init__(self): + self.args = {} + + def load(self): try: with open(Files.dot_env) as f: for line in f.read().splitlines(): if line == "" or line.startswith("#"): continue key, value = line.split("=") - args[key] = value + self.args[key] = value except FileNotFoundError: print(NO_ENV, file=sys.stderr) exit(1) else: - return args + return self.args + + def save(self): + with open(Files.dot_env, "w") as f: + for key, value in self.args.items(): + f.write(f"{key}={value}\n") class EnvDefault(argparse.Action): @@ -35,7 +41,7 @@ class EnvDefault(argparse.Action): def __init__( self, envvar, required=True, default=None, mandatory=False, **kwargs ): - self._args = EnvData.load() + self._args = EnvData().load() self._overrides = {} if required and not mandatory: default = self._args[envvar] @@ -229,6 +235,19 @@ class Arguments(argparse.ArgumentParser): "help": "number of folds", }, ], + "output": [ + ("-o", "--output"), + { + "type": str, + "default": "local", + "choices": ["local", "docker"], + "required": False, + "help": ( + "in be_flask tells if it is running in local or " + "in docker {local, docker}" + ), + }, + ], "platform": [ ("-P", "--platform"), { diff --git a/benchmark/Datasets.py b/benchmark/Datasets.py index 98e2709..ebb5e52 100644 --- a/benchmark/Datasets.py +++ b/benchmark/Datasets.py @@ -109,7 +109,7 @@ class DatasetsSurcov: class Datasets: def __init__(self, dataset_name=None, discretize=None): - env_data = EnvData.load() + env_data = EnvData().load() # DatasetsSurcov, DatasetsTanveer, DatasetsArff,... source_name = getattr( __import__(__name__), diff --git a/benchmark/Experiments.py b/benchmark/Experiments.py index fd2b739..e4dbeac 100644 --- a/benchmark/Experiments.py +++ b/benchmark/Experiments.py @@ -22,7 +22,7 @@ from .Arguments import EnvData class Randomized: @staticmethod def seeds(): - return json.loads(EnvData.load()["seeds"]) + return json.loads(EnvData().load()["seeds"]) class BestResults: @@ -117,7 +117,7 @@ class Experiment: discretize=None, folds=5, ): - env_data = EnvData.load() + env_data = EnvData().load() today = datetime.now() self.time = today.strftime("%H:%M:%S") self.date = today.strftime("%Y-%m-%d") diff --git a/benchmark/Models.py b/benchmark/Models.py index 0a808a9..2ef6fa4 100644 --- a/benchmark/Models.py +++ b/benchmark/Models.py @@ -9,7 +9,7 @@ from sklearn.ensemble import ( from sklearn.svm import SVC from stree import Stree -# from bayesclass.clfs import TAN, KDB, AODE, KDBNew, TANNew, AODENew +# from bayesclass.clfs import TAN, KDB, AODE, KDBNew, TANNew, AODENew, BoostAODE from wodt import Wodt from odte import Odte from xgboost import XGBClassifier @@ -46,6 +46,7 @@ class Models: # "KDBNew": KDBNew(k=2), # "AODENew": AODENew(random_state=random_state), # "AODE": AODE(random_state=random_state), + # "BoostAODE": BoostAODE(random_state=random_state), "Cart": DecisionTreeClassifier(random_state=random_state), "ExtraTree": ExtraTreeClassifier(random_state=random_state), "Wodt": Wodt(random_state=random_state), diff --git a/benchmark/Results.py b/benchmark/Results.py index 5669268..7a4317d 100644 --- a/benchmark/Results.py +++ b/benchmark/Results.py @@ -71,7 +71,6 @@ class Report(BaseReport): self._load_best_results( self.data["score_name"], self.data["model"] ) - self._compare_totals = {} self.header_line("*") self.header_line( f" {self.data['model']} ver. {self.data['version']}" diff --git a/benchmark/ResultsBase.py b/benchmark/ResultsBase.py index 0c3fde3..708f38a 100644 --- a/benchmark/ResultsBase.py +++ b/benchmark/ResultsBase.py @@ -52,10 +52,11 @@ class BaseReport(abc.ABC): self.score_name = self.data["score_name"] self.__load_env_data() self.__compute_best_results_ever() + self._compare_totals = {} def __load_env_data(self): # Set the labels for nodes, leaves, depth - env_data = EnvData.load() + env_data = EnvData().load() self.nodes_label = env_data["nodes"] self.leaves_label = env_data["leaves"] self.depth_label = env_data["depth"] @@ -148,8 +149,11 @@ class BaseReport(abc.ABC): class StubReport(BaseReport): - def __init__(self, file_name): + def __init__(self, file_name, compare=False): + self.compare = compare super().__init__(file_name=file_name, best_file=False) + if self.compare: + self._load_best_results(self.score_name, self.data["model"]) def print_line(self, line) -> None: pass @@ -165,7 +169,7 @@ class StubReport(BaseReport): class Summary: def __init__(self, hidden=False, compare=False) -> None: - self.results = Files().get_all_results(hidden=hidden) + self.results = Files.get_all_results(hidden=hidden) self.data = [] self.data_filtered = [] self.datasets = {} diff --git a/benchmark/ResultsFiles.py b/benchmark/ResultsFiles.py index 7e87d83..8499cc4 100644 --- a/benchmark/ResultsFiles.py +++ b/benchmark/ResultsFiles.py @@ -299,11 +299,11 @@ class ReportDatasets: color2 = "#FDE9D9" color3 = "#B1A0C7" - def __init__(self, excel=False, book=None): + def __init__(self, excel=False, book=None, output=True): self.excel = excel self.env = EnvData().load() self.close = False - self.output = True + self.output = output self.header_text = f"Datasets used in benchmark ver. {__version__}" if excel: self.max_length = 0 @@ -620,7 +620,7 @@ class Benchmark: self.__compute_best_results_ever() def __compute_best_results_ever(self): - args = EnvData.load() + args = EnvData().load() key = args["source_data"] best = BestResultsEver() _, self.best_score_value = best.get_name_value(key, self._score) diff --git a/benchmark/Utils.py b/benchmark/Utils.py index 177b49f..6f6eb31 100644 --- a/benchmark/Utils.py +++ b/benchmark/Utils.py @@ -15,6 +15,7 @@ class Folders: img = "img" excel = "excel" sql = "sql" + current = os.getcwd() @staticmethod def src(): @@ -108,7 +109,8 @@ class Files: ) return None - def get_all_results(self, hidden) -> list[str]: + @staticmethod + def get_all_results(hidden) -> list[str]: result_path = os.path.join( ".", Folders.hidden_results if hidden else Folders.results ) @@ -117,7 +119,7 @@ class Files: else: raise ValueError(f"{result_path} does not exist") result = [] - prefix, suffix = self.results_suffixes() + prefix, suffix = Files.results_suffixes() for result_file in files_list: if result_file.startswith(prefix) and result_file.endswith(suffix): result.append(result_file) diff --git a/benchmark/scripts/app/__init__.py b/benchmark/scripts/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmark/scripts/app/app.py b/benchmark/scripts/app/app.py new file mode 100755 index 0000000..b0f384c --- /dev/null +++ b/benchmark/scripts/app/app.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +from benchmark.Arguments import EnvData +from flask import Flask +from .main import main, OUTPUT + +FRAMEWORK = "framework" +FRAMEWORKS = "frameworks" +TEST = "test" + + +def create_app(output="local"): + app = Flask(__name__) + config = EnvData().load() + app.register_blueprint(main) + app.config[FRAMEWORK] = config[FRAMEWORK] + app.config[FRAMEWORKS] = ["bootstrap", "bulma"] + app.config[OUTPUT] = output + app.jinja_env.auto_reload = True + app.config["TEMPLATES_AUTO_RELOAD"] = True + return app diff --git a/benchmark/scripts/app/main.py b/benchmark/scripts/app/main.py new file mode 100755 index 0000000..446b9da --- /dev/null +++ b/benchmark/scripts/app/main.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python +import os +import json +import shutil +import xlsxwriter +from dotenv import dotenv_values +from benchmark.Utils import Files, Folders +from benchmark.Arguments import EnvData +from benchmark.ResultsBase import StubReport +from benchmark.ResultsFiles import Excel, ReportDatasets +from benchmark.Datasets import Datasets +from flask import Blueprint, current_app, send_file +from flask import render_template, request, redirect, url_for + + +main = Blueprint("main", __name__) +FRAMEWORK = "framework" +FRAMEWORKS = "frameworks" +OUTPUT = "output" +TEST = "test" + + +class AjaxResponse: + def __init__(self, success, file_name, code=200): + self.success = success + self.file_name = file_name + self.code = code + + def to_string(self): + return ( + json.dumps( + { + "success": self.success, + "file": self.file_name, + "output": current_app.config[OUTPUT], + } + ), + self.code, + {"ContentType": "application/json"}, + ) + + +def process_data(file_name, compare, data): + report = StubReport( + os.path.join(Folders.results, file_name), compare=compare + ) + new_list = [] + for result in data["results"]: + symbol = report._compute_status(result["dataset"], result["score"]) + result["symbol"] = symbol if symbol != " " else " " + new_list.append(result) + data["results"] = new_list + # Compute summary with explanation of symbols + summary = {} + for key, value in report._compare_totals.items(): + summary[key] = (report._status_meaning(key), value) + return summary + + +@main.route("/index/") +@main.route("/") +def index(compare="False"): + # Get a list of files in a directory + files = {} + names = Files.get_all_results(hidden=False) + for name in names: + report = StubReport(os.path.join(Folders.results, name)) + report.report() + files[name] = { + "duration": report.duration, + "score": report.score, + "title": report.title, + } + candidate = current_app.config[FRAMEWORKS].copy() + candidate.remove(current_app.config[FRAMEWORK]) + return render_template( + "select.html", + files=files, + candidate=candidate[0], + framework=current_app.config[FRAMEWORK], + compare=compare.capitalize() == "True", + ) + + +@main.route("/datasets/") +def datasets(compare): + dt = Datasets() + datos = [] + for dataset in dt: + datos.append(dt.get_attributes(dataset)) + return render_template( + "datasets.html", + datasets=datos, + compare=compare, + framework=current_app.config[FRAMEWORK], + ) + + +@main.route("/showfile//") +def showfile(file_name, compare, back=None): + compare = compare.capitalize() == "True" + back = request.args["url"] if back is None else back + app_config = dotenv_values(".env") + with open(os.path.join(Folders.results, file_name)) as f: + data = json.load(f) + try: + summary = process_data(file_name, compare, data) + except Exception as e: + return render_template("error.html", message=str(e), compare=compare) + return render_template( + "report.html", + data=data, + file=file_name, + summary=summary, + framework=current_app.config[FRAMEWORK], + back=back, + app_config=app_config, + ) + + +@main.route("/show", methods=["post"]) +def show(): + selected_file = request.form["selected-file"] + compare = request.form["compare"] + return showfile( + file_name=selected_file, + compare=compare, + back=url_for( + "main.index", compare=compare, output=current_app.config[OUTPUT] + ), + ) + + +@main.route("/excel", methods=["post"]) +def excel(): + selected_files = request.json["selectedFiles"] + compare = request.json["compare"] + book = None + if selected_files[0] == "datasets": + # Create a list of datasets + report = ReportDatasets(excel=True, output=False) + report.report() + excel_name = os.path.join(Folders.excel, Files.datasets_report_excel) + if current_app.config[OUTPUT] == "local": + Files.open(excel_name, test=current_app.config[TEST]) + return AjaxResponse(True, Files.datasets_report_excel).to_string() + try: + for file_name in selected_files: + file_name_result = os.path.join(Folders.results, file_name) + if book is None: + file_excel = os.path.join(Folders.excel, Files.be_list_excel) + book = xlsxwriter.Workbook( + file_excel, {"nan_inf_to_errors": True} + ) + excel = Excel( + file_name=file_name_result, + book=book, + compare=compare, + ) + excel.report() + except Exception as e: + if book is not None: + book.close() + return AjaxResponse( + False, "Could not create excel file, " + str(e) + ).to_string() + if book is not None: + book.close() + if current_app.config[OUTPUT] == "local": + Files.open(file_excel, test=current_app.config[TEST]) + return AjaxResponse(True, Files.be_list_excel).to_string() + + +@main.route("/download/") +def download(file_name): + src = os.path.join(Folders.current, Folders.excel, file_name) + dest = os.path.join( + Folders.src(), "scripts", "app", "static", "excel", file_name + ) + shutil.copyfile(src, dest) + return send_file(dest, as_attachment=True) + + +@main.route("/config//") +def config(framework, compare): + if framework not in current_app.config[FRAMEWORKS]: + message = f"framework {framework} not supported" + return render_template("error.html", message=message) + env = EnvData() + env.load() + env.args[FRAMEWORK] = framework + env.save() + current_app.config[FRAMEWORK] = framework + return redirect(url_for("main.index", compare=compare)) + + +@main.route("/best_results//") +def best_results(file, compare): + compare = compare.capitalize() == "True" + try: + with open(os.path.join(Folders.results, file)) as f: + data = json.load(f) + except Exception as e: + return render_template("error.html", message=str(e), compare=compare) + return render_template( + "report_best.html", + data=data, + compare=compare, + framework=current_app.config[FRAMEWORK], + ) diff --git a/benchmark/scripts/app/static/css/main.css b/benchmark/scripts/app/static/css/main.css new file mode 100644 index 0000000..4d68c79 --- /dev/null +++ b/benchmark/scripts/app/static/css/main.css @@ -0,0 +1,30 @@ +.alternate-font { + font-family: Arial; +} + +tbody { + font-family: Courier; +} + +.tag { + cursor: pointer; +} + +.ajaxLoading { + cursor: progress !important; +} + +#file-table tbody tr.selected td { + background-color: #0dcaf0; + color: white; +} + +#report-table tbody tr.selected td { + background-color: #0dcaf0; + color: white; +} + +.btn-small { + padding: 0.25rem 0.5rem; + font-size: 0.75rem; +} \ No newline at end of file diff --git a/benchmark/scripts/app/static/excel/.gitignore b/benchmark/scripts/app/static/excel/.gitignore new file mode 100644 index 0000000..10326c2 --- /dev/null +++ b/benchmark/scripts/app/static/excel/.gitignore @@ -0,0 +1 @@ +*.xlsx \ No newline at end of file diff --git a/benchmark/scripts/app/static/js/excelFiles.js b/benchmark/scripts/app/static/js/excelFiles.js new file mode 100644 index 0000000..930b44c --- /dev/null +++ b/benchmark/scripts/app/static/js/excelFiles.js @@ -0,0 +1,29 @@ +function excelFiles(selectedFiles, compare) { + var data = { + "selectedFiles": selectedFiles, + "compare": compare + }; + // send data to server with ajax post + $.ajax({ + type:'POST', + url:'/excel', + data: JSON.stringify(data), + contentType: "application/json", + dataType: 'json', + success: function(data){ + if (data.success) { + if (data.output == "local") { + alert("Se ha generado el archivo " + data.file); + } else { + window.open('/download/' + data.file, "_blank"); + } + } else { + alert(data.file); + } + }, + error: function (xhr, ajaxOptions, thrownError) { + var mensaje = JSON.parse(xhr.responseText || '{\"mensaje\": \"Error indeterminado\"}'); + alert(mensaje.mensaje); + } + }); +} \ No newline at end of file diff --git a/benchmark/scripts/app/templates/base_bootstrap.html b/benchmark/scripts/app/templates/base_bootstrap.html new file mode 100644 index 0000000..f81e603 --- /dev/null +++ b/benchmark/scripts/app/templates/base_bootstrap.html @@ -0,0 +1,20 @@ + + + + {{ title }} + + + + + + + + {% block content %} + + {% endblock %} + + + {% block jscript %} + + {% endblock %} + diff --git a/benchmark/scripts/app/templates/base_bulma.html b/benchmark/scripts/app/templates/base_bulma.html new file mode 100644 index 0000000..876a23d --- /dev/null +++ b/benchmark/scripts/app/templates/base_bulma.html @@ -0,0 +1,19 @@ + + + + {{ title }} + + + + + + + {% block content %} + + {% endblock %} + + + {% block jscript %} + + {% endblock %} + diff --git a/benchmark/scripts/app/templates/datasets.html b/benchmark/scripts/app/templates/datasets.html new file mode 100644 index 0000000..b028aae --- /dev/null +++ b/benchmark/scripts/app/templates/datasets.html @@ -0,0 +1,68 @@ +{% extends 'base_' ~ framework ~ '.html' %} +{% macro javascript(file) %} + +{% endmacro %} +{% if framework == 'bootstrap' %} + {% set button_class = 'btn btn-primary btn-small' %} + {% set h1_class = 'text-center' %} + {% set table_class = 'table table-striped table-hover table-bordered' %} + {% set head_class = 'bg-primary text-white' %} + {% set text_right = 'text-end' %} + {% set container = 'container' %} + {% set selected = 'selected' %} + {%- macro header(title, close, url) -%} +
+ {%- if close -%} + + {%- endif -%} +

{{ title }}

+
+ {%- endmacro -%} +{% else %} + {% set button_class = 'button is-primary is-small' %} + {% set h1_class = 'title is-1 has-text-centered' %} + {% set table_class = 'table is-striped is-hoverable cell-border is-bordered' %} + {% set head_class = 'is-selected' %} + {% set text_right = 'has-text-right' %} + {% set container = 'container' %} + {% set selected = 'is-selected' %} + {%- macro header(title, close, url) -%} +
+
+ {%- if close -%} + + {%- endif -%} +

{{ title }}

+
+
+ {%- endmacro -%} +{% endif %} +{% block content %} +
+ {{ header('Benchmark Datasets Report', True, url_for('main.index', compare = compare)) }} + + {% include 'partials/datasets_table.html' %} +
+{% endblock %} +{% block jscript %} +{{ javascript("js/excelFiles.js") }} + +{% endblock %} diff --git a/benchmark/scripts/app/templates/error.html b/benchmark/scripts/app/templates/error.html new file mode 100644 index 0000000..999cb09 --- /dev/null +++ b/benchmark/scripts/app/templates/error.html @@ -0,0 +1,20 @@ + + + + Error + + + +
+ +
+ + \ No newline at end of file diff --git a/benchmark/scripts/app/templates/js/report.js b/benchmark/scripts/app/templates/js/report.js new file mode 100644 index 0000000..2ad33c5 --- /dev/null +++ b/benchmark/scripts/app/templates/js/report.js @@ -0,0 +1,22 @@ +$(document).ready(function () { + // Check if row is selected + $('#report-table tbody').on('click', 'tr', function () { + if ($(this).hasClass('{{ selected }}')) { + $(this).removeClass('{{ selected }}'); + } else { + $('#report-table tbody tr.{{ selected }}').removeClass("{{ selected }}") + $(this).addClass('{{ selected }}'); + } + }); + $(document).ajaxStart(function(){ + $("body").addClass('ajaxLoading'); + }); + $(document).ajaxStop(function(){ + $("body").removeClass('ajaxLoading'); + }); + }); + function excelFile() { + var selectedFiles = ["{{ file }}"]; + var compare = "{{ compare }}" == "True"; + excelFiles(selectedFiles, compare) +} \ No newline at end of file diff --git a/benchmark/scripts/app/templates/js/select.js b/benchmark/scripts/app/templates/js/select.js new file mode 100644 index 0000000..d20a897 --- /dev/null +++ b/benchmark/scripts/app/templates/js/select.js @@ -0,0 +1,97 @@ +$(document).ready(function () { + var table = $("#file-table").DataTable({ + paging: true, + searching: true, + ordering: true, + info: true, + "select.items": "row", + pageLength: 25, + columnDefs: [ + { + targets: 8, + orderable: false, + }, + ], + //"language": { + // "lengthMenu": "_MENU_" + //} + }); + $('#file-table').on( 'draw.dt', function () { + enable_disable_best_buttons(); + } ); + // Check if row is selected + $("#file-table tbody").on("click", "tr", function () { + if ($(this).hasClass("{{ select.selected() }}")) { + $(this).removeClass("{{ select.selected() }}"); + } else { + table + .$("tr.{{ select.selected() }}") + .removeClass("{{ select.selected() }}"); + $(this).addClass("{{ select.selected() }}"); + } + }); + // Show file with doubleclick + $("#file-table tbody").on("dblclick", "tr", function () { + showFile($(this).attr("id")); + }); + $(document).ajaxStart(function () { + $("body").addClass("ajaxLoading"); + }); + $(document).ajaxStop(function () { + $("body").removeClass("ajaxLoading"); + }); + $('#compare').change(function() { + enable_disable_best_buttons(); + }); + enable_disable_best_buttons(); +}); +function enable_disable_best_buttons(){ + if ($('#compare').is(':checked')) { + $("[name='best_buttons']").addClass("tag is-link is-normal"); + $("[name='best_buttons']").removeAttr("hidden"); + } else { + $("[name='best_buttons']").removeClass("tag is-link is-normal"); + $("[name='best_buttons']").attr("hidden", true); + } +} +function showFile(selectedFile) { + var form = $( + '
' + + '' + + ' +{%- endif -%} +

{{ title }}

+ +{%- endmacro -%} +{%- macro get_table_class() -%} +table table-striped table-hover table-bordered +{%- endmacro -%} +{%- macro icon(icon_name) -%} + +{%- endmacro -%} +{%- macro get_button(text, action) -%} + +{%- endmacro -%} +{%- macro get_button_class() -%} + button btn-primary btn-small +{%- endmacro %} +{%- macro get_button_tag(icon_name, method, visible=True, name="") -%} + +{%- endmacro -%} +{%- macro get_button_reset() -%} + +{%- endmacro -%} +{%- macro get_button_all() -%} + +{%- endmacro -%} +{%- macro get_tag_class() -%} +badge bg-info bg-small +{%- endmacro -%} +{%- macro get_container_class() -%} +container-fluid +{%- endmacro -%} +{%- macro selected() -%} +selected +{%- endmacro -%} +{%- macro get_level_class() -%} +navbar +{%- endmacro -%} +{%- macro get_align_right() -%} +text-end +{%- endmacro -%} +{%- macro get_left_position() -%} +float-left +{%- endmacro -%} +{%- macro get_right_position() -%} +float-right +{%- endmacro -%} +{%- macro get_row_head_class() -%} +bg-primary text-white +{%- endmacro -%} +{%- macro get_align_center() -%} +text-center +{%- endmacro -%} \ No newline at end of file diff --git a/benchmark/scripts/app/templates/partials/cfg_select_bulma.jinja b/benchmark/scripts/app/templates/partials/cfg_select_bulma.jinja new file mode 100644 index 0000000..9c75baf --- /dev/null +++ b/benchmark/scripts/app/templates/partials/cfg_select_bulma.jinja @@ -0,0 +1,58 @@ +{%- macro header(title, close=False, url="") -%} +
+
+ {%- if close -%} + + {%- endif -%} +

{{ title }}

+
+
+{%- endmacro -%} +{%- macro get_table_class() -%} +table is-striped is-hoverable cell-border is-bordered +{%- endmacro -%} +{%- macro icon(icon_name) -%} + +{%- endmacro -%} +{%- macro get_button(text, action) -%} + +{%- endmacro -%} +{%- macro get_button_tag(icon_name, method, visible=True, name="") -%} + {{icon(icon_name)}} +{%- endmacro -%} +{%- macro get_button_reset() -%} + +{%- endmacro -%} +{%- macro get_button_all() -%} + +{%- endmacro -%} +{%- macro get_tag_class() -%} +tag is-info is-small +{%- endmacro -%} +{%- macro get_container_class() -%} +container is-fluid +{%- endmacro -%} +{%- macro selected() -%} +is-selected +{%- endmacro -%} +{%- macro get_level_class() -%} +level +{%- endmacro -%} +{%- macro get_align_right() -%} +has-text-right +{%- endmacro -%} +{%- macro get_align_center() -%} +has-text-center +{%- endmacro -%} +{%- macro get_left_position() -%} +float-left +{%- endmacro -%} +{%- macro get_right_position() -%} +float-right +{%- endmacro -%} +{%- macro get_row_head_class() -%} +is-selected +{%- endmacro -%} +{%- macro get_align_center() -%} +has-text-center +{%- endmacro -%} \ No newline at end of file diff --git a/benchmark/scripts/app/templates/partials/datasets_table.html b/benchmark/scripts/app/templates/partials/datasets_table.html new file mode 100644 index 0000000..234ac23 --- /dev/null +++ b/benchmark/scripts/app/templates/partials/datasets_table.html @@ -0,0 +1,27 @@ +{% extends "base_" ~ framework ~ ".html" %} +{% block content %} + + + + + + + + + + + + + {% for dataset in datasets %} + + + + + + + + + {% endfor %} + +
DatasetSamplesFeaturesCont. Feat.ClassesBalance
{{ dataset.dataset }}{{ "{:,}".format(dataset.samples) }}{{ "{:,}".format(dataset.features) }}{{ dataset.cont_features }}{{ dataset.classes }}{{ dataset.balance }}
+{% endblock %} \ No newline at end of file diff --git a/benchmark/scripts/app/templates/partials/table_report.html b/benchmark/scripts/app/templates/partials/table_report.html new file mode 100644 index 0000000..e572dd0 --- /dev/null +++ b/benchmark/scripts/app/templates/partials/table_report.html @@ -0,0 +1,14 @@ +{% for item in data.results %} + + {{ item.dataset }} + {{ '{:,}'.format(item.samples) }} + {{"%d" % item.features}} + {{"%d" % item.classes}} + {{ '{:,.2f}'.format(item.nodes|float) }} + {{ '{:,.2f}'.format(item.leaves|float) }} + {{ '{:,.2f}'.format(item.depth|float) }} + {{"%.6f±%.4f" % (item.score, item.score_std)}} {{ item.symbol|safe }} + {{"%.6f±%.4f" % (item.time, item.time_std)}} + {{ item.hyperparameters }} + +{% endfor %} diff --git a/benchmark/scripts/app/templates/partials/table_report_bootstrap.html b/benchmark/scripts/app/templates/partials/table_report_bootstrap.html new file mode 100644 index 0000000..8be1437 --- /dev/null +++ b/benchmark/scripts/app/templates/partials/table_report_bootstrap.html @@ -0,0 +1,102 @@ +
+
+
+
+ +

{{ data.title }}

+
+
+ + + + + + + + {% if data.duration > 7200 %} + {% set unit = "h" %} + {% set divider = 3600 %} + {% else %} + {% set unit = "min" %} + {% set divider = 60 %} + {% endif %} + + + + + + + + + + + + + + + + + + + + + + +
PlatformModelDateTimeDuration ({{ unit }})StratifiedDiscretized# Folds
{{ data.platform }}{{ data.model }} {{ data.version }}{{ data.date }}{{ data.time }}{{ "%.2f" % (data.duration/divider) }}{{ data.stratified }}{{ data.discretized }}{{ data.folds }}
Language{{ data.language }} {{ data.language_version }}Seeds{{ data.seeds }}
+
+ +
+ + + + + + + + + + + + + + + + + {% include "partials/table_report.html" %} + +
DatasetSamplesFeaturesClasses{{ app_config.nodes }}{{ app_config.leaves }}{{ app_config.depth }}{{ data.score_name|capitalize }}Timehyperparameters
+ {% if summary|length > 0 %} +
+ + + + + + + + + {% include "partials/table_summary.html" %} +
SymbolMeaningCount
+
+ {% endif %} + + + + Total score: {{ "%.6f" % (data.results | sum(attribute="score") ) }} + + + + Number of files: {{ data.results | length }} + +
+
+
diff --git a/benchmark/scripts/app/templates/partials/table_report_bulma.html b/benchmark/scripts/app/templates/partials/table_report_bulma.html new file mode 100644 index 0000000..07e87db --- /dev/null +++ b/benchmark/scripts/app/templates/partials/table_report_bulma.html @@ -0,0 +1,100 @@ +
+
+
+
+
+ +

{{ data.title }}

+
+
+
+
+
+
+
+ + + + + + + + {% if data.duration > 7200 %} + {% set unit = "h" %} + {% set divider = 3600 %} + {% else %} + {% set unit = "min" %} + {% set divider = 60 %} + {% endif %} + + + + + + + + + + + + + + + + + + + + + + +
PlatformModelDateTimeDuration ({{ unit }})StratifiedDiscretized# Folds
{{ data.platform }}{{ data.model }} {{ data.version }}{{ data.date }}{{ data.time }}{{ "%.2f" % (data.duration/divider) }}{{ data.stratified }}{{ data.discretized }}{{ data.folds }}
Language{{ data.language }} {{ data.language_version }}Seeds{{ data.seeds }}
+
+ +
+ + + + + + + + + + + + + + + + + {% include "partials/table_report.html" %} + +
DatasetSamplesFeaturesClasses{{ app_config.nodes }}{{ app_config.leaves }}{{ app_config.depth }}{{ data.score_name|capitalize }}Timehyperparameters
+ {% if summary|length > 0 %} +
+ + + + + + + + + {% include "partials/table_summary.html" %} +
SymbolMeaningCount
+
+ {% endif %} +

+ + + Total score: {{ "%.6f" % (data.results | sum(attribute="score") ) }} + +

+

Number of files: {{ data.results | length }}

+
+
+
diff --git a/benchmark/scripts/app/templates/partials/table_select.html b/benchmark/scripts/app/templates/partials/table_select.html new file mode 100644 index 0000000..9c74d04 --- /dev/null +++ b/benchmark/scripts/app/templates/partials/table_select.html @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + {% for file, data in files.items() %} + {% set parts = file.split('_') %} + {% set stratified = parts[6].split('.')[0] %} + + + + + + + + + + + + {% endfor %} + +
ModelMetricPlatformDateTimeStratifiedTitleScore{{ select.get_button_reset()|safe }} {{ select.get_button_all()|safe }}
{{ parts[2] }}{{ parts[1] }}{{ parts[3] }}{{ parts[4] }}{{ parts[5] }}{{ 'True' if stratified =='1' else 'False' }}{{ "%s" % data["title"] }}{{ "%.6f" % data["score"] }} + {{ select.get_button_tag("table-eye", "showFile('" ~ file ~ "')") | safe }} + {% set file_best = "best_results_" ~ parts[1] ~ "_" ~ parts[2] ~ ".json" %} + {{ select.get_button_tag("star-circle-outline", "redirectDouble('best_results', '" ~ file_best ~ "')", visible=False, name="best_buttons") | safe }} + +
diff --git a/benchmark/scripts/app/templates/partials/table_select_design.html b/benchmark/scripts/app/templates/partials/table_select_design.html new file mode 100644 index 0000000..b2050c0 --- /dev/null +++ b/benchmark/scripts/app/templates/partials/table_select_design.html @@ -0,0 +1,15 @@ +
+ {{ select.header("Benchmark Results") }} +
+
+ {{ select.get_button("Use " ~ candidate, "redirectDouble('config', '" ~ candidate ~ "')")|safe }} + {{ select.get_button(select.icon("excel") ~ " Excel", "excel()")|safe }} + {{ select.get_button(select.icon("database-eye") ~ " Datasets", "redirectSimple('datasets')")|safe }} +
+
+ + Comparing with best results +
+
+ {% include "partials/table_select.html" %} +
\ No newline at end of file diff --git a/benchmark/scripts/app/templates/partials/table_summary.html b/benchmark/scripts/app/templates/partials/table_summary.html new file mode 100644 index 0000000..dda3edf --- /dev/null +++ b/benchmark/scripts/app/templates/partials/table_summary.html @@ -0,0 +1,13 @@ +{% for key, value in summary.items() %} + + + {{key}} + + + {{value[0]}} + + + {{'{:,}'.format(value[1])}} + + +{% endfor %} \ No newline at end of file diff --git a/benchmark/scripts/app/templates/report.html b/benchmark/scripts/app/templates/report.html new file mode 100644 index 0000000..860e2c4 --- /dev/null +++ b/benchmark/scripts/app/templates/report.html @@ -0,0 +1,29 @@ +{% macro javascript(file) %} + +{% endmacro %} +{% set title = 'Report Viewer' %} +{% extends 'base_' ~ framework ~ '.html' %} +{% block content %} + {% if framework == 'bootstrap' %} + {% set center = 'text-center' %} + {% set right = 'text-end' %} + {% set button = 'btn btn-primary' %} + {% include 'partials/table_report_bootstrap.html' %} + {% else %} + {% set center = 'has-text-centered' %} + {% set right = 'has-text-right' %} + {% set button = 'button is-primary' %} + {% include 'partials/table_report_bulma.html' %} + {% endif %} +{% endblock %} +{% block jscript %} + {% if framework == 'bootstrap' %} + {% set selected = 'selected' %} + {% else %} + {% set selected = 'is-selected' %} + {% endif %} + + {{ javascript("js/excelFiles.js") }} +{% endblock %} diff --git a/benchmark/scripts/app/templates/report_best.html b/benchmark/scripts/app/templates/report_best.html new file mode 100644 index 0000000..442a74b --- /dev/null +++ b/benchmark/scripts/app/templates/report_best.html @@ -0,0 +1,47 @@ +{% set title = "Best Results" %} +{% extends "base_" ~ framework ~ ".html" %} +{% import "partials/cfg_select_" ~ framework ~ ".jinja" as select %} +{% block content %} +
+ {{ select.header(title, True, url_for("main.index", compare=compare)) }} + + + + + + + + + + + {% for dataset, info in data.items() %} + + + + + + + {% endfor %} + +
DatasetScoreHyperparametersFile
{{ dataset }}{{ '%9.7f' % info[0] }}{{ info[1] }} + {% set url = url_for(request.endpoint, **request.view_args)|urlencode %} + {{ info[2] }} +
+
+{% endblock %} +{% block jscript %} + +{% endblock %} + diff --git a/benchmark/scripts/app/templates/select.html b/benchmark/scripts/app/templates/select.html new file mode 100644 index 0000000..2411a46 --- /dev/null +++ b/benchmark/scripts/app/templates/select.html @@ -0,0 +1,20 @@ +{% macro javascript(file) %} + +{% endmacro %} +{% set title = 'Benchmark Results' %} +{% extends 'base_' ~ framework ~ '.html' %} +{% import 'partials/cfg_select_' ~ framework ~ '.jinja' as select %} +{% block content %} + {% include 'partials/table_select_design.html' %} +{% endblock %} + +{% block jscript %} + + {% if framework == 'bootstrap' %} + + {% endif %} + + {{ javascript('js/excelFiles.js') }} +{% endblock %} diff --git a/benchmark/scripts/be_flask.py b/benchmark/scripts/be_flask.py new file mode 100755 index 0000000..ef10285 --- /dev/null +++ b/benchmark/scripts/be_flask.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +import webbrowser +from benchmark.Arguments import Arguments +from benchmark.scripts.app.app import create_app, TEST, OUTPUT + +# Launch a flask server to serve the results + +def main(args_test=None): + arguments = Arguments(prog="be_flask") + arguments.xset("output") + args = arguments.parse(args_test) + app = create_app() + app.config[TEST] = args_test is not None + app.config[OUTPUT] = args.output + print("Output is ", args.output) + if args.output == "local": + webbrowser.open_new("http://127.0.0.1:1234/") + app.run(port=1234, host="0.0.0.0") diff --git a/benchmark/tests/Util_test.py b/benchmark/tests/Util_test.py index 3cca34d..81b4919 100644 --- a/benchmark/tests/Util_test.py +++ b/benchmark/tests/Util_test.py @@ -118,7 +118,7 @@ class UtilTest(TestBase): def test_Files_get_results(self): os.chdir(os.path.dirname(os.path.abspath(__file__))) self.assertCountEqual( - Files().get_all_results(hidden=False), + Files.get_all_results(hidden=False), [ "results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json", "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json", @@ -130,7 +130,7 @@ class UtilTest(TestBase): ], ) self.assertCountEqual( - Files().get_all_results(hidden=True), + Files.get_all_results(hidden=True), [ "results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json", "results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_" @@ -143,7 +143,7 @@ class UtilTest(TestBase): # check with results os.rename(Folders.results, f"{Folders.results}.test") try: - Files().get_all_results(hidden=False) + Files.get_all_results(hidden=False) except ValueError: pass else: @@ -153,7 +153,7 @@ class UtilTest(TestBase): # check with hidden_results os.rename(Folders.hidden_results, f"{Folders.hidden_results}.test") try: - Files().get_all_results(hidden=True) + Files.get_all_results(hidden=True) except ValueError: pass else: diff --git a/requirements.txt b/requirements.txt index 760042b..37dc400 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ xgboost graphviz Wodt @ git+ssh://git@github.com/doctorado-ml/Wodt.git#egg=Wodt unittest-xml-reporting +flask diff --git a/setup.py b/setup.py index ea9875c..e5693ac 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ def script_names(): "report", "summary", "init_project", + "flask", ] result = [] for script in scripts: