From 8c576f8f2ca51a1f0c119c2be92a4c0c8df2f63c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Tue, 6 Jun 2023 17:25:47 +0200 Subject: [PATCH] Fix Excel files --- app/app.db | Bin 28672 -> 28672 bytes app/config.py | 3 - app/env.dist | 4 +- app/main.py | 6 + app/results/main_select.py | 93 ++++++--- app/results/results.ant.py | 212 --------------------- app/results/templates/_table_datasets.html | 2 +- app/results/templates/_table_report.html | 40 ++-- app/results/templates/_table_select.html | 10 +- app/results/templates/best.html | 49 ++--- app/results/templates/datasets.html | 23 +-- app/results/templates/macros.html | 12 ++ app/results/templates/report.html | 1 + app/results/templates/select.html | 1 + app/static/excel/.gitignore | 1 + app/static/js/excelFiles.js | 32 ++-- app/static/js/select.js | 21 -- app/templates/_benchmarks.html | 8 +- app/templates/_nav.html | 3 +- 19 files changed, 182 insertions(+), 339 deletions(-) delete mode 100644 app/results/results.ant.py create mode 100644 app/results/templates/macros.html create mode 100644 app/static/excel/.gitignore diff --git a/app/app.db b/app/app.db index 74aceb9f23d6945bdd17d282da688a37364a6eb7..4d6ec5cf01253a29ccacc9650f5bb12b8fe76503 100644 GIT binary patch delta 63 zcmZp8z}WDBae@>RBilq7Cm^{oVWB-Y0|NsS-zx_GLcUj<1qJ;0SojMWCQOd=>tR!<&gRPC#;F!a{p){z3*Oz9d5IVQ*Xbu%(g Se&yHB!oa}Lzq!=ETLA!krxP9k diff --git a/app/config.py b/app/config.py index 027805f..216ad8a 100644 --- a/app/config.py +++ b/app/config.py @@ -7,9 +7,6 @@ load_dotenv(os.path.join(basedir, dotenv_file)) class Config(object): - FRAMEWORKS = ["bootstrap", "bulma"] - FRAMEWORK = os.environ.get("FRAMEWORK") or FRAMEWORKS[0] - OUTPUT = os.environ.get("OUTPUT") or "local" # local or docker COMPARE = os.environ.get("COMPARE") == "True" or False TEMPLATES_AUTO_RELOAD = True SECRET_KEY = os.environ.get("SECRET_KEY") or "really-hard-to-guess-key" diff --git a/app/env.dist b/app/env.dist index de335b0..3273d0c 100644 --- a/app/env.dist +++ b/app/env.dist @@ -1,3 +1 @@ -OUTPUT="local" -FRAMEWORK="bulma" -COMPARE="True" \ No newline at end of file +COMPARE=True \ No newline at end of file diff --git a/app/main.py b/app/main.py index 270f3d4..31fc48e 100644 --- a/app/main.py +++ b/app/main.py @@ -1,3 +1,5 @@ +import os +from benchmark.utils import Files from flask import ( Blueprint, render_template, @@ -24,6 +26,10 @@ def index(): benchmarks = Benchmark.query.all() else: benchmarks = [current_user.benchmark] + for benchmark in benchmarks: + os.chdir(benchmark.folder) + benchmark.num_files = len(Files.get_all_results(hidden=False)) + os.chdir(current_user.benchmark.folder) else: benchmarks = [] return render_template("index.html", benchmarks=benchmarks) diff --git a/app/results/main_select.py b/app/results/main_select.py index 91fdb59..bcfb67d 100644 --- a/app/results/main_select.py +++ b/app/results/main_select.py @@ -1,16 +1,21 @@ -import os import json -from benchmark.Utils import Files, Folders -from benchmark.ResultsBase import StubReport -from benchmark.Datasets import Datasets -from flask_login import current_user -from flask import Blueprint, current_app, send_file -from flask import render_template, current_app, request, redirect, url_for -from flask_login import login_required +import os +import shutil -# import shutil -# import xlsxwriter -# from benchmark.ResultsFiles import Excel, ReportDatasets +import xlsxwriter +from benchmark.Datasets import Datasets +from benchmark.ResultsBase import StubReport +from benchmark.ResultsFiles import Excel, ReportDatasets +from benchmark.Utils import Files, Folders +from flask import ( + Blueprint, + current_app, + render_template, + request, + send_file, + url_for, +) +from flask_login import current_user, login_required results = Blueprint("results", __name__, template_folder="templates") @@ -36,7 +41,7 @@ class AjaxResponse: @results.route("/select") @login_required -def select(compare="False"): +def select(): # Get a list of files in a directory files = {} os.chdir(current_user.benchmark.folder) @@ -49,14 +54,9 @@ def select(compare="False"): "score": report.score, "title": report.title, } - candidate = current_app.config["FRAMEWORKS"].copy() - candidate.remove(current_app.config["FRAMEWORK"]) return render_template( "select.html", files=files, - candidate=candidate[0], - framework=current_app.config["FRAMEWORK"], - compare=compare.capitalize() == "True", ) @@ -85,7 +85,10 @@ def best(file_name): "created yet! or...", error=str(e), ) - return render_template("best.html", data=data) + info = file_name.split("_") + model = info[3].split(".")[0] + title = f"Best results obtained with {model} on {info[2]}" + return render_template("best.html", data=data, title=title) @results.route("/set_compare", methods=["POST"]) @@ -99,6 +102,7 @@ def set_compare(): @login_required def report(file_name): os.chdir(current_user.benchmark.folder) + back = request.args.get("url") or "" with open(os.path.join(Folders.results, file_name)) as f: data = json.load(f) try: @@ -106,10 +110,7 @@ def report(file_name): except Exception as e: return render_template("error.html", message=str(e)) return render_template( - "report.html", - data=data, - file=file_name, - summary=summary, + "report.html", data=data, file=file_name, summary=summary, back=back ) @@ -128,3 +129,51 @@ def process_data(file_name, compare, data): for key, value in report._compare_totals.items(): summary[key] = (report._status_meaning(key), value) return summary + + +@results.route("/excel", methods=["post"]) +@login_required +def excel(): + os.chdir(current_user.benchmark.folder) + selected_files = request.json["selectedFiles"] + if selected_files[0] == "datasets": + # Create a list of datasets + report = ReportDatasets(excel=True, output=False) + report.report() + return AjaxResponse(True, Files.datasets_report_excel).to_string() + try: + # create a spreadsheet with the selected files + book = None + for file_name in selected_files: + file_name_result = os.path.join(Folders.results, file_name) + if book is None: + file_excel = os.path.join(Folders.excel, Files.be_list_excel) + book = xlsxwriter.Workbook( + file_excel, {"nan_inf_to_errors": True} + ) + excel = Excel( + file_name=file_name_result, + book=book, + compare=current_app.config["COMPARE"], + ) + excel.report() + except Exception as e: + if book is not None: + book.close() + return AjaxResponse( + False, "Could not create excel file, " + str(e) + ).to_string() + if book is not None: + book.close() + return AjaxResponse(True, Files.be_list_excel).to_string() + + +@results.route("/download/") +@login_required +def download(file_name): + os.chdir(current_user.benchmark.folder) + src = os.path.join(current_user.benchmark.folder, Folders.excel, file_name) + dest = os.path.join("static", "excel", file_name) + os.chdir(current_app.root_path) + shutil.copyfile(src, dest) + return send_file(dest, as_attachment=True) diff --git a/app/results/results.ant.py b/app/results/results.ant.py deleted file mode 100644 index 43a9c56..0000000 --- a/app/results/results.ant.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python -# import os -# import json -# import shutil -# import xlsxwriter -# from benchmark.Utils import Files, Folders -# from benchmark.Arguments import EnvData -# from benchmark.ResultsBase import StubReport -# from benchmark.ResultsFiles import Excel, ReportDatasets -# from benchmark.Datasets import Datasets -# from flask import Blueprint, current_app, send_file -# from flask import render_template, request, redirect, url_for -from flask import Blueprint, render_template - - -results = Blueprint("results", __name__, template_folder="results") -# FRAMEWORK = "framework" -# FRAMEWORKS = "frameworks" -# OUTPUT = "output" -# TEST = "test" - - -# class AjaxResponse: -# def __init__(self, success, file_name, code=200): -# self.success = success -# self.file_name = file_name -# self.code = code - -# def to_string(self): -# return ( -# json.dumps( -# { -# "success": self.success, -# "file": self.file_name, -# "output": current_app.config[OUTPUT], -# } -# ), -# self.code, -# {"ContentType": "application/json"}, -# ) - - -# def process_data(file_name, compare, data): -# report = StubReport( -# os.path.join(Folders.results, file_name), compare=compare -# ) -# new_list = [] -# for result in data["results"]: -# symbol = report._compute_status(result["dataset"], result["score"]) -# result["symbol"] = symbol if symbol != " " else " " -# new_list.append(result) -# data["results"] = new_list -# # Compute summary with explanation of symbols -# summary = {} -# for key, value in report._compare_totals.items(): -# summary[key] = (report._status_meaning(key), value) -# return summary - - -@results.route("/results/") -def results(compare="False"): - # # Get a list of files in a directory - # files = {} - # names = Files.get_all_results(hidden=False) - # for name in names: - # report = StubReport(os.path.join(Folders.results, name)) - # report.report() - # files[name] = { - # "duration": report.duration, - # "score": report.score, - # "title": report.title, - # } - # candidate = current_app.config[FRAMEWORKS].copy() - # candidate.remove(current_app.config[FRAMEWORK]) - # return render_template( - # "select.html", - # files=files, - # candidate=candidate[0], - # framework=current_app.config[FRAMEWORK], - # compare=compare.capitalize() == "True", - # ) - return render_template("test.html") - - -""" -@results.route("/datasets/") -@results.route("datasets") -def datasets(compare=False): - dt = Datasets() - datos = [] - for dataset in dt: - datos.append(dt.get_attributes(dataset)) - return render_template( - "datasets.html", - datasets=datos, - compare=compare, - framework=current_app.config[FRAMEWORK], - ) - - -@results.route("/showfile//") -def showfile(file_name, compare, back=None): - compare = compare.capitalize() == "True" - back = request.args["url"] if back is None else back - print(f"back [{back}]") - with open(os.path.join(Folders.results, file_name)) as f: - data = json.load(f) - try: - summary = process_data(file_name, compare, data) - except Exception as e: - return render_template("error.html", message=str(e), compare=compare) - return render_template( - "report.html", - data=data, - file=file_name, - summary=summary, - framework=current_app.config[FRAMEWORK], - back=back, - ) - - -@results.route("/show", methods=["post"]) -def show(): - selected_file = request.form["selected-file"] - compare = request.form["compare"] - return showfile( - file_name=selected_file, - compare=compare, - back=url_for( - "main.index", compare=compare, output=current_app.config[OUTPUT] - ), - ) - - -@results.route("/excel", methods=["post"]) -def excel(): - selected_files = request.json["selectedFiles"] - compare = request.json["compare"] - book = None - if selected_files[0] == "datasets": - # Create a list of datasets - report = ReportDatasets(excel=True, output=False) - report.report() - excel_name = os.path.join(Folders.excel, Files.datasets_report_excel) - if current_app.config[OUTPUT] == "local": - Files.open(excel_name, test=current_app.config[TEST]) - return AjaxResponse(True, Files.datasets_report_excel).to_string() - try: - for file_name in selected_files: - file_name_result = os.path.join(Folders.results, file_name) - if book is None: - file_excel = os.path.join(Folders.excel, Files.be_list_excel) - book = xlsxwriter.Workbook( - file_excel, {"nan_inf_to_errors": True} - ) - excel = Excel( - file_name=file_name_result, - book=book, - compare=compare, - ) - excel.report() - except Exception as e: - if book is not None: - book.close() - return AjaxResponse( - False, "Could not create excel file, " + str(e) - ).to_string() - if book is not None: - book.close() - if current_app.config[OUTPUT] == "local": - Files.open(file_excel, test=current_app.config[TEST]) - return AjaxResponse(True, Files.be_list_excel).to_string() - - -@results.route("/download/") -def download(file_name): - src = os.path.join(Folders.current, Folders.excel, file_name) - dest = os.path.join( - Folders.src(), "scripts", "app", "static", "excel", file_name - ) - shutil.copyfile(src, dest) - return send_file(dest, as_attachment=True) - - -@results.route("/config//") -def config(framework, compare): - if framework not in current_app.config[FRAMEWORKS]: - message = f"framework {framework} not supported" - return render_template("error.html", message=message) - env = EnvData() - env.load() - env.args[FRAMEWORK] = framework - env.save() - current_app.config[FRAMEWORK] = framework - return redirect(url_for("main.index", compare=compare)) - - -@results.route("/best_results//") -def best_results(file, compare): - compare = compare.capitalize() == "True" - try: - with open(os.path.join(Folders.results, file)) as f: - data = json.load(f) - except Exception as e: - return render_template("error.html", message=str(e), compare=compare) - return render_template( - "report_best.html", - data=data, - compare=compare, - framework=current_app.config[FRAMEWORK], - ) - """ diff --git a/app/results/templates/_table_datasets.html b/app/results/templates/_table_datasets.html index c0a7cde..88fa91c 100644 --- a/app/results/templates/_table_datasets.html +++ b/app/results/templates/_table_datasets.html @@ -1,4 +1,4 @@ - diff --git a/app/results/templates/_table_report.html b/app/results/templates/_table_report.html index 3d46ec3..30d666f 100644 --- a/app/results/templates/_table_report.html +++ b/app/results/templates/_table_report.html @@ -1,4 +1,15 @@ +{% from "macros.html" import excel_button %}
+ {% if back %} + + {% endif %}
@@ -39,11 +50,7 @@
-
- -
+
{{ excel_button([ file ]) }}
@@ -53,7 +60,7 @@ - + @@ -62,13 +69,14 @@ {% for item in data.results %} - - - - - - - + + + + + + + + {% endfor %} @@ -86,11 +94,9 @@ {% for key, value in summary.items() %} - + - + {% endfor %} diff --git a/app/results/templates/_table_select.html b/app/results/templates/_table_select.html index 9e8d5c6..4a2bbd3 100644 --- a/app/results/templates/_table_select.html +++ b/app/results/templates/_table_select.html @@ -1,6 +1,10 @@ -
- - Comparing with best results +{% from "macros.html" import excel_button %} +
+ {{ excel_button('', 'excel_select') }} +
+ + Comparing with best results +
Features Classes Nodes{{ data.score_name|capitalize }}{{ data.score_name|capitalize }} Time hyperparameters
{{ item.dataset }}{{ '{:,}'.format(item.samples) }}{{"%d" % item.features}}{{"%d" % item.classes}}{{ '{:,.2f}'.format(item.nodes) }}{{"%.6f±%.4f" % (item.score, item.score_std)}} {{ item.symbol|safe }}{{"%.6f±%.4f" % (item.time, item.time_std)}}{{ item.hyperparameters }}{{ '{:,}'.format(item.samples) }}{{"%d" % item.features}}{{"%d" % item.classes}}{{ '{:,.2f}'.format(item.nodes) }}{{"%.6f±%.4f" % (item.score, item.score_std)}}{{ item.symbol|safe }}{{"%.6f±%.4f" % (item.time, item.time_std)}}{{ item.hyperparameters }}
{{ key }}{{ key }} {{ value[0] }} - {{ '{:,}'.format(value[1]) }} - {{ '{:,}'.format(value[1]) }}
diff --git a/app/results/templates/best.html b/app/results/templates/best.html index be1977b..3692939 100644 --- a/app/results/templates/best.html +++ b/app/results/templates/best.html @@ -1,29 +1,32 @@ {% extends "base.html" %} {% block content %} -
- - - - - - - - - - {% for dataset, info in data.items() %} - - - - - +
DatasetScoreHyperparametersFile
{{ dataset }}{{ '%9.7f' % info[0] }}{{ info[1] }} - {% set url = url_for(request.endpoint, **request.view_args)|urlencode %} - {{ info[2] }} -
+ + + + + + - {% endfor %} - -
DatasetScoreHyperparametersFile
+ + + {% for dataset, info in data.items() %} + + {{ dataset }} + {{ '%9.7f' % info[0] }} + {{ info[1] }} + + {% set url = url_for(request.endpoint, **request.view_args)|urlencode %} + {{ info[2] }} + + + {% endfor %} + + + {% endblock %} {% block jscript %} {{ super() }} diff --git a/app/results/templates/datasets.html b/app/results/templates/datasets.html index c10e8f7..1b9f7f6 100644 --- a/app/results/templates/datasets.html +++ b/app/results/templates/datasets.html @@ -1,26 +1,11 @@ +{% from "macros.html" import excel_button %} {% extends "base.html" %} {% block content %} + {{ excel_button(["datasets"]) }} {% include "_table_datasets.html" %} {% endblock %} {% block jscript %} {{ super() }} - + + {% endblock %} diff --git a/app/results/templates/macros.html b/app/results/templates/macros.html new file mode 100644 index 0000000..561920e --- /dev/null +++ b/app/results/templates/macros.html @@ -0,0 +1,12 @@ +{% macro excel_button(files, function="excelSend") %} + {% if files %} + {% set call %} + {{ function }}({{ files }}) + {% endset %} + {% else %} + {% set call = function + "()" %} + {% endif %} + +{% endmacro %} diff --git a/app/results/templates/report.html b/app/results/templates/report.html index 3fde409..f77941e 100644 --- a/app/results/templates/report.html +++ b/app/results/templates/report.html @@ -5,4 +5,5 @@ {% block jscript %} {{ super() }} + {% endblock %} diff --git a/app/results/templates/select.html b/app/results/templates/select.html index b220b69..bc281df 100644 --- a/app/results/templates/select.html +++ b/app/results/templates/select.html @@ -13,4 +13,5 @@ + {% endblock %} diff --git a/app/static/excel/.gitignore b/app/static/excel/.gitignore new file mode 100644 index 0000000..10326c2 --- /dev/null +++ b/app/static/excel/.gitignore @@ -0,0 +1 @@ +*.xlsx \ No newline at end of file diff --git a/app/static/js/excelFiles.js b/app/static/js/excelFiles.js index 930b44c..abecae3 100644 --- a/app/static/js/excelFiles.js +++ b/app/static/js/excelFiles.js @@ -1,29 +1,37 @@ -function excelFiles(selectedFiles, compare) { +function excel_select() { + var checkbox = document.getElementsByName("selected_files"); + var selectedFiles = []; + for (var i = 0; i < checkbox.length; i++) { + if (checkbox[i].checked) { + selectedFiles.push(checkbox[i].value); + } + } + if (selectedFiles.length == 0) { + alert("Select at least one file"); + return; + } + excelSend(selectedFiles); +} +function excelSend(selectedFiles) { var data = { "selectedFiles": selectedFiles, - "compare": compare }; // send data to server with ajax post $.ajax({ type:'POST', - url:'/excel', + url:'/results/excel', data: JSON.stringify(data), contentType: "application/json", dataType: 'json', success: function(data){ if (data.success) { - if (data.output == "local") { - alert("Se ha generado el archivo " + data.file); - } else { - window.open('/download/' + data.file, "_blank"); - } + window.open('/results/download/' + data.file, "_blank"); } else { alert(data.file); } }, - error: function (xhr, ajaxOptions, thrownError) { - var mensaje = JSON.parse(xhr.responseText || '{\"mensaje\": \"Error indeterminado\"}'); - alert(mensaje.mensaje); - } + error: function (XMLHttpRequest, textStatus, errorThrown) { + alert(XMLHttpRequest.responseText); + } }); } \ No newline at end of file diff --git a/app/static/js/select.js b/app/static/js/select.js index 82ec58e..b432d28 100644 --- a/app/static/js/select.js +++ b/app/static/js/select.js @@ -73,30 +73,9 @@ $(document).ready(function () { function showFile(selectedFile) { location.href = "/results/show/" + selectedFile; } - function excel() { - var checkbox = document.getElementsByName("selected_files"); - var selectedFiles = []; - for (var i = 0; i < checkbox.length; i++) { - if (checkbox[i].checked) { - selectedFiles.push(checkbox[i].value); - } - } - if (selectedFiles.length == 0) { - alert("Select at least one file"); - return; - } - var compare = $("#compare").is(":checked"); - excelFiles(selectedFiles, compare); - } function setCheckBoxes(value) { var checkbox = document.getElementsByName("selected_files"); for (i = 0; i < checkbox.length; i++) { checkbox[i].checked = value; } - } - function redirectDouble(route, parameter) { - location.href = "/"+ route + "/" + parameter + "/" + $("#compare").is(":checked"); - } - function redirectSimple(route) { - location.href = "/" + route + "/" + $("#compare").is(":checked"); } \ No newline at end of file diff --git a/app/templates/_benchmarks.html b/app/templates/_benchmarks.html index 7ce43dc..a4407d7 100644 --- a/app/templates/_benchmarks.html +++ b/app/templates/_benchmarks.html @@ -3,7 +3,13 @@
{% for benchmark in benchmarks %} {{ benchmark.name + " - " + benchmark.description }} + class="list-group-item list-group-item-action flex-column align-items-start {% if benchmark == current_user.benchmark %}active{% endif %}"> +
+
{{ benchmark.name }}
+ {{ benchmark.num_files }} files +
+

{{ benchmark.description }}

+ {% endfor %}
diff --git a/app/templates/_nav.html b/app/templates/_nav.html index 62c081b..951cff6 100644 --- a/app/templates/_nav.html +++ b/app/templates/_nav.html @@ -28,8 +28,7 @@ {% else %} {{ render_nav_item('main.login', 'Login') }} {% endif %} - + -