From 9adac5d1fdbad0a77f179be82a8c2359817d56bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana?= Date: Mon, 27 Sep 2021 18:43:59 +0200 Subject: [PATCH] Add multy score to exreport --- .gitignore | 4 ++-- src/Results.py | 55 ++++++++++++++++++++++++++++-------------------- src/Utils.py | 15 ++++++++++--- src/benchmark.py | 10 ++++----- src/benchmark.r | 13 ++++++++---- src/report.py | 21 +++++++++++++++--- test.sh | 2 +- 7 files changed, 79 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 869f54a..880106d 100644 --- a/.gitignore +++ b/.gitignore @@ -135,6 +135,6 @@ results/* exreport/exreport_output/* exreport/.Rapp.history exreport/Rplots.pdf -exreport/exreport.xlsx -exreport/.~lock.exreport.xlsx# +exreport/*.xlsx +exreport/.~lock.*.xlsx# Rplots.pdf diff --git a/src/Results.py b/src/Results.py index 35d8bd5..78fa0b1 100644 --- a/src/Results.py +++ b/src/Results.py @@ -279,15 +279,15 @@ class Excel(BaseReport): header_cols = [ ("Dataset", 30), ("Samples", 10), - ("Variables", 7), + ("Features", 7), ("Classes", 7), ("Nodes", 7), ("Leaves", 7), ("Depth", 7), - ("Score", 10), - ("Score Std.", 10), - ("Time", 10), - ("Time Std.", 10), + ("Score", 12), + ("Score Std.", 12), + ("Time", 12), + ("Time Std.", 12), ("Parameters", 50), ] if self.compare: @@ -348,6 +348,7 @@ class Excel(BaseReport): self.sheet.write(self.row + 1, 0, message, bold) for c in range(self.row + 2): self.sheet.set_row(c, 20) + self.sheet.set_row(0, 25) self.book.close() @@ -452,12 +453,12 @@ class Benchmark: Benchmark._process_dataset(results, data) with open(result_file_name, "w") as f: - f.write("classifier, dataset, accuracy, stdev\n") + f.write(f"classifier, dataset, {score}, stdev\n") for (model, dataset), (accuracy, stdev) in results.items(): f.write(f"{model}, {dataset}, {accuracy}, {stdev}\n") @staticmethod - def exreport(): + def exreport(score): def end_message(message, file): length = 100 print("*" * length) @@ -477,25 +478,29 @@ class Benchmark: except OSError as e: print("Error: %s : %s" % (Folders.report, e.strerror)) # Compute Friedman & Holm Tests - fout = open(os.path.join(Folders.results, Files.exreport_output), "w") - ferr = open(os.path.join(Folders.results, Files.exreport_err), "w") + fout = open( + os.path.join(Folders.results, Files.exreport_output(score)), "w" + ) + ferr = open( + os.path.join(Folders.results, Files.exreport_err(score)), "w" + ) result = subprocess.run( - ["Rscript", os.path.join(Folders.src, Files.benchmark_r)], + ["Rscript", os.path.join(Folders.src, Files.benchmark_r), score], stdout=fout, stderr=ferr, ) fout.close() ferr.close() if result.returncode != 0: - end_message("Error computing benchmark", Files.exreport_err) + end_message("Error computing benchmark", Files.exreport_err(score)) else: - end_message("Benchmark Ok", Files.exreport_output) + end_message("Benchmark Ok", Files.exreport_output(score)) Files.open(Files.exreport_pdf) @staticmethod - def build_results(): + def build_results(score): # Build results data structure - file_name = Benchmark.get_result_file_name() + file_name = Benchmark.get_result_file_name(score) results = {} with open(file_name) as f: data = f.read().splitlines() @@ -508,7 +513,7 @@ class Benchmark: return results @staticmethod - def report(): + def report(score): def show(results): datasets = results[list(results)[0]] print(f"{'Dataset':30s} ", end="") @@ -524,18 +529,20 @@ class Benchmark: print(f"{float(results[model][dataset][1]):.3f} ", end="") print("") - show(Benchmark.build_results()) + print(f"* Score is: {score}") + show(Benchmark.build_results(score)) @staticmethod - def get_excel_file_name(): - return os.path.join(Folders.exreport, Files.exreport_excel) + def get_excel_file_name(score): + return os.path.join(Folders.exreport, Files.exreport_excel(score)) @staticmethod - def excel(): - results = Benchmark.build_results() - book = xlsxwriter.Workbook(Benchmark.get_excel_file_name()) + def excel(score): + results = Benchmark.build_results(score) + book = xlsxwriter.Workbook(Benchmark.get_excel_file_name(score)) sheet = book.add_worksheet("Benchmark") normal = book.add_format({"font_size": 14}) + bold = book.add_format({"bold": True, "font_size": 14}) decimal = book.add_format({"num_format": "0.000000", "font_size": 14}) merge_format = book.add_format( { @@ -545,12 +552,14 @@ class Benchmark: "font_size": 14, } ) - row = row_init = 3 + row = row_init = 4 def header(): nonlocal row sheet.merge_range(0, 0, 1, 0, "Benchmark of Models", merge_format) - # Set column width + sheet.write(1, 2, f"Score is {score}", bold) + sheet.set_row(1, 20) + # Set columns width sheet.set_column(0, 0, 40) for column in range(2 * len(results)): sheet.set_column(column + 1, column + 1, 15) diff --git a/src/Utils.py b/src/Utils.py index 9b6ba96..9e2c4d2 100644 --- a/src/Utils.py +++ b/src/Utils.py @@ -13,14 +13,23 @@ class Folders: class Files: index = "all.txt" - exreport_output = "exreport.txt" - exreport_err = "exreport_err.txt" - exreport_excel = "exreport.xlsx" cmd_open_macos = "/usr/bin/open" cmd_open_linux = "/usr/bin/xdg-open" exreport_pdf = "Rplots.pdf" benchmark_r = "benchmark.r" + @staticmethod + def exreport_output(score): + return f"exreport_{score}.txt" + + @staticmethod + def exreport_err(score): + return f"exreport_err_{score}.txt" + + @staticmethod + def exreport_excel(score): + return f"exreport_{score}.xlsx" + @staticmethod def exreport(score): return f"exreport_{score}.csv" diff --git a/src/benchmark.py b/src/benchmark.py index c369dca..345bc70 100644 --- a/src/benchmark.py +++ b/src/benchmark.py @@ -20,14 +20,14 @@ def parse_arguments(): help="Generate Excel File", ) args = ap.parse_args() - return (args, score, args.excel) + return (args.score, args.excel) (score, excel) = parse_arguments() benchmark = Benchmark() benchmark.compile_results(score) -benchmark.report() -benchmark.exreport() +benchmark.report(score) +benchmark.exreport(score) if excel: - benchmark.excel() - Files.open(benchmark.get_excel_file_name()) + benchmark.excel(score) + Files.open(benchmark.get_excel_file_name(score)) diff --git a/src/benchmark.r b/src/benchmark.r index 98e3419..71d8279 100644 --- a/src/benchmark.r +++ b/src/benchmark.r @@ -1,13 +1,18 @@ -csv_file <- "results/exreport.csv" +library(glue) +args = commandArgs(trailingOnly=TRUE) +if (length(args)!=1) { + stop("Only one argument must be supplied (score).n", call.=FALSE) +} +csv_file <- glue("results/exreport_{args[1]}.csv") destination <- "exreport/" results <- read.csv(csv_file) library(exreport) experiment <- expCreate(results, method="classifier", problem="dataset", name="Stree") -testAccuracy <- testMultipleControl(experiment, "accuracy", "max") +testAccuracy <- testMultipleControl(experiment, args[1], "max") summary(testAccuracy) table1 <- tabularTestSummary(testAccuracy, columns = c("pvalue", "rank", "wtl")) table1 -plot1 <- plotExpSummary(experiment, "accuracy", columns = 3) +plot1 <- plotExpSummary(experiment, args[1], columns = 3) plot2 <- plotCumulativeRank(testAccuracy) plot3 <- plotRankDistribution(testAccuracy) @@ -26,7 +31,7 @@ report <- exreportAdd(report, list(plot1,plot2,table1,plot3)) # We have decided to generate the table at this point of the tutorial to discusse some special formating parameters of this function. Concretely, some of the tabular outputs generated by exreport have some properties that are only useful when rendering the objets in a graphic report, and have no effect in the object representation in the R console. In this case, we will tell the function to boldface the method that maximices the result for each column, and to split the table into to pieces when rendering. # We create the table: -table2 <- tabularExpSummary(experiment, "accuracy", digits=4, format="f", boldfaceColumns="max", tableSplit=2) +table2 <- tabularExpSummary(experiment, args[1], digits=4, format="f", boldfaceColumns="max", tableSplit=2) # And add it to the report: report <- exreportAdd(report, table2) # Now that we have finished adding elements to the report it is time to render it. We want to generate an HTML report, so we call the appropiate function, by default it renders and opens the report in your browser using a temporary file, but you can optionally specify a folder in which the report will be saved for future use. diff --git a/src/report.py b/src/report.py index 037ae5c..379c64f 100644 --- a/src/report.py +++ b/src/report.py @@ -47,8 +47,23 @@ def parse_arguments(): required=False, help="best results of models", ) + ap.add_argument( + "-s", + "--score", + type=str, + required=False, + default="accuracy", + help="score used in best results model", + ) args = ap.parse_args() - return (args.file, args.excel, args.sql, args.compare, args.best) + return ( + args.file, + args.excel, + args.sql, + args.compare, + args.best, + args.score, + ) def default_report(): @@ -63,13 +78,13 @@ def default_report(): ) -(file, excel, sql, compare, best) = parse_arguments() +(file, excel, sql, compare, best, score) = parse_arguments() if file is None and best is None: default_report() else: if best is not None: - report = ReportBest(best) + report = ReportBest(score, best) report.report() else: report = Report(file, compare) diff --git a/test.sh b/test.sh index 00ab73c..6070436 100755 --- a/test.sh +++ b/test.sh @@ -10,5 +10,5 @@ for i in STree Wodt Cart SVC ExtraTree; do done done for a in accuracy f1_macro; do - ptyhon src/benchmark.py -s $a + python src/benchmark.py -s $a done \ No newline at end of file