From 93e4ff94db6d8e10f74a4f0c9dcac1b487083b65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana?= <rmontanana@gmail.com>
Date: Mon, 2 Oct 2023 15:46:40 +0200
Subject: [PATCH] Add significance level as parameter in best

---
 src/Platform/BestResults.cc |  1 -
 src/Platform/BestResults.h  |  3 ++-
 src/Platform/best.cc        | 18 +++++++++++++++++-
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/src/Platform/BestResults.cc b/src/Platform/BestResults.cc
index b8314ab..b18cc35 100644
--- a/src/Platform/BestResults.cc
+++ b/src/Platform/BestResults.cc
@@ -271,7 +271,6 @@ namespace platform {
     }
     void BestResults::reportAll(bool excel)
     {
-        double significance = 0.05;
         auto models = getModels();
         // Build the table of results
         json table = buildTableResults(models);
diff --git a/src/Platform/BestResults.h b/src/Platform/BestResults.h
index e9fbf6d..f830e01 100644
--- a/src/Platform/BestResults.h
+++ b/src/Platform/BestResults.h
@@ -7,7 +7,7 @@ using json = nlohmann::json;
 namespace platform {
     class BestResults {
     public:
-        explicit BestResults(const string& path, const string& score, const string& model, bool friedman) : path(path), score(score), model(model), friedman(friedman) {}
+        explicit BestResults(const string& path, const string& score, const string& model, bool friedman, double significance = 0.05) : path(path), score(score), model(model), friedman(friedman), significance(significance) {}
         string build();
         void reportSingle();
         void reportAll(bool excel);
@@ -24,6 +24,7 @@ namespace platform {
         string score;
         string model;
         bool friedman;
+        double significance;
         int maxModelName = 0;
         int maxDatasetName = 0;
     };
diff --git a/src/Platform/best.cc b/src/Platform/best.cc
index 8e0f47e..10aed7e 100644
--- a/src/Platform/best.cc
+++ b/src/Platform/best.cc
@@ -15,6 +15,20 @@ argparse::ArgumentParser manageArguments(int argc, char** argv)
     program.add_argument("--report").help("report of best score results file").default_value(false).implicit_value(true);
     program.add_argument("--friedman").help("Friedman test").default_value(false).implicit_value(true);
     program.add_argument("--excel").help("Output to excel").default_value(false).implicit_value(true);
+    program.add_argument("--level").help("significance level").default_value(0.05).scan<'g', double>().action([](const string& value) {
+        try {
+            auto k = stod(value);
+            if (k < 0.01 || k > 0.15) {
+                throw runtime_error("Significance level hast to be a number in [0.01, 0.15]");
+            }
+            return k;
+        }
+        catch (const runtime_error& err) {
+            throw runtime_error(err.what());
+        }
+        catch (...) {
+            throw runtime_error("Number of folds must be an decimal number");
+        }});
     try {
         program.parse_args(argc, argv);
         auto model = program.get<string>("model");
@@ -23,6 +37,7 @@ argparse::ArgumentParser manageArguments(int argc, char** argv)
         auto report = program.get<bool>("report");
         auto friedman = program.get<bool>("friedman");
         auto excel = program.get<bool>("excel");
+        auto level = program.get<double>("level");
         if (model == "" || score == "") {
             throw runtime_error("Model and score name must be supplied");
         }
@@ -59,7 +74,8 @@ int main(int argc, char** argv)
     auto report = program.get<bool>("report");
     auto friedman = program.get<bool>("friedman");
     auto excel = program.get<bool>("excel");
-    auto results = platform::BestResults(platform::Paths::results(), score, model, friedman);
+    auto level = program.get<double>("level");
+    auto results = platform::BestResults(platform::Paths::results(), score, model, friedman, level);
     if (build) {
         if (model == "any") {
             results.buildAll();