From cab8e14b2d09c6e06f2e03063ec43c49b8b3d4d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Tue, 26 Sep 2023 11:26:59 +0200 Subject: [PATCH] Add friedman hyperparameter --- src/Platform/BestResults.cc | 25 ++++++++++++------------- src/Platform/BestResults.h | 3 ++- src/Platform/best.cc | 10 +++++++++- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/Platform/BestResults.cc b/src/Platform/BestResults.cc index 232bce8..94ca802 100644 --- a/src/Platform/BestResults.cc +++ b/src/Platform/BestResults.cc @@ -237,11 +237,9 @@ namespace platform { cout << "Can't make the Friedman test with less than 3 models and/or less than 3 datasets." << endl; return; } - cout << Colors::BLUE() << "Friedman test: H0: 'There is no significant differences between all the classifiers.'" << endl; - cout << "N datasets: " << nDatasets << endl; - cout << "N models: " << nModels << endl; - cout << "Significance: " << significance << endl; - cout << "Nº Ranks: " << ranks.size() << endl; + cout << Colors::BLUE() << endl; + cout << "*************************************************************************************" << endl; + cout << "Friedman test: H0: 'There is no significant differences between all the classifiers.'" << endl; for (const auto& rank : ranks) { sum += rank.second; } @@ -250,22 +248,21 @@ namespace platform { for (const auto& rank : ranks) { sumSquared += rank.second * rank.second; } - cout << "Sum Squared: " << sumSquared << endl; - cout << "Degrees of freedom: " << degreesOfFreedom << endl; - double friedman = 12.0 / (nModels * nDatasets * (nModels + 1)) * sumSquared - 3 * nDatasets * (nModels + 1); - cout << "Friedman statistic: " << friedman << endl; + double friedmanQ = 12.0 / (nModels * nDatasets * (nModels + 1)) * sumSquared - 3 * nDatasets * (nModels + 1); + cout << "Friedman statistic: " << friedmanQ << endl; // Calculate the critical value boost::math::chi_squared chiSquared(degreesOfFreedom); - long double p_value = (long double)1.0 - cdf(chiSquared, friedman); + long double p_value = (long double)1.0 - cdf(chiSquared, friedmanQ); double criticalValue = quantile(chiSquared, 1 - significance); - std::cout << "Critical Chi-Square Value for df=" << degreesOfFreedom + std::cout << "Critical Chi-Square Value for df=" << fixed << (int)degreesOfFreedom << " and alpha=" << significance << ": " << criticalValue << std::endl; cout << "p-value: " << scientific << p_value << endl; - if (friedman > criticalValue) { + if (friedmanQ > criticalValue) { cout << Colors::MAGENTA() << "The null hypothesis H0 is rejected." << endl; } else { cout << Colors::GREEN() << "The null hypothesis H0 is accepted." << endl; } + cout << Colors::BLUE() << "*************************************************************************************" << endl; } void BestResults::printTableResults(set models, json table) { @@ -371,7 +368,9 @@ namespace platform { cout << efectiveColor << setw(12) << setprecision(9) << fixed << (double)ranksTotal[model] / (double)origin.size() << " "; } cout << endl; - friedmanTest(models.size(), table.begin().value().size(), ranksTotal, 0.05); + if (friedman) { + friedmanTest(models.size(), table.begin().value().size(), ranksTotal, 0.05); + } } void BestResults::reportAll() { diff --git a/src/Platform/BestResults.h b/src/Platform/BestResults.h index 3ba6b9d..5495222 100644 --- a/src/Platform/BestResults.h +++ b/src/Platform/BestResults.h @@ -8,7 +8,7 @@ using json = nlohmann::json; namespace platform { class BestResults { public: - explicit BestResults(const string& path, const string& score, const string& model) : path(path), score(score), model(model) {} + explicit BestResults(const string& path, const string& score, const string& model, bool friedman) : path(path), score(score), model(model), friedman(friedman) {} string build(); void reportSingle(); void reportAll(); @@ -23,6 +23,7 @@ namespace platform { string path; string score; string model; + bool friedman; }; } #endif //BESTRESULTS_H \ No newline at end of file diff --git a/src/Platform/best.cc b/src/Platform/best.cc index 6e6d432..dcd8d9b 100644 --- a/src/Platform/best.cc +++ b/src/Platform/best.cc @@ -13,12 +13,14 @@ argparse::ArgumentParser manageArguments(int argc, char** argv) program.add_argument("-s", "--score").default_value("").help("Filter results of the score name supplied"); program.add_argument("--build").help("build best score results file").default_value(false).implicit_value(true); program.add_argument("--report").help("report of best score results file").default_value(false).implicit_value(true); + program.add_argument("--friedman").help("Friedman test").default_value(false).implicit_value(true); try { program.parse_args(argc, argv); auto model = program.get("model"); auto score = program.get("score"); auto build = program.get("build"); auto report = program.get("report"); + auto friedman = program.get("friedman"); if (model == "" || score == "") { throw runtime_error("Model and score name must be supplied"); } @@ -38,12 +40,18 @@ int main(int argc, char** argv) auto score = program.get("score"); auto build = program.get("build"); auto report = program.get("report"); + auto friedman = program.get("friedman"); + if (friedman && model != "any") { + cerr << "Friedman test can only be used with all models" << endl; + cerr << program; + exit(1); + } if (!report && !build) { cerr << "Either build, report or both, have to be selected to do anything!" << endl; cerr << program; exit(1); } - auto results = platform::BestResults(platform::Paths::results(), score, model); + auto results = platform::BestResults(platform::Paths::results(), score, model, friedman); if (build) { if (model == "any") { results.buildAll();