From 6ae2b2182a375470433329cbe1d6074015dfbf73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana=20G=C3=B3mez?= Date: Tue, 3 Sep 2024 12:43:50 +0200 Subject: [PATCH] Complete Tex output with Holm test --- src/best/BestResults.cpp | 9 ++- src/best/BestResultsTex.cpp | 106 +++++++++++++++++++++++------------- src/best/BestResultsTex.h | 10 ++-- src/best/Statistics.cpp | 7 ++- src/best/Statistics.h | 2 +- src/common/Paths.h | 4 ++ src/common/Utils.h | 22 +++++++- src/grid/GridSearch.cpp | 22 +------- 8 files changed, 115 insertions(+), 67 deletions(-) diff --git a/src/best/BestResults.cpp b/src/best/BestResults.cpp index 8123661..d158436 100644 --- a/src/best/BestResults.cpp +++ b/src/best/BestResults.cpp @@ -220,9 +220,9 @@ namespace platform { std::cout << oss.str(); std::cout << std::string(oss.str().size() - 8, '-') << std::endl; std::cout << Colors::GREEN() << " # " << std::setw(maxDatasetName + 1) << std::left << std::string("Dataset"); - auto bestResultsTex = BestResultsTex(models, table.at("dateTable").get()); + auto bestResultsTex = BestResultsTex(); if (tex) { - bestResultsTex.results_header(); + bestResultsTex.results_header(models, table.at("dateTable").get()); } for (const auto& model : models) { std::cout << std::setw(maxModelName) << std::left << model << " "; @@ -333,11 +333,14 @@ namespace platform { if (friedman) { Statistics stats(models, datasets, table, significance); auto result = stats.friedmanTest(); - stats.postHocHolmTest(result); + stats.postHocHolmTest(result, tex); ranksModels = stats.getRanks(); } if (tex) { messageOutputFile("TeX", Paths::tex() + Paths::tex_output()); + if (friedman) { + messageOutputFile("TeX", Paths::tex() + Paths::tex_post_hoc()); + } } if (excel) { BestResultsExcel excel(score, datasets); diff --git a/src/best/BestResultsTex.cpp b/src/best/BestResultsTex.cpp index 24bd334..985d135 100644 --- a/src/best/BestResultsTex.cpp +++ b/src/best/BestResultsTex.cpp @@ -4,37 +4,39 @@ namespace platform { using json = nlohmann::ordered_json; - BestResultsTex::BestResultsTex(const std::vector& models, const std::string& date) : models(models), date(date) + void BestResultsTex::openTexFile(const std::string& name) { - } - void BestResultsTex::results_header() - { - auto file_name = Paths::tex() + Paths::tex_output(); - output_tex = fopen(file_name.c_str(), "w"); - if (output_tex == NULL) { - std::cerr << "Error opening file " << file_name << std::endl; + handler.open(name); + if (!handler.is_open()) { + std::cerr << "Error opening file " << name << std::endl; exit(1); } - fprintf(output_tex, "%% This file has been generated by the platform program\n"); - fprintf(output_tex, "%% Date: %s\n", date.c_str()); - fprintf(output_tex, "%%\n"); - fprintf(output_tex, "%% Table of results\n"); - fprintf(output_tex, "%%\n"); - fprintf(output_tex, "\\begin{table}[htbp] \n"); - fprintf(output_tex, "\\centering \n"); - fprintf(output_tex, "\\tiny \n"); - fprintf(output_tex, "\\renewcommand{\\arraystretch }{1.2} \n"); - fprintf(output_tex, "\\renewcommand{\\tabcolsep }{0.07cm} \n"); - fprintf(output_tex, "\\caption{Accuracy results(mean ± std) for all the algorithms and datasets} \n"); - fprintf(output_tex, "\\label{tab:results_accuracy}\n"); - fprintf(output_tex, "\\begin{tabular} {{r%s}}\n", std::string(models.size(), 'c').c_str()); - fprintf(output_tex, "\\hline \n"); - fprintf(output_tex, "Id"); + } + void BestResultsTex::results_header(const std::vector& models, const std::string& date) + { + this->models = models; + auto file_name = Paths::tex() + Paths::tex_output(); + openTexFile(file_name); + handler << "%% This file has been generated by the platform program" << std::endl; + handler << "%% Date: " << date.c_str() << std::endl; + handler << "%%" << std::endl; + handler << "%% Table of results" << std::endl; + handler << "%%" << std::endl; + handler << "\\begin{table}[htbp] " << std::endl; + handler << "\\centering " << std::endl; + handler << "\\tiny " << std::endl; + handler << "\\renewcommand{\\arraystretch }{1.2} " << std::endl; + handler << "\\renewcommand{\\tabcolsep }{0.07cm} " << std::endl; + handler << "\\caption{Accuracy results(mean ± std) for all the algorithms and datasets} " << std::endl; + handler << "\\label{tab:results_accuracy}" << std::endl; + handler << "\\begin{tabular} {{r" << std::string(models.size(), 'c').c_str() << "}}" << std::endl; + handler << "\\hline " << std::endl; + handler << "" << std::endl; for (const auto& model : models) { - fprintf(output_tex, "& %s ", model.c_str()); + handler << "& " << model.c_str(); } - fprintf(output_tex, "\\\\ \n"); - fprintf(output_tex, "\\hline \n"); + handler << "\\\\" << std::endl; + handler << "\\hline" << std::endl; } void BestResultsTex::results_body(const std::vector& datasets, json& table) { @@ -55,31 +57,61 @@ namespace platform { max_value = value; } } - fprintf(output_tex, "%d ", i); + handler << ++i << " "; for (const auto& model : models) { double value = table[model].at(dataset).at(0).get(); double std_value = table[model].at(dataset).at(3).get(); const char* bold = value == max_value ? "\\bfseries" : ""; - fprintf(output_tex, "& %s %0.4f±%0.3f", bold, value, std_value); + handler << "& " << bold << std::setprecision(4) << std::fixed << value << "±" << std::setprecision(3) << std_value; } - fprintf(output_tex, "\\\\\n"); + handler << "\\\\" << std::endl; } } void BestResultsTex::results_footer(const std::map>& totals, const std::string& best_model) { - fprintf(output_tex, "\\hline \n"); - fprintf(output_tex, "Average "); + handler << "\\hline" << std::endl; + handler << "Average "; int nDatasets = totals.begin()->second.size(); for (const auto& model : models) { double value = std::reduce(totals.at(model).begin(), totals.at(model).end()) / nDatasets; double std_value = compute_std(totals.at(model), value); const char* bold = model == best_model ? "\\bfseries" : ""; - fprintf(output_tex, "& %s %0.4f±%0.3f", bold, value, std_value); + handler << "& " << bold << std::setprecision(4) << std::fixed << value << "±" << std::setprecision(3) << std::fixed << std_value; } - fprintf(output_tex, "\\ \n"); - fprintf(output_tex, "\\hline \n"); - fprintf(output_tex, "\\end{tabular}\n"); - fprintf(output_tex, "\\end{table}\n"); - fclose(output_tex); + handler << "\\" << std::endl; + handler << "\\hline " << std::endl; + handler << "\\end{tabular}" << std::endl; + handler << "\\end{table}" << std::endl; + handler.close(); + } + void BestResultsTex::holm_test(struct HolmResult& holmResult, const std::string& date) + { + auto file_name = Paths::tex() + Paths::tex_post_hoc(); + openTexFile(file_name); + handler << "%% This file has been generated by the platform program" << std::endl; + handler << "%% Date: " << date.c_str() << std::endl; + handler << "%%" << std::endl; + handler << "%% Post-hoc handler test" << std::endl; + handler << "%%" << std::endl; + handler << "\\begin{table}[htbp]" << std::endl; + handler << "\\centering" << std::endl; + handler << "\\caption{Results of the post-hoc test for the mean accuracy of the algorithms.}\\label{tab:tests}" << std::endl; + handler << "\\begin{tabular}{lrrrrr}" << std::endl; + handler << "\\hline" << std::endl; + handler << "classifier & pvalue & rank & win & tie & loss\\\\" << std::endl; + handler << "\\hline" << std::endl; + for (auto const& line : holmResult.holmLines) { + auto textStatus = !line.reject ? "\\bf " : " "; + if (line.model == holmResult.model) { + handler << line.model << " & - & " << std::fixed << std::setprecision(2) << line.rank << " & - & - & - \\\\" << std::endl; + } else { + handler << line.model << " & " << std::scientific << std::setprecision(4) << line.pvalue << " & "; + handler << std::fixed << std::setprecision(2) << line.rank << " & " << line.wtl.win << " & " << line.wtl.tie << " & " << line.wtl.loss << "\\\\" << std::endl; + } + } + handler << "\\hline " << std::endl; + handler << "\\end{tabular}" << std::endl; + handler << "\\end{table}" << std::endl; + handler.close(); } } diff --git a/src/best/BestResultsTex.h b/src/best/BestResultsTex.h index 011291d..860e3e0 100644 --- a/src/best/BestResultsTex.h +++ b/src/best/BestResultsTex.h @@ -4,19 +4,21 @@ #include #include #include "common/Paths.h" +#include "Statistics.h" namespace platform { using json = nlohmann::ordered_json; class BestResultsTex { public: - BestResultsTex(const std::vector& models, const std::string& date); + BestResultsTex() = default; ~BestResultsTex() = default; - void results_header(); + void results_header(const std::vector& models, const std::string& date); void results_body(const std::vector& datasets, json& table); void results_footer(const std::map>& totals, const std::string& best_model); + void holm_test(struct HolmResult& holmResult, const std::string& date); private: - std::FILE* output_tex; + void openTexFile(const std::string& name); + std::ofstream handler; std::vector models; - std::string date; }; } #endif \ No newline at end of file diff --git a/src/best/Statistics.cpp b/src/best/Statistics.cpp index d5284e1..3f06e80 100644 --- a/src/best/Statistics.cpp +++ b/src/best/Statistics.cpp @@ -4,6 +4,7 @@ #include "common/Colors.h" #include "common/Symbols.h" #include "common/CLocale.h" +#include "BestResultsTex.h" #include "Statistics.h" @@ -113,7 +114,7 @@ namespace platform { } } - void Statistics::postHocHolmTest(bool friedmanResult) + void Statistics::postHocHolmTest(bool friedmanResult, bool tex) { if (!fitted) { fit(); @@ -195,6 +196,10 @@ namespace platform { if (output) { std::cout << oss.str(); } + if (tex) { + BestResultsTex bestResultsTex; + bestResultsTex.holm_test(holmResult, get_date() + " " + get_time()); + } } bool Statistics::friedmanTest() { diff --git a/src/best/Statistics.h b/src/best/Statistics.h index f082056..ee98c96 100644 --- a/src/best/Statistics.h +++ b/src/best/Statistics.h @@ -34,7 +34,7 @@ namespace platform { public: Statistics(const std::vector& models, const std::vector& datasets, const json& data, double significance = 0.05, bool output = true); bool friedmanTest(); - void postHocHolmTest(bool friedmanResult); + void postHocHolmTest(bool friedmanResult, bool tex=false); FriedmanResult& getFriedmanResult(); HolmResult& getHolmResult(); std::map>& getRanks(); diff --git a/src/common/Paths.h b/src/common/Paths.h index 47b17d9..dde0e2a 100644 --- a/src/common/Paths.h +++ b/src/common/Paths.h @@ -61,6 +61,10 @@ namespace platform { { return "results.tex"; } + static std::string tex_post_hoc() + { + return "post_hoc.tex"; + } }; } #endif \ No newline at end of file diff --git a/src/common/Utils.h b/src/common/Utils.h index 7b9a1c9..92e5a4a 100644 --- a/src/common/Utils.h +++ b/src/common/Utils.h @@ -36,7 +36,7 @@ namespace platform { } return result; } - double compute_std(std::vector values, double mean) + inline double compute_std(std::vector values, double mean) { // Compute standard devation of the values double sum = 0.0; @@ -46,5 +46,25 @@ namespace platform { double variance = sum / values.size(); return std::sqrt(variance); } + inline std::string get_date() + { + time_t rawtime; + tm* timeinfo; + time(&rawtime); + timeinfo = std::localtime(&rawtime); + std::ostringstream oss; + oss << std::put_time(timeinfo, "%Y-%m-%d"); + return oss.str(); + } + inline std::string get_time() + { + time_t rawtime; + tm* timeinfo; + time(&rawtime); + timeinfo = std::localtime(&rawtime); + std::ostringstream oss; + oss << std::put_time(timeinfo, "%H:%M:%S"); + return oss.str(); + } } #endif \ No newline at end of file diff --git a/src/grid/GridSearch.cpp b/src/grid/GridSearch.cpp index 2f79938..e9e8b64 100644 --- a/src/grid/GridSearch.cpp +++ b/src/grid/GridSearch.cpp @@ -5,29 +5,11 @@ #include "main/Models.h" #include "common/Paths.h" #include "common/Colors.h" +#include "common/Utils.h" #include "GridSearch.h" namespace platform { - std::string get_date() - { - time_t rawtime; - tm* timeinfo; - time(&rawtime); - timeinfo = std::localtime(&rawtime); - std::ostringstream oss; - oss << std::put_time(timeinfo, "%Y-%m-%d"); - return oss.str(); - } - std::string get_time() - { - time_t rawtime; - tm* timeinfo; - time(&rawtime); - timeinfo = std::localtime(&rawtime); - std::ostringstream oss; - oss << std::put_time(timeinfo, "%H:%M:%S"); - return oss.str(); - } + std::string get_color_rank(int rank) { auto colors = { Colors::WHITE(), Colors::RED(), Colors::GREEN(), Colors::BLUE(), Colors::MAGENTA(), Colors::CYAN() };