From ea13835701447a809eb44f660e54e62be594b3f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana=20G=C3=B3mez?= Date: Mon, 7 Oct 2024 18:08:42 +0200 Subject: [PATCH] Add Markdown best results output --- src/CMakeLists.txt | 2 +- src/best/BestResults.cpp | 7 +++ src/best/BestResultsMd.cpp | 103 ++++++++++++++++++++++++++++++++++++ src/best/BestResultsMd.h | 24 +++++++++ src/best/BestResultsTex.cpp | 6 +-- src/best/Statistics.cpp | 3 ++ src/common/Paths.h | 8 +++ 7 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 src/best/BestResultsMd.cpp create mode 100644 src/best/BestResultsMd.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ff5ea4a..a3fd165 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -20,7 +20,7 @@ include_directories( # b_best add_executable( b_best commands/b_best.cpp best/Statistics.cpp - best/BestResultsExcel.cpp best/BestResultsTex.cpp best/BestResults.cpp + best/BestResultsExcel.cpp best/BestResultsTex.cpp best/BestResultsMd.cpp best/BestResults.cpp common/Datasets.cpp common/Dataset.cpp common/Discretization.cpp main/Models.cpp main/Scores.cpp reports/ReportExcel.cpp reports/ReportBase.cpp reports/ExcelFile.cpp diff --git a/src/best/BestResults.cpp b/src/best/BestResults.cpp index d158436..56902b7 100644 --- a/src/best/BestResults.cpp +++ b/src/best/BestResults.cpp @@ -11,6 +11,7 @@ #include "results/Result.h" #include "BestResultsExcel.h" #include "BestResultsTex.h" +#include "BestResultsMd.h" #include "best/Statistics.h" #include "BestResults.h" @@ -221,8 +222,10 @@ namespace platform { std::cout << std::string(oss.str().size() - 8, '-') << std::endl; std::cout << Colors::GREEN() << " # " << std::setw(maxDatasetName + 1) << std::left << std::string("Dataset"); auto bestResultsTex = BestResultsTex(); + auto bestResultsMd = BestResultsMd(); if (tex) { bestResultsTex.results_header(models, table.at("dateTable").get()); + bestResultsMd.results_header(models, table.at("dateTable").get()); } for (const auto& model : models) { std::cout << std::setw(maxModelName) << std::left << model << " "; @@ -239,6 +242,7 @@ namespace platform { auto datasets = getDatasets(table.begin().value()); if (tex) { bestResultsTex.results_body(datasets, table); + bestResultsMd.results_body(datasets, table); } for (auto const& dataset_ : datasets) { auto color = (i % 2) ? Colors::BLUE() : Colors::CYAN(); @@ -297,6 +301,7 @@ namespace platform { } if (tex) { bestResultsTex.results_footer(totals, best_model); + bestResultsMd.results_footer(totals, best_model); } for (const auto& model : models) { std::string efectiveColor = model == best_model ? Colors::RED() : Colors::GREEN(); @@ -338,8 +343,10 @@ namespace platform { } if (tex) { messageOutputFile("TeX", Paths::tex() + Paths::tex_output()); + messageOutputFile("MarkDown", Paths::tex() + Paths::md_output()); if (friedman) { messageOutputFile("TeX", Paths::tex() + Paths::tex_post_hoc()); + messageOutputFile("MarkDown", Paths::tex() + Paths::md_post_hoc()); } } if (excel) { diff --git a/src/best/BestResultsMd.cpp b/src/best/BestResultsMd.cpp new file mode 100644 index 0000000..bfa0a9b --- /dev/null +++ b/src/best/BestResultsMd.cpp @@ -0,0 +1,103 @@ +#include +#include "BestResultsMd.h" +#include "common/Utils.h" // compute_std + +namespace platform { + using json = nlohmann::ordered_json; + void BestResultsMd::openMdFile(const std::string& name) + { + handler.open(name); + if (!handler.is_open()) { + std::cerr << "Error opening file " << name << std::endl; + exit(1); + } + } + void BestResultsMd::results_header(const std::vector& models, const std::string& date) + { + this->models = models; + auto file_name = Paths::tex() + Paths::md_output(); + openMdFile(file_name); + handler << "" << std::endl; + handler << "| # | Dataset |"; + for (const auto& model : models) { + handler << " " << model.c_str() << " |"; + } + handler << std::endl; + handler << "|--: | :--- |"; + for (const auto& model : models) { + handler << " :---: |"; + } + handler << std::endl; + } + void BestResultsMd::results_body(const std::vector& datasets, json& table) + { + int i = 0; + for (auto const& dataset : datasets) { + // Find out max value for this dataset + double max_value = 0; + // Find out the max value for this dataset + for (const auto& model : models) { + double value; + try { + value = table[model].at(dataset).at(0).get(); + } + catch (nlohmann::json_abi_v3_11_3::detail::out_of_range err) { + value = -1.0; + } + if (value > max_value) { + max_value = value; + } + } + handler << "| " << ++i << " | " << dataset.c_str() << " | "; + for (const auto& model : models) { + double value = table[model].at(dataset).at(0).get(); + double std_value = table[model].at(dataset).at(3).get(); + const char* bold = value == max_value ? "**" : ""; + handler << bold << std::setprecision(4) << std::fixed << value << "±" << std::setprecision(3) << std_value << bold << " | "; + } + handler << std::endl; + } + } + void BestResultsMd::results_footer(const std::map>& totals, const std::string& best_model) + { + handler << "| | **Average Score** | "; + int nDatasets = totals.begin()->second.size(); + for (const auto& model : models) { + double value = std::reduce(totals.at(model).begin(), totals.at(model).end()) / nDatasets; + double std_value = compute_std(totals.at(model), value); + const char* bold = model == best_model ? "**" : ""; + handler << bold << std::setprecision(4) << std::fixed << value << "±" << std::setprecision(3) << std::fixed << std_value << bold << " | "; + } + + handler.close(); + } + void BestResultsMd::holm_test(struct HolmResult& holmResult, const std::string& date) + { + auto file_name = Paths::tex() + Paths::md_post_hoc(); + openMdFile(file_name); + handler << "" << std::endl; + handler << "Post-hoc Holm test: H0: There is no significant differences between the control model and the other models." << std::endl << std::endl; + handler << "| classifier | pvalue | rank | win | tie | loss | H0 |" << std::endl; + handler << "| :-- | --: | --: | --:| --: | --: | :--: |" << std::endl; + for (auto const& line : holmResult.holmLines) { + auto textStatus = !line.reject ? "**" : " "; + if (line.model == holmResult.model) { + handler << "| " << line.model << " | - | " << std::fixed << std::setprecision(2) << line.rank << " | - | - | - |" << std::endl; + } else { + handler << "| " << line.model << " | " << textStatus << std::scientific << std::setprecision(4) << line.pvalue << textStatus << " |"; + handler << std::fixed << std::setprecision(2) << line.rank << " | " << line.wtl.win << " | " << line.wtl.tie << " | " << line.wtl.loss << " |"; + handler << (line.reject ? "rejected" : "**accepted**") << " |" << std::endl; + } + } + handler << std::endl; + handler.close(); + } +} diff --git a/src/best/BestResultsMd.h b/src/best/BestResultsMd.h new file mode 100644 index 0000000..253a54a --- /dev/null +++ b/src/best/BestResultsMd.h @@ -0,0 +1,24 @@ +#ifndef BEST_RESULTS_MD_H +#define BEST_RESULTS_MD_H +#include +#include +#include +#include "common/Paths.h" +#include "Statistics.h" +namespace platform { + using json = nlohmann::ordered_json; + class BestResultsMd { + public: + BestResultsMd() = default; + ~BestResultsMd() = default; + void results_header(const std::vector& models, const std::string& date); + void results_body(const std::vector& datasets, json& table); + void results_footer(const std::map>& totals, const std::string& best_model); + void holm_test(struct HolmResult& holmResult, const std::string& date); + private: + void openMdFile(const std::string& name); + std::ofstream handler; + std::vector models; + }; +} +#endif \ No newline at end of file diff --git a/src/best/BestResultsTex.cpp b/src/best/BestResultsTex.cpp index 612d5ec..d5b1eb8 100644 --- a/src/best/BestResultsTex.cpp +++ b/src/best/BestResultsTex.cpp @@ -27,7 +27,7 @@ namespace platform { handler << "\\tiny " << std::endl; handler << "\\renewcommand{\\arraystretch }{1.2} " << std::endl; handler << "\\renewcommand{\\tabcolsep }{0.07cm} " << std::endl; - handler << "\\caption{Accuracy results(mean ± std) for all the algorithms and datasets} " << std::endl; + handler << "\\caption{Accuracy results(mean $\pm$ std) for all the algorithms and datasets} " << std::endl; handler << "\\label{tab:results_accuracy}" << std::endl; handler << "\\begin{tabular} {{r" << std::string(models.size(), 'c').c_str() << "}}" << std::endl; handler << "\\hline " << std::endl; @@ -62,7 +62,7 @@ namespace platform { double value = table[model].at(dataset).at(0).get(); double std_value = table[model].at(dataset).at(3).get(); const char* bold = value == max_value ? "\\bfseries" : ""; - handler << "& " << bold << std::setprecision(4) << std::fixed << value << "±" << std::setprecision(3) << std_value; + handler << "& " << bold << std::setprecision(4) << std::fixed << value << "$\pm$" << std::setprecision(3) << std_value; } handler << "\\\\" << std::endl; } @@ -76,7 +76,7 @@ namespace platform { double value = std::reduce(totals.at(model).begin(), totals.at(model).end()) / nDatasets; double std_value = compute_std(totals.at(model), value); const char* bold = model == best_model ? "\\bfseries" : ""; - handler << "& " << bold << std::setprecision(4) << std::fixed << value << "±" << std::setprecision(3) << std::fixed << std_value; + handler << "& " << bold << std::setprecision(4) << std::fixed << value << "$\pm$" << std::setprecision(3) << std::fixed << std_value; } handler << "\\\\" << std::endl; handler << "\\hline " << std::endl; diff --git a/src/best/Statistics.cpp b/src/best/Statistics.cpp index 3f06e80..73f1edb 100644 --- a/src/best/Statistics.cpp +++ b/src/best/Statistics.cpp @@ -5,6 +5,7 @@ #include "common/Symbols.h" #include "common/CLocale.h" #include "BestResultsTex.h" +#include "BestResultsMd.h" #include "Statistics.h" @@ -198,7 +199,9 @@ namespace platform { } if (tex) { BestResultsTex bestResultsTex; + BestResultsMd bestResultsMd; bestResultsTex.holm_test(holmResult, get_date() + " " + get_time()); + bestResultsMd.holm_test(holmResult, get_date() + " " + get_time()); } } bool Statistics::friedmanTest() diff --git a/src/common/Paths.h b/src/common/Paths.h index 33ea909..15a42d1 100644 --- a/src/common/Paths.h +++ b/src/common/Paths.h @@ -61,10 +61,18 @@ namespace platform { { return "results.tex"; } + static std::string md_output() + { + return "results.md"; + } static std::string tex_post_hoc() { return "post_hoc.tex"; } + static std::string md_post_hoc() + { + return "post_hoc.md"; + } }; } #endif \ No newline at end of file