From 70d8022926ad576cae23b98fa778944a01dacb14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana=20G=C3=B3mez?= Date: Sat, 17 May 2025 18:12:57 +0200 Subject: [PATCH] Refactor postHoc --- CMakeLists.txt | 2 - src/CMakeLists.txt | 2 +- src/best/BestResults.cpp | 5 +- src/best/BestResultsExcel.cpp | 9 +-- src/best/BestResultsMd.cpp | 8 +-- src/best/BestResultsMd.h | 2 +- src/best/BestResultsTex.cpp | 16 ++--- src/best/BestResultsTex.h | 5 +- src/best/DeLong.cpp | 45 ++++++++++++++ src/best/DeLong.h | 24 ++++++++ src/best/Statistics.cpp | 107 ++++++++++++++++++++++++++++------ src/best/Statistics.h | 14 +++-- 12 files changed, 192 insertions(+), 47 deletions(-) create mode 100644 src/best/DeLong.cpp create mode 100644 src/best/DeLong.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 8875491..75b6900 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,8 +7,6 @@ project(Platform LANGUAGES CXX ) - - # Global CMake variables # ---------------------- set(CMAKE_CXX_STANDARD 20) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b89cebc..be63bee 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,7 +13,7 @@ include_directories( # b_best add_executable( b_best commands/b_best.cpp best/Statistics.cpp - best/BestResultsExcel.cpp best/BestResultsTex.cpp best/BestResultsMd.cpp best/BestResults.cpp + best/BestResultsExcel.cpp best/BestResultsTex.cpp best/BestResultsMd.cpp best/BestResults.cpp best/DeLong.cpp common/Datasets.cpp common/Dataset.cpp common/Discretization.cpp main/Models.cpp main/Scores.cpp reports/ReportExcel.cpp reports/ReportBase.cpp reports/ExcelFile.cpp diff --git a/src/best/BestResults.cpp b/src/best/BestResults.cpp index 09a2cf3..21da49e 100644 --- a/src/best/BestResults.cpp +++ b/src/best/BestResults.cpp @@ -222,7 +222,7 @@ namespace platform { std::cout << oss.str(); std::cout << std::string(oss.str().size() - 8, '-') << std::endl; std::cout << Colors::GREEN() << " # " << std::setw(maxDatasetName + 1) << std::left << std::string("Dataset"); - auto bestResultsTex = BestResultsTex(); + auto bestResultsTex = BestResultsTex(score); auto bestResultsMd = BestResultsMd(); if (tex) { bestResultsTex.results_header(models, table.at("dateTable").get(), index); @@ -339,7 +339,8 @@ namespace platform { if (friedman) { Statistics stats(models, datasets, table, significance); auto result = stats.friedmanTest(); - stats.postHocHolmTest(result, tex); + stats.postHocHolmTest(); + stats.postHocTestReport("Holm", score, result, tex); ranksModels = stats.getRanks(); } if (tex) { diff --git a/src/best/BestResultsExcel.cpp b/src/best/BestResultsExcel.cpp index 0bc961e..fb7b864 100644 --- a/src/best/BestResultsExcel.cpp +++ b/src/best/BestResultsExcel.cpp @@ -243,9 +243,10 @@ namespace platform { row = 2; Statistics stats(models, datasets, table, significance, false); auto result = stats.friedmanTest(); - stats.postHocHolmTest(result); + stats.postHocHolmTest(); + // stats.postHocTestReport("Holm", result, false); auto friedmanResult = stats.getFriedmanResult(); - auto holmResult = stats.getHolmResult(); + auto postHocResult = stats.getPostHocResult(); worksheet_merge_range(worksheet, row, 0, row, 7, "Null hypothesis: H0 'There is no significant differences between all the classifiers.'", styles["headerSmall"]); row += 2; writeString(row, 1, "Friedman Q", "bodyHeader"); @@ -264,7 +265,7 @@ namespace platform { row += 2; worksheet_merge_range(worksheet, row, 0, row, 7, "Null hypothesis: H0 'There is no significant differences between the control model and the other models.'", styles["headerSmall"]); row += 2; - std::string controlModel = "Control Model: " + holmResult.model; + std::string controlModel = "Control Model: " + postHocResult.model; worksheet_merge_range(worksheet, row, 1, row, 7, controlModel.c_str(), styles["bodyHeader_odd"]); row++; writeString(row, 1, "Model", "bodyHeader"); @@ -276,7 +277,7 @@ namespace platform { writeString(row, 7, "Reject H0", "bodyHeader"); row++; bool first = true; - for (const auto& item : holmResult.holmLines) { + for (const auto& item : postHocResult.postHocLines) { writeString(row, 1, item.model, "text"); if (first) { // Control model info diff --git a/src/best/BestResultsMd.cpp b/src/best/BestResultsMd.cpp index bfa0a9b..3c70901 100644 --- a/src/best/BestResultsMd.cpp +++ b/src/best/BestResultsMd.cpp @@ -75,7 +75,7 @@ namespace platform { handler.close(); } - void BestResultsMd::holm_test(struct HolmResult& holmResult, const std::string& date) + void BestResultsMd::postHoc_test(struct PostHocResult& postHocResult, const std::string& kind, const std::string& date) { auto file_name = Paths::tex() + Paths::md_post_hoc(); openMdFile(file_name); @@ -84,12 +84,12 @@ namespace platform { handler << std::endl; handler << " Post-hoc handler test" << std::endl; handler << "-->" << std::endl; - handler << "Post-hoc Holm test: H0: There is no significant differences between the control model and the other models." << std::endl << std::endl; + handler << "Post-hoc " << kind << " test: H0: There is no significant differences between the control model and the other models." << std::endl << std::endl; handler << "| classifier | pvalue | rank | win | tie | loss | H0 |" << std::endl; handler << "| :-- | --: | --: | --:| --: | --: | :--: |" << std::endl; - for (auto const& line : holmResult.holmLines) { + for (auto const& line : postHocResult.postHocLines) { auto textStatus = !line.reject ? "**" : " "; - if (line.model == holmResult.model) { + if (line.model == postHocResult.model) { handler << "| " << line.model << " | - | " << std::fixed << std::setprecision(2) << line.rank << " | - | - | - |" << std::endl; } else { handler << "| " << line.model << " | " << textStatus << std::scientific << std::setprecision(4) << line.pvalue << textStatus << " |"; diff --git a/src/best/BestResultsMd.h b/src/best/BestResultsMd.h index 253a54a..894ae80 100644 --- a/src/best/BestResultsMd.h +++ b/src/best/BestResultsMd.h @@ -14,7 +14,7 @@ namespace platform { void results_header(const std::vector& models, const std::string& date); void results_body(const std::vector& datasets, json& table); void results_footer(const std::map>& totals, const std::string& best_model); - void holm_test(struct HolmResult& holmResult, const std::string& date); + void postHoc_test(struct PostHocResult& postHocResult, const std::string& kind, const std::string& date); private: void openMdFile(const std::string& name); std::ofstream handler; diff --git a/src/best/BestResultsTex.cpp b/src/best/BestResultsTex.cpp index bf74c88..39e17d9 100644 --- a/src/best/BestResultsTex.cpp +++ b/src/best/BestResultsTex.cpp @@ -27,8 +27,10 @@ namespace platform { handler << "\\tiny " << std::endl; handler << "\\renewcommand{\\arraystretch }{1.2} " << std::endl; handler << "\\renewcommand{\\tabcolsep }{0.07cm} " << std::endl; - handler << "\\caption{Accuracy results(mean $\\pm$ std) for all the algorithms and datasets} " << std::endl; - handler << "\\label{tab:results_accuracy}" << std::endl; + auto umetric = metric; + umetric[0] = toupper(umetric[0]); + handler << "\\caption{" << umetric << " results(mean $\\pm$ std) for all the algorithms and datasets} " << std::endl; + handler << "\\label{tab:results_" << metric << "}" << std::endl; std::string header_dataset_name = index ? "r" : "l"; handler << "\\begin{tabular} {{" << header_dataset_name << std::string(models.size(), 'c').c_str() << "}}" << std::endl; handler << "\\hline " << std::endl; @@ -87,25 +89,25 @@ namespace platform { handler << "\\end{table}" << std::endl; handler.close(); } - void BestResultsTex::holm_test(struct HolmResult& holmResult, const std::string& date) + void BestResultsTex::postHoc_test(struct PostHocResult& postHocResult, const std::string& kind, const std::string& date) { auto file_name = Paths::tex() + Paths::tex_post_hoc(); openTexFile(file_name); handler << "%% This file has been generated by the platform program" << std::endl; handler << "%% Date: " << date.c_str() << std::endl; handler << "%%" << std::endl; - handler << "%% Post-hoc handler test" << std::endl; + handler << "%% Post-hoc " << kind << " test" << std::endl; handler << "%%" << std::endl; handler << "\\begin{table}[htbp]" << std::endl; handler << "\\centering" << std::endl; - handler << "\\caption{Results of the post-hoc test for the mean accuracy of the algorithms.}\\label{tab:tests}" << std::endl; + handler << "\\caption{Results of the post-hoc " << kind << " test for the mean " << metric << " of the algorithms.}\\label{ tab:tests }" << std::endl; handler << "\\begin{tabular}{lrrrrr}" << std::endl; handler << "\\hline" << std::endl; handler << "classifier & pvalue & rank & win & tie & loss\\\\" << std::endl; handler << "\\hline" << std::endl; - for (auto const& line : holmResult.holmLines) { + for (auto const& line : postHocResult.postHocLines) { auto textStatus = !line.reject ? "\\bf " : " "; - if (line.model == holmResult.model) { + if (line.model == postHocResult.model) { handler << line.model << " & - & " << std::fixed << std::setprecision(2) << line.rank << " & - & - & - \\\\" << std::endl; } else { handler << line.model << " & " << textStatus << std::scientific << std::setprecision(4) << line.pvalue << " & "; diff --git a/src/best/BestResultsTex.h b/src/best/BestResultsTex.h index ae88c6d..e587dec 100644 --- a/src/best/BestResultsTex.h +++ b/src/best/BestResultsTex.h @@ -9,13 +9,14 @@ namespace platform { using json = nlohmann::ordered_json; class BestResultsTex { public: - BestResultsTex(bool dataset_name = true) : dataset_name(dataset_name) {}; + BestResultsTex(const std::string metric_, bool dataset_name = true) : metric{ metric_ }, dataset_name{ dataset_name } {}; ~BestResultsTex() = default; void results_header(const std::vector& models, const std::string& date, bool index); void results_body(const std::vector& datasets, json& table, bool index); void results_footer(const std::map>& totals, const std::string& best_model); - void holm_test(struct HolmResult& holmResult, const std::string& date); + void postHoc_test(struct PostHocResult& postHocResult, const std::string& kind, const std::string& date); private: + std::string metric; bool dataset_name; void openTexFile(const std::string& name); std::ofstream handler; diff --git a/src/best/DeLong.cpp b/src/best/DeLong.cpp new file mode 100644 index 0000000..dbcc920 --- /dev/null +++ b/src/best/DeLong.cpp @@ -0,0 +1,45 @@ +// DeLong.cpp +// Integración del test de DeLong con la clase RocAuc y Statistics +// Basado en: X. Sun and W. Xu, "Fast Implementation of DeLong’s Algorithm for Comparing the Areas Under Correlated Receiver Operating Characteristic Curves," (2014), y algoritmos inspirados en sklearn/pROC + +#include "DeLong.h" +#include +#include +#include +#include +#include +#include + +namespace platform { + + DeLong::DeLongResult DeLong::compare(const std::vector& aucs_model1, + const std::vector& aucs_model2) + { + if (aucs_model1.size() != aucs_model2.size()) { + throw std::invalid_argument("AUC lists must have the same size"); + } + + size_t N = aucs_model1.size(); + if (N < 2) { + throw std::invalid_argument("At least two AUC values are required"); + } + + std::vector diffs(N); + for (size_t i = 0; i < N; ++i) { + diffs[i] = aucs_model1[i] - aucs_model2[i]; + } + + double mean_diff = std::accumulate(diffs.begin(), diffs.end(), 0.0) / N; + double var = 0.0; + for (size_t i = 0; i < N; ++i) { + var += (diffs[i] - mean_diff) * (diffs[i] - mean_diff); + } + var /= (N * (N - 1)); + if (var <= 0.0) var = 1e-10; + + double z = mean_diff / std::sqrt(var); + double p = 2.0 * (1.0 - std::erfc(std::abs(z) / std::sqrt(2.0)) / 2.0); + return { mean_diff, z, p }; + } + +} diff --git a/src/best/DeLong.h b/src/best/DeLong.h new file mode 100644 index 0000000..07e3cf3 --- /dev/null +++ b/src/best/DeLong.h @@ -0,0 +1,24 @@ +#ifndef DELONG_H +#define DELONG_H +/* ******************************************************************************************************************** +/* Integración del test de DeLong con la clase RocAuc y Statistics +/* Basado en: X. Sun and W. Xu, "Fast Implementation of DeLong’s Algorithm for Comparing the Areas Under Correlated +/* Receiver Operating Characteristic Curves," (2014), y algoritmos inspirados en sklearn/pROC +/* ********************************************************************************************************************/ +#include + +namespace platform { + class DeLong { + public: + struct DeLongResult { + double auc_diff; + double z_stat; + double p_value; + }; + // Compara dos vectores de AUCs por dataset y devuelve diferencia media, + // estadístico z y p-valor usando un test de rangos (DeLong simplificado) + static DeLongResult compare(const std::vector& aucs_model1, + const std::vector& aucs_model2); + }; +} +#endif // DELONG_H \ No newline at end of file diff --git a/src/best/Statistics.cpp b/src/best/Statistics.cpp index 397b679..cd40cc1 100644 --- a/src/best/Statistics.cpp +++ b/src/best/Statistics.cpp @@ -7,6 +7,7 @@ #include "BestResultsTex.h" #include "BestResultsMd.h" #include "Statistics.h" +#include "DeLong.h" namespace platform { @@ -114,8 +115,7 @@ namespace platform { } } } - - void Statistics::postHocHolmTest(bool friedmanResult, bool tex) + void Statistics::postHocHolmTest() { if (!fitted) { fit(); @@ -137,27 +137,33 @@ namespace platform { stats[i] = p_value; } // Sort the models by p-value - std::vector> statsOrder; for (const auto& stat : stats) { - statsOrder.push_back({ stat.first, stat.second }); + postHocData.push_back({ stat.first, stat.second }); } - std::sort(statsOrder.begin(), statsOrder.end(), [](const std::pair& a, const std::pair& b) { + std::sort(postHocData.begin(), postHocData.end(), [](const std::pair& a, const std::pair& b) { return a.second < b.second; }); // Holm adjustment - for (int i = 0; i < statsOrder.size(); ++i) { - auto item = statsOrder.at(i); - double before = i == 0 ? 0.0 : statsOrder.at(i - 1).second; + for (int i = 0; i < postHocData.size(); ++i) { + auto item = postHocData.at(i); + double before = i == 0 ? 0.0 : postHocData.at(i - 1).second; double p_value = std::min((double)1.0, item.second * (nModels - i)); p_value = std::max(before, p_value); - statsOrder[i] = { item.first, p_value }; + postHocData[i] = { item.first, p_value }; } - holmResult.model = models.at(controlIdx); + postHocResult.model = models.at(controlIdx); + } + + void Statistics::postHocTestReport(const std::string& kind, const std::string& metric, bool friedmanResult, bool tex) + { + + std::stringstream oss; + postHocResult.model = models.at(controlIdx); auto color = friedmanResult ? Colors::CYAN() : Colors::YELLOW(); oss << color; oss << " *************************************************************************************************************" << std::endl; - oss << " Post-hoc Holm test: H0: 'There is no significant differences between the control model and the other models.'" << std::endl; + oss << " Post-hoc " << kind << " test: H0: 'There is no significant differences between the control model and the other models.'" << std::endl; oss << " Control model: " << models.at(controlIdx) << std::endl; oss << " " << std::left << std::setw(maxModelName) << std::string("Model") << " p-value rank win tie loss Status" << std::endl; oss << " " << std::string(maxModelName, '=') << " ============ ========= === === ==== =============" << std::endl; @@ -175,12 +181,12 @@ namespace platform { for (const auto& item : ranksOrder) { auto idx = distance(models.begin(), find(models.begin(), models.end(), item.first)); double pvalue = 0.0; - for (const auto& stat : statsOrder) { + for (const auto& stat : postHocData) { if (stat.first == idx) { pvalue = stat.second; } } - holmResult.holmLines.push_back({ item.first, pvalue, item.second, wtl.at(idx), pvalue < significance }); + postHocResult.postHocLines.push_back({ item.first, pvalue, item.second, wtl.at(idx), pvalue < significance }); if (item.first == models.at(controlIdx)) { continue; } @@ -198,12 +204,77 @@ namespace platform { std::cout << oss.str(); } if (tex) { - BestResultsTex bestResultsTex; + BestResultsTex bestResultsTex(metric); BestResultsMd bestResultsMd; - bestResultsTex.holm_test(holmResult, get_date() + " " + get_time()); - bestResultsMd.holm_test(holmResult, get_date() + " " + get_time()); + bestResultsTex.postHoc_test(postHocResult, kind, get_date() + " " + get_time()); + bestResultsMd.postHoc_test(postHocResult, kind, get_date() + " " + get_time()); } } + // void Statistics::postHocDeLongTest(const std::vector>& y_trues, + // const std::vector>>& y_probas, + // bool tex) + // { + // std::map pvalues; + // postHocResult.model = models.at(controlIdx); + // postHocResult.postHocLines.clear(); + + // for (size_t i = 0; i < models.size(); ++i) { + // if ((int)i == controlIdx) continue; + // double acc_p = 0.0; + // int valid = 0; + // for (size_t d = 0; d < y_trues.size(); ++d) { + // try { + // auto result = compareModelsWithDeLong(y_probas[controlIdx][d], y_probas[i][d], y_trues[d]); + // acc_p += result.p_value; + // ++valid; + // } + // catch (...) {} + // } + // if (valid > 0) { + // pvalues[i] = acc_p / valid; + // } + // } + + // std::vector> sorted_pvalues(pvalues.begin(), pvalues.end()); + // std::sort(sorted_pvalues.begin(), sorted_pvalues.end(), [](const auto& a, const auto& b) { + // return a.second < b.second; + // }); + + // std::stringstream oss; + // oss << "\n*************************************************************************************************************\n"; + // oss << " Post-hoc DeLong-Holm test: H0: 'No significant differences in AUC with control model.'\n"; + // oss << " Control model: " << models[controlIdx] << "\n"; + // oss << " " << std::left << std::setw(maxModelName) << std::string("Model") << " p-value Adjusted Result\n"; + // oss << " " << std::string(maxModelName, '=') << " ============ ========== =============\n"; + + // double prev = 0.0; + // for (size_t i = 0; i < sorted_pvalues.size(); ++i) { + // int idx = sorted_pvalues[i].first; + // double raw = sorted_pvalues[i].second; + // double adj = std::min(1.0, raw * (models.size() - i - 1)); + // adj = std::max(prev, adj); + // prev = adj; + // bool reject = adj < significance; + + // postHocResult.postHocLines.push_back({ models[idx], adj, 0.0f, {}, reject }); + + // auto color = reject ? Colors::MAGENTA() : Colors::GREEN(); + // auto status = reject ? Symbols::cross : Symbols::check_mark; + // auto textStatus = reject ? " rejected H0" : " accepted H0"; + // oss << " " << color << std::left << std::setw(maxModelName) << models[idx] << " "; + // oss << std::setprecision(6) << std::scientific << raw << " "; + // oss << std::setprecision(6) << std::scientific << adj << " " << status << textStatus << "\n"; + // } + // oss << Colors::CYAN() << " *************************************************************************************************************\n"; + // oss << Colors::RESET(); + // if (output) std::cout << oss.str(); + // if (tex) { + // BestResultsTex bestResultsTex; + // BestResultsMd bestResultsMd; + // bestResultsTex.holm_test(postHocResult, get_date() + " " + get_time()); + // bestResultsMd.holm_test(postHocResult, get_date() + " " + get_time()); + // } + // } bool Statistics::friedmanTest() { if (!fitted) { @@ -249,9 +320,9 @@ namespace platform { { return friedmanResult; } - HolmResult& Statistics::getHolmResult() + PostHocResult& Statistics::getPostHocResult() { - return holmResult; + return postHocResult; } std::map>& Statistics::getRanks() { diff --git a/src/best/Statistics.h b/src/best/Statistics.h index ee98c96..285f34c 100644 --- a/src/best/Statistics.h +++ b/src/best/Statistics.h @@ -19,24 +19,25 @@ namespace platform { long double pvalue; bool reject; }; - struct HolmLine { + struct PostHocLine { std::string model; long double pvalue; double rank; WTL wtl; bool reject; }; - struct HolmResult { + struct PostHocResult { std::string model; - std::vector holmLines; + std::vector postHocLines; }; class Statistics { public: Statistics(const std::vector& models, const std::vector& datasets, const json& data, double significance = 0.05, bool output = true); bool friedmanTest(); - void postHocHolmTest(bool friedmanResult, bool tex=false); + void postHocHolmTest(); + void postHocTestReport(const std::string& kind, const std::string& metric, bool friedmanResult, bool tex); FriedmanResult& getFriedmanResult(); - HolmResult& getHolmResult(); + PostHocResult& getPostHocResult(); std::map>& getRanks(); private: void fit(); @@ -53,10 +54,11 @@ namespace platform { int controlIdx = 0; std::map wtl; std::map ranks; + std::vector> postHocData; int maxModelName = 0; int maxDatasetName = 0; FriedmanResult friedmanResult; - HolmResult holmResult; + PostHocResult postHocResult; std::map> ranksModels; }; }