Refactor postHoc

CMakeLists.txt

@@ -7,8 +7,6 @@ project(Platform
   LANGUAGES CXX
 )
 
-
-
 # Global CMake variables
 # ----------------------
 set(CMAKE_CXX_STANDARD 20)

src/CMakeLists.txt

@@ -13,7 +13,7 @@ include_directories(
 # b_best
 add_executable(
     b_best commands/b_best.cpp best/Statistics.cpp
-    best/BestResultsExcel.cpp best/BestResultsTex.cpp best/BestResultsMd.cpp best/BestResults.cpp
+    best/BestResultsExcel.cpp best/BestResultsTex.cpp best/BestResultsMd.cpp best/BestResults.cpp best/DeLong.cpp
     common/Datasets.cpp common/Dataset.cpp common/Discretization.cpp 
     main/Models.cpp main/Scores.cpp
     reports/ReportExcel.cpp reports/ReportBase.cpp reports/ExcelFile.cpp

src/best/BestResults.cpp

@@ -222,7 +222,7 @@ namespace platform {
         std::cout << oss.str();
         std::cout << std::string(oss.str().size() - 8, '-') << std::endl;
         std::cout << Colors::GREEN() << " #  " << std::setw(maxDatasetName + 1) << std::left << std::string("Dataset");
-        auto bestResultsTex = BestResultsTex();
+        auto bestResultsTex = BestResultsTex(score);
         auto bestResultsMd = BestResultsMd();
         if (tex) {
             bestResultsTex.results_header(models, table.at("dateTable").get<std::string>(), index);
@@ -339,7 +339,8 @@ namespace platform {
         if (friedman) {
             Statistics stats(models, datasets, table, significance);
             auto result = stats.friedmanTest();
-            stats.postHocHolmTest(result, tex);
+            stats.postHocHolmTest();
+            stats.postHocTestReport("Holm", score, result, tex);
             ranksModels = stats.getRanks();
         }
         if (tex) {

src/best/BestResultsExcel.cpp

@@ -243,9 +243,10 @@ namespace platform {
         row = 2;
         Statistics stats(models, datasets, table, significance, false);
         auto result = stats.friedmanTest();
-        stats.postHocHolmTest(result);
+        stats.postHocHolmTest();
+        // stats.postHocTestReport("Holm", result, false);
         auto friedmanResult = stats.getFriedmanResult();
-        auto holmResult = stats.getHolmResult();
+        auto postHocResult = stats.getPostHocResult();
         worksheet_merge_range(worksheet, row, 0, row, 7, "Null hypothesis: H0 'There is no significant differences between all the classifiers.'", styles["headerSmall"]);
         row += 2;
         writeString(row, 1, "Friedman Q", "bodyHeader");
@@ -264,7 +265,7 @@ namespace platform {
         row += 2;
         worksheet_merge_range(worksheet, row, 0, row, 7, "Null hypothesis: H0 'There is no significant differences between the control model and the other models.'", styles["headerSmall"]);
         row += 2;
-        std::string controlModel = "Control Model: " + holmResult.model;
+        std::string controlModel = "Control Model: " + postHocResult.model;
         worksheet_merge_range(worksheet, row, 1, row, 7, controlModel.c_str(), styles["bodyHeader_odd"]);
         row++;
         writeString(row, 1, "Model", "bodyHeader");
@@ -276,7 +277,7 @@ namespace platform {
         writeString(row, 7, "Reject H0", "bodyHeader");
         row++;
         bool first = true;
-        for (const auto& item : holmResult.holmLines) {
+        for (const auto& item : postHocResult.postHocLines) {
             writeString(row, 1, item.model, "text");
             if (first) {
                 // Control model info

src/best/BestResultsMd.cpp

@@ -75,7 +75,7 @@ namespace platform {
 
         handler.close();
     }
-    void BestResultsMd::holm_test(struct HolmResult& holmResult, const std::string& date)
+    void BestResultsMd::postHoc_test(struct PostHocResult& postHocResult, const std::string& kind, const std::string& date)
     {
         auto file_name = Paths::tex() + Paths::md_post_hoc();
         openMdFile(file_name);
@@ -84,12 +84,12 @@ namespace platform {
         handler << std::endl;
         handler << "  Post-hoc handler test" << std::endl;
         handler << "-->" << std::endl;
-        handler << "Post-hoc Holm test: H<sub>0</sub>: There is no significant differences between the control model and the other models." << std::endl << std::endl;
+        handler << "Post-hoc " << kind << " test: H<sub>0</sub>: There is no significant differences between the control model and the other models." << std::endl << std::endl;
         handler << "| classifier | pvalue | rank | win | tie | loss | H<sub>0</sub> |" << std::endl;
         handler << "| :-- | --: | --: | --:| --: | --: | :--: |" << std::endl;
-        for (auto const& line : holmResult.holmLines) {
+        for (auto const& line : postHocResult.postHocLines) {
             auto textStatus = !line.reject ? "**" : " ";
-            if (line.model == holmResult.model) {
+            if (line.model == postHocResult.model) {
                 handler << "| " << line.model << " | - | " << std::fixed << std::setprecision(2) << line.rank << " | - | - | - |" << std::endl;
             } else {
                 handler << "| " << line.model << " | " << textStatus << std::scientific << std::setprecision(4) << line.pvalue << textStatus << " |";

src/best/BestResultsMd.h

@@ -14,7 +14,7 @@ namespace platform {
         void results_header(const std::vector<std::string>& models, const std::string& date);
         void results_body(const std::vector<std::string>& datasets, json& table);
         void results_footer(const std::map<std::string, std::vector<double>>& totals, const std::string& best_model);
-        void holm_test(struct HolmResult& holmResult, const std::string& date);
+        void postHoc_test(struct PostHocResult& postHocResult, const std::string& kind, const std::string& date);
     private:
         void openMdFile(const std::string& name);
         std::ofstream handler;

src/best/BestResultsTex.cpp

@@ -27,8 +27,10 @@ namespace platform {
         handler << "\\tiny " << std::endl;
         handler << "\\renewcommand{\\arraystretch }{1.2} " << std::endl;
         handler << "\\renewcommand{\\tabcolsep }{0.07cm} " << std::endl;
-        handler << "\\caption{Accuracy results(mean $\\pm$ std) for all the algorithms and datasets} " << std::endl;
+        auto umetric = metric;
-        handler << "\\label{tab:results_accuracy}" << std::endl;
+        umetric[0] = toupper(umetric[0]);
+        handler << "\\caption{" << umetric << " results(mean $\\pm$ std) for all the algorithms and datasets} " << std::endl;
+        handler << "\\label{tab:results_" << metric << "}" << std::endl;
         std::string header_dataset_name = index ? "r" : "l";
         handler << "\\begin{tabular} {{" << header_dataset_name << std::string(models.size(), 'c').c_str() << "}}" << std::endl;
         handler << "\\hline " << std::endl;
@@ -87,25 +89,25 @@ namespace platform {
         handler << "\\end{table}" << std::endl;
         handler.close();
     }
-    void BestResultsTex::holm_test(struct HolmResult& holmResult, const std::string& date)
+    void BestResultsTex::postHoc_test(struct PostHocResult& postHocResult, const std::string& kind, const std::string& date)
     {
         auto file_name = Paths::tex() + Paths::tex_post_hoc();
         openTexFile(file_name);
         handler << "%% This file has been generated by the platform program" << std::endl;
         handler << "%% Date: " << date.c_str() << std::endl;
         handler << "%%" << std::endl;
-        handler << "%% Post-hoc handler test" << std::endl;
+        handler << "%% Post-hoc " << kind << " test" << std::endl;
         handler << "%%" << std::endl;
         handler << "\\begin{table}[htbp]" << std::endl;
         handler << "\\centering" << std::endl;
-        handler << "\\caption{Results of the post-hoc test for the mean accuracy of the algorithms.}\\label{tab:tests}" << std::endl;
+        handler << "\\caption{Results of the post-hoc " << kind << " test for the mean " << metric << " of the algorithms.}\\label{ tab:tests }" << std::endl;
         handler << "\\begin{tabular}{lrrrrr}" << std::endl;
         handler << "\\hline" << std::endl;
         handler << "classifier & pvalue & rank & win & tie & loss\\\\" << std::endl;
         handler << "\\hline" << std::endl;
-        for (auto const& line : holmResult.holmLines) {
+        for (auto const& line : postHocResult.postHocLines) {
             auto textStatus = !line.reject ? "\\bf " : " ";
-            if (line.model == holmResult.model) {
+            if (line.model == postHocResult.model) {
                 handler << line.model << " & - & " << std::fixed << std::setprecision(2) << line.rank << " & - & - & - \\\\" << std::endl;
             } else {
                 handler << line.model << " & " << textStatus << std::scientific << std::setprecision(4) << line.pvalue << " & ";

src/best/BestResultsTex.h

@@ -9,13 +9,14 @@ namespace platform {
     using json = nlohmann::ordered_json;
     class BestResultsTex {
     public:
-        BestResultsTex(bool dataset_name = true) : dataset_name(dataset_name) {};
+        BestResultsTex(const std::string metric_, bool dataset_name = true) : metric{ metric_ }, dataset_name{ dataset_name } {};
         ~BestResultsTex() = default;
         void results_header(const std::vector<std::string>& models, const std::string& date, bool index);
         void results_body(const std::vector<std::string>& datasets, json& table, bool index);
         void results_footer(const std::map<std::string, std::vector<double>>& totals, const std::string& best_model);
-        void holm_test(struct HolmResult& holmResult, const std::string& date);
+        void postHoc_test(struct PostHocResult& postHocResult, const std::string& kind, const std::string& date);
     private:
+        std::string metric;
         bool dataset_name;
         void openTexFile(const std::string& name);
         std::ofstream handler;

src/best/DeLong.cpp

@@ -0,0 +1,45 @@
+// DeLong.cpp
+// Integración del test de DeLong con la clase RocAuc y Statistics
+// Basado en: X. Sun and W. Xu, "Fast Implementation of DeLong’s Algorithm for Comparing the Areas Under Correlated Receiver Operating Characteristic Curves," (2014), y algoritmos inspirados en sklearn/pROC
+
+#include "DeLong.h"
+#include <vector>
+#include <cmath>
+#include <algorithm>
+#include <numeric>
+#include <stdexcept>
+#include <cassert>
+
+namespace platform {
+
+    DeLong::DeLongResult DeLong::compare(const std::vector<double>& aucs_model1,
+        const std::vector<double>& aucs_model2)
+    {
+        if (aucs_model1.size() != aucs_model2.size()) {
+            throw std::invalid_argument("AUC lists must have the same size");
+        }
+
+        size_t N = aucs_model1.size();
+        if (N < 2) {
+            throw std::invalid_argument("At least two AUC values are required");
+        }
+
+        std::vector<double> diffs(N);
+        for (size_t i = 0; i < N; ++i) {
+            diffs[i] = aucs_model1[i] - aucs_model2[i];
+        }
+
+        double mean_diff = std::accumulate(diffs.begin(), diffs.end(), 0.0) / N;
+        double var = 0.0;
+        for (size_t i = 0; i < N; ++i) {
+            var += (diffs[i] - mean_diff) * (diffs[i] - mean_diff);
+        }
+        var /= (N * (N - 1));
+        if (var <= 0.0) var = 1e-10;
+
+        double z = mean_diff / std::sqrt(var);
+        double p = 2.0 * (1.0 - std::erfc(std::abs(z) / std::sqrt(2.0)) / 2.0);
+        return { mean_diff, z, p };
+    }
+
+}

src/best/DeLong.h

@@ -0,0 +1,24 @@
+#ifndef DELONG_H
+#define DELONG_H
+/* ********************************************************************************************************************
+/* Integración del test de DeLong con la clase RocAuc y Statistics
+/* Basado en: X. Sun and W. Xu, "Fast Implementation of DeLong’s Algorithm for Comparing the Areas Under Correlated
+/* Receiver Operating Characteristic Curves," (2014), y algoritmos inspirados en sklearn/pROC
+/* ********************************************************************************************************************/
+#include <vector>
+
+namespace platform {
+    class DeLong {
+    public:
+        struct DeLongResult {
+            double auc_diff;
+            double z_stat;
+            double p_value;
+        };
+        // Compara dos vectores de AUCs por dataset y devuelve diferencia media,
+        // estadístico z y p-valor usando un test de rangos (DeLong simplificado)
+        static DeLongResult compare(const std::vector<double>& aucs_model1,
+            const std::vector<double>& aucs_model2);
+    };
+}
+#endif // DELONG_H

src/best/Statistics.cpp

@@ -7,6 +7,7 @@
 #include "BestResultsTex.h"
 #include "BestResultsMd.h"
 #include "Statistics.h"
+#include "DeLong.h"
 
 
 namespace platform {
@@ -114,8 +115,7 @@ namespace platform {
             }
         }
     }
-
+    void Statistics::postHocHolmTest()
-    void Statistics::postHocHolmTest(bool friedmanResult, bool tex)
     {
         if (!fitted) {
             fit();
@@ -137,27 +137,33 @@ namespace platform {
             stats[i] = p_value;
         }
         // Sort the models by p-value
-        std::vector<std::pair<int, double>> statsOrder;
         for (const auto& stat : stats) {
-            statsOrder.push_back({ stat.first, stat.second });
+            postHocData.push_back({ stat.first, stat.second });
         }
-        std::sort(statsOrder.begin(), statsOrder.end(), [](const std::pair<int, double>& a, const std::pair<int, double>& b) {
+        std::sort(postHocData.begin(), postHocData.end(), [](const std::pair<int, double>& a, const std::pair<int, double>& b) {
             return a.second < b.second;
             });
 
         // Holm adjustment
-        for (int i = 0; i < statsOrder.size(); ++i) {
+        for (int i = 0; i < postHocData.size(); ++i) {
-            auto item = statsOrder.at(i);
+            auto item = postHocData.at(i);
-            double before = i == 0 ? 0.0 : statsOrder.at(i - 1).second;
+            double before = i == 0 ? 0.0 : postHocData.at(i - 1).second;
             double p_value = std::min((double)1.0, item.second * (nModels - i));
             p_value = std::max(before, p_value);
-            statsOrder[i] = { item.first, p_value };
+            postHocData[i] = { item.first, p_value };
         }
-        holmResult.model = models.at(controlIdx);
+        postHocResult.model = models.at(controlIdx);
+    }
+
+    void Statistics::postHocTestReport(const std::string& kind, const std::string& metric, bool friedmanResult, bool tex)
+    {
+
+        std::stringstream oss;
+        postHocResult.model = models.at(controlIdx);
         auto color = friedmanResult ? Colors::CYAN() : Colors::YELLOW();
         oss << color;
         oss << "  *************************************************************************************************************" << std::endl;
-        oss << "  Post-hoc Holm test: H0: 'There is no significant differences between the control model and the other models.'" << std::endl;
+        oss << "  Post-hoc " << kind << " test: H0: 'There is no significant differences between the control model and the other models.'" << std::endl;
         oss << "  Control model: " << models.at(controlIdx) << std::endl;
         oss << "  " << std::left << std::setw(maxModelName) << std::string("Model") << " p-value      rank      win tie loss Status" << std::endl;
         oss << "  " << std::string(maxModelName, '=') << " ============ ========= === === ==== =============" << std::endl;
@@ -175,12 +181,12 @@ namespace platform {
         for (const auto& item : ranksOrder) {
             auto idx = distance(models.begin(), find(models.begin(), models.end(), item.first));
             double pvalue = 0.0;
-            for (const auto& stat : statsOrder) {
+            for (const auto& stat : postHocData) {
                 if (stat.first == idx) {
                     pvalue = stat.second;
                 }
             }
-            holmResult.holmLines.push_back({ item.first, pvalue, item.second, wtl.at(idx), pvalue < significance });
+            postHocResult.postHocLines.push_back({ item.first, pvalue, item.second, wtl.at(idx), pvalue < significance });
             if (item.first == models.at(controlIdx)) {
                 continue;
             }
@@ -198,12 +204,77 @@ namespace platform {
             std::cout << oss.str();
         }
         if (tex) {
-            BestResultsTex bestResultsTex;
+            BestResultsTex bestResultsTex(metric);
             BestResultsMd bestResultsMd;
-            bestResultsTex.holm_test(holmResult, get_date() + " " + get_time());
+            bestResultsTex.postHoc_test(postHocResult, kind, get_date() + " " + get_time());
-            bestResultsMd.holm_test(holmResult, get_date() + " " + get_time());
+            bestResultsMd.postHoc_test(postHocResult, kind, get_date() + " " + get_time());
         }
     }
+    // void Statistics::postHocDeLongTest(const std::vector<std::vector<int>>& y_trues,
+    //     const std::vector<std::vector<std::vector<double>>>& y_probas,
+    //     bool tex)
+    // {
+    //     std::map<int, double> pvalues;
+    //     postHocResult.model = models.at(controlIdx);
+    //     postHocResult.postHocLines.clear();
+
+    //     for (size_t i = 0; i < models.size(); ++i) {
+    //         if ((int)i == controlIdx) continue;
+    //         double acc_p = 0.0;
+    //         int valid = 0;
+    //         for (size_t d = 0; d < y_trues.size(); ++d) {
+    //             try {
+    //                 auto result = compareModelsWithDeLong(y_probas[controlIdx][d], y_probas[i][d], y_trues[d]);
+    //                 acc_p += result.p_value;
+    //                 ++valid;
+    //             }
+    //             catch (...) {}
+    //         }
+    //         if (valid > 0) {
+    //             pvalues[i] = acc_p / valid;
+    //         }
+    //     }
+
+    //     std::vector<std::pair<int, double>> sorted_pvalues(pvalues.begin(), pvalues.end());
+    //     std::sort(sorted_pvalues.begin(), sorted_pvalues.end(), [](const auto& a, const auto& b) {
+    //         return a.second < b.second;
+    //         });
+
+    //     std::stringstream oss;
+    //     oss << "\n*************************************************************************************************************\n";
+    //     oss << "  Post-hoc DeLong-Holm test: H0: 'No significant differences in AUC with control model.'\n";
+    //     oss << "  Control model: " << models[controlIdx] << "\n";
+    //     oss << "  " << std::left << std::setw(maxModelName) << std::string("Model") << " p-value      Adjusted    Result\n";
+    //     oss << "  " << std::string(maxModelName, '=') << " ============ ========== =============\n";
+
+    //     double prev = 0.0;
+    //     for (size_t i = 0; i < sorted_pvalues.size(); ++i) {
+    //         int idx = sorted_pvalues[i].first;
+    //         double raw = sorted_pvalues[i].second;
+    //         double adj = std::min(1.0, raw * (models.size() - i - 1));
+    //         adj = std::max(prev, adj);
+    //         prev = adj;
+    //         bool reject = adj < significance;
+
+    //         postHocResult.postHocLines.push_back({ models[idx], adj, 0.0f, {}, reject });
+
+    //         auto color = reject ? Colors::MAGENTA() : Colors::GREEN();
+    //         auto status = reject ? Symbols::cross : Symbols::check_mark;
+    //         auto textStatus = reject ? " rejected H0" : " accepted H0";
+    //         oss << "  " << color << std::left << std::setw(maxModelName) << models[idx] << " ";
+    //         oss << std::setprecision(6) << std::scientific << raw << " ";
+    //         oss << std::setprecision(6) << std::scientific << adj << " " << status << textStatus << "\n";
+    //     }
+    //     oss << Colors::CYAN() << "  *************************************************************************************************************\n";
+    //     oss << Colors::RESET();
+    //     if (output) std::cout << oss.str();
+    //     if (tex) {
+    //         BestResultsTex bestResultsTex;
+    //         BestResultsMd bestResultsMd;
+    //         bestResultsTex.holm_test(postHocResult, get_date() + " " + get_time());
+    //         bestResultsMd.holm_test(postHocResult, get_date() + " " + get_time());
+    //     }
+    // }
     bool Statistics::friedmanTest()
     {
         if (!fitted) {
@@ -249,9 +320,9 @@ namespace platform {
     {
         return friedmanResult;
     }
-    HolmResult& Statistics::getHolmResult()
+    PostHocResult& Statistics::getPostHocResult()
     {
-        return holmResult;
+        return postHocResult;
     }
     std::map<std::string, std::map<std::string, float>>& Statistics::getRanks()
     {

src/best/Statistics.h

@@ -19,24 +19,25 @@ namespace platform {
         long double pvalue;
         bool reject;
     };
-    struct HolmLine {
+    struct PostHocLine {
         std::string model;
         long double pvalue;
         double rank;
         WTL wtl;
         bool reject;
     };
-    struct HolmResult {
+    struct PostHocResult {
         std::string model;
-        std::vector<HolmLine> holmLines;
+        std::vector<PostHocLine> postHocLines;
     };
     class Statistics {
     public:
         Statistics(const std::vector<std::string>& models, const std::vector<std::string>& datasets, const json& data, double significance = 0.05, bool output = true);
         bool friedmanTest();
-        void postHocHolmTest(bool friedmanResult, bool tex=false);
+        void postHocHolmTest();
+        void postHocTestReport(const std::string& kind, const std::string& metric, bool friedmanResult, bool tex);
         FriedmanResult& getFriedmanResult();
-        HolmResult& getHolmResult();
+        PostHocResult& getPostHocResult();
         std::map<std::string, std::map<std::string, float>>& getRanks();
     private:
         void fit();
@@ -53,10 +54,11 @@ namespace platform {
         int controlIdx = 0;
         std::map<int, WTL> wtl;
         std::map<std::string, float> ranks;
+        std::vector<std::pair<int, double>> postHocData;
         int maxModelName = 0;
         int maxDatasetName = 0;
         FriedmanResult friedmanResult;
-        HolmResult holmResult;
+        PostHocResult postHocResult;
         std::map<std::string, std::map<std::string, float>> ranksModels;
     };
 }