rmontanana/Platform
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor postHoc

Browse Source
This commit is contained in:
Ricardo Montañana Gómez
2025-05-17 18:12:57 +02:00
parent f5107abea7
commit 70d8022926
12 changed files with 192 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@@ -7,8 +7,6 @@ project(Platform
LANGUAGES CXX
)
# Global CMake variables
# ----------------------
set(CMAKE_CXX_STANDARD 20)

2
src/CMakeLists.txt
View File

@@ -13,7 +13,7 @@ include_directories(
# b_best
add_executable(
b_best commands/b_best.cpp best/Statistics.cpp
best/BestResultsExcel.cpp best/BestResultsTex.cpp best/BestResultsMd.cpp best/BestResults.cpp
best/BestResultsExcel.cpp best/BestResultsTex.cpp best/BestResultsMd.cpp best/BestResults.cpp best/DeLong.cpp
common/Datasets.cpp common/Dataset.cpp common/Discretization.cpp
main/Models.cpp main/Scores.cpp
reports/ReportExcel.cpp reports/ReportBase.cpp reports/ExcelFile.cpp

5
src/best/BestResults.cpp
View File

@@ -222,7 +222,7 @@ namespace platform {
std::cout << oss.str();
std::cout << std::string(oss.str().size() - 8, '-') << std::endl;
std::cout << Colors::GREEN() << " # " << std::setw(maxDatasetName + 1) << std::left << std::string("Dataset");
auto bestResultsTex = BestResultsTex();
auto bestResultsTex = BestResultsTex(score);
auto bestResultsMd = BestResultsMd();
if (tex) {
bestResultsTex.results_header(models, table.at("dateTable").get<std::string>(), index);
@@ -339,7 +339,8 @@ namespace platform {
if (friedman) {
Statistics stats(models, datasets, table, significance);
auto result = stats.friedmanTest();
stats.postHocHolmTest(result, tex);
stats.postHocHolmTest();
stats.postHocTestReport("Holm", score, result, tex);
ranksModels = stats.getRanks();
}
if (tex) {

9
src/best/BestResultsExcel.cpp
View File

@@ -243,9 +243,10 @@ namespace platform {
row = 2;
Statistics stats(models, datasets, table, significance, false);
auto result = stats.friedmanTest();
stats.postHocHolmTest(result);
stats.postHocHolmTest();
// stats.postHocTestReport("Holm", result, false);
auto friedmanResult = stats.getFriedmanResult();
auto holmResult = stats.getHolmResult();
auto postHocResult = stats.getPostHocResult();
worksheet_merge_range(worksheet, row, 0, row, 7, "Null hypothesis: H0 'There is no significant differences between all the classifiers.'", styles["headerSmall"]);
row += 2;
writeString(row, 1, "Friedman Q", "bodyHeader");
@@ -264,7 +265,7 @@ namespace platform {
row += 2;
worksheet_merge_range(worksheet, row, 0, row, 7, "Null hypothesis: H0 'There is no significant differences between the control model and the other models.'", styles["headerSmall"]);
row += 2;
std::string controlModel = "Control Model: " + holmResult.model;
std::string controlModel = "Control Model: " + postHocResult.model;
worksheet_merge_range(worksheet, row, 1, row, 7, controlModel.c_str(), styles["bodyHeader_odd"]);
row++;
writeString(row, 1, "Model", "bodyHeader");
@@ -276,7 +277,7 @@ namespace platform {
writeString(row, 7, "Reject H0", "bodyHeader");
row++;
bool first = true;
for (const auto& item : holmResult.holmLines) {
for (const auto& item : postHocResult.postHocLines) {
writeString(row, 1, item.model, "text");
if (first) {
// Control model info

8
src/best/BestResultsMd.cpp
View File

@@ -75,7 +75,7 @@ namespace platform {
handler.close();
}
void BestResultsMd::holm_test(struct HolmResult& holmResult, const std::string& date)
void BestResultsMd::postHoc_test(struct PostHocResult& postHocResult, const std::string& kind, const std::string& date)
{
auto file_name = Paths::tex() + Paths::md_post_hoc();
openMdFile(file_name);
@@ -84,12 +84,12 @@ namespace platform {
handler << std::endl;
handler << " Post-hoc handler test" << std::endl;
handler << "-->" << std::endl;
handler << "Post-hoc Holm test: H<sub>0</sub>: There is no significant differences between the control model and the other models." << std::endl << std::endl;
handler << "Post-hoc " << kind << " test: H<sub>0</sub>: There is no significant differences between the control model and the other models." << std::endl << std::endl;
handler << "| classifier | pvalue | rank | win | tie | loss | H<sub>0</sub> |" << std::endl;
handler << "| :-- | --: | --: | --:| --: | --: | :--: |" << std::endl;
for (auto const& line : holmResult.holmLines) {
for (auto const& line : postHocResult.postHocLines) {
auto textStatus = !line.reject ? "**" : " ";
if (line.model == holmResult.model) {
if (line.model == postHocResult.model) {
handler << "| " << line.model << " | - | " << std::fixed << std::setprecision(2) << line.rank << " | - | - | - |" << std::endl;
} else {
handler << "| " << line.model << " | " << textStatus << std::scientific << std::setprecision(4) << line.pvalue << textStatus << " |";

2
src/best/BestResultsMd.h
View File

@@ -14,7 +14,7 @@ namespace platform {
void results_header(const std::vector<std::string>& models, const std::string& date);
void results_body(const std::vector<std::string>& datasets, json& table);
void results_footer(const std::map<std::string, std::vector<double>>& totals, const std::string& best_model);
void holm_test(struct HolmResult& holmResult, const std::string& date);
void postHoc_test(struct PostHocResult& postHocResult, const std::string& kind, const std::string& date);
private:
void openMdFile(const std::string& name);
std::ofstream handler;

16
src/best/BestResultsTex.cpp
View File

@@ -27,8 +27,10 @@ namespace platform {
handler << "\\tiny " << std::endl;
handler << "\\renewcommand{\\arraystretch }{1.2} " << std::endl;
handler << "\\renewcommand{\\tabcolsep }{0.07cm} " << std::endl;
handler << "\\caption{Accuracy results(mean $\\pm$ std) for all the algorithms and datasets} " << std::endl;
handler << "\\label{tab:results_accuracy}" << std::endl;
auto umetric = metric;
umetric[0] = toupper(umetric[0]);
handler << "\\caption{" << umetric << " results(mean $\\pm$ std) for all the algorithms and datasets} " << std::endl;
handler << "\\label{tab:results_" << metric << "}" << std::endl;
std::string header_dataset_name = index ? "r" : "l";
handler << "\\begin{tabular} {{" << header_dataset_name << std::string(models.size(), 'c').c_str() << "}}" << std::endl;
handler << "\\hline " << std::endl;
@@ -87,25 +89,25 @@ namespace platform {
handler << "\\end{table}" << std::endl;
handler.close();
}
void BestResultsTex::holm_test(struct HolmResult& holmResult, const std::string& date)
void BestResultsTex::postHoc_test(struct PostHocResult& postHocResult, const std::string& kind, const std::string& date)
{
auto file_name = Paths::tex() + Paths::tex_post_hoc();
openTexFile(file_name);
handler << "%% This file has been generated by the platform program" << std::endl;
handler << "%% Date: " << date.c_str() << std::endl;
handler << "%%" << std::endl;
handler << "%% Post-hoc handler test" << std::endl;
handler << "%% Post-hoc " << kind << " test" << std::endl;
handler << "%%" << std::endl;
handler << "\\begin{table}[htbp]" << std::endl;
handler << "\\centering" << std::endl;
handler << "\\caption{Results of the post-hoc test for the mean accuracy of the algorithms.}\\label{tab:tests}" << std::endl;
handler << "\\caption{Results of the post-hoc " << kind << " test for the mean " << metric << " of the algorithms.}\\label{ tab:tests }" << std::endl;
handler << "\\begin{tabular}{lrrrrr}" << std::endl;
handler << "\\hline" << std::endl;
handler << "classifier & pvalue & rank & win & tie & loss\\\\" << std::endl;
handler << "\\hline" << std::endl;
for (auto const& line : holmResult.holmLines) {
for (auto const& line : postHocResult.postHocLines) {
auto textStatus = !line.reject ? "\\bf " : " ";
if (line.model == holmResult.model) {
if (line.model == postHocResult.model) {
handler << line.model << " & - & " << std::fixed << std::setprecision(2) << line.rank << " & - & - & - \\\\" << std::endl;
} else {
handler << line.model << " & " << textStatus << std::scientific << std::setprecision(4) << line.pvalue << " & ";

5
src/best/BestResultsTex.h
View File

@@ -9,13 +9,14 @@ namespace platform {
using json = nlohmann::ordered_json;
class BestResultsTex {
public:
BestResultsTex(bool dataset_name = true) : dataset_name(dataset_name) {};
BestResultsTex(const std::string metric_, bool dataset_name = true) : metric{ metric_ }, dataset_name{ dataset_name } {};
~BestResultsTex() = default;
void results_header(const std::vector<std::string>& models, const std::string& date, bool index);
void results_body(const std::vector<std::string>& datasets, json& table, bool index);
void results_footer(const std::map<std::string, std::vector<double>>& totals, const std::string& best_model);
void holm_test(struct HolmResult& holmResult, const std::string& date);
void postHoc_test(struct PostHocResult& postHocResult, const std::string& kind, const std::string& date);
private:
std::string metric;
bool dataset_name;
void openTexFile(const std::string& name);
std::ofstream handler;

45
src/best/DeLong.cpp Normal file
View File

@@ -0,0 +1,45 @@
// DeLong.cpp
// Integración del test de DeLong con la clase RocAuc y Statistics
// Basado en: X. Sun and W. Xu, "Fast Implementation of DeLong’s Algorithm for Comparing the Areas Under Correlated Receiver Operating Characteristic Curves," (2014), y algoritmos inspirados en sklearn/pROC
#include "DeLong.h"
#include <vector>
#include <cmath>
#include <algorithm>
#include <numeric>
#include <stdexcept>
#include <cassert>
namespace platform {
DeLong::DeLongResult DeLong::compare(const std::vector<double>& aucs_model1,
const std::vector<double>& aucs_model2)
{
if (aucs_model1.size() != aucs_model2.size()) {
throw std::invalid_argument("AUC lists must have the same size");
}
size_t N = aucs_model1.size();
if (N < 2) {
throw std::invalid_argument("At least two AUC values are required");
}
std::vector<double> diffs(N);
for (size_t i = 0; i < N; ++i) {
diffs[i] = aucs_model1[i] - aucs_model2[i];
}
double mean_diff = std::accumulate(diffs.begin(), diffs.end(), 0.0) / N;
double var = 0.0;
for (size_t i = 0; i < N; ++i) {
var += (diffs[i] - mean_diff) * (diffs[i] - mean_diff);
}
var /= (N * (N - 1));
if (var <= 0.0) var = 1e-10;
double z = mean_diff / std::sqrt(var);
double p = 2.0 * (1.0 - std::erfc(std::abs(z) / std::sqrt(2.0)) / 2.0);
return { mean_diff, z, p };
}
}

24
src/best/DeLong.h Normal file
View File

@@ -0,0 +1,24 @@
#ifndef DELONG_H
#define DELONG_H
/* ********************************************************************************************************************
/* Integración del test de DeLong con la clase RocAuc y Statistics
/* Basado en: X. Sun and W. Xu, "Fast Implementation of DeLong’s Algorithm for Comparing the Areas Under Correlated
/* Receiver Operating Characteristic Curves," (2014), y algoritmos inspirados en sklearn/pROC
/* ********************************************************************************************************************/
#include <vector>
namespace platform {
class DeLong {
public:
struct DeLongResult {
double auc_diff;
double z_stat;
double p_value;
};
// Compara dos vectores de AUCs por dataset y devuelve diferencia media,
// estadístico z y p-valor usando un test de rangos (DeLong simplificado)
static DeLongResult compare(const std::vector<double>& aucs_model1,
const std::vector<double>& aucs_model2);
};
}
#endif // DELONG_H

107
src/best/Statistics.cpp
View File

@@ -7,6 +7,7 @@
#include "BestResultsTex.h"
#include "BestResultsMd.h"
#include "Statistics.h"
#include "DeLong.h"
namespace platform {
@@ -114,8 +115,7 @@ namespace platform {
}
}
}
void Statistics::postHocHolmTest(bool friedmanResult, bool tex)
void Statistics::postHocHolmTest()
{
if (!fitted) {
fit();
@@ -137,27 +137,33 @@ namespace platform {
stats[i] = p_value;
}
// Sort the models by p-value
std::vector<std::pair<int, double>> statsOrder;
for (const auto& stat : stats) {
statsOrder.push_back({ stat.first, stat.second });
postHocData.push_back({ stat.first, stat.second });
}
std::sort(statsOrder.begin(), statsOrder.end(), [](const std::pair<int, double>& a, const std::pair<int, double>& b) {
std::sort(postHocData.begin(), postHocData.end(), [](const std::pair<int, double>& a, const std::pair<int, double>& b) {
return a.second < b.second;
});
// Holm adjustment
for (int i = 0; i < statsOrder.size(); ++i) {
auto item = statsOrder.at(i);
double before = i == 0 ? 0.0 : statsOrder.at(i - 1).second;
for (int i = 0; i < postHocData.size(); ++i) {
auto item = postHocData.at(i);
double before = i == 0 ? 0.0 : postHocData.at(i - 1).second;
double p_value = std::min((double)1.0, item.second * (nModels - i));
p_value = std::max(before, p_value);
statsOrder[i] = { item.first, p_value };
postHocData[i] = { item.first, p_value };
}
holmResult.model = models.at(controlIdx);
postHocResult.model = models.at(controlIdx);
}
void Statistics::postHocTestReport(const std::string& kind, const std::string& metric, bool friedmanResult, bool tex)
{
std::stringstream oss;
postHocResult.model = models.at(controlIdx);
auto color = friedmanResult ? Colors::CYAN() : Colors::YELLOW();
oss << color;
oss << " *************************************************************************************************************" << std::endl;
oss << " Post-hoc Holm test: H0: 'There is no significant differences between the control model and the other models.'" << std::endl;
oss << " Post-hoc " << kind << " test: H0: 'There is no significant differences between the control model and the other models.'" << std::endl;
oss << " Control model: " << models.at(controlIdx) << std::endl;
oss << " " << std::left << std::setw(maxModelName) << std::string("Model") << " p-value rank win tie loss Status" << std::endl;
oss << " " << std::string(maxModelName, '=') << " ============ ========= === === ==== =============" << std::endl;
@@ -175,12 +181,12 @@ namespace platform {
for (const auto& item : ranksOrder) {
auto idx = distance(models.begin(), find(models.begin(), models.end(), item.first));
double pvalue = 0.0;
for (const auto& stat : statsOrder) {
for (const auto& stat : postHocData) {
if (stat.first == idx) {
pvalue = stat.second;
}
}
holmResult.holmLines.push_back({ item.first, pvalue, item.second, wtl.at(idx), pvalue < significance });
postHocResult.postHocLines.push_back({ item.first, pvalue, item.second, wtl.at(idx), pvalue < significance });
if (item.first == models.at(controlIdx)) {
continue;
}
@@ -198,12 +204,77 @@ namespace platform {
std::cout << oss.str();
}
if (tex) {
BestResultsTex bestResultsTex;
BestResultsTex bestResultsTex(metric);
BestResultsMd bestResultsMd;
bestResultsTex.holm_test(holmResult, get_date() + " " + get_time());
bestResultsMd.holm_test(holmResult, get_date() + " " + get_time());
bestResultsTex.postHoc_test(postHocResult, kind, get_date() + " " + get_time());
bestResultsMd.postHoc_test(postHocResult, kind, get_date() + " " + get_time());
}
}
// void Statistics::postHocDeLongTest(const std::vector<std::vector<int>>& y_trues,
// const std::vector<std::vector<std::vector<double>>>& y_probas,
// bool tex)
// {
// std::map<int, double> pvalues;
// postHocResult.model = models.at(controlIdx);
// postHocResult.postHocLines.clear();
// for (size_t i = 0; i < models.size(); ++i) {
// if ((int)i == controlIdx) continue;
// double acc_p = 0.0;
// int valid = 0;
// for (size_t d = 0; d < y_trues.size(); ++d) {
// try {
// auto result = compareModelsWithDeLong(y_probas[controlIdx][d], y_probas[i][d], y_trues[d]);
// acc_p += result.p_value;
// ++valid;
// }
// catch (...) {}
// }
// if (valid > 0) {
// pvalues[i] = acc_p / valid;
// }
// }
// std::vector<std::pair<int, double>> sorted_pvalues(pvalues.begin(), pvalues.end());
// std::sort(sorted_pvalues.begin(), sorted_pvalues.end(), [](const auto& a, const auto& b) {
// return a.second < b.second;
// });
// std::stringstream oss;
// oss << "\n*************************************************************************************************************\n";
// oss << " Post-hoc DeLong-Holm test: H0: 'No significant differences in AUC with control model.'\n";
// oss << " Control model: " << models[controlIdx] << "\n";
// oss << " " << std::left << std::setw(maxModelName) << std::string("Model") << " p-value Adjusted Result\n";
// oss << " " << std::string(maxModelName, '=') << " ============ ========== =============\n";
// double prev = 0.0;
// for (size_t i = 0; i < sorted_pvalues.size(); ++i) {
// int idx = sorted_pvalues[i].first;
// double raw = sorted_pvalues[i].second;
// double adj = std::min(1.0, raw * (models.size() - i - 1));
// adj = std::max(prev, adj);
// prev = adj;
// bool reject = adj < significance;
// postHocResult.postHocLines.push_back({ models[idx], adj, 0.0f, {}, reject });
// auto color = reject ? Colors::MAGENTA() : Colors::GREEN();
// auto status = reject ? Symbols::cross : Symbols::check_mark;
// auto textStatus = reject ? " rejected H0" : " accepted H0";
// oss << " " << color << std::left << std::setw(maxModelName) << models[idx] << " ";
// oss << std::setprecision(6) << std::scientific << raw << " ";
// oss << std::setprecision(6) << std::scientific << adj << " " << status << textStatus << "\n";
// }
// oss << Colors::CYAN() << " *************************************************************************************************************\n";
// oss << Colors::RESET();
// if (output) std::cout << oss.str();
// if (tex) {
// BestResultsTex bestResultsTex;
// BestResultsMd bestResultsMd;
// bestResultsTex.holm_test(postHocResult, get_date() + " " + get_time());
// bestResultsMd.holm_test(postHocResult, get_date() + " " + get_time());
// }
// }
bool Statistics::friedmanTest()
{
if (!fitted) {
@@ -249,9 +320,9 @@ namespace platform {
{
return friedmanResult;
}
HolmResult& Statistics::getHolmResult()
PostHocResult& Statistics::getPostHocResult()
{
return holmResult;
return postHocResult;
}
std::map<std::string, std::map<std::string, float>>& Statistics::getRanks()
{

14
src/best/Statistics.h
View File

@@ -19,24 +19,25 @@ namespace platform {
long double pvalue;
bool reject;
};
struct HolmLine {
struct PostHocLine {
std::string model;
long double pvalue;
double rank;
WTL wtl;
bool reject;
};
struct HolmResult {
struct PostHocResult {
std::string model;
std::vector<HolmLine> holmLines;
std::vector<PostHocLine> postHocLines;
};
class Statistics {
public:
Statistics(const std::vector<std::string>& models, const std::vector<std::string>& datasets, const json& data, double significance = 0.05, bool output = true);
bool friedmanTest();
void postHocHolmTest(bool friedmanResult, bool tex=false);
void postHocHolmTest();
void postHocTestReport(const std::string& kind, const std::string& metric, bool friedmanResult, bool tex);
FriedmanResult& getFriedmanResult();
HolmResult& getHolmResult();
PostHocResult& getPostHocResult();
std::map<std::string, std::map<std::string, float>>& getRanks();
private:
void fit();
@@ -53,10 +54,11 @@ namespace platform {
int controlIdx = 0;
std::map<int, WTL> wtl;
std::map<std::string, float> ranks;
std::vector<std::pair<int, double>> postHocData;
int maxModelName = 0;
int maxDatasetName = 0;
FriedmanResult friedmanResult;
HolmResult holmResult;
PostHocResult postHocResult;
std::map<std::string, std::map<std::string, float>> ranksModels;
};
}