rmontanana/BayesNet
rmontanana/BayesNet
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases 3 Wiki Activity

Complete Excel output for bestResults with Friedman test

Browse Source
This commit is contained in:
Ricardo Montañana Gómez 2023-09-28 18:52:37 +02:00
parent cfcf3c16df
commit 9d3d9cc6c6
Signed by: rmontanana
GPG Key ID: 46064262FD9A7ADE
6 changed files with 139 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/Platform/BestResults.cc
View File

@ -283,6 +283,7 @@ namespace platform {
}
void BestResults::reportAll(bool excel)
{
double significance = 0.05;
auto models = getModels();
// Build the table of results
json table = buildTableResults(models);
@ -295,13 +296,12 @@ namespace platform {
printTableResults(models, table);
// Compute the Friedman test
if (friedman) {
double significance = 0.05;
Statistics stats(models, datasets, table, significance);
auto result = stats.friedmanTest();
stats.postHocHolmTest(result);
}
if (excel) {
BestResultsExcel excel(score, models, datasets, table, friedman);
BestResultsExcel excel(score, models, datasets, table, friedman, significance);
excel.build();
}
}

59
src/Platform/BestResultsExcel.cc
View File

@ -1,10 +1,10 @@
#include <sstream>
#include "BestResultsExcel.h"
#include "Paths.h"
#include <iostream>
#include "Statistics.h"
namespace platform {
BestResultsExcel::BestResultsExcel(string score, vector<string> models, vector<string> datasets, json table, bool friedman) : score(score), models(models), datasets(datasets), table(table), friedman(friedman)
BestResultsExcel::BestResultsExcel(string score, vector<string> models, vector<string> datasets, json table, bool friedman, double significance) : score(score), models(models), datasets(datasets), table(table), friedman(friedman), significance(significance)
{
workbook = workbook_new((Paths::excel() + fileName).c_str());
worksheet = workbook_add_worksheet(workbook, "Best Results");
@ -79,7 +79,60 @@ namespace platform {
void BestResultsExcel::footer()
{
if (friedman) {
worksheet = workbook_add_worksheet(workbook, "Friedman");
vector<int> columns_sizes = { 5, datasetNameSize };
for (int i = 0; i < models.size(); ++i) {
columns_sizes.push_back(modelNameSize);
}
for (int i = 0; i < columns_sizes.size(); ++i) {
worksheet_set_column(worksheet, i, i, columns_sizes.at(i), NULL);
}
worksheet_merge_range(worksheet, 0, 0, 0, 1 + models.size(), "Friedman Test", styles["headerFirst"]);
row = 2;
Statistics stats(models, datasets, table, significance, false);
auto result = stats.friedmanTest();
stats.postHocHolmTest(result);
auto friedmanResult = stats.getFriedmanResult();
auto holmResult = stats.getHolmResult();
worksheet_merge_range(worksheet, row, 0, row, 1 + models.size(), "Null hypothesis: H0 'There is no significant differences between all the classifiers.'", styles["headerSmall"]);
row += 2;
writeString(row, 1, "Friedman Q", "bodyHeader");
writeDouble(row, 2, friedmanResult.statistic, "bodyHeader");
row++;
writeString(row, 1, "Critical χ2 value", "bodyHeader");
writeDouble(row, 2, friedmanResult.criticalValue, "bodyHeader");
row++;
writeString(row, 1, "p-value", "bodyHeader");
writeDouble(row, 2, friedmanResult.pvalue, "bodyHeader");
writeString(row, 3, friedmanResult.reject ? "<" : ">", "bodyHeader");
writeDouble(row, 4, significance, "bodyHeader");
writeString(row, 5, friedmanResult.reject ? "Reject H0" : "Accept H0", "bodyHeader");
row += 3;
worksheet_merge_range(worksheet, row, 0, row, 1 + models.size(), "Holm Test", styles["headerFirst"]);
row += 2;
worksheet_merge_range(worksheet, row, 0, row, 1 + models.size(), "Null hypothesis: H0 'There is no significant differences between the control model and the other models.'", styles["headerSmall"]);
row += 2;
string controlModel = "Control Model: " + holmResult.model;
worksheet_merge_range(worksheet, row, 1, row, 7, controlModel.c_str(), styles["bodyHeader_odd"]);
row++;
writeString(row, 1, "Model", "bodyHeader");
writeString(row, 2, "p-value", "bodyHeader");
writeString(row, 3, "Rank", "bodyHeader");
writeString(row, 4, "Win", "bodyHeader");
writeString(row, 5, "Tie", "bodyHeader");
writeString(row, 6, "Loss", "bodyHeader");
writeString(row, 7, "Reject H0", "bodyHeader");
row++;
for (const auto& item : holmResult.holmLines) {
writeString(row, 1, item.model, "text");
writeDouble(row, 2, item.pvalue, "result");
writeDouble(row, 3, item.rank, "result");
writeInt(row, 4, item.wtl.win, "ints");
writeInt(row, 5, item.wtl.tie, "ints");
writeInt(row, 6, item.wtl.loss, "ints");
writeString(row, 7, item.reject ? "Yes" : "No", "textCentered");
row++;
}
}
}
}

4
src/Platform/BestResultsExcel.h
View File

@ -8,9 +8,10 @@ using namespace std;
using json = nlohmann::json;
namespace platform {
class BestResultsExcel : ExcelFile {
public:
BestResultsExcel(string score, vector<string> models, vector<string> datasets, json table, bool friedman);
BestResultsExcel(string score, vector<string> models, vector<string> datasets, json table, bool friedman, double significance);
~BestResultsExcel();
void build();
private:
@ -24,6 +25,7 @@ namespace platform {
vector<string> datasets;
json table;
bool friedman;
double significance;
int modelNameSize = 12; // Min size of the column
int datasetNameSize = 25; // Min size of the column
};

18
src/Platform/ExcelFile.cc
View File

@ -38,12 +38,20 @@ namespace platform {
}
lxw_format* ExcelFile::efectiveStyle(const string& style)
{
lxw_format* efectiveStyle;
if (style == "") {
efectiveStyle = NULL;
} else {
lxw_format* efectiveStyle = NULL;
if (style != "") {
string suffix = row % 2 ? "_odd" : "_even";
efectiveStyle = styles.at(style + suffix);
try {
efectiveStyle = styles.at(style + suffix);
}
catch (const out_of_range& oor) {
try {
efectiveStyle = styles.at(style);
}
catch (const out_of_range& oor) {
throw invalid_argument("Style " + style + " not found");
}
}
}
return efectiveStyle;
}

63
src/Platform/Statistics.cc
View File

@ -1,3 +1,4 @@
#include <sstream>
#include "Statistics.h"
#include "Colors.h"
#include "Symbols.h"
@ -6,7 +7,8 @@
namespace platform {
Statistics::Statistics(vector<string>& models, vector<string>& datasets, json data, double significance) : models(models), datasets(datasets), data(data), significance(significance)
Statistics::Statistics(vector<string>& models, vector<string>& datasets, json data, double significance, bool output) :
models(models), datasets(datasets), data(data), significance(significance), output(output)
{
nModels = models.size();
nDatasets = datasets.size();
@ -110,6 +112,7 @@ namespace platform {
if (!fitted) {
fit();
}
stringstream oss;
// Reference https://link.springer.com/article/10.1007/s44196-022-00083-8
// Post-hoc Holm test
// Calculate the p-value for the models paired with the control model
@ -142,13 +145,14 @@ namespace platform {
p_value = max(before, p_value);
statsOrder[i] = { item.first, p_value };
}
holmResult.model = models.at(controlIdx);
auto color = friedmanResult ? Colors::CYAN() : Colors::YELLOW();
cout << color;
cout << " *************************************************************************************************************" << endl;
cout << " Post-hoc Holm test: H0: 'There is no significant differences between the control model and the other models.'" << endl;
cout << " Control model: " << models[controlIdx] << endl;
cout << " " << left << setw(maxModelName) << string("Model") << " p-value rank win tie loss Status" << endl;
cout << " " << string(maxModelName, '=') << " ============ ========= === === ==== =============" << endl;
oss << color;
oss << " *************************************************************************************************************" << endl;
oss << " Post-hoc Holm test: H0: 'There is no significant differences between the control model and the other models.'" << endl;
oss << " Control model: " << models.at(controlIdx) << endl;
oss << " " << left << setw(maxModelName) << string("Model") << " p-value rank win tie loss Status" << endl;
oss << " " << string(maxModelName, '=') << " ============ ========= === === ==== =============" << endl;
// sort ranks from lowest to highest
vector<pair<string, float>> ranksOrder;
for (const auto& rank : ranks) {
@ -171,23 +175,28 @@ namespace platform {
auto colorStatus = pvalue > significance ? Colors::GREEN() : Colors::MAGENTA();
auto status = pvalue > significance ? Symbols::check_mark : Symbols::cross;
auto textStatus = pvalue > significance ? " accepted H0" : " rejected H0";
cout << " " << colorStatus << left << setw(maxModelName) << item.first << " " << setprecision(6) << scientific << pvalue << setprecision(7) << fixed << " " << item.second;
cout << " " << right << setw(3) << wtl.at(idx).win << " " << setw(3) << wtl.at(idx).tie << " " << setw(4) << wtl.at(idx).loss;
cout << " " << status << textStatus << endl;
oss << " " << colorStatus << left << setw(maxModelName) << item.first << " " << setprecision(6) << scientific << pvalue << setprecision(7) << fixed << " " << item.second;
oss << " " << right << setw(3) << wtl.at(idx).win << " " << setw(3) << wtl.at(idx).tie << " " << setw(4) << wtl.at(idx).loss;
oss << " " << status << textStatus << endl;
holmResult.holmLines.push_back({ item.first, pvalue, item.second, wtl.at(idx), pvalue < significance });
}
oss << color << " *************************************************************************************************************" << endl;
oss << Colors::RESET();
if (output) {
cout << oss.str();
}
cout << color << " *************************************************************************************************************" << endl;
cout << Colors::RESET();
}
bool Statistics::friedmanTest()
{
if (!fitted) {
fit();
}
stringstream oss;
// Friedman test
// Calculate the Friedman statistic
cout << Colors::BLUE() << endl;
cout << "***************************************************************************************************************" << endl;
cout << Colors::GREEN() << "Friedman test: H0: 'There is no significant differences between all the classifiers.'" << Colors::BLUE() << endl;
oss << Colors::BLUE() << endl;
oss << "***************************************************************************************************************" << endl;
oss << Colors::GREEN() << "Friedman test: H0: 'There is no significant differences between all the classifiers.'" << Colors::BLUE() << endl;
double degreesOfFreedom = nModels - 1.0;
double sumSquared = 0;
for (const auto& rank : ranks) {
@ -195,23 +204,35 @@ namespace platform {
}
// Compute the Friedman statistic as in https://link.springer.com/article/10.1007/s44196-022-00083-8
double friedmanQ = 12.0 * nDatasets / (nModels * (nModels + 1)) * (sumSquared - (nModels * pow(nModels + 1, 2)) / 4);
cout << "Friedman statistic: " << friedmanQ << endl;
// Calculate the critical value
boost::math::chi_squared chiSquared(degreesOfFreedom);
long double p_value = (long double)1.0 - cdf(chiSquared, friedmanQ);
double criticalValue = quantile(chiSquared, 1 - significance);
std::cout << "Critical Chi-Square Value for df=" << fixed << (int)degreesOfFreedom
oss << "Friedman statistic: " << friedmanQ << endl;
oss << "Critical χ2 Value for df=" << fixed << (int)degreesOfFreedom
<< " and alpha=" << setprecision(2) << fixed << significance << ": " << setprecision(7) << scientific << criticalValue << std::endl;
cout << "p-value: " << scientific << p_value << " is " << (p_value < significance ? "less" : "greater") << " than " << setprecision(2) << fixed << significance << endl;
oss << "p-value: " << scientific << p_value << " is " << (p_value < significance ? "less" : "greater") << " than " << setprecision(2) << fixed << significance << endl;
bool result;
if (p_value < significance) {
cout << Colors::GREEN() << "The null hypothesis H0 is rejected." << endl;
oss << Colors::GREEN() << "The null hypothesis H0 is rejected." << endl;
result = true;
} else {
cout << Colors::YELLOW() << "The null hypothesis H0 is accepted. Computed p-values will not be significant." << endl;
oss << Colors::YELLOW() << "The null hypothesis H0 is accepted. Computed p-values will not be significant." << endl;
result = false;
}
cout << Colors::BLUE() << "***************************************************************************************************************" << Colors::RESET() << endl;
oss << Colors::BLUE() << "***************************************************************************************************************" << Colors::RESET() << endl;
if (output) {
cout << oss.str();
}
friedmanResult = { friedmanQ, criticalValue, p_value, result };
return result;
}
FriedmanResult& Statistics::getFriedmanResult()
{
return friedmanResult;
}
HolmResult& Statistics::getHolmResult()
{
return holmResult;
}
} // namespace platform

24
src/Platform/Statistics.h
View File

@ -13,11 +13,30 @@ namespace platform {
int tie;
int loss;
};
struct FriedmanResult {
double statistic;
double criticalValue;
long double pvalue;
bool reject;
};
struct HolmLine {
string model;
long double pvalue;
double rank;
WTL wtl;
bool reject;
};
struct HolmResult {
string model;
vector<HolmLine> holmLines;
};
class Statistics {
public:
Statistics(vector<string>& models, vector<string>& datasets, json data, double significance = 0.05);
Statistics(vector<string>& models, vector<string>& datasets, json data, double significance = 0.05, bool output = true);
bool friedmanTest();
void postHocHolmTest(bool friedmanResult);
FriedmanResult& getFriedmanResult();
HolmResult& getHolmResult();
private:
void fit();
void computeRanks();
@ -26,6 +45,7 @@ namespace platform {
vector<string> datasets;
json data;
double significance;
bool output;
bool fitted = false;
int nModels = 0;
int nDatasets = 0;
@ -34,6 +54,8 @@ namespace platform {
map<string, float> ranks;
int maxModelName = 0;
int maxDatasetName = 0;
FriedmanResult friedmanResult;
HolmResult holmResult;
};
}
#endif // !STATISTICS_H