diff --git a/src/Platform/BestResults.cc b/src/Platform/BestResults.cc index 013dbea..ed5a19b 100644 --- a/src/Platform/BestResults.cc +++ b/src/Platform/BestResults.cc @@ -283,6 +283,7 @@ namespace platform { } void BestResults::reportAll(bool excel) { + double significance = 0.05; auto models = getModels(); // Build the table of results json table = buildTableResults(models); @@ -295,13 +296,12 @@ namespace platform { printTableResults(models, table); // Compute the Friedman test if (friedman) { - double significance = 0.05; Statistics stats(models, datasets, table, significance); auto result = stats.friedmanTest(); stats.postHocHolmTest(result); } if (excel) { - BestResultsExcel excel(score, models, datasets, table, friedman); + BestResultsExcel excel(score, models, datasets, table, friedman, significance); excel.build(); } } diff --git a/src/Platform/BestResultsExcel.cc b/src/Platform/BestResultsExcel.cc index 768063e..3fb75ba 100644 --- a/src/Platform/BestResultsExcel.cc +++ b/src/Platform/BestResultsExcel.cc @@ -1,10 +1,10 @@ #include #include "BestResultsExcel.h" #include "Paths.h" -#include +#include "Statistics.h" namespace platform { - BestResultsExcel::BestResultsExcel(string score, vector models, vector datasets, json table, bool friedman) : score(score), models(models), datasets(datasets), table(table), friedman(friedman) + BestResultsExcel::BestResultsExcel(string score, vector models, vector datasets, json table, bool friedman, double significance) : score(score), models(models), datasets(datasets), table(table), friedman(friedman), significance(significance) { workbook = workbook_new((Paths::excel() + fileName).c_str()); worksheet = workbook_add_worksheet(workbook, "Best Results"); @@ -79,7 +79,60 @@ namespace platform { void BestResultsExcel::footer() { if (friedman) { - + worksheet = workbook_add_worksheet(workbook, "Friedman"); + vector columns_sizes = { 5, datasetNameSize }; + for (int i = 0; i < models.size(); ++i) { + columns_sizes.push_back(modelNameSize); + } + for (int i = 0; i < columns_sizes.size(); ++i) { + worksheet_set_column(worksheet, i, i, columns_sizes.at(i), NULL); + } + worksheet_merge_range(worksheet, 0, 0, 0, 1 + models.size(), "Friedman Test", styles["headerFirst"]); + row = 2; + Statistics stats(models, datasets, table, significance, false); + auto result = stats.friedmanTest(); + stats.postHocHolmTest(result); + auto friedmanResult = stats.getFriedmanResult(); + auto holmResult = stats.getHolmResult(); + worksheet_merge_range(worksheet, row, 0, row, 1 + models.size(), "Null hypothesis: H0 'There is no significant differences between all the classifiers.'", styles["headerSmall"]); + row += 2; + writeString(row, 1, "Friedman Q", "bodyHeader"); + writeDouble(row, 2, friedmanResult.statistic, "bodyHeader"); + row++; + writeString(row, 1, "Critical χ2 value", "bodyHeader"); + writeDouble(row, 2, friedmanResult.criticalValue, "bodyHeader"); + row++; + writeString(row, 1, "p-value", "bodyHeader"); + writeDouble(row, 2, friedmanResult.pvalue, "bodyHeader"); + writeString(row, 3, friedmanResult.reject ? "<" : ">", "bodyHeader"); + writeDouble(row, 4, significance, "bodyHeader"); + writeString(row, 5, friedmanResult.reject ? "Reject H0" : "Accept H0", "bodyHeader"); + row += 3; + worksheet_merge_range(worksheet, row, 0, row, 1 + models.size(), "Holm Test", styles["headerFirst"]); + row += 2; + worksheet_merge_range(worksheet, row, 0, row, 1 + models.size(), "Null hypothesis: H0 'There is no significant differences between the control model and the other models.'", styles["headerSmall"]); + row += 2; + string controlModel = "Control Model: " + holmResult.model; + worksheet_merge_range(worksheet, row, 1, row, 7, controlModel.c_str(), styles["bodyHeader_odd"]); + row++; + writeString(row, 1, "Model", "bodyHeader"); + writeString(row, 2, "p-value", "bodyHeader"); + writeString(row, 3, "Rank", "bodyHeader"); + writeString(row, 4, "Win", "bodyHeader"); + writeString(row, 5, "Tie", "bodyHeader"); + writeString(row, 6, "Loss", "bodyHeader"); + writeString(row, 7, "Reject H0", "bodyHeader"); + row++; + for (const auto& item : holmResult.holmLines) { + writeString(row, 1, item.model, "text"); + writeDouble(row, 2, item.pvalue, "result"); + writeDouble(row, 3, item.rank, "result"); + writeInt(row, 4, item.wtl.win, "ints"); + writeInt(row, 5, item.wtl.tie, "ints"); + writeInt(row, 6, item.wtl.loss, "ints"); + writeString(row, 7, item.reject ? "Yes" : "No", "textCentered"); + row++; + } } } } \ No newline at end of file diff --git a/src/Platform/BestResultsExcel.h b/src/Platform/BestResultsExcel.h index f4d3743..703298b 100644 --- a/src/Platform/BestResultsExcel.h +++ b/src/Platform/BestResultsExcel.h @@ -8,9 +8,10 @@ using namespace std; using json = nlohmann::json; namespace platform { + class BestResultsExcel : ExcelFile { public: - BestResultsExcel(string score, vector models, vector datasets, json table, bool friedman); + BestResultsExcel(string score, vector models, vector datasets, json table, bool friedman, double significance); ~BestResultsExcel(); void build(); private: @@ -24,6 +25,7 @@ namespace platform { vector datasets; json table; bool friedman; + double significance; int modelNameSize = 12; // Min size of the column int datasetNameSize = 25; // Min size of the column }; diff --git a/src/Platform/ExcelFile.cc b/src/Platform/ExcelFile.cc index c8c999b..04970ab 100644 --- a/src/Platform/ExcelFile.cc +++ b/src/Platform/ExcelFile.cc @@ -38,12 +38,20 @@ namespace platform { } lxw_format* ExcelFile::efectiveStyle(const string& style) { - lxw_format* efectiveStyle; - if (style == "") { - efectiveStyle = NULL; - } else { + lxw_format* efectiveStyle = NULL; + if (style != "") { string suffix = row % 2 ? "_odd" : "_even"; - efectiveStyle = styles.at(style + suffix); + try { + efectiveStyle = styles.at(style + suffix); + } + catch (const out_of_range& oor) { + try { + efectiveStyle = styles.at(style); + } + catch (const out_of_range& oor) { + throw invalid_argument("Style " + style + " not found"); + } + } } return efectiveStyle; } diff --git a/src/Platform/Statistics.cc b/src/Platform/Statistics.cc index a153eb4..b3ae878 100644 --- a/src/Platform/Statistics.cc +++ b/src/Platform/Statistics.cc @@ -1,3 +1,4 @@ +#include #include "Statistics.h" #include "Colors.h" #include "Symbols.h" @@ -6,7 +7,8 @@ namespace platform { - Statistics::Statistics(vector& models, vector& datasets, json data, double significance) : models(models), datasets(datasets), data(data), significance(significance) + Statistics::Statistics(vector& models, vector& datasets, json data, double significance, bool output) : + models(models), datasets(datasets), data(data), significance(significance), output(output) { nModels = models.size(); nDatasets = datasets.size(); @@ -110,6 +112,7 @@ namespace platform { if (!fitted) { fit(); } + stringstream oss; // Reference https://link.springer.com/article/10.1007/s44196-022-00083-8 // Post-hoc Holm test // Calculate the p-value for the models paired with the control model @@ -142,13 +145,14 @@ namespace platform { p_value = max(before, p_value); statsOrder[i] = { item.first, p_value }; } + holmResult.model = models.at(controlIdx); auto color = friedmanResult ? Colors::CYAN() : Colors::YELLOW(); - cout << color; - cout << " *************************************************************************************************************" << endl; - cout << " Post-hoc Holm test: H0: 'There is no significant differences between the control model and the other models.'" << endl; - cout << " Control model: " << models[controlIdx] << endl; - cout << " " << left << setw(maxModelName) << string("Model") << " p-value rank win tie loss Status" << endl; - cout << " " << string(maxModelName, '=') << " ============ ========= === === ==== =============" << endl; + oss << color; + oss << " *************************************************************************************************************" << endl; + oss << " Post-hoc Holm test: H0: 'There is no significant differences between the control model and the other models.'" << endl; + oss << " Control model: " << models.at(controlIdx) << endl; + oss << " " << left << setw(maxModelName) << string("Model") << " p-value rank win tie loss Status" << endl; + oss << " " << string(maxModelName, '=') << " ============ ========= === === ==== =============" << endl; // sort ranks from lowest to highest vector> ranksOrder; for (const auto& rank : ranks) { @@ -171,23 +175,28 @@ namespace platform { auto colorStatus = pvalue > significance ? Colors::GREEN() : Colors::MAGENTA(); auto status = pvalue > significance ? Symbols::check_mark : Symbols::cross; auto textStatus = pvalue > significance ? " accepted H0" : " rejected H0"; - cout << " " << colorStatus << left << setw(maxModelName) << item.first << " " << setprecision(6) << scientific << pvalue << setprecision(7) << fixed << " " << item.second; - cout << " " << right << setw(3) << wtl.at(idx).win << " " << setw(3) << wtl.at(idx).tie << " " << setw(4) << wtl.at(idx).loss; - cout << " " << status << textStatus << endl; + oss << " " << colorStatus << left << setw(maxModelName) << item.first << " " << setprecision(6) << scientific << pvalue << setprecision(7) << fixed << " " << item.second; + oss << " " << right << setw(3) << wtl.at(idx).win << " " << setw(3) << wtl.at(idx).tie << " " << setw(4) << wtl.at(idx).loss; + oss << " " << status << textStatus << endl; + holmResult.holmLines.push_back({ item.first, pvalue, item.second, wtl.at(idx), pvalue < significance }); + } + oss << color << " *************************************************************************************************************" << endl; + oss << Colors::RESET(); + if (output) { + cout << oss.str(); } - cout << color << " *************************************************************************************************************" << endl; - cout << Colors::RESET(); } bool Statistics::friedmanTest() { if (!fitted) { fit(); } + stringstream oss; // Friedman test // Calculate the Friedman statistic - cout << Colors::BLUE() << endl; - cout << "***************************************************************************************************************" << endl; - cout << Colors::GREEN() << "Friedman test: H0: 'There is no significant differences between all the classifiers.'" << Colors::BLUE() << endl; + oss << Colors::BLUE() << endl; + oss << "***************************************************************************************************************" << endl; + oss << Colors::GREEN() << "Friedman test: H0: 'There is no significant differences between all the classifiers.'" << Colors::BLUE() << endl; double degreesOfFreedom = nModels - 1.0; double sumSquared = 0; for (const auto& rank : ranks) { @@ -195,23 +204,35 @@ namespace platform { } // Compute the Friedman statistic as in https://link.springer.com/article/10.1007/s44196-022-00083-8 double friedmanQ = 12.0 * nDatasets / (nModels * (nModels + 1)) * (sumSquared - (nModels * pow(nModels + 1, 2)) / 4); - cout << "Friedman statistic: " << friedmanQ << endl; // Calculate the critical value boost::math::chi_squared chiSquared(degreesOfFreedom); long double p_value = (long double)1.0 - cdf(chiSquared, friedmanQ); double criticalValue = quantile(chiSquared, 1 - significance); - std::cout << "Critical Chi-Square Value for df=" << fixed << (int)degreesOfFreedom + oss << "Friedman statistic: " << friedmanQ << endl; + oss << "Critical χ2 Value for df=" << fixed << (int)degreesOfFreedom << " and alpha=" << setprecision(2) << fixed << significance << ": " << setprecision(7) << scientific << criticalValue << std::endl; - cout << "p-value: " << scientific << p_value << " is " << (p_value < significance ? "less" : "greater") << " than " << setprecision(2) << fixed << significance << endl; + oss << "p-value: " << scientific << p_value << " is " << (p_value < significance ? "less" : "greater") << " than " << setprecision(2) << fixed << significance << endl; bool result; if (p_value < significance) { - cout << Colors::GREEN() << "The null hypothesis H0 is rejected." << endl; + oss << Colors::GREEN() << "The null hypothesis H0 is rejected." << endl; result = true; } else { - cout << Colors::YELLOW() << "The null hypothesis H0 is accepted. Computed p-values will not be significant." << endl; + oss << Colors::YELLOW() << "The null hypothesis H0 is accepted. Computed p-values will not be significant." << endl; result = false; } - cout << Colors::BLUE() << "***************************************************************************************************************" << Colors::RESET() << endl; + oss << Colors::BLUE() << "***************************************************************************************************************" << Colors::RESET() << endl; + if (output) { + cout << oss.str(); + } + friedmanResult = { friedmanQ, criticalValue, p_value, result }; return result; } + FriedmanResult& Statistics::getFriedmanResult() + { + return friedmanResult; + } + HolmResult& Statistics::getHolmResult() + { + return holmResult; + } } // namespace platform diff --git a/src/Platform/Statistics.h b/src/Platform/Statistics.h index 8304ab7..a3f9b3c 100644 --- a/src/Platform/Statistics.h +++ b/src/Platform/Statistics.h @@ -13,11 +13,30 @@ namespace platform { int tie; int loss; }; + struct FriedmanResult { + double statistic; + double criticalValue; + long double pvalue; + bool reject; + }; + struct HolmLine { + string model; + long double pvalue; + double rank; + WTL wtl; + bool reject; + }; + struct HolmResult { + string model; + vector holmLines; + }; class Statistics { public: - Statistics(vector& models, vector& datasets, json data, double significance = 0.05); + Statistics(vector& models, vector& datasets, json data, double significance = 0.05, bool output = true); bool friedmanTest(); void postHocHolmTest(bool friedmanResult); + FriedmanResult& getFriedmanResult(); + HolmResult& getHolmResult(); private: void fit(); void computeRanks(); @@ -26,6 +45,7 @@ namespace platform { vector datasets; json data; double significance; + bool output; bool fitted = false; int nModels = 0; int nDatasets = 0; @@ -34,6 +54,8 @@ namespace platform { map ranks; int maxModelName = 0; int maxDatasetName = 0; + FriedmanResult friedmanResult; + HolmResult holmResult; }; } #endif // !STATISTICS_H \ No newline at end of file