Complete Excel output for bestResults with Friedman test

src/Platform/BestResults.cc

@@ -283,6 +283,7 @@ namespace platform {
     }
     void BestResults::reportAll(bool excel)
     {
+        double significance = 0.05;
         auto models = getModels();
         // Build the table of results
         json table = buildTableResults(models);
@@ -295,13 +296,12 @@ namespace platform {
         printTableResults(models, table);
         // Compute the Friedman test
         if (friedman) {
-            double significance = 0.05;
             Statistics stats(models, datasets, table, significance);
             auto result = stats.friedmanTest();
             stats.postHocHolmTest(result);
         }
         if (excel) {
-            BestResultsExcel excel(score, models, datasets, table, friedman);
+            BestResultsExcel excel(score, models, datasets, table, friedman, significance);
             excel.build();
         }
     }

src/Platform/BestResultsExcel.cc

@@ -1,10 +1,10 @@
 #include <sstream>
 #include "BestResultsExcel.h"
 #include "Paths.h"
-#include <iostream>
+#include "Statistics.h"
 
 namespace platform {
-    BestResultsExcel::BestResultsExcel(string score, vector<string> models, vector<string> datasets, json table, bool friedman) : score(score), models(models), datasets(datasets), table(table), friedman(friedman)
+    BestResultsExcel::BestResultsExcel(string score, vector<string> models, vector<string> datasets, json table, bool friedman, double significance) : score(score), models(models), datasets(datasets), table(table), friedman(friedman), significance(significance)
     {
         workbook = workbook_new((Paths::excel() + fileName).c_str());
         worksheet = workbook_add_worksheet(workbook, "Best Results");
@@ -79,7 +79,60 @@ namespace platform {
     void BestResultsExcel::footer()
     {
         if (friedman) {
-
+            worksheet = workbook_add_worksheet(workbook, "Friedman");
+            vector<int> columns_sizes = { 5, datasetNameSize };
+            for (int i = 0; i < models.size(); ++i) {
+                columns_sizes.push_back(modelNameSize);
+            }
+            for (int i = 0; i < columns_sizes.size(); ++i) {
+                worksheet_set_column(worksheet, i, i, columns_sizes.at(i), NULL);
+            }
+            worksheet_merge_range(worksheet, 0, 0, 0, 1 + models.size(), "Friedman Test", styles["headerFirst"]);
+            row = 2;
+            Statistics stats(models, datasets, table, significance, false);
+            auto result = stats.friedmanTest();
+            stats.postHocHolmTest(result);
+            auto friedmanResult = stats.getFriedmanResult();
+            auto holmResult = stats.getHolmResult();
+            worksheet_merge_range(worksheet, row, 0, row, 1 + models.size(), "Null hypothesis: H0 'There is no significant differences between all the classifiers.'", styles["headerSmall"]);
+            row += 2;
+            writeString(row, 1, "Friedman Q", "bodyHeader");
+            writeDouble(row, 2, friedmanResult.statistic, "bodyHeader");
+            row++;
+            writeString(row, 1, "Critical χ2 value", "bodyHeader");
+            writeDouble(row, 2, friedmanResult.criticalValue, "bodyHeader");
+            row++;
+            writeString(row, 1, "p-value", "bodyHeader");
+            writeDouble(row, 2, friedmanResult.pvalue, "bodyHeader");
+            writeString(row, 3, friedmanResult.reject ? "<" : ">", "bodyHeader");
+            writeDouble(row, 4, significance, "bodyHeader");
+            writeString(row, 5, friedmanResult.reject ? "Reject H0" : "Accept H0", "bodyHeader");
+            row += 3;
+            worksheet_merge_range(worksheet, row, 0, row, 1 + models.size(), "Holm Test", styles["headerFirst"]);
+            row += 2;
+            worksheet_merge_range(worksheet, row, 0, row, 1 + models.size(), "Null hypothesis: H0 'There is no significant differences between the control model and the other models.'", styles["headerSmall"]);
+            row += 2;
+            string controlModel = "Control Model: " + holmResult.model;
+            worksheet_merge_range(worksheet, row, 1, row, 7, controlModel.c_str(), styles["bodyHeader_odd"]);
+            row++;
+            writeString(row, 1, "Model", "bodyHeader");
+            writeString(row, 2, "p-value", "bodyHeader");
+            writeString(row, 3, "Rank", "bodyHeader");
+            writeString(row, 4, "Win", "bodyHeader");
+            writeString(row, 5, "Tie", "bodyHeader");
+            writeString(row, 6, "Loss", "bodyHeader");
+            writeString(row, 7, "Reject H0", "bodyHeader");
+            row++;
+            for (const auto& item : holmResult.holmLines) {
+                writeString(row, 1, item.model, "text");
+                writeDouble(row, 2, item.pvalue, "result");
+                writeDouble(row, 3, item.rank, "result");
+                writeInt(row, 4, item.wtl.win, "ints");
+                writeInt(row, 5, item.wtl.tie, "ints");
+                writeInt(row, 6, item.wtl.loss, "ints");
+                writeString(row, 7, item.reject ? "Yes" : "No", "textCentered");
+                row++;
+            }
         }
     }
 }

src/Platform/BestResultsExcel.h

@@ -8,9 +8,10 @@ using namespace std;
 using json = nlohmann::json;
 
 namespace platform {
+
     class BestResultsExcel : ExcelFile {
     public:
-        BestResultsExcel(string score, vector<string> models, vector<string> datasets, json table, bool friedman);
+        BestResultsExcel(string score, vector<string> models, vector<string> datasets, json table, bool friedman, double significance);
         ~BestResultsExcel();
         void build();
     private:
@@ -24,6 +25,7 @@ namespace platform {
         vector<string> datasets;
         json table;
         bool friedman;
+        double significance;
         int modelNameSize = 12; // Min size of the column
         int datasetNameSize = 25; // Min size of the column
     };

src/Platform/ExcelFile.cc

@@ -38,12 +38,20 @@ namespace platform {
     }
     lxw_format* ExcelFile::efectiveStyle(const string& style)
     {
-        lxw_format* efectiveStyle;
+        lxw_format* efectiveStyle = NULL;
-        if (style == "") {
+        if (style != "") {
-            efectiveStyle = NULL;
-        } else {
             string suffix = row % 2 ? "_odd" : "_even";
-            efectiveStyle = styles.at(style + suffix);
+            try {
+                efectiveStyle = styles.at(style + suffix);
+            }
+            catch (const out_of_range& oor) {
+                try {
+                    efectiveStyle = styles.at(style);
+                }
+                catch (const out_of_range& oor) {
+                    throw invalid_argument("Style " + style + " not found");
+                }
+            }
         }
         return efectiveStyle;
     }

src/Platform/Statistics.cc

@@ -1,3 +1,4 @@
+#include <sstream>
 #include "Statistics.h"
 #include "Colors.h"
 #include "Symbols.h"
@@ -6,7 +7,8 @@
 
 namespace platform {
 
-    Statistics::Statistics(vector<string>& models, vector<string>& datasets, json data, double significance) : models(models), datasets(datasets), data(data), significance(significance)
+    Statistics::Statistics(vector<string>& models, vector<string>& datasets, json data, double significance, bool output) :
+        models(models), datasets(datasets), data(data), significance(significance), output(output)
     {
         nModels = models.size();
         nDatasets = datasets.size();
@@ -110,6 +112,7 @@ namespace platform {
         if (!fitted) {
             fit();
         }
+        stringstream oss;
         // Reference https://link.springer.com/article/10.1007/s44196-022-00083-8
         // Post-hoc Holm test
         // Calculate the p-value for the models paired with the control model
@@ -142,13 +145,14 @@ namespace platform {
             p_value = max(before, p_value);
             statsOrder[i] = { item.first, p_value };
         }
+        holmResult.model = models.at(controlIdx);
         auto color = friedmanResult ? Colors::CYAN() : Colors::YELLOW();
-        cout << color;
+        oss << color;
-        cout << "  *************************************************************************************************************" << endl;
+        oss << "  *************************************************************************************************************" << endl;
-        cout << "  Post-hoc Holm test: H0: 'There is no significant differences between the control model and the other models.'" << endl;
+        oss << "  Post-hoc Holm test: H0: 'There is no significant differences between the control model and the other models.'" << endl;
-        cout << "  Control model: " << models[controlIdx] << endl;
+        oss << "  Control model: " << models.at(controlIdx) << endl;
-        cout << "  " << left << setw(maxModelName) << string("Model") << " p-value      rank      win tie loss Status" << endl;
+        oss << "  " << left << setw(maxModelName) << string("Model") << " p-value      rank      win tie loss Status" << endl;
-        cout << "  " << string(maxModelName, '=') << " ============ ========= === === ==== =============" << endl;
+        oss << "  " << string(maxModelName, '=') << " ============ ========= === === ==== =============" << endl;
         // sort ranks from lowest to highest
         vector<pair<string, float>> ranksOrder;
         for (const auto& rank : ranks) {
@@ -171,23 +175,28 @@ namespace platform {
             auto colorStatus = pvalue > significance ? Colors::GREEN() : Colors::MAGENTA();
             auto status = pvalue > significance ? Symbols::check_mark : Symbols::cross;
             auto textStatus = pvalue > significance ? " accepted H0" : " rejected H0";
-            cout << "  " << colorStatus << left << setw(maxModelName) << item.first << " " << setprecision(6) << scientific << pvalue << setprecision(7) << fixed << " " << item.second;
+            oss << "  " << colorStatus << left << setw(maxModelName) << item.first << " " << setprecision(6) << scientific << pvalue << setprecision(7) << fixed << " " << item.second;
-            cout << " " << right << setw(3) << wtl.at(idx).win << " " << setw(3) << wtl.at(idx).tie << " " << setw(4) << wtl.at(idx).loss;
+            oss << " " << right << setw(3) << wtl.at(idx).win << " " << setw(3) << wtl.at(idx).tie << " " << setw(4) << wtl.at(idx).loss;
-            cout << " " << status << textStatus << endl;
+            oss << " " << status << textStatus << endl;
+            holmResult.holmLines.push_back({ item.first, pvalue, item.second, wtl.at(idx), pvalue < significance });
+        }
+        oss << color << "  *************************************************************************************************************" << endl;
+        oss << Colors::RESET();
+        if (output) {
+            cout << oss.str();
         }
-        cout << color << "  *************************************************************************************************************" << endl;
-        cout << Colors::RESET();
     }
     bool Statistics::friedmanTest()
     {
         if (!fitted) {
             fit();
         }
+        stringstream oss;
         // Friedman test
         // Calculate the Friedman statistic
-        cout << Colors::BLUE() << endl;
+        oss << Colors::BLUE() << endl;
-        cout << "***************************************************************************************************************" << endl;
+        oss << "***************************************************************************************************************" << endl;
-        cout << Colors::GREEN() << "Friedman test: H0: 'There is no significant differences between all the classifiers.'" << Colors::BLUE() << endl;
+        oss << Colors::GREEN() << "Friedman test: H0: 'There is no significant differences between all the classifiers.'" << Colors::BLUE() << endl;
         double degreesOfFreedom = nModels - 1.0;
         double sumSquared = 0;
         for (const auto& rank : ranks) {
@@ -195,23 +204,35 @@ namespace platform {
         }
         // Compute the Friedman statistic as in https://link.springer.com/article/10.1007/s44196-022-00083-8
         double friedmanQ = 12.0 * nDatasets / (nModels * (nModels + 1)) * (sumSquared - (nModels * pow(nModels + 1, 2)) / 4);
-        cout << "Friedman statistic: " << friedmanQ << endl;
         // Calculate the critical value
         boost::math::chi_squared chiSquared(degreesOfFreedom);
         long double p_value = (long double)1.0 - cdf(chiSquared, friedmanQ);
         double criticalValue = quantile(chiSquared, 1 - significance);
-        std::cout << "Critical Chi-Square Value for df=" << fixed << (int)degreesOfFreedom
+        oss << "Friedman statistic: " << friedmanQ << endl;
+        oss << "Critical χ2 Value for df=" << fixed << (int)degreesOfFreedom
             << " and alpha=" << setprecision(2) << fixed << significance << ": " << setprecision(7) << scientific << criticalValue << std::endl;
-        cout << "p-value: " << scientific << p_value << " is " << (p_value < significance ? "less" : "greater") << " than " << setprecision(2) << fixed << significance << endl;
+        oss << "p-value: " << scientific << p_value << " is " << (p_value < significance ? "less" : "greater") << " than " << setprecision(2) << fixed << significance << endl;
         bool result;
         if (p_value < significance) {
-            cout << Colors::GREEN() << "The null hypothesis H0 is rejected." << endl;
+            oss << Colors::GREEN() << "The null hypothesis H0 is rejected." << endl;
             result = true;
         } else {
-            cout << Colors::YELLOW() << "The null hypothesis H0 is accepted. Computed p-values will not be significant." << endl;
+            oss << Colors::YELLOW() << "The null hypothesis H0 is accepted. Computed p-values will not be significant." << endl;
             result = false;
         }
-        cout << Colors::BLUE() << "***************************************************************************************************************" << Colors::RESET() << endl;
+        oss << Colors::BLUE() << "***************************************************************************************************************" << Colors::RESET() << endl;
+        if (output) {
+            cout << oss.str();
+        }
+        friedmanResult = { friedmanQ, criticalValue, p_value, result };
         return result;
     }
+    FriedmanResult& Statistics::getFriedmanResult()
+    {
+        return friedmanResult;
+    }
+    HolmResult& Statistics::getHolmResult()
+    {
+        return holmResult;
+    }
 } // namespace platform

src/Platform/Statistics.h

@@ -13,11 +13,30 @@ namespace platform {
         int tie;
         int loss;
     };
+    struct FriedmanResult {
+        double statistic;
+        double criticalValue;
+        long double pvalue;
+        bool reject;
+    };
+    struct HolmLine {
+        string model;
+        long double pvalue;
+        double rank;
+        WTL wtl;
+        bool reject;
+    };
+    struct HolmResult {
+        string model;
+        vector<HolmLine> holmLines;
+    };
     class Statistics {
     public:
-        Statistics(vector<string>& models, vector<string>& datasets, json data, double significance = 0.05);
+        Statistics(vector<string>& models, vector<string>& datasets, json data, double significance = 0.05, bool output = true);
         bool friedmanTest();
         void postHocHolmTest(bool friedmanResult);
+        FriedmanResult& getFriedmanResult();
+        HolmResult& getHolmResult();
     private:
         void fit();
         void computeRanks();
@@ -26,6 +45,7 @@ namespace platform {
         vector<string> datasets;
         json data;
         double significance;
+        bool output;
         bool fitted = false;
         int nModels = 0;
         int nDatasets = 0;
@@ -34,6 +54,8 @@ namespace platform {
         map<string, float> ranks;
         int maxModelName = 0;
         int maxDatasetName = 0;
+        FriedmanResult friedmanResult;
+        HolmResult holmResult;
     };
 }
 #endif // !STATISTICS_H