Begin b_list excel

src/best/BestResults.cc

@@ -0,0 +1,355 @@
+#include <filesystem>
+#include <set>
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <algorithm>
+#include "BestResults.h"
+#include "Result.h"
+#include "Colors.h"
+#include "Statistics.h"
+#include "BestResultsExcel.h"
+#include "CLocale.h"
+
+
+namespace fs = std::filesystem;
+// function ftime_to_std::string, Code taken from 
+// https://stackoverflow.com/a/58237530/1389271
+template <typename TP>
+std::string ftime_to_string(TP tp)
+{
+    auto sctp = std::chrono::time_point_cast<std::chrono::system_clock::duration>(tp - TP::clock::now()
+        + std::chrono::system_clock::now());
+    auto tt = std::chrono::system_clock::to_time_t(sctp);
+    std::tm* gmt = std::gmtime(&tt);
+    std::stringstream buffer;
+    buffer << std::put_time(gmt, "%Y-%m-%d %H:%M");
+    return buffer.str();
+}
+namespace platform {
+    std::string BestResults::build()
+    {
+        auto files = loadResultFiles();
+        if (files.size() == 0) {
+            std::cerr << Colors::MAGENTA() << "No result files were found!" << Colors::RESET() << std::endl;
+            exit(1);
+        }
+        json bests;
+        for (const auto& file : files) {
+            auto result = Result();
+            result.load(path, file);
+            auto data = result.getJson();
+            for (auto const& item : data.at("results")) {
+                bool update = true;
+                auto datasetName = item.at("dataset").get<std::string>();
+                if (bests.contains(datasetName)) {
+                    if (item.at("score").get<double>() < bests[datasetName].at(0).get<double>()) {
+                        update = false;
+                    }
+                }
+                if (update) {
+                    bests[datasetName] = { item.at("score").get<double>(), item.at("hyperparameters"), file };
+                }
+            }
+        }
+        std::string bestFileName = path + bestResultFile();
+        std::ofstream file(bestFileName);
+        file << bests;
+        file.close();
+        return bestFileName;
+    }
+    std::string BestResults::bestResultFile()
+    {
+        return "best_results_" + score + "_" + model + ".json";
+    }
+    std::pair<std::string, std::string> getModelScore(std::string name)
+    {
+        // results_accuracy_BoostAODE_MacBookpro16_2023-09-06_12:27:00_1.json
+        auto pos = name.find("_");
+        auto pos2 = name.find("_", pos + 1);
+        std::string score = name.substr(pos + 1, pos2 - pos - 1);
+        pos = name.find("_", pos2 + 1);
+        std::string model = name.substr(pos2 + 1, pos - pos2 - 1);
+        return { model, score };
+    }
+    std::vector<std::string> BestResults::loadResultFiles()
+    {
+        std::vector<std::string> files;
+        using std::filesystem::directory_iterator;
+        std::string fileModel, fileScore;
+        for (const auto& file : directory_iterator(path)) {
+            auto fileName = file.path().filename().string();
+            if (fileName.find(".json") != std::string::npos && fileName.find("results_") == 0) {
+                tie(fileModel, fileScore) = getModelScore(fileName);
+                if (score == fileScore && (model == fileModel || model == "any")) {
+                    files.push_back(fileName);
+                }
+            }
+        }
+        std::sort(files.begin(), files.end());
+        return files;
+    }
+    json BestResults::loadFile(const std::string& fileName)
+    {
+        std::ifstream resultData(fileName);
+        if (resultData.is_open()) {
+            json data = json::parse(resultData);
+            return data;
+        }
+        throw std::invalid_argument("Unable to open result file. [" + fileName + "]");
+    }
+    std::vector<std::string> BestResults::getModels()
+    {
+        std::set<std::string> models;
+        std::vector<std::string> result;
+        auto files = loadResultFiles();
+        if (files.size() == 0) {
+            std::cerr << Colors::MAGENTA() << "No result files were found!" << Colors::RESET() << std::endl;
+            exit(1);
+        }
+        std::string fileModel, fileScore;
+        for (const auto& file : files) {
+            // extract the model from the file name
+            tie(fileModel, fileScore) = getModelScore(file);
+            // add the model to the std::vector of models
+            models.insert(fileModel);
+        }
+        result = std::vector<std::string>(models.begin(), models.end());
+        maxModelName = (*max_element(result.begin(), result.end(), [](const std::string& a, const std::string& b) { return a.size() < b.size(); })).size();
+        maxModelName = std::max(12, maxModelName);
+        return result;
+    }
+    std::vector<std::string> BestResults::getDatasets(json table)
+    {
+        std::vector<std::string> datasets;
+        for (const auto& dataset : table.items()) {
+            datasets.push_back(dataset.key());
+        }
+        maxDatasetName = (*max_element(datasets.begin(), datasets.end(), [](const std::string& a, const std::string& b) { return a.size() < b.size(); })).size();
+        maxDatasetName = std::max(7, maxDatasetName);
+        return datasets;
+    }
+    void BestResults::buildAll()
+    {
+        auto models = getModels();
+        std::cout << "Building best results for model: ";
+        for (const auto& model : models) {
+            this->model = model;
+            std::cout << model << ", ";
+            build();
+        }
+        std::cout << "end." << std::endl << std::endl;
+        model = "any";
+    }
+    void BestResults::listFile()
+    {
+        std::string bestFileName = path + bestResultFile();
+        if (FILE* fileTest = fopen(bestFileName.c_str(), "r")) {
+            fclose(fileTest);
+        } else {
+            std::cerr << Colors::MAGENTA() << "File " << bestFileName << " doesn't exist." << Colors::RESET() << std::endl;
+            exit(1);
+        }
+        auto temp = ConfigLocale();
+        auto date = ftime_to_string(std::filesystem::last_write_time(bestFileName));
+        auto data = loadFile(bestFileName);
+        auto datasets = getDatasets(data);
+        int maxFileName = 0;
+        int maxHyper = 15;
+        for (auto const& item : data.items()) {
+            maxHyper = std::max(maxHyper, (int)item.value().at(1).dump().size());
+            maxFileName = std::max(maxFileName, (int)item.value().at(2).get<std::string>().size());
+        }
+        std::stringstream oss;
+        oss << Colors::GREEN() << "Best results for " << model << " as of " << date << std::endl;
+        std::cout << oss.str();
+        std::cout << std::string(oss.str().size() - 8, '-') << std::endl;
+        std::cout << Colors::GREEN() << " #  " << std::setw(maxDatasetName + 1) << std::left << "Dataset" << "Score       " << std::setw(maxFileName) << "File" << " Hyperparameters" << std::endl;
+        std::cout << "=== " << std::string(maxDatasetName, '=') << " =========== " << std::string(maxFileName, '=') << " " << std::string(maxHyper, '=') << std::endl;
+        auto i = 0;
+        bool odd = true;
+        double total = 0;
+        for (auto const& item : data.items()) {
+            auto color = odd ? Colors::BLUE() : Colors::CYAN();
+            double value = item.value().at(0).get<double>();
+            std::cout << color << std::setw(3) << std::fixed << std::right << i++ << " ";
+            std::cout << std::setw(maxDatasetName) << std::left << item.key() << " ";
+            std::cout << std::setw(11) << std::setprecision(9) << std::fixed << value << " ";
+            std::cout << std::setw(maxFileName) << item.value().at(2).get<std::string>() << " ";
+            std::cout << item.value().at(1) << " ";
+            std::cout << std::endl;
+            total += value;
+            odd = !odd;
+        }
+        std::cout << Colors::GREEN() << "=== " << std::string(maxDatasetName, '=') << " ===========" << std::endl;
+        std::cout << Colors::GREEN() << "    Total" << std::string(maxDatasetName - 5, '.') << " " << std::setw(11) << std::setprecision(8) << std::fixed << total << std::endl;
+
+    }
+    json BestResults::buildTableResults(std::vector<std::string> models)
+    {
+        json table;
+        auto maxDate = std::filesystem::file_time_type::max();
+        for (const auto& model : models) {
+            this->model = model;
+            std::string bestFileName = path + bestResultFile();
+            if (FILE* fileTest = fopen(bestFileName.c_str(), "r")) {
+                fclose(fileTest);
+            } else {
+                std::cerr << Colors::MAGENTA() << "File " << bestFileName << " doesn't exist." << Colors::RESET() << std::endl;
+                exit(1);
+            }
+            auto dateWrite = std::filesystem::last_write_time(bestFileName);
+            if (dateWrite < maxDate) {
+                maxDate = dateWrite;
+            }
+            auto data = loadFile(bestFileName);
+            table[model] = data;
+        }
+        table["dateTable"] = ftime_to_string(maxDate);
+        return table;
+    }
+    void BestResults::printTableResults(std::vector<std::string> models, json table)
+    {
+        std::stringstream oss;
+        oss << Colors::GREEN() << "Best results for " << score << " as of " << table.at("dateTable").get<std::string>() << std::endl;
+        std::cout << oss.str();
+        std::cout << std::string(oss.str().size() - 8, '-') << std::endl;
+        std::cout << Colors::GREEN() << " #  " << std::setw(maxDatasetName + 1) << std::left << std::string("Dataset");
+        for (const auto& model : models) {
+            std::cout << std::setw(maxModelName) << std::left << model << " ";
+        }
+        std::cout << std::endl;
+        std::cout << "=== " << std::string(maxDatasetName, '=') << " ";
+        for (const auto& model : models) {
+            std::cout << std::string(maxModelName, '=') << " ";
+        }
+        std::cout << std::endl;
+        auto i = 0;
+        bool odd = true;
+        std::map<std::string, double> totals;
+        int nDatasets = table.begin().value().size();
+        for (const auto& model : models) {
+            totals[model] = 0.0;
+        }
+        auto datasets = getDatasets(table.begin().value());
+        for (auto const& dataset : datasets) {
+            auto color = odd ? Colors::BLUE() : Colors::CYAN();
+            std::cout << color << std::setw(3) << std::fixed << std::right << i++ << " ";
+            std::cout << std::setw(maxDatasetName) << std::left << dataset << " ";
+            double maxValue = 0;
+            // Find out the max value for this dataset
+            for (const auto& model : models) {
+                double value;
+                try {
+                    value = table[model].at(dataset).at(0).get<double>();
+                }
+                catch (nlohmann::json_abi_v3_11_3::detail::out_of_range err) {
+                    value = -1.0;
+                }
+                if (value > maxValue) {
+                    maxValue = value;
+                }
+            }
+            // Print the row with red colors on max values
+            for (const auto& model : models) {
+                std::string efectiveColor = color;
+                double value;
+                try {
+                    value = table[model].at(dataset).at(0).get<double>();
+                }
+                catch (nlohmann::json_abi_v3_11_3::detail::out_of_range err) {
+                    value = -1.0;
+                }
+                if (value == maxValue) {
+                    efectiveColor = Colors::RED();
+                }
+                if (value == -1) {
+                    std::cout << Colors::YELLOW() << std::setw(maxModelName) << std::right << "N/A" << " ";
+                } else {
+                    totals[model] += value;
+                    std::cout << efectiveColor << std::setw(maxModelName) << std::setprecision(maxModelName - 2) << std::fixed << value << " ";
+                }
+            }
+            std::cout << std::endl;
+            odd = !odd;
+        }
+        std::cout << Colors::GREEN() << "=== " << std::string(maxDatasetName, '=') << " ";
+        for (const auto& model : models) {
+            std::cout << std::string(maxModelName, '=') << " ";
+        }
+        std::cout << std::endl;
+        std::cout << Colors::GREEN() << "    Totals" << std::string(maxDatasetName - 6, '.') << " ";
+        double max_value = 0.0;
+        for (const auto& total : totals) {
+            if (total.second > max_value) {
+                max_value = total.second;
+            }
+        }
+        for (const auto& model : models) {
+            std::string efectiveColor = Colors::GREEN();
+            if (totals[model] == max_value) {
+                efectiveColor = Colors::RED();
+            }
+            std::cout << efectiveColor << std::right << std::setw(maxModelName) << std::setprecision(maxModelName - 4) << std::fixed << totals[model] << " ";
+        }
+        std::cout << std::endl;
+    }
+    void BestResults::reportSingle(bool excel)
+    {
+        listFile();
+        if (excel) {
+            auto models = getModels();
+            // Build the table of results
+            json table = buildTableResults(models);
+            std::vector<std::string> datasets = getDatasets(table.begin().value());
+            BestResultsExcel excel_report(score, datasets);
+            excel_report.reportSingle(model, path + bestResultFile());
+            messageExcelFile(excel_report.getFileName());
+        }
+    }
+    void BestResults::reportAll(bool excel)
+    {
+        auto models = getModels();
+        // Build the table of results
+        json table = buildTableResults(models);
+        std::vector<std::string> datasets = getDatasets(table.begin().value());
+        // Print the table of results
+        printTableResults(models, table);
+        // Compute the Friedman test
+        std::map<std::string, std::map<std::string, float>> ranksModels;
+        if (friedman) {
+            Statistics stats(models, datasets, table, significance);
+            auto result = stats.friedmanTest();
+            stats.postHocHolmTest(result);
+            ranksModels = stats.getRanks();
+        }
+        if (excel) {
+            BestResultsExcel excel(score, datasets);
+            excel.reportAll(models, table, ranksModels, friedman, significance);
+            if (friedman) {
+                int idx = -1;
+                double min = 2000;
+                // Find out the control model
+                auto totals = std::vector<double>(models.size(), 0.0);
+                for (const auto& dataset : datasets) {
+                    for (int i = 0; i < models.size(); ++i) {
+                        totals[i] += ranksModels[dataset][models[i]];
+                    }
+                }
+                for (int i = 0; i < models.size(); ++i) {
+                    if (totals[i] < min) {
+                        min = totals[i];
+                        idx = i;
+                    }
+                }
+                model = models.at(idx);
+                excel.reportSingle(model, path + bestResultFile());
+            }
+            messageExcelFile(excel.getFileName());
+        }
+    }
+    void BestResults::messageExcelFile(const std::string& fileName)
+    {
+        std::cout << Colors::YELLOW() << "** Excel file generated: " << fileName << Colors::RESET() << std::endl;
+    }
+}

src/best/BestResults.h

@@ -0,0 +1,36 @@
+#ifndef BESTRESULTS_H
+#define BESTRESULTS_H
+#include <string>
+#include <nlohmann/json.hpp>
+using json = nlohmann::json;
+namespace platform {
+    class BestResults {
+    public:
+        explicit BestResults(const std::string& path, const std::string& score, const std::string& model, bool friedman, double significance = 0.05)
+            : path(path), score(score), model(model), friedman(friedman), significance(significance)
+        {
+        }
+        std::string build();
+        void reportSingle(bool excel);
+        void reportAll(bool excel);
+        void buildAll();
+    private:
+        std::vector<std::string> getModels();
+        std::vector<std::string> getDatasets(json table);
+        std::vector<std::string> loadResultFiles();
+        void messageExcelFile(const std::string& fileName);
+        json buildTableResults(std::vector<std::string> models);
+        void printTableResults(std::vector<std::string> models, json table);
+        std::string bestResultFile();
+        json loadFile(const std::string& fileName);
+        void listFile();
+        std::string path;
+        std::string score;
+        std::string model;
+        bool friedman;
+        double significance;
+        int maxModelName = 0;
+        int maxDatasetName = 0;
+    };
+}
+#endif //BESTRESULTS_H

src/best/BestResultsExcel.cc

@@ -0,0 +1,300 @@
+#include <sstream>
+#include "BestResultsExcel.h"
+#include "Paths.h"
+#include <map>
+#include <nlohmann/json.hpp>
+#include "Statistics.h"
+#include "ReportExcel.h"
+
+namespace platform {
+    json loadResultData(const std::string& fileName)
+    {
+        json data;
+        std::ifstream resultData(fileName);
+        if (resultData.is_open()) {
+            data = json::parse(resultData);
+        } else {
+            throw std::invalid_argument("Unable to open result file. [" + fileName + "]");
+        }
+        return data;
+    }
+    std::string getColumnName(int colNum)
+    {
+        std::string columnName = "";
+        if (colNum == 0)
+            return "A";
+        while (colNum > 0) {
+            int modulo = colNum % 26;
+            columnName = char(65 + modulo) + columnName;
+            colNum = (int)((colNum - modulo) / 26);
+        }
+        return columnName;
+    }
+    BestResultsExcel::BestResultsExcel(const std::string& score, const std::vector<std::string>& datasets) : score(score), datasets(datasets)
+    {
+        file_name = "BestResults.xlsx";
+        workbook = workbook_new(getFileName().c_str());
+        setProperties("Best Results");
+        int maxDatasetName = (*max_element(datasets.begin(), datasets.end(), [](const std::string& a, const std::string& b) { return a.size() < b.size(); })).size();
+        datasetNameSize = std::max(datasetNameSize, maxDatasetName);
+    }
+    void BestResultsExcel::reportAll(const std::vector<std::string>& models, const json& table, const std::map<std::string, std::map<std::string, float>>& ranks, bool friedman, double significance)
+    {
+        this->table = table;
+        this->models = models;
+        ranksModels = ranks;
+        this->friedman = friedman;
+        this->significance = significance;
+        worksheet = workbook_add_worksheet(workbook, "Best Results");
+        int maxModelName = (*std::max_element(models.begin(), models.end(), [](const std::string& a, const std::string& b) { return a.size() < b.size(); })).size();
+        modelNameSize = std::max(modelNameSize, maxModelName);
+        formatColumns();
+        build();
+    }
+    void BestResultsExcel::reportSingle(const std::string& model, const std::string& fileName)
+    {
+        worksheet = workbook_add_worksheet(workbook, "Report");
+        if (FILE* fileTest = fopen(fileName.c_str(), "r")) {
+            fclose(fileTest);
+        } else {
+            std::cerr << "File " << fileName << " doesn't exist." << std::endl;
+            exit(1);
+        }
+        json data = loadResultData(fileName);
+
+        std::string title = "Best results for " + model;
+        worksheet_merge_range(worksheet, 0, 0, 0, 4, title.c_str(), styles["headerFirst"]);
+        // Body header
+        row = 3;
+        int col = 1;
+        writeString(row, 0, "Nº", "bodyHeader");
+        writeString(row, 1, "Dataset", "bodyHeader");
+        writeString(row, 2, "Score", "bodyHeader");
+        writeString(row, 3, "File", "bodyHeader");
+        writeString(row, 4, "Hyperparameters", "bodyHeader");
+        auto i = 0;
+        std::string hyperparameters;
+        int hypSize = 22;
+        std::map<std::string, std::string> files; // map of files imported and their tabs
+        for (auto const& item : data.items()) {
+            row++;
+            writeInt(row, 0, i++, "ints");
+            writeString(row, 1, item.key().c_str(), "text");
+            writeDouble(row, 2, item.value().at(0).get<double>(), "result");
+            auto fileName = item.value().at(2).get<std::string>();
+            std::string hyperlink = "";
+            try {
+                hyperlink = files.at(fileName);
+            }
+            catch (const std::out_of_range& oor) {
+                auto tabName = "table_" + std::to_string(i);
+                auto worksheetNew = workbook_add_worksheet(workbook, tabName.c_str());
+                json data = loadResultData(Paths::results() + fileName);
+                auto report = ReportExcel(data, false, workbook, worksheetNew);
+                report.show();
+                hyperlink = "#table_" + std::to_string(i);
+                files[fileName] = hyperlink;
+            }
+            hyperlink += "!H" + std::to_string(i + 6);
+            std::string fileNameText = "=HYPERLINK(\"" + hyperlink + "\",\"" + fileName + "\")";
+            worksheet_write_formula(worksheet, row, 3, fileNameText.c_str(), efectiveStyle("text"));
+            hyperparameters = item.value().at(1).dump();
+            if (hyperparameters.size() > hypSize) {
+                hypSize = hyperparameters.size();
+            }
+            writeString(row, 4, hyperparameters, "text");
+        }
+        row++;
+        // Set Totals
+        writeString(row, 1, "Total", "bodyHeader");
+        std::stringstream oss;
+        auto colName = getColumnName(2);
+        oss << "=sum(" << colName << "5:" << colName << row << ")";
+        worksheet_write_formula(worksheet, row, 2, oss.str().c_str(), styles["bodyHeader_odd"]);
+        // Set format
+        worksheet_freeze_panes(worksheet, 4, 2);
+        std::vector<int> columns_sizes = { 5, datasetNameSize, modelNameSize, 66, hypSize + 1 };
+        for (int i = 0; i < columns_sizes.size(); ++i) {
+            worksheet_set_column(worksheet, i, i, columns_sizes.at(i), NULL);
+        }
+    }
+    BestResultsExcel::~BestResultsExcel()
+    {
+        workbook_close(workbook);
+    }
+    void BestResultsExcel::formatColumns()
+    {
+        worksheet_freeze_panes(worksheet, 4, 2);
+        std::vector<int> columns_sizes = { 5, datasetNameSize };
+        for (int i = 0; i < models.size(); ++i) {
+            columns_sizes.push_back(modelNameSize);
+        }
+        for (int i = 0; i < columns_sizes.size(); ++i) {
+            worksheet_set_column(worksheet, i, i, columns_sizes.at(i), NULL);
+        }
+    }
+    void BestResultsExcel::addConditionalFormat(std::string formula)
+    {
+        // Add conditional format for max/min values in scores/ranks sheets
+        lxw_format* custom_format = workbook_add_format(workbook);
+        format_set_bg_color(custom_format, 0xFFC7CE);
+        format_set_font_color(custom_format, 0x9C0006);
+        // Create a conditional format object. A static object would also work.
+        lxw_conditional_format* conditional_format = (lxw_conditional_format*)calloc(1, sizeof(lxw_conditional_format));
+        conditional_format->type = LXW_CONDITIONAL_TYPE_FORMULA;
+        std::string col = getColumnName(models.size() + 1);
+        std::stringstream oss;
+        oss << "=C5=" << formula << "($C5:$" << col << "5)";
+        auto formulaValue = oss.str();
+        conditional_format->value_string = formulaValue.c_str();
+        conditional_format->format = custom_format;
+        worksheet_conditional_format_range(worksheet, 4, 2, datasets.size() + 3, models.size() + 1, conditional_format);
+    }
+    void BestResultsExcel::build()
+    {
+        // Create Sheet with scores
+        header(false);
+        body(false);
+        // Add conditional format for max values
+        addConditionalFormat("max");
+        footer(false);
+        if (friedman) {
+            // Create Sheet with ranks
+            worksheet = workbook_add_worksheet(workbook, "Ranks");
+            formatColumns();
+            header(true);
+            body(true);
+            addConditionalFormat("min");
+            footer(true);
+            // Create Sheet with Friedman Test
+            doFriedman();
+        }
+    }
+
+    void BestResultsExcel::header(bool ranks)
+    {
+        row = 0;
+        std::string message = ranks ? "Ranks for score " + score : "Best results for " + score;
+        worksheet_merge_range(worksheet, 0, 0, 0, 1 + models.size(), message.c_str(), styles["headerFirst"]);
+        // Body header
+        row = 3;
+        int col = 1;
+        writeString(row, 0, "Nº", "bodyHeader");
+        writeString(row, 1, "Dataset", "bodyHeader");
+        for (const auto& model : models) {
+            writeString(row, ++col, model.c_str(), "bodyHeader");
+        }
+    }
+    void BestResultsExcel::body(bool ranks)
+    {
+        row = 4;
+        int i = 0;
+        json origin = table.begin().value();
+        for (auto const& item : origin.items()) {
+            writeInt(row, 0, i++, "ints");
+            writeString(row, 1, item.key().c_str(), "text");
+            int col = 1;
+            for (const auto& model : models) {
+                double value = ranks ? ranksModels[item.key()][model] : table[model].at(item.key()).at(0).get<double>();
+                writeDouble(row, ++col, value, "result");
+            }
+            ++row;
+        }
+    }
+    void BestResultsExcel::footer(bool ranks)
+    {
+        // Set Totals
+        writeString(row, 1, "Total", "bodyHeader");
+        int col = 1;
+        for (const auto& model : models) {
+            std::stringstream oss;
+            auto colName = getColumnName(col + 1);
+            oss << "=SUM(" << colName << "5:" << colName << row << ")";
+            worksheet_write_formula(worksheet, row, ++col, oss.str().c_str(), styles["bodyHeader_odd"]);
+        }
+        if (ranks) {
+            row++;
+            writeString(row, 1, "Average ranks", "bodyHeader");
+            int col = 1;
+            for (const auto& model : models) {
+                auto colName = getColumnName(col + 1);
+                std::stringstream oss;
+                oss << "=SUM(" << colName << "5:" << colName << row - 1 << ")/" << datasets.size();
+                worksheet_write_formula(worksheet, row, ++col, oss.str().c_str(), styles["bodyHeader_odd"]);
+            }
+        }
+    }
+    void BestResultsExcel::doFriedman()
+    {
+        worksheet = workbook_add_worksheet(workbook, "Friedman");
+        std::vector<int> columns_sizes = { 5, datasetNameSize };
+        for (int i = 0; i < models.size(); ++i) {
+            columns_sizes.push_back(modelNameSize);
+        }
+        for (int i = 0; i < columns_sizes.size(); ++i) {
+            worksheet_set_column(worksheet, i, i, columns_sizes.at(i), NULL);
+        }
+        worksheet_merge_range(worksheet, 0, 0, 0, 7, "Friedman Test", styles["headerFirst"]);
+        row = 2;
+        Statistics stats(models, datasets, table, significance, false);
+        auto result = stats.friedmanTest();
+        stats.postHocHolmTest(result);
+        auto friedmanResult = stats.getFriedmanResult();
+        auto holmResult = stats.getHolmResult();
+        worksheet_merge_range(worksheet, row, 0, row, 7, "Null hypothesis: H0 'There is no significant differences between all the classifiers.'", styles["headerSmall"]);
+        row += 2;
+        writeString(row, 1, "Friedman Q", "bodyHeader");
+        writeDouble(row, 2, friedmanResult.statistic, "bodyHeader");
+        row++;
+        writeString(row, 1, "Critical χ2 value", "bodyHeader");
+        writeDouble(row, 2, friedmanResult.criticalValue, "bodyHeader");
+        row++;
+        writeString(row, 1, "p-value", "bodyHeader");
+        writeDouble(row, 2, friedmanResult.pvalue, "bodyHeader");
+        writeString(row, 3, friedmanResult.reject ? "<" : ">", "bodyHeader");
+        writeDouble(row, 4, significance, "bodyHeader");
+        writeString(row, 5, friedmanResult.reject ? "Reject H0" : "Accept H0", "bodyHeader");
+        row += 3;
+        worksheet_merge_range(worksheet, row, 0, row, 7, "Holm Test", styles["headerFirst"]);
+        row += 2;
+        worksheet_merge_range(worksheet, row, 0, row, 7, "Null hypothesis: H0 'There is no significant differences between the control model and the other models.'", styles["headerSmall"]);
+        row += 2;
+        std::string controlModel = "Control Model: " + holmResult.model;
+        worksheet_merge_range(worksheet, row, 1, row, 7, controlModel.c_str(), styles["bodyHeader_odd"]);
+        row++;
+        writeString(row, 1, "Model", "bodyHeader");
+        writeString(row, 2, "p-value", "bodyHeader");
+        writeString(row, 3, "Rank", "bodyHeader");
+        writeString(row, 4, "Win", "bodyHeader");
+        writeString(row, 5, "Tie", "bodyHeader");
+        writeString(row, 6, "Loss", "bodyHeader");
+        writeString(row, 7, "Reject H0", "bodyHeader");
+        row++;
+        bool first = true;
+        for (const auto& item : holmResult.holmLines) {
+            writeString(row, 1, item.model, "text");
+            if (first) {
+                // Control model info
+                first = false;
+                writeString(row, 2, "", "text");
+                writeDouble(row, 3, item.rank, "result");
+                writeString(row, 4, "", "text");
+                writeString(row, 5, "", "text");
+                writeString(row, 6, "", "text");
+                writeString(row, 7, "", "textCentered");
+            } else {
+                // Rest of the models info
+                writeDouble(row, 2, item.pvalue, "result");
+                writeDouble(row, 3, item.rank, "result");
+                writeInt(row, 4, item.wtl.win, "ints");
+                writeInt(row, 5, item.wtl.tie, "ints");
+                writeInt(row, 6, item.wtl.loss, "ints");
+                writeString(row, 7, item.reject ? "Yes" : "No", "textCentered");
+            }
+            row++;
+        }
+        // set column width for the 5th and the 7th column
+        worksheet_set_column(worksheet, 4, 5, 10, NULL);
+        worksheet_set_column(worksheet, 6, 7, 10, NULL);
+    }
+}

src/best/BestResultsExcel.h

@@ -0,0 +1,37 @@
+#ifndef BESTRESULTS_EXCEL_H
+#define BESTRESULTS_EXCEL_H
+#include "ExcelFile.h"
+#include <vector>
+#include <map>
+#include <nlohmann/json.hpp>
+
+using json = nlohmann::json;
+
+namespace platform {
+
+    class BestResultsExcel : public ExcelFile {
+    public:
+        BestResultsExcel(const std::string& score, const std::vector<std::string>& datasets);
+        ~BestResultsExcel();
+        void reportAll(const std::vector<std::string>& models, const json& table, const std::map<std::string, std::map<std::string, float>>& ranks, bool friedman, double significance);
+        void reportSingle(const std::string& model, const std::string& fileName);
+    private:
+        void build();
+        void header(bool ranks);
+        void body(bool ranks);
+        void footer(bool ranks);
+        void formatColumns();
+        void doFriedman();
+        void addConditionalFormat(std::string formula);
+        std::string score;
+        std::vector<std::string> models;
+        std::vector<std::string> datasets;
+        json table;
+        std::map<std::string, std::map<std::string, float>> ranksModels;
+        bool friedman;
+        double significance;
+        int modelNameSize = 12; // Min size of the column
+        int datasetNameSize = 25; // Min size of the column
+    };
+}
+#endif //BESTRESULTS_EXCEL_H

src/best/BestScore.h

@@ -0,0 +1,28 @@
+#ifndef BESTSCORE_H
+#define BESTSCORE_H
+#include <string>
+#include <map>
+#include <utility>
+#include "DotEnv.h"
+namespace platform {
+    class BestScore {
+    public:
+        static std::pair<std::string, double> getScore(const std::string& metric)
+        {
+            static std::map<std::pair<std::string, std::string>, std::pair<std::string, double>> data = {
+               {{"discretiz", "accuracy"}, {"STree_default (linear-ovo)",  22.109799}},
+               {{"odte", "accuracy"}, {"STree_default (linear-ovo)",  22.109799}},
+            };
+            auto env = platform::DotEnv();
+            std::string experiment = env.get("experiment");
+            try {
+                return data[{experiment, metric}];
+            }
+            catch (...) {
+                return { "", 0.0 };
+            }
+        }
+    };
+}
+
+#endif

src/best/Statistics.cc

@@ -0,0 +1,252 @@
+#include <sstream>
+#include "Statistics.h"
+#include "Colors.h"
+#include "Symbols.h"
+#include <boost/math/distributions/chi_squared.hpp>
+#include <boost/math/distributions/normal.hpp>
+#include "CLocale.h"
+
+
+namespace platform {
+
+    Statistics::Statistics(const std::vector<std::string>& models, const std::vector<std::string>& datasets, const json& data, double significance, bool output) :
+        models(models), datasets(datasets), data(data), significance(significance), output(output)
+    {
+        nModels = models.size();
+        nDatasets = datasets.size();
+        auto temp = ConfigLocale();
+    }
+
+    void Statistics::fit()
+    {
+        if (nModels < 3 || nDatasets < 3) {
+            std::cerr << "nModels: " << nModels << std::endl;
+            std::cerr << "nDatasets: " << nDatasets << std::endl;
+            throw std::runtime_error("Can't make the Friedman test with less than 3 models and/or less than 3 datasets.");
+        }
+        ranksModels.clear();
+        computeRanks();
+        // Set the control model as the one with the lowest average rank
+        controlIdx = distance(ranks.begin(), min_element(ranks.begin(), ranks.end(), [](const auto& l, const auto& r) { return l.second < r.second; }));
+        computeWTL();
+        maxModelName = (*std::max_element(models.begin(), models.end(), [](const std::string& a, const std::string& b) { return a.size() < b.size(); })).size();
+        maxDatasetName = (*std::max_element(datasets.begin(), datasets.end(), [](const std::string& a, const std::string& b) { return a.size() < b.size(); })).size();
+        fitted = true;
+    }
+    std::map<std::string, float> assignRanks(std::vector<std::pair<std::string, double>>& ranksOrder)
+    {
+        // sort the ranksOrder std::vector by value
+        std::sort(ranksOrder.begin(), ranksOrder.end(), [](const std::pair<std::string, double>& a, const std::pair<std::string, double>& b) {
+            return a.second > b.second;
+            });
+        //Assign ranks to  values and if they are the same they share the same averaged rank
+        std::map<std::string, float> ranks;
+        for (int i = 0; i < ranksOrder.size(); i++) {
+            ranks[ranksOrder[i].first] = i + 1.0;
+        }
+        int i = 0;
+        while (i < static_cast<int>(ranksOrder.size())) {
+            int j = i + 1;
+            int sumRanks = ranks[ranksOrder[i].first];
+            while (j < static_cast<int>(ranksOrder.size()) && ranksOrder[i].second == ranksOrder[j].second) {
+                sumRanks += ranks[ranksOrder[j++].first];
+            }
+            if (j > i + 1) {
+                float averageRank = (float)sumRanks / (j - i);
+                for (int k = i; k < j; k++) {
+                    ranks[ranksOrder[k].first] = averageRank;
+                }
+            }
+            i = j;
+        }
+        return ranks;
+    }
+    void Statistics::computeRanks()
+    {
+        std::map<std::string, float> ranksLine;
+        for (const auto& dataset : datasets) {
+            std::vector<std::pair<std::string, double>> ranksOrder;
+            for (const auto& model : models) {
+                double value = data[model].at(dataset).at(0).get<double>();
+                ranksOrder.push_back({ model, value });
+            }
+            // Assign the ranks
+            ranksLine = assignRanks(ranksOrder);
+            // Store the ranks of the dataset
+            ranksModels[dataset] = ranksLine;
+            if (ranks.size() == 0) {
+                ranks = ranksLine;
+            } else {
+                for (const auto& rank : ranksLine) {
+                    ranks[rank.first] += rank.second;
+                }
+            }
+        }
+        // Average the ranks
+        for (const auto& rank : ranks) {
+            ranks[rank.first] /= nDatasets;
+        }
+    }
+    void Statistics::computeWTL()
+    {
+        // Compute the WTL matrix
+        for (int i = 0; i < nModels; ++i) {
+            wtl[i] = { 0, 0, 0 };
+        }
+        json origin = data.begin().value();
+        for (auto const& item : origin.items()) {
+            auto controlModel = models.at(controlIdx);
+            double controlValue = data[controlModel].at(item.key()).at(0).get<double>();
+            for (int i = 0; i < nModels; ++i) {
+                if (i == controlIdx) {
+                    continue;
+                }
+                double value = data[models[i]].at(item.key()).at(0).get<double>();
+                if (value < controlValue) {
+                    wtl[i].win++;
+                } else if (value == controlValue) {
+                    wtl[i].tie++;
+                } else {
+                    wtl[i].loss++;
+                }
+            }
+        }
+    }
+
+    void Statistics::postHocHolmTest(bool friedmanResult)
+    {
+        if (!fitted) {
+            fit();
+        }
+        std::stringstream oss;
+        // Reference https://link.springer.com/article/10.1007/s44196-022-00083-8
+        // Post-hoc Holm test
+        // Calculate the p-value for the models paired with the control model
+        std::map<int, double> stats; // p-value of each model paired with the control model
+        boost::math::normal dist(0.0, 1.0);
+        double diff = sqrt(nModels * (nModels + 1) / (6.0 * nDatasets));
+        for (int i = 0; i < nModels; i++) {
+            if (i == controlIdx) {
+                stats[i] = 0.0;
+                continue;
+            }
+            double z = abs(ranks.at(models[controlIdx]) - ranks.at(models[i])) / diff;
+            double p_value = (long double)2 * (1 - cdf(dist, z));
+            stats[i] = p_value;
+        }
+        // Sort the models by p-value
+        std::vector<std::pair<int, double>> statsOrder;
+        for (const auto& stat : stats) {
+            statsOrder.push_back({ stat.first, stat.second });
+        }
+        std::sort(statsOrder.begin(), statsOrder.end(), [](const std::pair<int, double>& a, const std::pair<int, double>& b) {
+            return a.second < b.second;
+            });
+
+        // Holm adjustment
+        for (int i = 0; i < statsOrder.size(); ++i) {
+            auto item = statsOrder.at(i);
+            double before = i == 0 ? 0.0 : statsOrder.at(i - 1).second;
+            double p_value = std::min((double)1.0, item.second * (nModels - i));
+            p_value = std::max(before, p_value);
+            statsOrder[i] = { item.first, p_value };
+        }
+        holmResult.model = models.at(controlIdx);
+        auto color = friedmanResult ? Colors::CYAN() : Colors::YELLOW();
+        oss << color;
+        oss << "  *************************************************************************************************************" << std::endl;
+        oss << "  Post-hoc Holm test: H0: 'There is no significant differences between the control model and the other models.'" << std::endl;
+        oss << "  Control model: " << models.at(controlIdx) << std::endl;
+        oss << "  " << std::left << std::setw(maxModelName) << std::string("Model") << " p-value      rank      win tie loss Status" << std::endl;
+        oss << "  " << std::string(maxModelName, '=') << " ============ ========= === === ==== =============" << std::endl;
+        // sort ranks from lowest to highest
+        std::vector<std::pair<std::string, float>> ranksOrder;
+        for (const auto& rank : ranks) {
+            ranksOrder.push_back({ rank.first, rank.second });
+        }
+        std::sort(ranksOrder.begin(), ranksOrder.end(), [](const std::pair<std::string, float>& a, const std::pair<std::string, float>& b) {
+            return a.second < b.second;
+            });
+        // Show the control model info.
+        oss << "  " << Colors::BLUE() << std::left << std::setw(maxModelName) << ranksOrder.at(0).first << " ";
+        oss << std::setw(12) << " " << std::setprecision(7) << std::fixed << " " << ranksOrder.at(0).second << std::endl;
+        for (const auto& item : ranksOrder) {
+            auto idx = distance(models.begin(), find(models.begin(), models.end(), item.first));
+            double pvalue = 0.0;
+            for (const auto& stat : statsOrder) {
+                if (stat.first == idx) {
+                    pvalue = stat.second;
+                }
+            }
+            holmResult.holmLines.push_back({ item.first, pvalue, item.second, wtl.at(idx), pvalue < significance });
+            if (item.first == models.at(controlIdx)) {
+                continue;
+            }
+            auto colorStatus = pvalue > significance ? Colors::GREEN() : Colors::MAGENTA();
+            auto status = pvalue > significance ? Symbols::check_mark : Symbols::cross;
+            auto textStatus = pvalue > significance ? " accepted H0" : " rejected H0";
+            oss << "  " << colorStatus << std::left << std::setw(maxModelName) << item.first << " ";
+            oss << std::setprecision(6) << std::scientific << pvalue << std::setprecision(7) << std::fixed << " " << item.second;
+            oss << " " << std::right << std::setw(3) << wtl.at(idx).win << " " << std::setw(3) << wtl.at(idx).tie << " " << std::setw(4) << wtl.at(idx).loss;
+            oss << " " << status << textStatus << std::endl;
+        }
+        oss << color << "  *************************************************************************************************************" << std::endl;
+        oss << Colors::RESET();
+        if (output) {
+            std::cout << oss.str();
+        }
+    }
+    bool Statistics::friedmanTest()
+    {
+        if (!fitted) {
+            fit();
+        }
+        std::stringstream oss;
+        // Friedman test
+        // Calculate the Friedman statistic
+        oss << Colors::BLUE() << std::endl;
+        oss << "***************************************************************************************************************" << std::endl;
+        oss << Colors::GREEN() << "Friedman test: H0: 'There is no significant differences between all the classifiers.'" << Colors::BLUE() << std::endl;
+        double degreesOfFreedom = nModels - 1.0;
+        double sumSquared = 0;
+        for (const auto& rank : ranks) {
+            sumSquared += pow(rank.second, 2);
+        }
+        // Compute the Friedman statistic as in https://link.springer.com/article/10.1007/s44196-022-00083-8
+        double friedmanQ = 12.0 * nDatasets / (nModels * (nModels + 1)) * (sumSquared - (nModels * pow(nModels + 1, 2)) / 4);
+        // Calculate the critical value
+        boost::math::chi_squared chiSquared(degreesOfFreedom);
+        long double p_value = (long double)1.0 - cdf(chiSquared, friedmanQ);
+        double criticalValue = quantile(chiSquared, 1 - significance);
+        oss << "Friedman statistic: " << friedmanQ << std::endl;
+        oss << "Critical χ2 Value for df=" << std::fixed << (int)degreesOfFreedom
+            << " and alpha=" << std::setprecision(2) << std::fixed << significance << ": " << std::setprecision(7) << std::scientific << criticalValue << std::endl;
+        oss << "p-value: " << std::scientific << p_value << " is " << (p_value < significance ? "less" : "greater") << " than " << std::setprecision(2) << std::fixed << significance << std::endl;
+        bool result;
+        if (p_value < significance) {
+            oss << Colors::GREEN() << "The null hypothesis H0 is rejected." << std::endl;
+            result = true;
+        } else {
+            oss << Colors::YELLOW() << "The null hypothesis H0 is accepted. Computed p-values will not be significant." << std::endl;
+            result = false;
+        }
+        oss << Colors::BLUE() << "***************************************************************************************************************" << Colors::RESET() << std::endl;
+        if (output) {
+            std::cout << oss.str();
+        }
+        friedmanResult = { friedmanQ, criticalValue, p_value, result };
+        return result;
+    }
+    FriedmanResult& Statistics::getFriedmanResult()
+    {
+        return friedmanResult;
+    }
+    HolmResult& Statistics::getHolmResult()
+    {
+        return holmResult;
+    }
+    std::map<std::string, std::map<std::string, float>>& Statistics::getRanks()
+    {
+        return ranksModels;
+    }
+} // namespace platform

src/best/Statistics.h

@@ -0,0 +1,63 @@
+#ifndef STATISTICS_H
+#define STATISTICS_H
+#include <iostream>
+#include <vector>
+#include <map>
+#include <nlohmann/json.hpp>
+
+using json = nlohmann::json;
+
+namespace platform {
+    struct WTL {
+        int win;
+        int tie;
+        int loss;
+    };
+    struct FriedmanResult {
+        double statistic;
+        double criticalValue;
+        long double pvalue;
+        bool reject;
+    };
+    struct HolmLine {
+        std::string model;
+        long double pvalue;
+        double rank;
+        WTL wtl;
+        bool reject;
+    };
+    struct HolmResult {
+        std::string model;
+        std::vector<HolmLine> holmLines;
+    };
+    class Statistics {
+    public:
+        Statistics(const std::vector<std::string>& models, const std::vector<std::string>& datasets, const json& data, double significance = 0.05, bool output = true);
+        bool friedmanTest();
+        void postHocHolmTest(bool friedmanResult);
+        FriedmanResult& getFriedmanResult();
+        HolmResult& getHolmResult();
+        std::map<std::string, std::map<std::string, float>>& getRanks();
+    private:
+        void fit();
+        void computeRanks();
+        void computeWTL();
+        const std::vector<std::string>& models;
+        const std::vector<std::string>& datasets;
+        const json& data;
+        double significance;
+        bool output;
+        bool fitted = false;
+        int nModels = 0;
+        int nDatasets = 0;
+        int controlIdx = 0;
+        std::map<int, WTL> wtl;
+        std::map<std::string, float> ranks;
+        int maxModelName = 0;
+        int maxDatasetName = 0;
+        FriedmanResult friedmanResult;
+        HolmResult holmResult;
+        std::map<std::string, std::map<std::string, float>> ranksModels;
+    };
+}
+#endif // !STATISTICS_H

src/best/b_best.cc

@@ -0,0 +1,70 @@
+#include <iostream>
+#include <argparse/argparse.hpp>
+#include "Paths.h"
+#include "BestResults.h"
+#include "Colors.h"
+#include "config.h"
+
+void manageArguments(argparse::ArgumentParser& program)
+{
+    program.add_argument("-m", "--model").default_value("").help("Filter results of the selected model) (any for all models)");
+    program.add_argument("-s", "--score").default_value("accuracy").help("Filter results of the score name supplied");
+    program.add_argument("--friedman").help("Friedman test").default_value(false).implicit_value(true);
+    program.add_argument("--excel").help("Output to excel").default_value(false).implicit_value(true);
+    program.add_argument("--level").help("significance level").default_value(0.05).scan<'g', double>().action([](const std::string& value) {
+        try {
+            auto k = std::stod(value);
+            if (k < 0.01 || k > 0.15) {
+                throw std::runtime_error("Significance level hast to be a number in [0.01, 0.15]");
+            }
+            return k;
+        }
+        catch (const std::runtime_error& err) {
+            throw std::runtime_error(err.what());
+        }
+        catch (...) {
+            throw std::runtime_error("Number of folds must be an decimal number");
+        }});
+}
+
+int main(int argc, char** argv)
+{
+    argparse::ArgumentParser program("b_best", { project_version.begin(), project_version.end() });
+    manageArguments(program);
+    std::string model, score;
+    bool build, report, friedman, excel;
+    double level;
+    try {
+        program.parse_args(argc, argv);
+        model = program.get<std::string>("model");
+        score = program.get<std::string>("score");
+        friedman = program.get<bool>("friedman");
+        excel = program.get<bool>("excel");
+        level = program.get<double>("level");
+        if (model == "" || score == "") {
+            throw std::runtime_error("Model and score name must be supplied");
+        }
+        if (friedman && model != "any") {
+            std::cerr << "Friedman test can only be used with all models" << std::endl;
+            std::cerr << program;
+            exit(1);
+        }
+    }
+    catch (const std::exception& err) {
+        std::cerr << err.what() << std::endl;
+        std::cerr << program;
+        exit(1);
+    }
+    // Generate report
+    auto results = platform::BestResults(platform::Paths::results(), score, model, friedman, level);
+    if (model == "any") {
+        results.buildAll();
+        results.reportAll(excel);
+    } else {
+        std::string fileName = results.build();
+        std::cout << Colors::GREEN() << fileName << " created!" << Colors::RESET() << std::endl;
+        results.reportSingle(excel);
+    }
+    std::cout << Colors::RESET();
+    return 0;
+}