Add date/time to header of report best

.vscode/launch.json

@@ -37,6 +37,20 @@
             ],
             "cwd": "/Users/rmontanana/Code/discretizbench",
         },
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "best",
+            "program": "${workspaceFolder}/build/src/Platform/best",
+            "args": [
+                "-m",
+                "BoostAODE",
+                "-s",
+                "accuracy",
+                "--build",
+            ],
+            "cwd": "/Users/rmontanana/Code/discretizbench",
+        },
         {
             "type": "lldb",
             "request": "launch",

src/Platform/BestResults.cc

@@ -1,18 +1,38 @@
 #include <filesystem>
 #include <fstream>
 #include <iostream>
-#include "platformUtils.h"
+#include <sstream>
+#include <set>
 #include "BestResults.h"
-#include "Results.h"
+#include "Result.h"
 #include "Colors.h"
 
+
+
+namespace fs = std::filesystem;
+// function ftime_to_string, Code taken from 
+// https://stackoverflow.com/a/58237530/1389271
+template <typename TP>
+std::string ftime_to_string(TP tp)
+{
+    using namespace std::chrono;
+    auto sctp = time_point_cast<system_clock::duration>(tp - TP::clock::now()
+        + system_clock::now());
+    auto tt = system_clock::to_time_t(sctp);
+    std::tm* gmt = std::gmtime(&tt);
+    std::stringstream buffer;
+    buffer << std::put_time(gmt, "%Y-%m-%d %H:%M");
+    return buffer.str();
+}
+
 namespace platform {
 
-    void BestResults::build()
+    string BestResults::build()
     {
-        auto files = loadFiles();
+        auto files = loadResultFiles();
         if (files.size() == 0) {
-            throw runtime_error("No result files were found!");
+            cerr << Colors::MAGENTA() << "No result files were found!" << Colors::RESET() << endl;
+            exit(1);
         }
         json bests;
         for (const auto& file : files) {
@@ -21,7 +41,7 @@ namespace platform {
             for (auto const& item : data.at("results")) {
                 bool update = false;
                 if (bests.contains(item.at("dataset").get<string>())) {
-                    if (item.at("score").get<double>() > bests["dataset"].at(0).get<double>()) {
+                    if (item.at("score").get<double>() > bests[item.at("dataset").get<string>()].at(0).get<double>()) {
                         update = true;
                     }
                 } else {
@@ -32,13 +52,15 @@ namespace platform {
                 }
             }
         }
-        string bestFileName = path + "/" + bestResultFile();
-        if (file_exists(bestFileName)) {
-            cout << Colors::MAGENTA() << "File " << bestFileName << " already exists and it shall be overwritten." << Colors::RESET();
+        string bestFileName = path + bestResultFile();
+        if (FILE* fileTest = fopen(bestFileName.c_str(), "r")) {
+            fclose(fileTest);
+            cout << Colors::MAGENTA() << "File " << bestFileName << " already exists and it shall be overwritten." << Colors::RESET() << endl;
         }
         ofstream file(bestFileName);
         file << bests;
         file.close();
+        return bestFileName;
     }
 
     string BestResults::bestResultFile()
@@ -46,23 +68,238 @@ namespace platform {
         return "best_results_" + score + "_" + model + ".json";
     }
 
-    vector<string> BestResults::loadFiles()
+    pair<string, string> getModelScore(string name)
+    {
+        // results_accuracy_BoostAODE_MacBookpro16_2023-09-06_12:27:00_1.json
+        int i = 0;
+        auto pos = name.find("_");
+        auto pos2 = name.find("_", pos + 1);
+        string score = name.substr(pos + 1, pos2 - pos - 1);
+        pos = name.find("_", pos2 + 1);
+        string model = name.substr(pos2 + 1, pos - pos2 - 1);
+        return { model, score };
+    }
+
+    vector<string> BestResults::loadResultFiles()
     {
         vector<string> files;
         using std::filesystem::directory_iterator;
+        string fileModel, fileScore;
         for (const auto& file : directory_iterator(path)) {
             auto fileName = file.path().filename().string();
-            if (fileName.find(".json") != string::npos && fileName.find("results_") == 0
-                && fileName.find("_" + score + "_") != string::npos
-                && fileName.find("_" + model + "_") != string::npos) {
-                files.push_back(fileName);
+            if (fileName.find(".json") != string::npos && fileName.find("results_") == 0) {
+                tie(fileModel, fileScore) = getModelScore(fileName);
+                if (score == fileScore && (model == fileModel || model == "any")) {
+                    files.push_back(fileName);
+                }
             }
         }
         return files;
     }
 
-    void BestResults::report()
+    json BestResults::loadFile(const string& fileName)
     {
+        ifstream resultData(fileName);
+        if (resultData.is_open()) {
+            json data = json::parse(resultData);
+            return data;
+        }
+        throw invalid_argument("Unable to open result file. [" + fileName + "]");
+    }
+    set<string> BestResults::getModels()
+    {
+        set<string> models;
+        auto files = loadResultFiles();
+        if (files.size() == 0) {
+            cerr << Colors::MAGENTA() << "No result files were found!" << Colors::RESET() << endl;
+            exit(1);
+        }
+        string fileModel, fileScore;
+        for (const auto& file : files) {
+            // extract the model from the file name
+            tie(fileModel, fileScore) = getModelScore(file);
+            // add the model to the vector of models
+            models.insert(fileModel);
+        }
+        return models;
+    }
 
+    void BestResults::buildAll()
+    {
+        auto models = getModels();
+        for (const auto& model : models) {
+            cout << "Building best results for model: " << model << endl;
+            this->model = model;
+            build();
+        }
+        model = "any";
+    }
+
+    void BestResults::reportSingle()
+    {
+        string bestFileName = path + bestResultFile();
+        if (FILE* fileTest = fopen(bestFileName.c_str(), "r")) {
+            fclose(fileTest);
+        } else {
+            cerr << Colors::MAGENTA() << "File " << bestFileName << " doesn't exist." << Colors::RESET() << endl;
+            exit(1);
+        }
+        auto date = ftime_to_string(filesystem::last_write_time(bestFileName));
+        auto data = loadFile(bestFileName);
+        cout << Colors::GREEN() << "Best results for " << model << " and " << score << " as of " << date << endl;
+        cout << "--------------------------------------------------------" << endl;
+        cout << Colors::GREEN() << " #  Dataset                   Score       File                                                               Hyperparameters" << endl;
+        cout << "=== ========================= =========== ================================================================== ================================================= " << endl;
+        auto i = 0;
+        bool odd = true;
+        for (auto const& item : data.items()) {
+            auto color = odd ? Colors::BLUE() : Colors::CYAN();
+            cout << color << setw(3) << fixed << right << i++ << " ";
+            cout << setw(25) << left << item.key() << " ";
+            cout << setw(11) << setprecision(9) << fixed << item.value().at(0).get<double>() << " ";
+            cout << setw(66) << item.value().at(2).get<string>() << " ";
+            cout << item.value().at(1) << " ";
+            cout << endl;
+            odd = !odd;
+        }
+    }
+    json BestResults::buildTableResults(set<string> models)
+    {
+        int numberOfDatasets = 0;
+        bool first = true;
+        json origin;
+        json table;
+        auto maxDate = filesystem::file_time_type::max();
+        for (const auto& model : models) {
+            this->model = model;
+            string bestFileName = path + bestResultFile();
+            if (FILE* fileTest = fopen(bestFileName.c_str(), "r")) {
+                fclose(fileTest);
+            } else {
+                cerr << Colors::MAGENTA() << "File " << bestFileName << " doesn't exist." << Colors::RESET() << endl;
+                exit(1);
+            }
+            auto dateWrite = filesystem::last_write_time(bestFileName);
+            if (dateWrite < maxDate) {
+                maxDate = dateWrite;
+            }
+            auto data = loadFile(bestFileName);
+            if (first) {
+                // Get the number of datasets of the first file and check that is the same for all the models
+                first = false;
+                numberOfDatasets = data.size();
+                origin = data;
+            } else {
+                if (numberOfDatasets != data.size()) {
+                    cerr << Colors::MAGENTA() << "The number of datasets in the best results files is not the same for all the models." << Colors::RESET() << endl;
+                    exit(1);
+                }
+            }
+            table[model] = data;
+        }
+        table["dateTable"] = ftime_to_string(maxDate);
+        return table;
+    }
+    void BestResults::printTableResults(set<string> models, json table)
+    {
+        cout << Colors::GREEN() << "Best results for " << score << " as of " << table.at("dateTable").get<string>() << endl;
+        cout << "------------------------------------------------" << endl;
+        cout << Colors::GREEN() << " #  Dataset                   ";
+        for (const auto& model : models) {
+            cout << setw(12) << left << model << " ";
+        }
+        cout << endl;
+        cout << "=== ========================= ";
+        for (const auto& model : models) {
+            cout << "============ ";
+        }
+        cout << endl;
+        auto i = 0;
+        bool odd = true;
+        map<string, double> totals;
+        map<string, int> ranks;
+        for (const auto& model : models) {
+            totals[model] = 0.0;
+        }
+        json origin = table.begin().value();
+        for (auto const& item : origin.items()) {
+            auto color = odd ? Colors::BLUE() : Colors::CYAN();
+            cout << color << setw(3) << fixed << right << i++ << " ";
+            cout << setw(25) << left << item.key() << " ";
+            double maxValue = 0;
+            vector<pair<string, double>> ranksOrder;
+            // Find out the max value for this dataset
+            for (const auto& model : models) {
+                double value = table[model].at(item.key()).at(0).get<double>();
+                if (value > maxValue) {
+                    maxValue = value;
+                }
+                ranksOrder.push_back({ model, value });
+            }
+            // sort the ranksOrder vector by value
+            sort(ranksOrder.begin(), ranksOrder.end(), [](const pair<string, double>& a, const pair<string, double>& b) {
+                return a.second > b.second;
+                });
+            // Assign the ranks
+            for (int i = 0; i < ranksOrder.size(); i++) {
+                ranks[ranksOrder[i].first] = i + 1;
+            }
+            // Print the row with red colors on max values
+            for (const auto& model : models) {
+                string efectiveColor = color;
+                double value = table[model].at(item.key()).at(0).get<double>();
+                if (value == maxValue) {
+                    efectiveColor = Colors::RED();
+                }
+                totals[model] += value;
+                cout << efectiveColor << setw(12) << setprecision(10) << fixed << value << " ";
+            }
+            cout << endl;
+            odd = !odd;
+        }
+        cout << Colors::GREEN() << "=== ========================= ";
+        for (const auto& model : models) {
+            cout << "============ ";
+        }
+        cout << endl;
+        cout << Colors::GREEN() << setw(30) << "    Totals...................";
+        double max = 0.0;
+        for (const auto& total : totals) {
+            if (total.second > max) {
+                max = total.second;
+            }
+        }
+        for (const auto& model : models) {
+            string efectiveColor = Colors::GREEN();
+            if (totals[model] == max) {
+                efectiveColor = Colors::RED();
+            }
+            cout << efectiveColor << setw(12) << setprecision(9) << fixed << totals[model] << " ";
+        }
+        // Output the averaged ranks
+        cout << endl;
+        int min = 1;
+        for (const auto& rank : ranks) {
+            if (rank.second < min) {
+                min = rank.second;
+            }
+        }
+        cout << Colors::GREEN() << setw(30) << "    Averaged ranks...........";
+        for (const auto& model : models) {
+            string efectiveColor = Colors::GREEN();
+            if (ranks[model] == min) {
+                efectiveColor = Colors::RED();
+            }
+            cout << efectiveColor << setw(12) << setprecision(10) << fixed << (double)ranks[model] / (double)origin.size() << " ";
+        }
+        cout << endl;
+    }
+    void BestResults::reportAll()
+    {
+        auto models = getModels();
+        // Build the table of results
+        json table = buildTableResults(models);
+        // Print the table of results
+        printTableResults(models, table);
     }
 }

src/Platform/BestResults.h

@@ -1,17 +1,25 @@
 #ifndef BESTRESULTS_H
 #define BESTRESULTS_H
 #include <string>
+#include <set>
+#include <nlohmann/json.hpp>
 using namespace std;
-
+using json = nlohmann::json;
 namespace platform {
     class BestResults {
     public:
         explicit BestResults(const string& path, const string& score, const string& model) : path(path), score(score), model(model) {}
-        void build();
-        void report();
+        string build();
+        void reportSingle();
+        void reportAll();
+        void buildAll();
     private:
-        vector<string> loadFiles();
+        set<string> getModels();
+        vector<string> loadResultFiles();
+        json buildTableResults(set<string> models);
+        void printTableResults(set<string> models, json table);
         string bestResultFile();
+        json loadFile(const string& fileName);
         string path;
         string score;
         string model;

src/Platform/CMakeLists.txt

@@ -6,15 +6,14 @@ include_directories(${BayesNet_SOURCE_DIR}/lib/argparse/include)
 include_directories(${BayesNet_SOURCE_DIR}/lib/json/include)
 include_directories(${BayesNet_SOURCE_DIR}/lib/libxlsxwriter/include)
 add_executable(main main.cc Folding.cc platformUtils.cc Experiment.cc Datasets.cc Models.cc ReportConsole.cc ReportBase.cc)
-add_executable(manage manage.cc Results.cc ReportConsole.cc ReportExcel.cc ReportBase.cc Datasets.cc platformUtils.cc)
+add_executable(manage manage.cc Results.cc Result.cc ReportConsole.cc ReportExcel.cc ReportBase.cc Datasets.cc platformUtils.cc)
 add_executable(list list.cc platformUtils Datasets.cc)
-add_executable(best best.cc BestResults.cc Results.cc ReportBase.cc ReportExcel.cc platformUtils.cc)
+add_executable(best best.cc BestResults.cc Result.cc)
 target_link_libraries(main BayesNet ArffFiles mdlp "${TORCH_LIBRARIES}")
 if (${CMAKE_HOST_SYSTEM_NAME} MATCHES "Linux")
     target_link_libraries(manage "${TORCH_LIBRARIES}" libxlsxwriter.so ArffFiles mdlp stdc++fs)
-    target_link_libraries(best "${TORCH_LIBRARIES}" libxlsxwriter.so stdc++fs)
+    target_link_libraries(best stdc++fs)
 else()
     target_link_libraries(manage "${TORCH_LIBRARIES}" "${XLSXWRITER_LIB}" ArffFiles mdlp)
-    target_link_libraries(best "${TORCH_LIBRARIES}" "${XLSXWRITER_LIB}")
 endif()
 target_link_libraries(list ArffFiles mdlp "${TORCH_LIBRARIES}")

src/Platform/Result.cc

@@ -0,0 +1,51 @@
+#include <filesystem>
+#include <fstream>
+#include <sstream>
+#include "Result.h"
+#include "Colors.h"
+#include "BestScore.h"
+namespace platform {
+    Result::Result(const string& path, const string& filename)
+        : path(path)
+        , filename(filename)
+    {
+        auto data = load();
+        date = data["date"];
+        score = 0;
+        for (const auto& result : data["results"]) {
+            score += result["score"].get<double>();
+        }
+        scoreName = data["score_name"];
+        if (scoreName == BestScore::scoreName()) {
+            score /= BestScore::score();
+        }
+        title = data["title"];
+        duration = data["duration"];
+        model = data["model"];
+        complete = data["results"].size() > 1;
+    }
+
+    json Result::load() const
+    {
+        ifstream resultData(path + "/" + filename);
+        if (resultData.is_open()) {
+            json data = json::parse(resultData);
+            return data;
+        }
+        throw invalid_argument("Unable to open result file. [" + path + "/" + filename + "]");
+    }
+
+    string Result::to_string() const
+    {
+        stringstream oss;
+        oss << date << " ";
+        oss << setw(12) << left << model << " ";
+        oss << setw(11) << left << scoreName << " ";
+        oss << right << setw(11) << setprecision(7) << fixed << score << " ";
+        auto completeString = isComplete() ? "C" : "P";
+        oss << setw(1) << " " << completeString << "  ";
+        oss << setw(9) << setprecision(3) << fixed << duration << " ";
+        oss << setw(50) << left << title << " ";
+        return  oss.str();
+    }
+}

src/Platform/Result.h

@@ -0,0 +1,37 @@
+#ifndef RESULT_H
+#define RESULT_H
+#include <map>
+#include <vector>
+#include <string>
+#include <nlohmann/json.hpp>
+namespace platform {
+    using namespace std;
+    using json = nlohmann::json;
+
+    class Result {
+    public:
+        Result(const string& path, const string& filename);
+        json load() const;
+        string to_string() const;
+        string getFilename() const { return filename; };
+        string getDate() const { return date; };
+        double getScore() const { return score; };
+        string getTitle() const { return title; };
+        double getDuration() const { return duration; };
+        string getModel() const { return model; };
+        string getScoreName() const { return scoreName; };
+        bool isComplete() const { return complete; };
+    private:
+        string path;
+        string filename;
+        string date;
+        double score;
+        string title;
+        double duration;
+        string model;
+        string scoreName;
+        bool complete;
+    };
+};
+
+#endif

src/Platform/Results.cc

@@ -6,34 +6,6 @@
 #include "BestScore.h"
 #include "Colors.h"
 namespace platform {
-    Result::Result(const string& path, const string& filename)
-        : path(path)
-        , filename(filename)
-    {
-        auto data = load();
-        date = data["date"];
-        score = 0;
-        for (const auto& result : data["results"]) {
-            score += result["score"].get<double>();
-        }
-        scoreName = data["score_name"];
-        if (scoreName == BestScore::scoreName()) {
-            score /= BestScore::score();
-        }
-        title = data["title"];
-        duration = data["duration"];
-        model = data["model"];
-        complete = data["results"].size() > 1;
-    }
-    json Result::load() const
-    {
-        ifstream resultData(path + "/" + filename);
-        if (resultData.is_open()) {
-            json data = json::parse(resultData);
-            return data;
-        }
-        throw invalid_argument("Unable to open result file. [" + path + "/" + filename + "]");
-    }
     void Results::load()
     {
         using std::filesystem::directory_iterator;
@@ -52,19 +24,6 @@ namespace platform {
             max = files.size();
         }
     }
-    string Result::to_string() const
-    {
-        stringstream oss;
-        oss << date << " ";
-        oss << setw(12) << left << model << " ";
-        oss << setw(11) << left << scoreName << " ";
-        oss << right << setw(11) << setprecision(7) << fixed << score << " ";
-        auto completeString = isComplete() ? "C" : "P";
-        oss << setw(1) << " " << completeString << "  ";
-        oss << setw(9) << setprecision(3) << fixed << duration << " ";
-        oss << setw(50) << left << title << " ";
-        return  oss.str();
-    }
     void Results::show() const
     {
         cout << Colors::GREEN() << "Results found: " << files.size() << endl;

src/Platform/Results.h

@@ -5,34 +5,11 @@
 #include <vector>
 #include <string>
 #include <nlohmann/json.hpp>
+#include "Result.h"
 namespace platform {
     using namespace std;
     using json = nlohmann::json;
 
-    class Result {
-    public:
-        Result(const string& path, const string& filename);
-        json load() const;
-        string to_string() const;
-        string getFilename() const { return filename; };
-        string getDate() const { return date; };
-        double getScore() const { return score; };
-        string getTitle() const { return title; };
-        double getDuration() const { return duration; };
-        string getModel() const { return model; };
-        string getScoreName() const { return scoreName; };
-        bool isComplete() const { return complete; };
-    private:
-        string path;
-        string filename;
-        string date;
-        double score;
-        string title;
-        double duration;
-        string model;
-        string scoreName;
-        bool complete;
-    };
     class Results {
     public:
         Results(const string& path, const int max, const string& model, const string& score, bool complete, bool partial, bool compare) :

src/Platform/best.cc

@@ -2,14 +2,15 @@
 #include <argparse/argparse.hpp>
 #include "Paths.h"
 #include "BestResults.h"
+#include "Colors.h"
 
 using namespace std;
 
 argparse::ArgumentParser manageArguments(int argc, char** argv)
 {
     argparse::ArgumentParser program("best");
-    program.add_argument("-m", "--model").default_value("any").help("Filter results of the selected model)");
-    program.add_argument("-s", "--score").default_value("any").help("Filter results of the score name supplied");
+    program.add_argument("-m", "--model").default_value("").help("Filter results of the selected model) (any for all models)");
+    program.add_argument("-s", "--score").default_value("").help("Filter results of the score name supplied");
     program.add_argument("--build").help("build best score results file").default_value(false).implicit_value(true);
     program.add_argument("--report").help("report of best score results file").default_value(false).implicit_value(true);
     try {
@@ -18,6 +19,9 @@ argparse::ArgumentParser manageArguments(int argc, char** argv)
         auto score = program.get<string>("score");
         auto build = program.get<bool>("build");
         auto report = program.get<bool>("report");
+        if (model == "" || score == "") {
+            throw runtime_error("Model and score name must be supplied");
+        }
     }
     catch (const exception& err) {
         cerr << err.what() << endl;
@@ -35,15 +39,25 @@ int main(int argc, char** argv)
     auto build = program.get<bool>("build");
     auto report = program.get<bool>("report");
     if (!report && !build) {
-        cout << "Either build, report or both, have to be selected to do anything!" << endl;
+        cerr << "Either build, report or both, have to be selected to do anything!" << endl;
+        cerr << program;
         exit(1);
     }
-    auto results = platform::BestResults(platform::Paths::results(), model, score);
+    auto results = platform::BestResults(platform::Paths::results(), score, model);
     if (build) {
-        results.build();
+        if (model == "any") {
+            results.buildAll();
+        } else {
+            string fileName = results.build();
+            cout << Colors::GREEN() << fileName << " created!" << Colors::RESET() << endl;
+        }
     }
     if (report) {
-        results.report();
+        if (model == "any") {
+            results.reportAll();
+        } else {
+            results.reportSingle();
+        }
     }
     return 0;
 }