Add date/time to header of report best

Add multiple sheets to excel file
Add format and color to sheets
Add comparison with ZeroR
Add comparison with Best Results
Separate contextual menu from general in manage

.gitignore

@@ -36,3 +36,4 @@ build/
 cmake-build*/**
 .idea
 puml/**
+.vscode/settings.json

.gitmodules

@@ -10,6 +10,6 @@
 [submodule "lib/json"]
 	path = lib/json
 	url = https://github.com/nlohmann/json.git
-[submodule "lib/openXLSX"]
-	path = lib/openXLSX
-	url = https://github.com/troldal/OpenXLSX.git
+[submodule "lib/libxlsxwriter"]
+	path = lib/libxlsxwriter
+	url = https://github.com/jmcnamara/libxlsxwriter.git

.vscode/launch.json

@@ -25,9 +25,9 @@
             "program": "${workspaceFolder}/build/src/Platform/main",
             "args": [
                 "-m",
-                "AODE",
+                "BoostAODE",
                 "-p",
-                "/home/rmontanana/Code/discretizbench/datasets",
+                "/Users/rmontanana/Code/discretizbench/datasets",
                 "--stratified",
                 "-d",
                 "mfeat-morphological",
@@ -35,7 +35,21 @@
                 // "--hyperparameters",
                 // "{\"repeatSparent\": true, \"maxModels\": 12}"
             ],
-            "cwd": "/home/rmontanana/Code/discretizbench",
+            "cwd": "/Users/rmontanana/Code/discretizbench",
+        },
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "best",
+            "program": "${workspaceFolder}/build/src/Platform/best",
+            "args": [
+                "-m",
+                "BoostAODE",
+                "-s",
+                "accuracy",
+                "--build",
+            ],
+            "cwd": "/Users/rmontanana/Code/discretizbench",
         },
         {
             "type": "lldb",

.vscode/settings.json

@@ -1,109 +0,0 @@
-{
-    "files.associations": {
-        "*.rmd": "markdown",
-        "*.py": "python",
-        "vector": "cpp",
-        "__bit_reference": "cpp",
-        "__bits": "cpp",
-        "__config": "cpp",
-        "__debug": "cpp",
-        "__errc": "cpp",
-        "__hash_table": "cpp",
-        "__locale": "cpp",
-        "__mutex_base": "cpp",
-        "__node_handle": "cpp",
-        "__nullptr": "cpp",
-        "__split_buffer": "cpp",
-        "__string": "cpp",
-        "__threading_support": "cpp",
-        "__tuple": "cpp",
-        "array": "cpp",
-        "atomic": "cpp",
-        "bitset": "cpp",
-        "cctype": "cpp",
-        "chrono": "cpp",
-        "clocale": "cpp",
-        "cmath": "cpp",
-        "compare": "cpp",
-        "complex": "cpp",
-        "concepts": "cpp",
-        "cstdarg": "cpp",
-        "cstddef": "cpp",
-        "cstdint": "cpp",
-        "cstdio": "cpp",
-        "cstdlib": "cpp",
-        "cstring": "cpp",
-        "ctime": "cpp",
-        "cwchar": "cpp",
-        "cwctype": "cpp",
-        "exception": "cpp",
-        "initializer_list": "cpp",
-        "ios": "cpp",
-        "iosfwd": "cpp",
-        "istream": "cpp",
-        "limits": "cpp",
-        "locale": "cpp",
-        "memory": "cpp",
-        "mutex": "cpp",
-        "new": "cpp",
-        "optional": "cpp",
-        "ostream": "cpp",
-        "ratio": "cpp",
-        "sstream": "cpp",
-        "stdexcept": "cpp",
-        "streambuf": "cpp",
-        "string": "cpp",
-        "string_view": "cpp",
-        "system_error": "cpp",
-        "tuple": "cpp",
-        "type_traits": "cpp",
-        "typeinfo": "cpp",
-        "unordered_map": "cpp",
-        "variant": "cpp",
-        "algorithm": "cpp",
-        "iostream": "cpp",
-        "iomanip": "cpp",
-        "numeric": "cpp",
-        "set": "cpp",
-        "__tree": "cpp",
-        "deque": "cpp",
-        "list": "cpp",
-        "map": "cpp",
-        "unordered_set": "cpp",
-        "any": "cpp",
-        "condition_variable": "cpp",
-        "forward_list": "cpp",
-        "fstream": "cpp",
-        "stack": "cpp",
-        "thread": "cpp",
-        "__memory": "cpp",
-        "filesystem": "cpp",
-        "*.toml": "toml",
-        "utility": "cpp",
-        "__verbose_abort": "cpp",
-        "bit": "cpp",
-        "random": "cpp",
-        "*.tcc": "cpp",
-        "functional": "cpp",
-        "iterator": "cpp",
-        "memory_resource": "cpp",
-        "format": "cpp",
-        "valarray": "cpp",
-        "regex": "cpp",
-        "span": "cpp",
-        "cfenv": "cpp",
-        "cinttypes": "cpp",
-        "csetjmp": "cpp",
-        "future": "cpp",
-        "queue": "cpp",
-        "typeindex": "cpp",
-        "shared_mutex": "cpp",
-        "*.ipp": "cpp",
-        "cassert": "cpp",
-        "charconv": "cpp",
-        "source_location": "cpp",
-        "ranges": "cpp"
-    },
-    "cmake.configureOnOpen": false,
-    "C_Cpp.default.configurationProvider": "ms-vscode.cmake-tools"
-}

CMakeLists.txt

@@ -40,7 +40,7 @@ if (CODE_COVERAGE)
     enable_testing()
     include(CodeCoverage)
     MESSAGE("Code coverage enabled")
-    set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage -O0")
+    set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage -O0 -g")
     SET(GCC_COVERAGE_LINK_FLAGS " ${GCC_COVERAGE_LINK_FLAGS} -lgcov --coverage")
 endif (CODE_COVERAGE)
 
@@ -54,7 +54,7 @@ endif (ENABLE_CLANG_TIDY)
 add_git_submodule("lib/mdlp")
 add_git_submodule("lib/argparse")
 add_git_submodule("lib/json")
-add_git_submodule("lib/openXLSX")
+find_library(XLSXWRITER_LIB libxlsxwriter.dylib PATHS /usr/local/lib)
 
 # Subdirectories
 # --------------
@@ -73,8 +73,7 @@ file(GLOB Platform_SOURCES CONFIGURE_DEPENDS ${BayesNet_SOURCE_DIR}/src/Platform
 
 if (ENABLE_TESTING)
   MESSAGE("Testing enabled")
-  add_git_submodule("lib/catch2")
-  
+  add_git_submodule("lib/catch2")  
   include(CTest)
   add_subdirectory(tests)
 endif (ENABLE_TESTING)

Makefile

@@ -19,13 +19,14 @@ copy: ## Copy binary files to selected folder
 	@cp build/src/Platform/main $(dest)
 	@cp build/src/Platform/list $(dest)
 	@cp build/src/Platform/manage $(dest)
+	@cp build/src/Platform/best $(dest)
 	@echo ">>> Done"
 
 dependency: ## Create a dependency graph diagram of the project (build/dependency.png)
 	cd build && cmake .. --graphviz=dependency.dot && dot -Tpng dependency.dot -o dependency.png
 
 build: ## Build the main and BayesNetSample
-	cmake --build build -t main -t BayesNetSample -t manage -t list -j 32
+	cmake --build build -t main -t BayesNetSample -t manage -t list -t best -j 32
 
 clean: ## Clean the debug info
 	@echo ">>> Cleaning Debug BayesNet ...";
@@ -40,7 +41,7 @@ debug: ## Build a debug version of the project
 	@if [ -d ./build ]; then rm -rf ./build; fi
 	@mkdir build; 
 	cmake -S . -B build -D CMAKE_BUILD_TYPE=Debug -D ENABLE_TESTING=ON -D CODE_COVERAGE=ON; \
-	cmake --build build -j 32;
+	cmake --build build -t main -t BayesNetSample -t manage -t list -t best -t unit_tests -j 32;
 	@echo ">>> Done";
 
 release: ## Build a Release version of the project
@@ -48,7 +49,7 @@ release: ## Build a Release version of the project
 	@if [ -d ./build ]; then rm -rf ./build; fi
 	@mkdir build; 
 	cmake -S . -B build -D CMAKE_BUILD_TYPE=Release; \
-	cmake --build build -t main -t BayesNetSample -t manage -t list -j 32;
+	cmake --build build -t main -t BayesNetSample -t manage -t list -t best -j 32;
 	@echo ">>> Done";	
 
 test: ## Run tests

README.md

@@ -2,4 +2,36 @@
 
 Bayesian Network Classifier with libtorch from scratch
 
+## 0. Setup
+
+### libxlswriter
+
+Before compiling BayesNet.
+
+```bash
+cd lib/libxlsxwriter
+make
+sudo make install
+```
+
+It has to be installed in /usr/local/lib otherwise CMakeLists.txt has to be modified accordingly
+
+Environment variable has to be set:
+
+```bash
+ export LD_LIBRARY_PATH=/usr/local/lib
+ ```
+
+### Release
+
+```bash
+make release
+```
+
+### Debug & Tests
+
+```bash
+make debug
+```
+
 ## 1. Introduction

sample/sample.cc

@@ -58,6 +58,52 @@ pair<vector<vector<int>>, vector<int>> extract_indices(vector<int> indices, vect
 
 int main(int argc, char** argv)
 {
+    torch::Tensor weights_ = torch::full({ 10 }, 1.0 / 10, torch::kFloat64);
+    torch::Tensor y_ = torch::tensor({ 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 }, torch::kInt32);
+    torch::Tensor ypred = torch::tensor({ 1, 1, 1, 0, 0, 1, 1, 1, 1, 0 }, torch::kInt32);
+    cout << "Initial weights_: " << endl;
+    for (int i = 0; i < 10; i++) {
+        cout << weights_.index({ i }).item<double>() << ", ";
+    }
+    cout << "end." << endl;
+    cout << "y_: " << endl;
+    for (int i = 0; i < 10; i++) {
+        cout << y_.index({ i }).item<int>() << ", ";
+    }
+    cout << "end." << endl;
+    cout << "ypred: " << endl;
+    for (int i = 0; i < 10; i++) {
+        cout << ypred.index({ i }).item<int>() << ", ";
+    }
+    cout << "end." << endl;
+    auto mask_wrong = ypred != y_;
+    auto mask_right = ypred == y_;
+    auto masked_weights = weights_ * mask_wrong.to(weights_.dtype());
+    double epsilon_t = masked_weights.sum().item<double>();
+    cout << "epsilon_t: " << epsilon_t << endl;
+    double wt = (1 - epsilon_t) / epsilon_t;
+    cout << "wt: " << wt << endl;
+    double alpha_t = epsilon_t == 0 ? 1 : 0.5 * log(wt);
+    cout << "alpha_t: " << alpha_t << endl;
+    // Step 3.2: Update weights for next classifier
+    // Step 3.2.1: Update weights of wrong samples
+    cout << "exp(alpha_t): " << exp(alpha_t) << endl;
+    cout << "exp(-alpha_t): " << exp(-alpha_t) << endl;
+    weights_ += mask_wrong.to(weights_.dtype()) * exp(alpha_t) * weights_;
+    // Step 3.2.2: Update weights of right samples
+    weights_ += mask_right.to(weights_.dtype()) * exp(-alpha_t) * weights_;
+    // Step 3.3: Normalise the weights
+    double totalWeights = torch::sum(weights_).item<double>();
+    cout << "totalWeights: " << totalWeights << endl;
+    cout << "Before normalization: " << endl;
+    for (int i = 0; i < 10; i++) {
+        cout << weights_.index({ i }).item<double>() << endl;
+    }
+    weights_ = weights_ / totalWeights;
+    cout << "After normalization: " << endl;
+    for (int i = 0; i < 10; i++) {
+        cout << weights_.index({ i }).item<double>() << endl;
+    }
     map<string, bool> datasets = {
             {"diabetes",           true},
             {"ecoli",              true},

src/BayesNet/BaseClassifier.h

@@ -5,6 +5,7 @@
 #include <vector>
 namespace bayesnet {
     using namespace std;
+    enum status_t { NORMAL, WARNING, ERROR };
     class BaseClassifier {
     protected:
         virtual void trainModel(const torch::Tensor& weights) = 0;
@@ -18,6 +19,7 @@ namespace bayesnet {
         virtual ~BaseClassifier() = default;
         torch::Tensor virtual predict(torch::Tensor& X) = 0;
         vector<int> virtual predict(vector<vector<int>>& X) = 0;
+        status_t virtual getStatus() const = 0;
         float virtual score(vector<vector<int>>& X, vector<int>& y) = 0;
         float virtual score(torch::Tensor& X, torch::Tensor& y) = 0;
         int virtual getNumberOfNodes()const = 0;

src/BayesNet/BoostAODE.cc

@@ -1,6 +1,9 @@
 #include "BoostAODE.h"
 #include <set>
 #include "BayesMetrics.h"
+#include "Colors.h"
+#include "Folding.h"
+#include <limits.h>
 
 namespace bayesnet {
     BoostAODE::BoostAODE() : Ensemble() {}
@@ -11,7 +14,7 @@ namespace bayesnet {
     void BoostAODE::setHyperparameters(nlohmann::json& hyperparameters)
     {
         // Check if hyperparameters are valid
-        const vector<string> validKeys = { "repeatSparent", "maxModels", "ascending" };
+        const vector<string> validKeys = { "repeatSparent", "maxModels", "ascending", "convergence" };
         checkHyperparameters(validKeys, hyperparameters);
         if (hyperparameters.contains("repeatSparent")) {
             repeatSparent = hyperparameters["repeatSparent"];
@@ -22,6 +25,38 @@ namespace bayesnet {
         if (hyperparameters.contains("ascending")) {
             ascending = hyperparameters["ascending"];
         }
+        if (hyperparameters.contains("convergence")) {
+            convergence = hyperparameters["convergence"];
+        }
+    }
+    void BoostAODE::validationInit()
+    {
+        auto y_ = dataset.index({ -1, "..." });
+        if (convergence) {
+            // Prepare train & validation sets from train data
+            auto fold = platform::StratifiedKFold(5, y_, 271);
+            dataset_ = torch::clone(dataset);
+            // save input dataset
+            auto [train, test] = fold.getFold(0);
+            auto train_t = torch::tensor(train);
+            auto test_t = torch::tensor(test);
+            // Get train and validation sets
+            X_train = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), train_t });
+            y_train = dataset.index({ -1, train_t });
+            X_test = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), test_t });
+            y_test = dataset.index({ -1, test_t });
+            dataset = X_train;
+            m = X_train.size(1);
+            auto n_classes = states.at(className).size();
+            metrics = Metrics(dataset, features, className, n_classes);
+            // Build dataset with train data
+            buildDataset(y_train);
+        } else {
+            // Use all data to train
+            X_train = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), "..." });
+            y_train = y_;
+        }
+
     }
     void BoostAODE::trainModel(const torch::Tensor& weights)
     {
@@ -29,14 +64,22 @@ namespace bayesnet {
         n_models = 0;
         if (maxModels == 0)
             maxModels = .1 * n > 10 ? .1 * n : n;
+        validationInit();
         Tensor weights_ = torch::full({ m }, 1.0 / m, torch::kFloat64);
-        auto X_ = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), "..." });
-        auto y_ = dataset.index({ -1, "..." });
         bool exitCondition = false;
         unordered_set<int> featuresUsed;
+        // Variables to control the accuracy finish condition
+        double priorAccuracy = 0.0;
+        double delta = 1.0;
+        double threshold = 1e-4;
+        int tolerance = 5; // number of times the accuracy can be lower than the threshold
+        int count = 0; // number of times the accuracy is lower than the threshold
+        fitted = true; // to enable predict
         // Step 0: Set the finish condition
         // if not repeatSparent a finish condition is run out of features
         // n_models == maxModels
+        // epsiolon sub t > 0.5 => inverse the weights policy
+        // validation error is not decreasing
         while (!exitCondition) {
             // Step 1: Build ranking with mutual information
             auto featureSelection = metrics.SelectKBestWeighted(weights_, ascending, n); // Get all the features sorted
@@ -59,29 +102,44 @@ namespace bayesnet {
             }
             featuresUsed.insert(feature);
             model = std::make_unique<SPODE>(feature);
-            n_models++;
             model->fit(dataset, features, className, states, weights_);
-            auto ypred = model->predict(X_);
+            auto ypred = model->predict(X_train);
             // Step 3.1: Compute the classifier amout of say
-            auto mask_wrong = ypred != y_;
+            auto mask_wrong = ypred != y_train;
+            auto mask_right = ypred == y_train;
             auto masked_weights = weights_ * mask_wrong.to(weights_.dtype());
-            double wrongWeights = masked_weights.sum().item<double>();
-            double significance = wrongWeights == 0 ? 1 : 0.5 * log((1 - wrongWeights) / wrongWeights);
+            double epsilon_t = masked_weights.sum().item<double>();
+            double wt = (1 - epsilon_t) / epsilon_t;
+            double alpha_t = epsilon_t == 0 ? 1 : 0.5 * log(wt);
             // Step 3.2: Update weights for next classifier
             // Step 3.2.1: Update weights of wrong samples
-            weights_ += mask_wrong.to(weights_.dtype()) * exp(significance) * weights_;
+            weights_ += mask_wrong.to(weights_.dtype()) * exp(alpha_t) * weights_;
+            // Step 3.2.2: Update weights of right samples
+            weights_ += mask_right.to(weights_.dtype()) * exp(-alpha_t) * weights_;
             // Step 3.3: Normalise the weights
             double totalWeights = torch::sum(weights_).item<double>();
             weights_ = weights_ / totalWeights;
             // Step 3.4: Store classifier and its accuracy to weigh its future vote
             models.push_back(std::move(model));
-            significanceModels.push_back(significance);
-            exitCondition = n_models == maxModels && repeatSparent;
+            significanceModels.push_back(alpha_t);
+            n_models++;
+            if (convergence) {
+                auto y_val_predict = predict(X_test);
+                double accuracy = (y_val_predict == y_test).sum().item<double>() / (double)y_test.size(0);
+                if (priorAccuracy == 0) {
+                    priorAccuracy = accuracy;
+                } else {
+                    delta = accuracy - priorAccuracy;
+                }
+                if (delta < threshold) {
+                    count++;
+                }
+            }
+            exitCondition = n_models == maxModels && repeatSparent || epsilon_t > 0.5 || count > tolerance;
         }
         if (featuresUsed.size() != features.size()) {
-            cout << "Warning: BoostAODE did not use all the features" << endl;
+            status = WARNING;
         }
-        weights.copy_(weights_);
     }
     vector<string> BoostAODE::graph(const string& title) const
     {

src/BayesNet/BoostAODE.h

@@ -13,9 +13,13 @@ namespace bayesnet {
         void buildModel(const torch::Tensor& weights) override;
         void trainModel(const torch::Tensor& weights) override;
     private:
-        bool repeatSparent=false;
-        int maxModels=0;
-        bool ascending=false; //Process KBest features ascending or descending order
+        torch::Tensor dataset_;
+        torch::Tensor X_train, y_train, X_test, y_test;
+        void validationInit();
+        bool repeatSparent = false;
+        int maxModels = 0;
+        bool ascending = false; //Process KBest features ascending or descending order
+        bool convergence = false; //if true, stop when the model does not improve
     };
 }
 #endif

src/BayesNet/Classifier.cc

@@ -75,7 +75,7 @@ namespace bayesnet {
             throw invalid_argument("dataset (X, y) must be of type Integer");
         }
         if (n != features.size()) {
-            throw invalid_argument("X " + to_string(n) + " and features " + to_string(features.size()) + " must have the same number of features");
+            throw invalid_argument("Classifier: X " + to_string(n) + " and features " + to_string(features.size()) + " must have the same number of features");
         }
         if (states.find(className) == states.end()) {
             throw invalid_argument("className not found in states");

src/BayesNet/Classifier.h

@@ -10,7 +10,6 @@ using namespace torch;
 namespace bayesnet {
     class Classifier : public BaseClassifier {
     private:
-        void buildDataset(torch::Tensor& y);
         Classifier& build(const vector<string>& features, const string& className, map<string, vector<int>>& states, const torch::Tensor& weights);
     protected:
         bool fitted;
@@ -21,10 +20,12 @@ namespace bayesnet {
         string className;
         map<string, vector<int>> states;
         Tensor dataset; // (n+1)xm tensor
+        status_t status = NORMAL;
         void checkFitParameters();
         virtual void buildModel(const torch::Tensor& weights) = 0;
         void trainModel(const torch::Tensor& weights) override;
         void checkHyperparameters(const vector<string>& validKeys, nlohmann::json& hyperparameters);
+        void buildDataset(torch::Tensor& y);
     public:
         Classifier(Network model);
         virtual ~Classifier() = default;
@@ -37,6 +38,7 @@ namespace bayesnet {
         int getNumberOfEdges() const override;
         int getNumberOfStates() const override;
         Tensor predict(Tensor& X) override;
+        status_t getStatus() const override { return status; }
         vector<int> predict(vector<vector<int>>& X) override;
         float score(Tensor& X, Tensor& y) override;
         float score(vector<vector<int>>& X, vector<int>& y) override;

src/BayesNet/Ensemble.cc

@@ -24,7 +24,7 @@ namespace bayesnet {
             // i.e. votes[0] contains how much value has the value 0 of class. That value is generated by the models predictions
             vector<double> votes(numClasses, 0.0);
             for (int j = 0; j < n_models; ++j) {
-                votes[y_pred_[i][j]] += significanceModels[j];
+                votes[y_pred_[i][j]] += significanceModels.at(j);
             }
             // argsort in descending order
             auto indices = argsort(votes);

src/BayesNet/Network.cc

@@ -132,10 +132,10 @@ namespace bayesnet {
     void Network::setStates(const map<string, vector<int>>& states)
     {
         // Set states to every Node in the network
-        for (int i = 0; i < features.size(); ++i) {
-            nodes[features[i]]->setNumStates(states.at(features[i]).size());
-        }
-        classNumStates = nodes[className]->getNumStates();
+        for_each(features.begin(), features.end(), [this, &states](const string& feature) {
+            nodes.at(feature)->setNumStates(states.at(feature).size());
+            });
+        classNumStates = nodes.at(className)->getNumStates();
     }
     // X comes in nxm, where n is the number of features and m the number of samples
     void Network::fit(const torch::Tensor& X, const torch::Tensor& y, const torch::Tensor& weights, const vector<string>& featureNames, const string& className, const map<string, vector<int>>& states)
@@ -174,10 +174,16 @@ namespace bayesnet {
     {
         setStates(states);
         laplaceSmoothing = 1.0 / samples.size(1); // To use in CPT computation
+        vector<thread> threads;
         for (auto& node : nodes) {
-            node.second->computeCPT(samples, features, laplaceSmoothing, weights);
-            fitted = true;
+            threads.emplace_back([this, &node, &weights]() {
+                node.second->computeCPT(samples, features, laplaceSmoothing, weights);
+                });
         }
+        for (auto& thread : threads) {
+            thread.join();
+        }
+        fitted = true;
     }
     torch::Tensor Network::predict_tensor(const torch::Tensor& samples, const bool proba)
     {

src/Platform/BestResults.cc

@@ -0,0 +1,305 @@
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <set>
+#include "BestResults.h"
+#include "Result.h"
+#include "Colors.h"
+
+
+
+namespace fs = std::filesystem;
+// function ftime_to_string, Code taken from 
+// https://stackoverflow.com/a/58237530/1389271
+template <typename TP>
+std::string ftime_to_string(TP tp)
+{
+    using namespace std::chrono;
+    auto sctp = time_point_cast<system_clock::duration>(tp - TP::clock::now()
+        + system_clock::now());
+    auto tt = system_clock::to_time_t(sctp);
+    std::tm* gmt = std::gmtime(&tt);
+    std::stringstream buffer;
+    buffer << std::put_time(gmt, "%Y-%m-%d %H:%M");
+    return buffer.str();
+}
+
+namespace platform {
+
+    string BestResults::build()
+    {
+        auto files = loadResultFiles();
+        if (files.size() == 0) {
+            cerr << Colors::MAGENTA() << "No result files were found!" << Colors::RESET() << endl;
+            exit(1);
+        }
+        json bests;
+        for (const auto& file : files) {
+            auto result = Result(path, file);
+            auto data = result.load();
+            for (auto const& item : data.at("results")) {
+                bool update = false;
+                if (bests.contains(item.at("dataset").get<string>())) {
+                    if (item.at("score").get<double>() > bests[item.at("dataset").get<string>()].at(0).get<double>()) {
+                        update = true;
+                    }
+                } else {
+                    update = true;
+                }
+                if (update) {
+                    bests[item.at("dataset").get<string>()] = { item.at("score").get<double>(), item.at("hyperparameters"), file };
+                }
+            }
+        }
+        string bestFileName = path + bestResultFile();
+        if (FILE* fileTest = fopen(bestFileName.c_str(), "r")) {
+            fclose(fileTest);
+            cout << Colors::MAGENTA() << "File " << bestFileName << " already exists and it shall be overwritten." << Colors::RESET() << endl;
+        }
+        ofstream file(bestFileName);
+        file << bests;
+        file.close();
+        return bestFileName;
+    }
+
+    string BestResults::bestResultFile()
+    {
+        return "best_results_" + score + "_" + model + ".json";
+    }
+
+    pair<string, string> getModelScore(string name)
+    {
+        // results_accuracy_BoostAODE_MacBookpro16_2023-09-06_12:27:00_1.json
+        int i = 0;
+        auto pos = name.find("_");
+        auto pos2 = name.find("_", pos + 1);
+        string score = name.substr(pos + 1, pos2 - pos - 1);
+        pos = name.find("_", pos2 + 1);
+        string model = name.substr(pos2 + 1, pos - pos2 - 1);
+        return { model, score };
+    }
+
+    vector<string> BestResults::loadResultFiles()
+    {
+        vector<string> files;
+        using std::filesystem::directory_iterator;
+        string fileModel, fileScore;
+        for (const auto& file : directory_iterator(path)) {
+            auto fileName = file.path().filename().string();
+            if (fileName.find(".json") != string::npos && fileName.find("results_") == 0) {
+                tie(fileModel, fileScore) = getModelScore(fileName);
+                if (score == fileScore && (model == fileModel || model == "any")) {
+                    files.push_back(fileName);
+                }
+            }
+        }
+        return files;
+    }
+
+    json BestResults::loadFile(const string& fileName)
+    {
+        ifstream resultData(fileName);
+        if (resultData.is_open()) {
+            json data = json::parse(resultData);
+            return data;
+        }
+        throw invalid_argument("Unable to open result file. [" + fileName + "]");
+    }
+    set<string> BestResults::getModels()
+    {
+        set<string> models;
+        auto files = loadResultFiles();
+        if (files.size() == 0) {
+            cerr << Colors::MAGENTA() << "No result files were found!" << Colors::RESET() << endl;
+            exit(1);
+        }
+        string fileModel, fileScore;
+        for (const auto& file : files) {
+            // extract the model from the file name
+            tie(fileModel, fileScore) = getModelScore(file);
+            // add the model to the vector of models
+            models.insert(fileModel);
+        }
+        return models;
+    }
+
+    void BestResults::buildAll()
+    {
+        auto models = getModels();
+        for (const auto& model : models) {
+            cout << "Building best results for model: " << model << endl;
+            this->model = model;
+            build();
+        }
+        model = "any";
+    }
+
+    void BestResults::reportSingle()
+    {
+        string bestFileName = path + bestResultFile();
+        if (FILE* fileTest = fopen(bestFileName.c_str(), "r")) {
+            fclose(fileTest);
+        } else {
+            cerr << Colors::MAGENTA() << "File " << bestFileName << " doesn't exist." << Colors::RESET() << endl;
+            exit(1);
+        }
+        auto date = ftime_to_string(filesystem::last_write_time(bestFileName));
+        auto data = loadFile(bestFileName);
+        cout << Colors::GREEN() << "Best results for " << model << " and " << score << " as of " << date << endl;
+        cout << "--------------------------------------------------------" << endl;
+        cout << Colors::GREEN() << " #  Dataset                   Score       File                                                               Hyperparameters" << endl;
+        cout << "=== ========================= =========== ================================================================== ================================================= " << endl;
+        auto i = 0;
+        bool odd = true;
+        for (auto const& item : data.items()) {
+            auto color = odd ? Colors::BLUE() : Colors::CYAN();
+            cout << color << setw(3) << fixed << right << i++ << " ";
+            cout << setw(25) << left << item.key() << " ";
+            cout << setw(11) << setprecision(9) << fixed << item.value().at(0).get<double>() << " ";
+            cout << setw(66) << item.value().at(2).get<string>() << " ";
+            cout << item.value().at(1) << " ";
+            cout << endl;
+            odd = !odd;
+        }
+    }
+    json BestResults::buildTableResults(set<string> models)
+    {
+        int numberOfDatasets = 0;
+        bool first = true;
+        json origin;
+        json table;
+        auto maxDate = filesystem::file_time_type::max();
+        for (const auto& model : models) {
+            this->model = model;
+            string bestFileName = path + bestResultFile();
+            if (FILE* fileTest = fopen(bestFileName.c_str(), "r")) {
+                fclose(fileTest);
+            } else {
+                cerr << Colors::MAGENTA() << "File " << bestFileName << " doesn't exist." << Colors::RESET() << endl;
+                exit(1);
+            }
+            auto dateWrite = filesystem::last_write_time(bestFileName);
+            if (dateWrite < maxDate) {
+                maxDate = dateWrite;
+            }
+            auto data = loadFile(bestFileName);
+            if (first) {
+                // Get the number of datasets of the first file and check that is the same for all the models
+                first = false;
+                numberOfDatasets = data.size();
+                origin = data;
+            } else {
+                if (numberOfDatasets != data.size()) {
+                    cerr << Colors::MAGENTA() << "The number of datasets in the best results files is not the same for all the models." << Colors::RESET() << endl;
+                    exit(1);
+                }
+            }
+            table[model] = data;
+        }
+        table["dateTable"] = ftime_to_string(maxDate);
+        return table;
+    }
+    void BestResults::printTableResults(set<string> models, json table)
+    {
+        cout << Colors::GREEN() << "Best results for " << score << " as of " << table.at("dateTable").get<string>() << endl;
+        cout << "------------------------------------------------" << endl;
+        cout << Colors::GREEN() << " #  Dataset                   ";
+        for (const auto& model : models) {
+            cout << setw(12) << left << model << " ";
+        }
+        cout << endl;
+        cout << "=== ========================= ";
+        for (const auto& model : models) {
+            cout << "============ ";
+        }
+        cout << endl;
+        auto i = 0;
+        bool odd = true;
+        map<string, double> totals;
+        map<string, int> ranks;
+        for (const auto& model : models) {
+            totals[model] = 0.0;
+        }
+        json origin = table.begin().value();
+        for (auto const& item : origin.items()) {
+            auto color = odd ? Colors::BLUE() : Colors::CYAN();
+            cout << color << setw(3) << fixed << right << i++ << " ";
+            cout << setw(25) << left << item.key() << " ";
+            double maxValue = 0;
+            vector<pair<string, double>> ranksOrder;
+            // Find out the max value for this dataset
+            for (const auto& model : models) {
+                double value = table[model].at(item.key()).at(0).get<double>();
+                if (value > maxValue) {
+                    maxValue = value;
+                }
+                ranksOrder.push_back({ model, value });
+            }
+            // sort the ranksOrder vector by value
+            sort(ranksOrder.begin(), ranksOrder.end(), [](const pair<string, double>& a, const pair<string, double>& b) {
+                return a.second > b.second;
+                });
+            // Assign the ranks
+            for (int i = 0; i < ranksOrder.size(); i++) {
+                ranks[ranksOrder[i].first] = i + 1;
+            }
+            // Print the row with red colors on max values
+            for (const auto& model : models) {
+                string efectiveColor = color;
+                double value = table[model].at(item.key()).at(0).get<double>();
+                if (value == maxValue) {
+                    efectiveColor = Colors::RED();
+                }
+                totals[model] += value;
+                cout << efectiveColor << setw(12) << setprecision(10) << fixed << value << " ";
+            }
+            cout << endl;
+            odd = !odd;
+        }
+        cout << Colors::GREEN() << "=== ========================= ";
+        for (const auto& model : models) {
+            cout << "============ ";
+        }
+        cout << endl;
+        cout << Colors::GREEN() << setw(30) << "    Totals...................";
+        double max = 0.0;
+        for (const auto& total : totals) {
+            if (total.second > max) {
+                max = total.second;
+            }
+        }
+        for (const auto& model : models) {
+            string efectiveColor = Colors::GREEN();
+            if (totals[model] == max) {
+                efectiveColor = Colors::RED();
+            }
+            cout << efectiveColor << setw(12) << setprecision(9) << fixed << totals[model] << " ";
+        }
+        // Output the averaged ranks
+        cout << endl;
+        int min = 1;
+        for (const auto& rank : ranks) {
+            if (rank.second < min) {
+                min = rank.second;
+            }
+        }
+        cout << Colors::GREEN() << setw(30) << "    Averaged ranks...........";
+        for (const auto& model : models) {
+            string efectiveColor = Colors::GREEN();
+            if (ranks[model] == min) {
+                efectiveColor = Colors::RED();
+            }
+            cout << efectiveColor << setw(12) << setprecision(10) << fixed << (double)ranks[model] / (double)origin.size() << " ";
+        }
+        cout << endl;
+    }
+    void BestResults::reportAll()
+    {
+        auto models = getModels();
+        // Build the table of results
+        json table = buildTableResults(models);
+        // Print the table of results
+        printTableResults(models, table);
+    }
+}

src/Platform/BestResults.h

@@ -0,0 +1,28 @@
+#ifndef BESTRESULTS_H
+#define BESTRESULTS_H
+#include <string>
+#include <set>
+#include <nlohmann/json.hpp>
+using namespace std;
+using json = nlohmann::json;
+namespace platform {
+    class BestResults {
+    public:
+        explicit BestResults(const string& path, const string& score, const string& model) : path(path), score(score), model(model) {}
+        string build();
+        void reportSingle();
+        void reportAll();
+        void buildAll();
+    private:
+        set<string> getModels();
+        vector<string> loadResultFiles();
+        json buildTableResults(set<string> models);
+        void printTableResults(set<string> models, json table);
+        string bestResultFile();
+        json loadFile(const string& fileName);
+        string path;
+        string score;
+        string model;
+    };
+}
+#endif //BESTRESULTS_H

src/Platform/BestScore.h

@@ -1,7 +1,7 @@
-#ifndef BESTRESULT_H
-#define BESTRESULT_H
+#ifndef BESTSCORE_H
+#define BESTSCORE_H
 #include <string>
-class BestResult {
+class BestScore {
 public:
     static std::string title() { return "STree_default (linear-ovo)"; }
     static double score() { return 22.109799; }

src/Platform/CMakeLists.txt

@@ -4,9 +4,16 @@ include_directories(${BayesNet_SOURCE_DIR}/lib/Files)
 include_directories(${BayesNet_SOURCE_DIR}/lib/mdlp)
 include_directories(${BayesNet_SOURCE_DIR}/lib/argparse/include)
 include_directories(${BayesNet_SOURCE_DIR}/lib/json/include)
+include_directories(${BayesNet_SOURCE_DIR}/lib/libxlsxwriter/include)
 add_executable(main main.cc Folding.cc platformUtils.cc Experiment.cc Datasets.cc Models.cc ReportConsole.cc ReportBase.cc)
-add_executable(manage manage.cc Results.cc ReportConsole.cc ReportExcel.cc ReportBase.cc)
+add_executable(manage manage.cc Results.cc Result.cc ReportConsole.cc ReportExcel.cc ReportBase.cc Datasets.cc platformUtils.cc)
 add_executable(list list.cc platformUtils Datasets.cc)
+add_executable(best best.cc BestResults.cc Result.cc)
 target_link_libraries(main BayesNet ArffFiles mdlp "${TORCH_LIBRARIES}")
-target_link_libraries(manage "${TORCH_LIBRARIES}" OpenXLSX::OpenXLSX)
+if (${CMAKE_HOST_SYSTEM_NAME} MATCHES "Linux")
+    target_link_libraries(manage "${TORCH_LIBRARIES}" libxlsxwriter.so ArffFiles mdlp stdc++fs)
+    target_link_libraries(best stdc++fs)
+else()
+    target_link_libraries(manage "${TORCH_LIBRARIES}" "${XLSXWRITER_LIB}" ArffFiles mdlp)
+endif()
 target_link_libraries(list ArffFiles mdlp "${TORCH_LIBRARIES}")

src/Platform/Experiment.cc

@@ -111,6 +111,26 @@ namespace platform {
         }
     }
 
+    string getColor(bayesnet::status_t status)
+    {
+        switch (status) {
+            case bayesnet::NORMAL:
+                return Colors::GREEN();
+            case bayesnet::WARNING:
+                return Colors::YELLOW();
+            case bayesnet::ERROR:
+                return Colors::RED();
+            default:
+                return Colors::RESET();
+        }
+    }
+
+    void showProgress(int fold, const string& color, const string& phase)
+    {
+        string prefix = phase == "a" ? "" : "\b\b\b\b";
+        cout << prefix << color << fold << Colors::RESET() << "(" << color << phase << Colors::RESET() << ")" << flush;
+
+    }
     void Experiment::cross_validation(const string& path, const string& fileName)
     {
         auto datasets = platform::Datasets(path, discretized, platform::ARFF);
@@ -159,20 +179,24 @@ namespace platform {
                 auto y_train = y.index({ train_t });
                 auto X_test = X.index({ "...", test_t });
                 auto y_test = y.index({ test_t });
-                cout << nfold + 1 << ", " << flush;
+                showProgress(nfold + 1, getColor(clf->getStatus()), "a");
                 // Train model
                 clf->fit(X_train, y_train, features, className, states);
+                showProgress(nfold + 1, getColor(clf->getStatus()), "b");
                 nodes[item] = clf->getNumberOfNodes();
                 edges[item] = clf->getNumberOfEdges();
                 num_states[item] = clf->getNumberOfStates();
                 train_time[item] = train_timer.getDuration();
+                // Score train
                 auto accuracy_train_value = clf->score(X_train, y_train);
                 // Test model
+                showProgress(nfold + 1, getColor(clf->getStatus()), "c");
                 test_timer.start();
                 auto accuracy_test_value = clf->score(X_test, y_test);
                 test_time[item] = test_timer.getDuration();
                 accuracy_train[item] = accuracy_train_value;
                 accuracy_test[item] = accuracy_test_value;
+                cout << "\b\b\b, " << flush;
                 // Store results and times in vector
                 result.addScoreTrain(accuracy_train_value);
                 result.addScoreTest(accuracy_test_value);

src/Platform/ReportBase.cc

@@ -1,10 +1,22 @@
 #include <sstream>
 #include <locale>
+#include "Datasets.h"
 #include "ReportBase.h"
-#include "BestResult.h"
+#include "BestScore.h"
 
 
 namespace platform {
+    ReportBase::ReportBase(json data_, bool compare) : data(data_), compare(compare), margin(0.1)
+    {
+        stringstream oss;
+        oss << "Better than ZeroR + " << setprecision(1) << fixed << margin * 100 << "%";
+        meaning = {
+            {Symbols::equal_best, "Equal to best"},
+            {Symbols::better_best, "Better than best"},
+            {Symbols::cross, "Less than or equal to ZeroR"},
+            {Symbols::upward_arrow, oss.str()}
+        };
+    }
     string ReportBase::fromVector(const string& key)
     {
         stringstream oss;
@@ -34,4 +46,69 @@ namespace platform {
         header();
         body();
     }
+    string ReportBase::compareResult(const string& dataset, double result)
+    {
+        string status = " ";
+        if (compare) {
+            double best = bestResult(dataset, data["model"].get<string>());
+            if (result == best) {
+                status = Symbols::equal_best;
+            } else if (result > best) {
+                status = Symbols::better_best;
+            }
+        } else {
+            if (data["score_name"].get<string>() == "accuracy") {
+                auto dt = Datasets(Paths::datasets(), false);
+                dt.loadDataset(dataset);
+                auto numClasses = dt.getNClasses(dataset);
+                if (numClasses == 2) {
+                    vector<int> distribution = dt.getClassesCounts(dataset);
+                    double nSamples = dt.getNSamples(dataset);
+                    vector<int>::iterator maxValue = max_element(distribution.begin(), distribution.end());
+                    double mark = *maxValue / nSamples * (1 + margin);
+                    if (mark > 1) {
+                        mark = 0.9995;
+                    }
+                    status = result < mark ? Symbols::cross : result > mark ? Symbols::upward_arrow : "=";
+                }
+            }
+        }
+        if (status != " ") {
+            auto item = summary.find(status);
+            if (item != summary.end()) {
+                summary[status]++;
+            } else {
+                summary[status] = 1;
+            }
+        }
+        return status;
+    }
+    double ReportBase::bestResult(const string& dataset, const string& model)
+    {
+        double value = 0.0;
+        if (bestResults.size() == 0) {
+            // try to load the best results
+            string score = data["score_name"];
+            replace(score.begin(), score.end(), '_', '-');
+            string fileName = "best_results_" + score + "_" + model + ".json";
+            ifstream resultData(Paths::results() + "/" + fileName);
+            if (resultData.is_open()) {
+                bestResults = json::parse(resultData);
+            } else {
+                existBestFile = false;
+            }
+        }
+        try {
+            value = bestResults.at(dataset).at(0);
+        }
+        catch (exception) {
+            value = 1.0;
+
+        }
+        return value;
+    }
+    bool ReportBase::getExistBestFile()
+    {
+        return existBestFile;
+    }
 }

src/Platform/ReportBase.h

@@ -2,22 +2,45 @@
 #define REPORTBASE_H
 #include <string>
 #include <iostream>
+#include "Paths.h"
 #include <nlohmann/json.hpp>
 
 using json = nlohmann::json;
 namespace platform {
     using namespace std;
+    class Symbols {
+    public:
+        inline static const string check_mark{ "\u2714" };
+        inline static const string exclamation{ "\u2757" };
+        inline static const string black_star{ "\u2605" };
+        inline static const string cross{ "\u2717" };
+        inline static const string upward_arrow{ "\u27B6" };
+        inline static const string down_arrow{ "\u27B4" };
+        inline static const string equal_best{ check_mark };
+        inline static const string better_best{ black_star };
+    };
     class ReportBase {
     public:
-        explicit ReportBase(json data_) { data = data_; };
+        explicit ReportBase(json data_, bool compare);
         virtual ~ReportBase() = default;
         void show();
     protected:
         json data;
         string fromVector(const string& key);
         string fVector(const string& title, const json& data, const int width, const int precision);
+        bool getExistBestFile();
         virtual void header() = 0;
         virtual void body() = 0;
+        virtual void showSummary() = 0;
+        string compareResult(const string& dataset, double result);
+        map<string, int> summary;
+        double margin;
+        map<string, string> meaning;
+        bool compare;
+    private:
+        double bestResult(const string& dataset, const string& model);
+        json bestResults;
+        bool existBestFile = true;
     };
 };
 #endif

src/Platform/ReportConsole.cc

@@ -1,7 +1,7 @@
 #include <sstream>
 #include <locale>
 #include "ReportConsole.h"
-#include "BestResult.h"
+#include "BestScore.h"
 
 
 namespace platform {
@@ -10,14 +10,14 @@ namespace platform {
         char do_thousands_sep() const { return '.'; }
         string do_grouping() const { return "\03"; }
     };
-    
-    string ReportConsole::headerLine(const string& text)
+
+    string ReportConsole::headerLine(const string& text, int utf = 0)
     {
         int n = MAXL - text.length() - 3;
         n = n < 0 ? 0 : n;
-        return "* " + text + string(n, ' ') + "*\n";
+        return "* " + text + string(n + utf, ' ') + "*\n";
     }
-    
+
     void ReportConsole::header()
     {
         locale mylocale(cout.getloc(), new separated);
@@ -36,22 +36,31 @@ namespace platform {
     }
     void ReportConsole::body()
     {
-        cout << Colors::GREEN() << "Dataset                        Sampl. Feat. Cls Nodes     Edges     States    Score           Time               Hyperparameters" << endl;
-        cout << "============================== ====== ===== === ========= ========= ========= =============== ================== ===============" << endl;
+        cout << Colors::GREEN() << " #  Dataset                   Sampl. Feat. Cls Nodes     Edges     States    Score           Time                Hyperparameters" << endl;
+        cout << "=== ========================= ====== ===== === ========= ========= ========= =============== =================== ====================" << endl;
         json lastResult;
         double totalScore = 0.0;
         bool odd = true;
+        int index = 0;
         for (const auto& r : data["results"]) {
+            if (selectedIndex != -1 && index != selectedIndex) {
+                index++;
+                continue;
+            }
             auto color = odd ? Colors::CYAN() : Colors::BLUE();
-            cout << color << setw(30) << left << r["dataset"].get<string>() << " ";
+            cout << color;
+            cout << setw(3) << index++ << " ";
+            cout << setw(25) << left << r["dataset"].get<string>() << " ";
             cout << setw(6) << right << r["samples"].get<int>() << " ";
             cout << setw(5) << right << r["features"].get<int>() << " ";
             cout << setw(3) << right << r["classes"].get<int>() << " ";
             cout << setw(9) << setprecision(2) << fixed << r["nodes"].get<float>() << " ";
             cout << setw(9) << setprecision(2) << fixed << r["leaves"].get<float>() << " ";
             cout << setw(9) << setprecision(2) << fixed << r["depth"].get<float>() << " ";
-            cout << setw(8) << right << setprecision(6) << fixed << r["score"].get<double>() << "±" << setw(6) << setprecision(4) << fixed << r["score_std"].get<double>() << " ";
-            cout << setw(11) << right << setprecision(6) << fixed << r["time"].get<double>() << "±" << setw(6) << setprecision(4) << fixed << r["time_std"].get<double>() << " ";
+            cout << setw(8) << right << setprecision(6) << fixed << r["score"].get<double>() << "±" << setw(6) << setprecision(4) << fixed << r["score_std"].get<double>();
+            const string status = compareResult(r["dataset"].get<string>(), r["score"].get<double>());
+            cout << status;
+            cout << setw(12) << right << setprecision(6) << fixed << r["time"].get<double>() << "±" << setw(6) << setprecision(4) << fixed << r["time_std"].get<double>() << " ";
             try {
                 cout << r["hyperparameters"].get<string>();
             }
@@ -63,7 +72,7 @@ namespace platform {
             totalScore += r["score"].get<double>();
             odd = !odd;
         }
-        if (data["results"].size() == 1) {
+        if (data["results"].size() == 1 || selectedIndex != -1) {
             cout << string(MAXL, '*') << endl;
             cout << headerLine(fVector("Train scores: ", lastResult["scores_train"], 14, 12));
             cout << headerLine(fVector("Test  scores: ", lastResult["scores_test"], 14, 12));
@@ -74,15 +83,30 @@ namespace platform {
             footer(totalScore);
         }
     }
+    void ReportConsole::showSummary()
+    {
+        for (const auto& item : summary) {
+            stringstream oss;
+            oss << setw(3) << left << item.first;
+            oss << setw(3) << right << item.second << " ";
+            oss << left << meaning.at(item.first);
+            cout << headerLine(oss.str(), 2);
+        }
+    }
+
     void ReportConsole::footer(double totalScore)
     {
         cout << Colors::MAGENTA() << string(MAXL, '*') << endl;
+        showSummary();
         auto score = data["score_name"].get<string>();
-        if (score == BestResult::scoreName()) {
+        if (score == BestScore::scoreName()) {
             stringstream oss;
-            oss << score << " compared to " << BestResult::title() << " .:  " << totalScore / BestResult::score();
+            oss << score << " compared to " << BestScore::title() << " .:  " << totalScore / BestScore::score();
             cout << headerLine(oss.str());
         }
+        if (!getExistBestFile() && compare) {
+            cout << headerLine("*** Best Results File not found. Couldn't compare any result!");
+        }
         cout << string(MAXL, '*') << endl << Colors::RESET();
     }
 }

src/Platform/ReportConsole.h

@@ -7,16 +7,18 @@
 
 namespace platform {
     using namespace std;
-    const int MAXL = 128;
-    class ReportConsole : public ReportBase{
+    const int MAXL = 133;
+    class ReportConsole : public ReportBase {
     public:
-        explicit ReportConsole(json data_) : ReportBase(data_) {};
+        explicit ReportConsole(json data_, bool compare = false, int index = -1) : ReportBase(data_, compare), selectedIndex(index) {};
         virtual ~ReportConsole() = default;
     private:
-        string headerLine(const string& text);
+        int selectedIndex;
+        string headerLine(const string& text, int utf);
         void header() override;
         void body() override;
         void footer(double totalScore);
+        void showSummary() override;
     };
 };
 #endif

src/Platform/ReportExcel.cc

@@ -1,7 +1,7 @@
 #include <sstream>
 #include <locale>
 #include "ReportExcel.h"
-#include "BestResult.h"
+#include "BestScore.h"
 
 
 namespace platform {
@@ -13,17 +13,195 @@ namespace platform {
         string do_grouping() const { return "\03"; }
     };
 
+    ReportExcel::ReportExcel(json data_, bool compare, lxw_workbook* workbook) : ReportBase(data_, compare), row(0), workbook(workbook)
+    {
+        normalSize = 14; //font size for report body
+        colorTitle = 0xB1A0C7;
+        colorOdd = 0xDCE6F1;
+        colorEven = 0xFDE9D9;
+        createFile();
+    }
+
+    lxw_workbook* ReportExcel::getWorkbook()
+    {
+        return workbook;
+    }
+
+    lxw_format* ReportExcel::efectiveStyle(const string& style)
+    {
+        lxw_format* efectiveStyle;
+        if (style == "") {
+            efectiveStyle = NULL;
+        } else {
+            string suffix = row % 2 ? "_odd" : "_even";
+            efectiveStyle = styles.at(style + suffix);
+        }
+        return efectiveStyle;
+    }
+
+    void ReportExcel::writeString(int row, int col, const string& text, const string& style)
+    {
+        worksheet_write_string(worksheet, row, col, text.c_str(), efectiveStyle(style));
+    }
+    void ReportExcel::writeInt(int row, int col, const int number, const string& style)
+    {
+        worksheet_write_number(worksheet, row, col, number, efectiveStyle(style));
+    }
+    void ReportExcel::writeDouble(int row, int col, const double number, const string& style)
+    {
+        worksheet_write_number(worksheet, row, col, number, efectiveStyle(style));
+    }
+
+    void ReportExcel::formatColumns()
+    {
+        worksheet_freeze_panes(worksheet, 6, 1);
+        vector<int> columns_sizes = { 22, 10, 9, 7, 12, 12, 12, 12, 12, 3, 15, 12, 23 };
+        for (int i = 0; i < columns_sizes.size(); ++i) {
+            worksheet_set_column(worksheet, i, i, columns_sizes.at(i), NULL);
+        }
+    }
+
+    void ReportExcel::addColor(lxw_format* style, bool odd)
+    {
+        uint32_t efectiveColor = odd ? colorEven : colorOdd;
+        format_set_bg_color(style, lxw_color_t(efectiveColor));
+    }
+    void ReportExcel::createStyle(const string& name, lxw_format* style, bool odd)
+    {
+        addColor(style, odd);
+        if (name == "textCentered") {
+            format_set_align(style, LXW_ALIGN_CENTER);
+            format_set_font_size(style, normalSize);
+            format_set_border(style, LXW_BORDER_THIN);
+        } else if (name == "text") {
+            format_set_font_size(style, normalSize);
+            format_set_border(style, LXW_BORDER_THIN);
+        } else if (name == "bodyHeader") {
+            format_set_bold(style);
+            format_set_font_size(style, normalSize);
+            format_set_align(style, LXW_ALIGN_CENTER);
+            format_set_align(style, LXW_ALIGN_VERTICAL_CENTER);
+            format_set_border(style, LXW_BORDER_THIN);
+            format_set_bg_color(style, lxw_color_t(colorTitle));
+        } else if (name == "result") {
+            format_set_font_size(style, normalSize);
+            format_set_border(style, LXW_BORDER_THIN);
+            format_set_num_format(style, "0.0000000");
+        } else if (name == "time") {
+            format_set_font_size(style, normalSize);
+            format_set_border(style, LXW_BORDER_THIN);
+            format_set_num_format(style, "#,##0.000000");
+        } else if (name == "ints") {
+            format_set_font_size(style, normalSize);
+            format_set_num_format(style, "###,##0");
+            format_set_border(style, LXW_BORDER_THIN);
+        } else if (name == "floats") {
+            format_set_border(style, LXW_BORDER_THIN);
+            format_set_font_size(style, normalSize);
+            format_set_num_format(style, "#,##0.00");
+        }
+    }
+
+    void ReportExcel::createFormats()
+    {
+        auto styleNames = { "text", "textCentered", "bodyHeader", "result", "time", "ints", "floats" };
+        lxw_format* style;
+        for (string name : styleNames) {
+            lxw_format* style = workbook_add_format(workbook);
+            style = workbook_add_format(workbook);
+            createStyle(name, style, true);
+            styles[name + "_odd"] = style;
+            style = workbook_add_format(workbook);
+            createStyle(name, style, false);
+            styles[name + "_even"] = style;
+        }
+
+        // Header 1st line
+        lxw_format* headerFirst = workbook_add_format(workbook);
+        format_set_bold(headerFirst);
+        format_set_font_size(headerFirst, 18);
+        format_set_align(headerFirst, LXW_ALIGN_CENTER);
+        format_set_align(headerFirst, LXW_ALIGN_VERTICAL_CENTER);
+        format_set_border(headerFirst, LXW_BORDER_THIN);
+        format_set_bg_color(headerFirst, lxw_color_t(colorTitle));
+
+        // Header rest
+        lxw_format* headerRest = workbook_add_format(workbook);
+        format_set_bold(headerRest);
+        format_set_align(headerRest, LXW_ALIGN_CENTER);
+        format_set_font_size(headerRest, 16);
+        format_set_align(headerRest, LXW_ALIGN_VERTICAL_CENTER);
+        format_set_border(headerRest, LXW_BORDER_THIN);
+        format_set_bg_color(headerRest, lxw_color_t(colorOdd));
+
+        // Header small
+        lxw_format* headerSmall = workbook_add_format(workbook);
+        format_set_bold(headerSmall);
+        format_set_align(headerSmall, LXW_ALIGN_LEFT);
+        format_set_font_size(headerSmall, 12);
+        format_set_border(headerSmall, LXW_BORDER_THIN);
+        format_set_align(headerSmall, LXW_ALIGN_VERTICAL_CENTER);
+        format_set_bg_color(headerSmall, lxw_color_t(colorOdd));
+
+        // Summary style
+        lxw_format* summaryStyle = workbook_add_format(workbook);
+        format_set_bold(summaryStyle);
+        format_set_font_size(summaryStyle, 16);
+        format_set_border(summaryStyle, LXW_BORDER_THIN);
+        format_set_align(summaryStyle, LXW_ALIGN_VERTICAL_CENTER);
+
+        styles["headerFirst"] = headerFirst;
+        styles["headerRest"] = headerRest;
+        styles["headerSmall"] = headerSmall;
+        styles["summaryStyle"] = summaryStyle;
+    }
+
+    void ReportExcel::setProperties()
+    {
+        char line[data["title"].get<string>().size() + 1];
+        strcpy(line, data["title"].get<string>().c_str());
+        lxw_doc_properties properties = {
+            .title = line,
+            .subject = (char*)"Machine learning results",
+            .author = (char*)"Ricardo Montañana Gómez",
+            .manager = (char*)"Dr. J. A. Gámez, Dr. J. M. Puerta",
+            .company = (char*)"UCLM",
+            .comments = (char*)"Created with libxlsxwriter and c++",
+        };
+        workbook_set_properties(workbook, &properties);
+    }
+
     void ReportExcel::createFile()
     {
-        doc.create(Paths::excel() + "some_results.xlsx");
-        wks = doc.workbook().worksheet("Sheet1");
-        wks.setName(data["model"].get<string>());
+        if (workbook == NULL) {
+            workbook = workbook_new((Paths::excel() + fileName).c_str());
+        }
+        const string name = data["model"].get<string>();
+        string suffix = "";
+        string efectiveName;
+        int num = 1;
+        // Create a sheet with the name of the model
+        while (true) {
+            efectiveName = name + suffix;
+            if (workbook_get_worksheet_by_name(workbook, efectiveName.c_str())) {
+                suffix = to_string(++num);
+            } else {
+                worksheet = workbook_add_worksheet(workbook, efectiveName.c_str());
+                break;
+            }
+            if (num > 100) {
+                throw invalid_argument("Couldn't create sheet " + efectiveName);
+            }
+        }
+        cout << "Adding sheet " << efectiveName << " to " << Paths::excel() + fileName << endl;
+        setProperties();
+        createFormats();
+        formatColumns();
     }
 
     void ReportExcel::closeFile()
     {
-        doc.save();
-        doc.close();
+        workbook_close(workbook);
     }
 
     void ReportExcel::header()
@@ -32,45 +210,62 @@ namespace platform {
         locale::global(mylocale);
         cout.imbue(mylocale);
         stringstream oss;
-        wks.cell("A1").value().set(
-            "Report " + data["model"].get<string>() + " ver. " + data["version"].get<string>() + " with " +
-            to_string(data["folds"].get<int>()) + " Folds cross validation and " + to_string(data["seeds"].size()) +
-            " random seeds. " + data["date"].get<string>() + " " + data["time"].get<string>());
-        wks.cell("A2").value() = data["title"].get<string>();
-        wks.cell("A3").value() = "Random seeds: " + fromVector("seeds") + " Stratified: " +
-            (data["stratified"].get<bool>() ? "True" : "False");
-        oss << "Execution took  " << setprecision(2) << fixed << data["duration"].get<float>() << " seconds,   "
-            << data["duration"].get<float>() / 3600 << " hours, on " << data["platform"].get<string>();
-        wks.cell("A4").value() = oss.str();
-        wks.cell("A5").value() = "Score is " + data["score_name"].get<string>();
+        string message = data["model"].get<string>() + " ver. " + data["version"].get<string>() + " " +
+            data["language"].get<string>() + " ver. " + data["language_version"].get<string>() +
+            " with " + to_string(data["folds"].get<int>()) + " Folds cross validation and " + to_string(data["seeds"].size()) +
+            " random seeds. " + data["date"].get<string>() + " " + data["time"].get<string>();
+        worksheet_merge_range(worksheet, 0, 0, 0, 12, message.c_str(), styles["headerFirst"]);
+        worksheet_merge_range(worksheet, 1, 0, 1, 12, data["title"].get<string>().c_str(), styles["headerRest"]);
+        worksheet_merge_range(worksheet, 2, 0, 3, 0, ("Score is " + data["score_name"].get<string>()).c_str(), styles["headerRest"]);
+        worksheet_merge_range(worksheet, 2, 1, 3, 3, "Execution time", styles["headerRest"]);
+        oss << setprecision(2) << fixed << data["duration"].get<float>() << " s";
+        worksheet_merge_range(worksheet, 2, 4, 2, 5, oss.str().c_str(), styles["headerRest"]);
+        oss.str("");
+        oss.clear();
+        oss << setprecision(2) << fixed << data["duration"].get<float>() / 3600 << " h";
+        worksheet_merge_range(worksheet, 3, 4, 3, 5, oss.str().c_str(), styles["headerRest"]);
+        worksheet_merge_range(worksheet, 2, 6, 3, 7, "Platform", styles["headerRest"]);
+        worksheet_merge_range(worksheet, 2, 8, 3, 9, data["platform"].get<string>().c_str(), styles["headerRest"]);
+        worksheet_merge_range(worksheet, 2, 10, 2, 12, ("Random seeds: " + fromVector("seeds")).c_str(), styles["headerSmall"]);
+        oss.str("");
+        oss.clear();
+        oss << "Stratified: " << (data["stratified"].get<bool>() ? "True" : "False");
+        worksheet_merge_range(worksheet, 3, 10, 3, 11, oss.str().c_str(), styles["headerSmall"]);
+        oss.str("");
+        oss.clear();
+        oss << "Discretized: " << (data["discretized"].get<bool>() ? "True" : "False");
+        worksheet_write_string(worksheet, 3, 12, oss.str().c_str(), styles["headerSmall"]);
     }
 
     void ReportExcel::body()
     {
         auto head = vector<string>(
-            { "Dataset", "Samples", "Features", "Classes", "Nodes", "Edges", "States", "Score", "Score Std.", "Time",
+            { "Dataset", "Samples", "Features", "Classes", "Nodes", "Edges", "States", "Score", "Score Std.", "St.", "Time",
              "Time Std.", "Hyperparameters" });
-        int col = 1;
+        int col = 0;
         for (const auto& item : head) {
-            wks.cell(8, col++).value() = item;
+            writeString(5, col++, item, "bodyHeader");
         }
-        int row = 9;
-        col = 1;
+        row = 6;
+        col = 0;
+        int hypSize = 22;
         json lastResult;
         double totalScore = 0.0;
         string hyperparameters;
         for (const auto& r : data["results"]) {
-            wks.cell(row, col).value() = r["dataset"].get<string>();
-            wks.cell(row, col + 1).value() = r["samples"].get<int>();
-            wks.cell(row, col + 2).value() = r["features"].get<int>();
-            wks.cell(row, col + 3).value() = r["classes"].get<int>();
-            wks.cell(row, col + 4).value() = r["nodes"].get<float>();
-            wks.cell(row, col + 5).value() = r["leaves"].get<float>();
-            wks.cell(row, col + 6).value() = r["depth"].get<float>();
-            wks.cell(row, col + 7).value() = r["score"].get<double>();
-            wks.cell(row, col + 8).value() = r["score_std"].get<double>();
-            wks.cell(row, col + 9).value() = r["time"].get<double>();
-            wks.cell(row, col + 10).value() = r["time_std"].get<double>();
+            writeString(row, col, r["dataset"].get<string>(), "text");
+            writeInt(row, col + 1, r["samples"].get<int>(), "ints");
+            writeInt(row, col + 2, r["features"].get<int>(), "ints");
+            writeInt(row, col + 3, r["classes"].get<int>(), "ints");
+            writeDouble(row, col + 4, r["nodes"].get<float>(), "floats");
+            writeDouble(row, col + 5, r["leaves"].get<float>(), "floats");
+            writeDouble(row, col + 6, r["depth"].get<double>(), "floats");
+            writeDouble(row, col + 7, r["score"].get<double>(), "result");
+            writeDouble(row, col + 8, r["score_std"].get<double>(), "result");
+            const string status = compareResult(r["dataset"].get<string>(), r["score"].get<double>());
+            writeString(row, col + 9, status, "textCentered");
+            writeDouble(row, col + 10, r["time"].get<double>(), "time");
+            writeDouble(row, col + 11, r["time_std"].get<double>(), "time");
             try {
                 hyperparameters = r["hyperparameters"].get<string>();
             }
@@ -79,31 +274,60 @@ namespace platform {
                 oss << r["hyperparameters"];
                 hyperparameters = oss.str();
             }
-            wks.cell(row, col + 11).value() = hyperparameters;
+            if (hyperparameters.size() > hypSize) {
+                hypSize = hyperparameters.size();
+            }
+            writeString(row, col + 12, hyperparameters, "text");
             lastResult = r;
             totalScore += r["score"].get<double>();
             row++;
+
         }
+        // Set the right column width of hyperparameters with the maximum length
+        worksheet_set_column(worksheet, 12, 12, hypSize + 5, NULL);
+        // Show totals if only one dataset is present in the result
         if (data["results"].size() == 1) {
             for (const string& group : { "scores_train", "scores_test", "times_train", "times_test" }) {
                 row++;
                 col = 1;
-                wks.cell(row, col).value() = group;
+                writeString(row, col, group, "text");
                 for (double item : lastResult[group]) {
-                    wks.cell(row, ++col).value() = item;
+                    string style = group.find("scores") != string::npos ? "result" : "time";
+                    writeDouble(row, ++col, item, style);
                 }
             }
+            // Set with of columns to show those totals completely
+            worksheet_set_column(worksheet, 1, 1, 12, NULL);
+            for (int i = 2; i < 7; ++i) {
+                // doesn't work with from col to col, so...
+                worksheet_set_column(worksheet, i, i, 15, NULL);
+            }
         } else {
             footer(totalScore, row);
         }
     }
 
+    void ReportExcel::showSummary()
+    {
+        for (const auto& item : summary) {
+            worksheet_write_string(worksheet, row + 2, 1, item.first.c_str(), styles["summaryStyle"]);
+            worksheet_write_number(worksheet, row + 2, 2, item.second, styles["summaryStyle"]);
+            worksheet_merge_range(worksheet, row + 2, 3, row + 2, 5, meaning.at(item.first).c_str(), styles["summaryStyle"]);
+            row += 1;
+        }
+    }
+
     void ReportExcel::footer(double totalScore, int row)
     {
+        showSummary();
+        row += 4 + summary.size();
         auto score = data["score_name"].get<string>();
-        if (score == BestResult::scoreName()) {
-            wks.cell(row + 2, 1).value() = score + " compared to " + BestResult::title() + " .:  ";
-            wks.cell(row + 2, 5).value() = totalScore / BestResult::score();
+        if (score == BestScore::scoreName()) {
+            worksheet_merge_range(worksheet, row, 1, row, 5, (score + " compared to " + BestScore::title() + " .:").c_str(), efectiveStyle("text"));
+            writeDouble(row, 6, totalScore / BestScore::score(), "result");
+        }
+        if (!getExistBestFile() && compare) {
+            worksheet_write_string(worksheet, row + 1, 0, "*** Best Results File not found. Couldn't compare any result!", styles["summaryStyle"]);
         }
     }
 }

src/Platform/ReportExcel.h

@@ -1,25 +1,42 @@
 #ifndef REPORTEXCEL_H
 #define REPORTEXCEL_H
-#include <OpenXLSX.hpp>
+#include<map>
+#include "xlsxwriter.h"
 #include "ReportBase.h"
-#include "Paths.h"
 #include "Colors.h"
 namespace platform {
     using namespace std;
-    using namespace OpenXLSX;
     const int MAXLL = 128;
-    class ReportExcel : public ReportBase{
+
+    class ReportExcel : public ReportBase {
     public:
-        explicit ReportExcel(json data_) : ReportBase(data_) {createFile();};
-        virtual ~ReportExcel() {closeFile();};
+        explicit ReportExcel(json data_, bool compare, lxw_workbook* workbook);
+        lxw_workbook* getWorkbook();
     private:
+        void writeString(int row, int col, const string& text, const string& style = "");
+        void writeInt(int row, int col, const int number, const string& style = "");
+        void writeDouble(int row, int col, const double number, const string& style = "");
+        void formatColumns();
+        void createFormats();
+        void setProperties();
         void createFile();
         void closeFile();
-        XLDocument doc;
-        XLWorksheet wks;
+        lxw_workbook* workbook;
+        lxw_worksheet* worksheet;
+        map<string, lxw_format*> styles;
+        int row;
+        int normalSize; //font size for report body
+        uint32_t colorTitle;
+        uint32_t colorOdd;
+        uint32_t colorEven;
+        const string fileName = "some_results.xlsx";
         void header() override;
         void body() override;
+        void showSummary() override;
         void footer(double totalScore, int row);
+        void createStyle(const string& name, lxw_format* style, bool odd);
+        void addColor(lxw_format* style, bool odd);
+        lxw_format* efectiveStyle(const string& name);
     };
 };
 #endif // !REPORTEXCEL_H

src/Platform/Result.cc

@@ -0,0 +1,51 @@
+#include <filesystem>
+#include <fstream>
+#include <sstream>
+#include "Result.h"
+#include "Colors.h"
+#include "BestScore.h"
+namespace platform {
+    Result::Result(const string& path, const string& filename)
+        : path(path)
+        , filename(filename)
+    {
+        auto data = load();
+        date = data["date"];
+        score = 0;
+        for (const auto& result : data["results"]) {
+            score += result["score"].get<double>();
+        }
+        scoreName = data["score_name"];
+        if (scoreName == BestScore::scoreName()) {
+            score /= BestScore::score();
+        }
+        title = data["title"];
+        duration = data["duration"];
+        model = data["model"];
+        complete = data["results"].size() > 1;
+    }
+
+    json Result::load() const
+    {
+        ifstream resultData(path + "/" + filename);
+        if (resultData.is_open()) {
+            json data = json::parse(resultData);
+            return data;
+        }
+        throw invalid_argument("Unable to open result file. [" + path + "/" + filename + "]");
+    }
+
+    string Result::to_string() const
+    {
+        stringstream oss;
+        oss << date << " ";
+        oss << setw(12) << left << model << " ";
+        oss << setw(11) << left << scoreName << " ";
+        oss << right << setw(11) << setprecision(7) << fixed << score << " ";
+        auto completeString = isComplete() ? "C" : "P";
+        oss << setw(1) << " " << completeString << "  ";
+        oss << setw(9) << setprecision(3) << fixed << duration << " ";
+        oss << setw(50) << left << title << " ";
+        return  oss.str();
+    }
+}

src/Platform/Result.h

@@ -0,0 +1,37 @@
+#ifndef RESULT_H
+#define RESULT_H
+#include <map>
+#include <vector>
+#include <string>
+#include <nlohmann/json.hpp>
+namespace platform {
+    using namespace std;
+    using json = nlohmann::json;
+
+    class Result {
+    public:
+        Result(const string& path, const string& filename);
+        json load() const;
+        string to_string() const;
+        string getFilename() const { return filename; };
+        string getDate() const { return date; };
+        double getScore() const { return score; };
+        string getTitle() const { return title; };
+        double getDuration() const { return duration; };
+        string getModel() const { return model; };
+        string getScoreName() const { return scoreName; };
+        bool isComplete() const { return complete; };
+    private:
+        string path;
+        string filename;
+        string date;
+        double score;
+        string title;
+        double duration;
+        string model;
+        string scoreName;
+        bool complete;
+    };
+};
+
+#endif

src/Platform/Results.cc

@@ -3,36 +3,9 @@
 #include "Results.h"
 #include "ReportConsole.h"
 #include "ReportExcel.h"
-#include "BestResult.h"
+#include "BestScore.h"
 #include "Colors.h"
 namespace platform {
-    Result::Result(const string& path, const string& filename)
-        : path(path)
-        , filename(filename)
-    {
-        auto data = load();
-        date = data["date"];
-        score = 0;
-        for (const auto& result : data["results"]) {
-            score += result["score"].get<double>();
-        }
-        scoreName = data["score_name"];
-        if (scoreName == BestResult::scoreName()) {
-            score /= BestResult::score();
-        }
-        title = data["title"];
-        duration = data["duration"];
-        model = data["model"];
-    }
-    json Result::load() const
-    {
-        ifstream resultData(path + "/" + filename);
-        if (resultData.is_open()) {
-            json data = json::parse(resultData);
-            return data;
-        }
-        throw invalid_argument("Unable to open result file. [" + path + "/" + filename + "]");
-    }
     void Results::load()
     {
         using std::filesystem::directory_iterator;
@@ -41,31 +14,29 @@ namespace platform {
             if (filename.find(".json") != string::npos && filename.find("results_") == 0) {
                 auto result = Result(path, filename);
                 bool addResult = true;
-                if (model != "any" && result.getModel() != model || scoreName != "any" && scoreName != result.getScoreName())
+                if (model != "any" && result.getModel() != model || scoreName != "any" && scoreName != result.getScoreName() || complete && !result.isComplete() || partial && result.isComplete())
                     addResult = false;
                 if (addResult)
                     files.push_back(result);
             }
         }
-    }
-    string Result::to_string() const
-    {
-        stringstream oss;
-        oss << date << " ";
-        oss << setw(12) << left << model << " ";
-        oss << setw(11) << left << scoreName << " ";
-        oss << right << setw(11) << setprecision(7) << fixed << score << " ";
-        oss << setw(9) << setprecision(3) << fixed << duration << " ";
-        oss << setw(50) << left << title << " ";
-        return  oss.str();
+        if (max == 0) {
+            max = files.size();
+        }
     }
     void Results::show() const
     {
         cout << Colors::GREEN() << "Results found: " << files.size() << endl;
         cout << "-------------------" << endl;
+        if (complete) {
+            cout << Colors::MAGENTA() << "Only listing complete results" << endl;
+        }
+        if (partial) {
+            cout << Colors::MAGENTA() << "Only listing partial results" << endl;
+        }
         auto i = 0;
-        cout << " #  Date       Model        Score Name  Score       Duration  Title" << endl;
-        cout << "=== ========== ============ =========== =========== ========= =============================================================" << endl;
+        cout << Colors::GREEN() << " #  Date       Model        Score Name  Score       C/P Duration  Title" << endl;
+        cout << "=== ========== ============ =========== =========== === ========= =============================================================" << endl;
         bool odd = true;
         for (const auto& result : files) {
             auto color = odd ? Colors::BLUE() : Colors::CYAN();
@@ -95,26 +66,51 @@ namespace platform {
         cout << "Invalid index" << endl;
         return -1;
     }
-    void Results::report(const int index, const bool excelReport) const
+    void Results::report(const int index, const bool excelReport)
     {
         cout << Colors::YELLOW() << "Reporting " << files.at(index).getFilename() << endl;
         auto data = files.at(index).load();
         if (excelReport) {
-            ReportExcel reporter(data);
+            ReportExcel reporter(data, compare, workbook);
             reporter.show();
+            openExcel = true;
+            workbook = reporter.getWorkbook();
         } else {
-            ReportConsole reporter(data);
+            ReportConsole reporter(data, compare);
             reporter.show();
         }
     }
+    void Results::showIndex(const int index, const int idx) const
+    {
+        auto data = files.at(index).load();
+        if (idx < 0 or idx >= static_cast<int>(data["results"].size())) {
+            cout << "Invalid index" << endl;
+            return;
+        }
+        cout << Colors::YELLOW() << "Showing " << files.at(index).getFilename() << endl;
+        ReportConsole reporter(data, compare, idx);
+        reporter.show();
+    }
     void Results::menu()
     {
         char option;
         int index;
         bool finished = false;
+        string color, context;
         string filename, line, options = "qldhsre";
         while (!finished) {
-            cout << Colors::RESET() << "Choose option (quit='q', list='l', delete='d', hide='h', sort='s', report='r', excel='e'): ";
+            if (indexList) {
+                color = Colors::GREEN();
+                context = " (quit='q', list='l', delete='d', hide='h', sort='s', report='r', excel='e'): ";
+                options = "qldhsre";
+            } else {
+                color = Colors::MAGENTA();
+                context = " (quit='q', list='l'): ";
+                options = "ql";
+            }
+            cout << Colors::RESET() << color;
+
+            cout << "Choose option " << context;
             getline(cin, line);
             if (line.size() == 0)
                 continue;
@@ -126,9 +122,18 @@ namespace platform {
                 option = line[0];
             } else {
                 if (all_of(line.begin(), line.end(), ::isdigit)) {
-                    index = stoi(line);
-                    if (index >= 0 && index < files.size()) {
-                        report(index, false);
+                    int idx = stoi(line);
+                    if (indexList) {
+                        // The value is about the files list
+                        index = idx;
+                        if (index >= 0 && index < max) {
+                            report(index, false);
+                            indexList = false;
+                            continue;
+                        }
+                    } else {
+                        // The value is about the result showed on screen
+                        showIndex(index, idx);
                         continue;
                     }
                 }
@@ -141,6 +146,7 @@ namespace platform {
                     break;
                 case 'l':
                     show();
+                    indexList = true;
                     break;
                 case 'd':
                     index = getIndex("delete");
@@ -152,6 +158,7 @@ namespace platform {
                     files.erase(files.begin() + index);
                     cout << "File: " + filename + " deleted!" << endl;
                     show();
+                    indexList = true;
                     break;
                 case 'h':
                     index = getIndex("hide");
@@ -163,21 +170,25 @@ namespace platform {
                     files.erase(files.begin() + index);
                     show();
                     menu();
+                    indexList = true;
                     break;
                 case 's':
                     sortList();
+                    indexList = true;
                     show();
                     break;
                 case 'r':
                     index = getIndex("report");
                     if (index == -1)
                         break;
+                    indexList = false;
                     report(index, false);
                     break;
                 case 'e':
                     index = getIndex("excel");
                     if (index == -1)
                         break;
+                    indexList = true;
                     report(index, true);
                     break;
                 default:
@@ -248,7 +259,10 @@ namespace platform {
         sortDate();
         show();
         menu();
-        cout << "Done!" << endl;
+        if (openExcel) {
+            workbook_close(workbook);
+        }
+        cout << Colors::RESET() << "Done!" << endl;
     }
 
 }

src/Platform/Results.h

@@ -1,48 +1,39 @@
 #ifndef RESULTS_H
 #define RESULTS_H
+#include "xlsxwriter.h"
 #include <map>
 #include <vector>
 #include <string>
 #include <nlohmann/json.hpp>
+#include "Result.h"
 namespace platform {
     using namespace std;
     using json = nlohmann::json;
 
-    class Result {
-    public:
-        Result(const string& path, const string& filename);
-        json load() const;
-        string to_string() const;
-        string getFilename() const { return filename; };
-        string getDate() const { return date; };
-        double getScore() const { return score; };
-        string getTitle() const { return title; };
-        double getDuration() const { return duration; };
-        string getModel() const { return model; };
-        string getScoreName() const { return scoreName; };
-    private:
-        string path;
-        string filename;
-        string date;
-        double score;
-        string title;
-        double duration;
-        string model;
-        string scoreName;
-    };
     class Results {
     public:
-        Results(const string& path, const int max, const string& model, const string& score) : path(path), max(max), model(model), scoreName(score) { load(); };
+        Results(const string& path, const int max, const string& model, const string& score, bool complete, bool partial, bool compare) :
+            path(path), max(max), model(model), scoreName(score), complete(complete), partial(partial), compare(compare)
+        {
+            load();
+        };
         void manage();
     private:
         string path;
         int max;
         string model;
         string scoreName;
+        bool complete;
+        bool partial;
+        bool indexList = true;
+        bool openExcel = false;
+        bool compare;
+        lxw_workbook* workbook = NULL;
         vector<Result> files;
         void load(); // Loads the list of results
         void show() const;
-        void report(const int index, const bool excelReport) const;
+        void report(const int index, const bool excelReport);
+        void showIndex(const int index, const int idx) const;
         int getIndex(const string& intent) const;
         void menu();
         void sortList();

src/Platform/best.cc

@@ -0,0 +1,63 @@
+#include <iostream>
+#include <argparse/argparse.hpp>
+#include "Paths.h"
+#include "BestResults.h"
+#include "Colors.h"
+
+using namespace std;
+
+argparse::ArgumentParser manageArguments(int argc, char** argv)
+{
+    argparse::ArgumentParser program("best");
+    program.add_argument("-m", "--model").default_value("").help("Filter results of the selected model) (any for all models)");
+    program.add_argument("-s", "--score").default_value("").help("Filter results of the score name supplied");
+    program.add_argument("--build").help("build best score results file").default_value(false).implicit_value(true);
+    program.add_argument("--report").help("report of best score results file").default_value(false).implicit_value(true);
+    try {
+        program.parse_args(argc, argv);
+        auto model = program.get<string>("model");
+        auto score = program.get<string>("score");
+        auto build = program.get<bool>("build");
+        auto report = program.get<bool>("report");
+        if (model == "" || score == "") {
+            throw runtime_error("Model and score name must be supplied");
+        }
+    }
+    catch (const exception& err) {
+        cerr << err.what() << endl;
+        cerr << program;
+        exit(1);
+    }
+    return program;
+}
+
+int main(int argc, char** argv)
+{
+    auto program = manageArguments(argc, argv);
+    auto model = program.get<string>("model");
+    auto score = program.get<string>("score");
+    auto build = program.get<bool>("build");
+    auto report = program.get<bool>("report");
+    if (!report && !build) {
+        cerr << "Either build, report or both, have to be selected to do anything!" << endl;
+        cerr << program;
+        exit(1);
+    }
+    auto results = platform::BestResults(platform::Paths::results(), score, model);
+    if (build) {
+        if (model == "any") {
+            results.buildAll();
+        } else {
+            string fileName = results.build();
+            cout << Colors::GREEN() << fileName << " created!" << Colors::RESET() << endl;
+        }
+    }
+    if (report) {
+        if (model == "any") {
+            results.reportAll();
+        } else {
+            results.reportSingle();
+        }
+    }
+    return 0;
+}

src/Platform/main.cc

@@ -87,7 +87,7 @@ int main(int argc, char** argv)
     auto stratified = program.get<bool>("stratified");
     auto n_folds = program.get<int>("folds");
     auto seeds = program.get<vector<int>>("seeds");
-    auto hyperparameters =program.get<string>("hyperparameters");
+    auto hyperparameters = program.get<string>("hyperparameters");
     vector<string> filesToTest;
     auto datasets = platform::Datasets(path, true, platform::ARFF);
     auto title = program.get<string>("title");
@@ -102,7 +102,7 @@ int main(int argc, char** argv)
         }
         filesToTest.push_back(file_name);
     } else {
-        filesToTest = platform::Datasets(path, true, platform::ARFF).getNames();
+        filesToTest = datasets.getNames();
         saveResults = true;
     }
     /*

src/Platform/manage.cc

@@ -12,6 +12,9 @@ argparse::ArgumentParser manageArguments(int argc, char** argv)
     program.add_argument("-n", "--number").default_value(0).help("Number of results to show (0 = all)").scan<'i', int>();
     program.add_argument("-m", "--model").default_value("any").help("Filter results of the selected model)");
     program.add_argument("-s", "--score").default_value("any").help("Filter results of the score name supplied");
+    program.add_argument("--complete").help("Show only results with all datasets").default_value(false).implicit_value(true);
+    program.add_argument("--partial").help("Show only partial results").default_value(false).implicit_value(true);
+    program.add_argument("--compare").help("Compare with best results").default_value(false).implicit_value(true);
     try {
         program.parse_args(argc, argv);
         auto number = program.get<int>("number");
@@ -20,6 +23,9 @@ argparse::ArgumentParser manageArguments(int argc, char** argv)
         }
         auto model = program.get<string>("model");
         auto score = program.get<string>("score");
+        auto complete = program.get<bool>("complete");
+        auto partial = program.get<bool>("partial");
+        auto compare = program.get<bool>("compare");
     }
     catch (const exception& err) {
         cerr << err.what() << endl;
@@ -35,7 +41,12 @@ int main(int argc, char** argv)
     auto number = program.get<int>("number");
     auto model = program.get<string>("model");
     auto score = program.get<string>("score");
-    auto results = platform::Results(platform::Paths::results(), number, model, score);
+    auto complete = program.get<bool>("complete");
+    auto partial = program.get<bool>("partial");
+    auto compare = program.get<bool>("compare");
+    if (complete)
+        partial = false;
+    auto results = platform::Results(platform::Paths::results(), number, model, score, complete, partial, compare);
     results.manage();
     return 0;
 }

src/Platform/platformUtils.h

@@ -8,7 +8,6 @@
 #include "ArffFiles.h"
 #include "CPPFImdlp.h"
 using namespace std;
-const string PATH = "../../data/";
 
 bool file_exists(const std::string& name);
 vector<string> split(const string& text, char delimiter);

tests/CMakeLists.txt

@@ -4,6 +4,7 @@ if(ENABLE_TESTING)
     include_directories(${BayesNet_SOURCE_DIR}/src/Platform)
     include_directories(${BayesNet_SOURCE_DIR}/lib/Files)
     include_directories(${BayesNet_SOURCE_DIR}/lib/mdlp)
+    include_directories(${BayesNet_SOURCE_DIR}/lib/json/include)
     set(TEST_SOURCES BayesModels.cc BayesNetwork.cc ${BayesNet_SOURCE_DIR}/src/Platform/platformUtils.cc ${BayesNet_SOURCES})
     add_executable(${TEST_MAIN} ${TEST_SOURCES})
     target_link_libraries(${TEST_MAIN} PUBLIC "${TORCH_LIBRARIES}" ArffFiles mdlp Catch2::Catch2WithMain)