Add openmpi instructions for Oracle Linux

Reviewed-on: #13

.gitmodules

@@ -1,15 +1,25 @@
 [submodule "lib/mdlp"]
 	path = lib/mdlp
 	url = https://github.com/rmontanana/mdlp
+	main = main
+	update = merge
 [submodule "lib/catch2"]
 	path = lib/catch2
+        main  = v2.x
+	update = merge
 	url = https://github.com/catchorg/Catch2.git
 [submodule "lib/argparse"]
 	path = lib/argparse
 	url = https://github.com/p-ranav/argparse
+	master = master
+	update = merge
 [submodule "lib/json"]
 	path = lib/json
 	url = https://github.com/nlohmann/json.git
+        master = master
+	update = merge
 [submodule "lib/libxlsxwriter"]
 	path = lib/libxlsxwriter
 	url = https://github.com/jmcnamara/libxlsxwriter.git
+        main = main
+	update = merge

.vscode/launch.json

@@ -14,7 +14,7 @@
                 "-s",
                 "271",
                 "-p",
-                "/home/rmontanana/Code/discretizbench/datasets/",
+                "/Users/rmontanana/Code/discretizbench/datasets/",
             ],
             //"cwd": "${workspaceFolder}/build/sample/",
         },
@@ -33,7 +33,23 @@
                 // "--hyperparameters",
                 // "{\"repeatSparent\": true, \"maxModels\": 12}"
             ],
-            "cwd": "/home/rmontanana/Code/discretizbench",
+            "cwd": "${workspaceFolder}/../discretizbench",
+        },
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "gridsearch",
+            "program": "${workspaceFolder}/build_debug/src/Platform/b_grid",
+            "args": [
+                "-m",
+                "KDB",
+                "--discretize",
+                "--continue",
+                "glass",
+                "--only",
+                "--compute"
+            ],
+            "cwd": "${workspaceFolder}/../discretizbench",
         },
         {
             "type": "lldb",
@@ -64,7 +80,7 @@
                 "accuracy",
                 "--build",
             ],
-            "cwd": "/home/rmontanana/Code/discretizbench",
+            "cwd": "${workspaceFolder}/../discretizbench",
         },
         {
             "type": "lldb",
@@ -75,7 +91,7 @@
                 "-n",
                 "20"
             ],
-            "cwd": "/home/rmontanana/Code/discretizbench",
+            "cwd": "${workspaceFolder}/../discretizbench",
         },
         {
             "type": "lldb",
@@ -84,7 +100,7 @@
             "program": "${workspaceFolder}/build_debug/src/Platform/b_list",
             "args": [],
             //"cwd": "/Users/rmontanana/Code/discretizbench",
-            "cwd": "/home/rmontanana/Code/covbench",
+            "cwd": "${workspaceFolder}/../discretizbench",
         },
         {
             "type": "lldb",

CMakeLists.txt

@@ -25,12 +25,18 @@ set(CMAKE_CXX_EXTENSIONS                  OFF)
 set(CMAKE_EXPORT_COMPILE_COMMANDS          ON)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")
 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
-
 # Options
 # -------
 option(ENABLE_CLANG_TIDY "Enable to add clang tidy."              OFF)
 option(ENABLE_TESTING "Unit testing build"                        OFF)
 option(CODE_COVERAGE "Collect coverage from test library"         OFF)
+option(MPI_ENABLED "Enable MPI options"                           ON)
+
+if (MPI_ENABLED)
+  find_package(MPI REQUIRED)
+  message("MPI_CXX_LIBRARIES=${MPI_CXX_LIBRARIES}")
+  message("MPI_CXX_INCLUDE_DIRS=${MPI_CXX_INCLUDE_DIRS}")
+endif (MPI_ENABLED)
 
 # Boost Library
 set(Boost_USE_STATIC_LIBS OFF) 

Makefile

@@ -49,10 +49,10 @@ dependency: ## Create a dependency graph diagram of the project (build/dependenc
 	cd $(f_debug) && cmake .. --graphviz=dependency.dot && dot -Tpng dependency.dot -o dependency.png
 
 buildd: ## Build the debug targets
-	cmake --build $(f_debug) -t $(app_targets) $(n_procs)
+	cmake --build $(f_debug) -t $(app_targets) BayesNetSample $(n_procs)
 
 buildr: ## Build the release targets
-	cmake --build $(f_release) -t $(app_targets) $(n_procs)
+	cmake --build $(f_release) -t $(app_targets) BayesNetSample $(n_procs)
 
 clean: ## Clean the tests info
 	@echo ">>> Cleaning Debug BayesNet tests...";

README.md

@@ -8,6 +8,20 @@ Bayesian Network Classifier with libtorch from scratch
 
 Before compiling BayesNet.
 
+### MPI
+
+In Linux just install openmpi & openmpi-devel packages. Only cmake can't find openmpi install (like in Oracle Linux) set the following variable:
+
+```bash
+export MPI_HOME="/usr/lib64/openmpi"
+```
+
+In Mac OS X, install mpich with brew and if cmake doesn't find it, edit mpicxx wrapper to remove the ",-commons,use_dylibs" from final_ldflags
+
+```bash
+vi /opt/homebrew/bin/mpicx
+```
+
 ### boost library
 
 [Getting Started](<https://www.boost.org/doc/libs/1_83_0/more/getting_started/index.html>)

sample/CMakeLists.txt

@@ -1,8 +1,10 @@
 include_directories(${BayesNet_SOURCE_DIR}/src/Platform)
 include_directories(${BayesNet_SOURCE_DIR}/src/BayesNet)
+include_directories(${BayesNet_SOURCE_DIR}/src/PyClassifiers)
+include_directories(${Python3_INCLUDE_DIRS})
 include_directories(${BayesNet_SOURCE_DIR}/lib/Files)
 include_directories(${BayesNet_SOURCE_DIR}/lib/mdlp)
 include_directories(${BayesNet_SOURCE_DIR}/lib/argparse/include)
 include_directories(${BayesNet_SOURCE_DIR}/lib/json/include)
 add_executable(BayesNetSample sample.cc ${BayesNet_SOURCE_DIR}/src/Platform/Folding.cc ${BayesNet_SOURCE_DIR}/src/Platform/Models.cc) 
-target_link_libraries(BayesNetSample BayesNet ArffFiles mdlp "${TORCH_LIBRARIES}")
+target_link_libraries(BayesNetSample BayesNet ArffFiles mdlp "${TORCH_LIBRARIES}" PyWrap)

sample/sample.cc

@@ -29,7 +29,7 @@ pair<std::vector<mdlp::labels_t>, map<std::string, int>> discretize(std::vector<
     return { Xd, maxes };
 }
 
-bool file_exists(const std::std::std::string& name)
+bool file_exists(const std::string& name)
 {
     if (FILE* file = fopen(name.c_str(), "r")) {
         fclose(file);
@@ -72,7 +72,7 @@ int main(int argc, char** argv)
     argparse::ArgumentParser program("BayesNetSample");
     program.add_argument("-d", "--dataset")
         .help("Dataset file name")
-        .action([valid_datasets](const std::std::std::string& value) {
+        .action([valid_datasets](const std::string& value) {
         if (find(valid_datasets.begin(), valid_datasets.end(), value) != valid_datasets.end()) {
             return value;
         }
@@ -84,20 +84,20 @@ int main(int argc, char** argv)
         .default_value(std::string{ PATH }
     );
     program.add_argument("-m", "--model")
-        .help("Model to use " + platform::Models::instance()->tostd::string())
+        .help("Model to use " + platform::Models::instance()->tostring())
-        .action([](const std::std::std::string& value) {
+        .action([](const std::string& value) {
         static const std::vector<std::string> choices = platform::Models::instance()->getNames();
         if (find(choices.begin(), choices.end(), value) != choices.end()) {
             return value;
         }
-        throw runtime_error("Model must be one of " + platform::Models::instance()->tostd::string());
+        throw runtime_error("Model must be one of " + platform::Models::instance()->tostring());
             }
     );
     program.add_argument("--discretize").help("Discretize input dataset").default_value(false).implicit_value(true);
     program.add_argument("--dumpcpt").help("Dump CPT Tables").default_value(false).implicit_value(true);
     program.add_argument("--stratified").help("If Stratified KFold is to be done").default_value(false).implicit_value(true);
     program.add_argument("--tensors").help("Use tensors to store samples").default_value(false).implicit_value(true);
-    program.add_argument("-f", "--folds").help("Number of folds").default_value(5).scan<'i', int>().action([](const std::std::string& value) {
+    program.add_argument("-f", "--folds").help("Number of folds").default_value(5).scan<'i', int>().action([](const std::string& value) {
         try {
             auto k = stoi(value);
             if (k < 2) {
@@ -184,8 +184,8 @@ int main(int argc, char** argv)
     file.close();
     std::cout << "Graph saved in " << model_name << "_" << file_name << ".dot" << std::endl;
     std::cout << "dot -Tpng -o " + dot_file + ".png " + dot_file + ".dot " << std::endl;
-    std::string stratified_std::string = stratified ? " Stratified" : "";
+    std::string stratified_string = stratified ? " Stratified" : "";
-    std::cout << nFolds << " Folds" << stratified_std::string << " Cross validation" << std::endl;
+    std::cout << nFolds << " Folds" << stratified_string << " Cross validation" << std::endl;
     std::cout << "==========================================" << std::endl;
     torch::Tensor Xt = torch::zeros({ static_cast<int>(Xd.size()), static_cast<int>(Xd[0].size()) }, torch::kInt32);
     torch::Tensor yt = torch::tensor(y, torch::kInt32);

src/BayesNet/BoostAODE.cc

@@ -12,7 +12,7 @@
 namespace bayesnet {
     BoostAODE::BoostAODE() : Ensemble()
     {
-        validHyperparameters = { "repeatSparent", "maxModels", "ascending", "convergence", "threshold", "select_features" };
+        validHyperparameters = { "repeatSparent", "maxModels", "ascending", "convergence", "threshold", "select_features", "tolerance" };
 
     }
     void BoostAODE::buildModel(const torch::Tensor& weights)
@@ -47,22 +47,32 @@ namespace bayesnet {
             y_train = y_;
         }
     }
-    void BoostAODE::setHyperparameters(const nlohmann::json& hyperparameters)
+    void BoostAODE::setHyperparameters(const nlohmann::json& hyperparameters_)
     {
+        auto hyperparameters = hyperparameters_;
         if (hyperparameters.contains("repeatSparent")) {
             repeatSparent = hyperparameters["repeatSparent"];
+            hyperparameters.erase("repeatSparent");
         }
         if (hyperparameters.contains("maxModels")) {
             maxModels = hyperparameters["maxModels"];
+            hyperparameters.erase("maxModels");
         }
         if (hyperparameters.contains("ascending")) {
             ascending = hyperparameters["ascending"];
+            hyperparameters.erase("ascending");
         }
         if (hyperparameters.contains("convergence")) {
             convergence = hyperparameters["convergence"];
+            hyperparameters.erase("convergence");
         }
         if (hyperparameters.contains("threshold")) {
             threshold = hyperparameters["threshold"];
+            hyperparameters.erase("threshold");
+        }
+        if (hyperparameters.contains("tolerance")) {
+            tolerance = hyperparameters["tolerance"];
+            hyperparameters.erase("tolerance");
         }
         if (hyperparameters.contains("select_features")) {
             auto selectedAlgorithm = hyperparameters["select_features"];
@@ -72,6 +82,10 @@ namespace bayesnet {
             if (std::find(algos.begin(), algos.end(), selectedAlgorithm) == algos.end()) {
                 throw std::invalid_argument("Invalid selectFeatures value [IWSS, FCBF, CFS]");
             }
+            hyperparameters.erase("select_features");
+        }
+        if (!hyperparameters.empty()) {
+            throw std::invalid_argument("Invalid hyperparameters" + hyperparameters.dump());
         }
     }
     std::unordered_set<int> BoostAODE::initializeModels()
@@ -109,10 +123,8 @@ namespace bayesnet {
     void BoostAODE::trainModel(const torch::Tensor& weights)
     {
         std::unordered_set<int> featuresUsed;
-        int tolerance = 5; // number of times the accuracy can be lower than the threshold
         if (selectFeatures) {
             featuresUsed = initializeModels();
-            tolerance = 0; // Remove tolerance if features are selected
         }
         if (maxModels == 0)
             maxModels = .1 * n > 10 ? .1 * n : n;

src/BayesNet/BoostAODE.h

@@ -21,6 +21,7 @@ namespace bayesnet {
         // Hyperparameters
         bool repeatSparent = false; // if true, a feature can be selected more than once
         int maxModels = 0;
+        int tolerance = 0;
         bool ascending = false; //Process KBest features ascending or descending order
         bool convergence = false; //if true, stop when the model does not improve
         bool selectFeatures = false; // if true, use feature selection

src/Platform/CMakeLists.txt

@@ -7,6 +7,7 @@ include_directories(${BayesNet_SOURCE_DIR}/lib/argparse/include)
 include_directories(${BayesNet_SOURCE_DIR}/lib/json/include)
 include_directories(${BayesNet_SOURCE_DIR}/lib/libxlsxwriter/include)
 include_directories(${Python3_INCLUDE_DIRS})
+include_directories(${MPI_CXX_INCLUDE_DIRS})
 
 add_executable(b_best b_best.cc BestResults.cc Result.cc Statistics.cc BestResultsExcel.cc ReportExcel.cc ReportBase.cc Datasets.cc Dataset.cc ExcelFile.cc)
 add_executable(b_grid b_grid.cc GridSearch.cc GridData.cc HyperParameters.cc Folding.cc Datasets.cc Dataset.cc)
@@ -15,7 +16,7 @@ add_executable(b_main b_main.cc Folding.cc Experiment.cc Datasets.cc Dataset.cc
 add_executable(b_manage b_manage.cc Results.cc ManageResults.cc CommandParser.cc Result.cc ReportConsole.cc ReportExcel.cc ReportBase.cc Datasets.cc Dataset.cc ExcelFile.cc)
 
 target_link_libraries(b_best Boost::boost "${XLSXWRITER_LIB}" "${TORCH_LIBRARIES}" ArffFiles mdlp)
-target_link_libraries(b_grid BayesNet  PyWrap)
+target_link_libraries(b_grid BayesNet PyWrap ${MPI_CXX_LIBRARIES})
 target_link_libraries(b_list ArffFiles mdlp "${TORCH_LIBRARIES}")
 target_link_libraries(b_main BayesNet ArffFiles mdlp "${TORCH_LIBRARIES}" PyWrap)
 target_link_libraries(b_manage "${TORCH_LIBRARIES}" "${XLSXWRITER_LIB}" ArffFiles mdlp)

src/Platform/Colors.h

@@ -9,6 +9,7 @@ public:
     static std::string YELLOW() { return "\033[1;33m"; }
     static std::string RED() { return "\033[1;31m"; }
     static std::string WHITE() { return "\033[1;37m"; }
+    static std::string IBLUE() { return "\033[0;94m"; }
     static std::string RESET() { return "\033[0m"; }
 };
 #endif // COLORS_H

src/Platform/GridData.cc

@@ -4,12 +4,19 @@
 namespace platform {
     GridData::GridData(const std::string& fileName)
     {
+        json grid_file;
         std::ifstream resultData(fileName);
         if (resultData.is_open()) {
-            grid = json::parse(resultData);
+            grid_file = json::parse(resultData);
         } else {
             throw std::invalid_argument("Unable to open input file. [" + fileName + "]");
         }
+        for (const auto& item : grid_file.items()) {
+            auto key = item.key();
+            auto value = item.value();
+            grid[key] = value;
+        }
+
     }
     int GridData::computeNumCombinations(const json& line)
     {
@@ -19,10 +26,11 @@ namespace platform {
         }
         return numCombinations;
     }
-    int GridData::getNumCombinations()
+    int GridData::getNumCombinations(const std::string& dataset)
     {
         int numCombinations = 0;
-        for (const auto& line : grid) {
+        auto selected = decide_dataset(dataset);
+        for (const auto& line : grid.at(selected)) {
             numCombinations += computeNumCombinations(line);
         }
         return numCombinations;
@@ -44,12 +52,24 @@ namespace platform {
         }
         return currentCombination;
     }
-    std::vector<json> GridData::getGrid()
+    std::vector<json> GridData::getGrid(const std::string& dataset)
     {
+        auto selected = decide_dataset(dataset);
         auto result = std::vector<json>();
-        for (json line : grid) {
+        for (json line : grid.at(selected)) {
             generateCombinations(line.begin(), line.end(), result, json({}));
         }
         return result;
     }
+    json& GridData::getInputGrid(const std::string& dataset)
+    {
+        auto selected = decide_dataset(dataset);
+        return grid.at(selected);
+    }
+    std::string GridData::decide_dataset(const std::string& dataset)
+    {
+        if (grid.find(dataset) != grid.end())
+            return dataset;
+        return ALL_DATASETS;
+    }
 } /* namespace platform */

src/Platform/GridData.h

@@ -7,16 +7,20 @@
 
 namespace platform {
     using json = nlohmann::json;
+    const std::string ALL_DATASETS = "all";
     class GridData {
     public:
         explicit GridData(const std::string& fileName);
         ~GridData() = default;
-        std::vector<json> getGrid();
+        std::vector<json> getGrid(const std::string& dataset = ALL_DATASETS);
-        int getNumCombinations();
+        int getNumCombinations(const std::string& dataset = ALL_DATASETS);
+        json& getInputGrid(const std::string& dataset = ALL_DATASETS);
+        std::map<std::string, json>& getGridFile() { return grid; }
     private:
+        std::string decide_dataset(const std::string& dataset);
         json generateCombinations(json::iterator index, const json::iterator last, std::vector<json>& output, json currentCombination);
         int computeNumCombinations(const json& line);
-        json grid;
+        std::map<std::string, json> grid;
     };
 } /* namespace platform */
 #endif /* GRIDDATA_H */

src/Platform/GridSearch.cc

@@ -7,15 +7,74 @@
 #include "Colors.h"
 
 namespace platform {
+    std::string get_date()
+    {
+        time_t rawtime;
+        tm* timeinfo;
+        time(&rawtime);
+        timeinfo = std::localtime(&rawtime);
+        std::ostringstream oss;
+        oss << std::put_time(timeinfo, "%Y-%m-%d");
+        return oss.str();
+    }
+    std::string get_time()
+    {
+        time_t rawtime;
+        tm* timeinfo;
+        time(&rawtime);
+        timeinfo = std::localtime(&rawtime);
+        std::ostringstream oss;
+        oss << std::put_time(timeinfo, "%H:%M:%S");
+        return oss.str();
+    }
     GridSearch::GridSearch(struct ConfigGrid& config) : config(config)
     {
-        this->config.output_file = config.path + "grid_" + config.model + "_output.json";
-        this->config.input_file = config.path + "grid_" + config.model + "_input.json";
     }
-    void showProgressComb(const int num, const int total, const std::string& color)
+    json GridSearch::getResults()
+    {
+        std::ifstream file(Paths::grid_output(config.model));
+        if (file.is_open()) {
+            return json::parse(file);
+        }
+        return json();
+    }
+    vector<std::string> GridSearch::processDatasets(Datasets& datasets)
+    {
+        // Load datasets
+        auto datasets_names = datasets.getNames();
+        if (config.continue_from != NO_CONTINUE()) {
+            // Continue previous execution:
+            if (std::find(datasets_names.begin(), datasets_names.end(), config.continue_from) == datasets_names.end()) {
+                throw std::invalid_argument("Dataset " + config.continue_from + " not found");
+            }
+            // Remove datasets already processed
+            vector< string >::iterator it = datasets_names.begin();
+            while (it != datasets_names.end()) {
+                if (*it != config.continue_from) {
+                    it = datasets_names.erase(it);
+                } else {
+                    if (config.only)
+                        ++it;
+                    else
+                        break;
+                }
+            }
+        }
+        // Exclude datasets
+        for (const auto& name : config.excluded) {
+            auto dataset = name.get<std::string>();
+            auto it = std::find(datasets_names.begin(), datasets_names.end(), dataset);
+            if (it == datasets_names.end()) {
+                throw std::invalid_argument("Dataset " + dataset + " already excluded or doesn't exist!");
+            }
+            datasets_names.erase(it);
+        }
+        return datasets_names;
+    }
+    void showProgressComb(const int num, const int n_folds, const int total, const std::string& color)
     {
         int spaces = int(log(total) / log(10)) + 1;
-        int magic = 37 + 2 * spaces;
+        int magic = n_folds * 3 + 22 + 2 * spaces;
         std::string prefix = num == 1 ? "" : string(magic, '\b') + string(magic + 1, ' ') + string(magic + 1, '\b');
         std::cout << prefix << color << "(" << setw(spaces) << num << "/" << setw(spaces) << total << ") " << Colors::RESET() << flush;
     }
@@ -37,13 +96,315 @@ namespace platform {
                 return Colors::RESET();
         }
     }
-    double GridSearch::processFile(std::string fileName, Datasets& datasets, HyperParameters& hyperparameters)
+    json GridSearch::build_tasks_mpi()
     {
+        auto tasks = json::array();
+        auto grid = GridData(Paths::grid_input(config.model));
+        auto datasets = Datasets(false, Paths::datasets());
+        auto datasets_names = processDatasets(datasets);
+        for (const auto& dataset : datasets_names) {
+            for (const auto& seed : config.seeds) {
+                auto combinations = grid.getGrid(dataset);
+                for (int n_fold = 0; n_fold < config.n_folds; n_fold++) {
+                    json task = {
+                        { "dataset", dataset },
+                        { "seed", seed },
+                        { "fold", n_fold}
+                    };
+                    tasks.push_back(task);
+                }
+            }
+        }
+        // It's important to shuffle the array so heavy datasets are spread across the Workers
+        std::mt19937 g{ 271 }; // Use fixed seed to obtain the same shuffle
+        std::shuffle(tasks.begin(), tasks.end(), g);
+        std::cout << "Tasks size: " << tasks.size() << std::endl;
+        std::cout << "|";
+        for (int i = 0; i < tasks.size(); ++i) {
+            std::cout << (i + 1) % 10;
+        }
+        std::cout << "|" << std::endl << "|" << std::flush;
+        return tasks;
+    }
+    std::pair<int, int> GridSearch::part_range_mpi(int n_tasks, int nprocs, int rank)
+    {
+        int assigned = 0;
+        int remainder = n_tasks % nprocs;
+        int start = 0;
+        if (rank < remainder) {
+            assigned = n_tasks / nprocs + 1;
+        } else {
+            assigned = n_tasks / nprocs;
+            start = remainder;
+        }
+        start += rank * assigned;
+        int end = start + assigned;
+        if (rank == nprocs - 1) {
+            end = n_tasks;
+        }
+        return { start, end };
+    }
+    std::string get_color_rank(int rank)
+    {
+        auto colors = { Colors::RED(), Colors::GREEN(),  Colors::BLUE(), Colors::MAGENTA(), Colors::CYAN() };
+        return *(colors.begin() + rank % colors.size());
+    }
+    void GridSearch::process_task_mpi(struct ConfigMPI& config_mpi, json& task, Datasets& datasets, json& results)
+    {
+        // Process the task and store the result in the results json
+        Timer timer;
+        timer.start();
+        auto grid = GridData(Paths::grid_input(config.model));
+        auto dataset = task["dataset"].get<std::string>();
+        auto seed = task["seed"].get<int>();
+        auto n_fold = task["fold"].get<int>();
+        // Generate the hyperparamters combinations
+        auto combinations = grid.getGrid(dataset);
+        auto [X, y] = datasets.getTensors(dataset);
+        auto states = datasets.getStates(dataset);
+        auto features = datasets.getFeatures(dataset);
+        auto className = datasets.getClassName(dataset);
+        //
+        // Start working on task
+        //
+        Fold* fold;
+        if (config.stratified)
+            fold = new StratifiedKFold(config.n_folds, y, seed);
+        else
+            fold = new KFold(config.n_folds, y.size(0), seed);
+        auto [train, test] = fold->getFold(n_fold);
+        auto train_t = torch::tensor(train);
+        auto test_t = torch::tensor(test);
+        auto X_train = X.index({ "...", train_t });
+        auto y_train = y.index({ train_t });
+        auto X_test = X.index({ "...", test_t });
+        auto y_test = y.index({ test_t });
+        auto num = 0;
+        double best_fold_score = 0.0;
+        json best_fold_hyper;
+        for (const auto& hyperparam_line : combinations) {
+            auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line);
+            Fold* nested_fold;
+            if (config.stratified)
+                nested_fold = new StratifiedKFold(config.nested, y_train, seed);
+            else
+                nested_fold = new KFold(config.nested, y_train.size(0), seed);
+            double score = 0.0;
+            for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) {
+                // Nested level fold
+                auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold);
+                auto train_nested_t = torch::tensor(train_nested);
+                auto test_nested_t = torch::tensor(test_nested);
+                auto X_nested_train = X_train.index({ "...", train_nested_t });
+                auto y_nested_train = y_train.index({ train_nested_t });
+                auto X_nested_test = X_train.index({ "...", test_nested_t });
+                auto y_nested_test = y_train.index({ test_nested_t });
+                // Build Classifier with selected hyperparameters
+                auto clf = Models::instance()->create(config.model);
+                auto valid = clf->getValidHyperparameters();
+                hyperparameters.check(valid, dataset);
+                clf->setHyperparameters(hyperparameters.get(dataset));
+                // Train model
+                clf->fit(X_nested_train, y_nested_train, features, className, states);
+                // Test model
+                score += clf->score(X_nested_test, y_nested_test);
+            }
+            delete nested_fold;
+            score /= config.nested;
+            if (score > best_fold_score) {
+                best_fold_score = score;
+                best_fold_hyper = hyperparam_line;
+            }
+        }
+        delete fold;
+        // Build Classifier with the best hyperparameters to obtain the best score
+        auto hyperparameters = platform::HyperParameters(datasets.getNames(), best_fold_hyper);
+        auto clf = Models::instance()->create(config.model);
+        auto valid = clf->getValidHyperparameters();
+        hyperparameters.check(valid, dataset);
+        clf->setHyperparameters(best_fold_hyper);
+        clf->fit(X_train, y_train, features, className, states);
+        best_fold_score = clf->score(X_test, y_test);
+        // Save results
+        results[dataset][std::to_string(n_fold)]["score"] = best_fold_score;
+        results[dataset][std::to_string(n_fold)]["hyperparameters"] = best_fold_hyper;
+        results[dataset][std::to_string(n_fold)]["seed"] = seed;
+        results[dataset][std::to_string(n_fold)]["duration"] = timer.getDuration();
+        std::cout << get_color_rank(config_mpi.rank) << "*" << std::flush;
+    }
+    void GridSearch::go_mpi(struct ConfigMPI& config_mpi)
+    {
+        /*
+         * Each task is a json object with the following structure:
+         * {
+         *   "dataset": "dataset_name",
+         *   "seed": # of seed to use,
+         *   "model": "model_name",
+         *   "Fold": # of fold to process
+         * }
+         *
+         * The overall process consists in these steps:
+            * 1. Manager will broadcast the tasks to all the processes
+            * 1.1 Broadcast the number of tasks
+            * 1.2 Broadcast the length of the following string
+            * 1.2 Broadcast the tasks as a char* string
+            * 2. Workers will receive the tasks and start the process
+            * 2.1 A method will tell each worker the range of tasks to process
+            * 2.2 Each worker will process the tasks and generate the best score for each task
+            * 3. Manager gather the scores from all the workers and find out the best hyperparameters for each dataset
+            * 3.1 Obtain the maximum size of the results message of all the workers
+            * 3.2 Gather all the results from the workers into the manager
+            * 3.3 Compile the results from all the workers
+            * 3.4 Filter the best hyperparameters for each dataset
+         */
+        char* msg;
+        int tasks_size;
+        if (config_mpi.rank == config_mpi.manager) {
+            timer.start();
+            auto tasks = build_tasks_mpi();
+            auto tasks_str = tasks.dump();
+            tasks_size = tasks_str.size();
+            msg = new char[tasks_size + 1];
+            strcpy(msg, tasks_str.c_str());
+        }
+        //
+        // 1. Manager will broadcast the tasks to all the processes
+        //
+        MPI_Bcast(&tasks_size, 1, MPI_INT, config_mpi.manager, MPI_COMM_WORLD);
+        if (config_mpi.rank != config_mpi.manager) {
+            msg = new char[tasks_size + 1];
+        }
+        MPI_Bcast(msg, tasks_size + 1, MPI_CHAR, config_mpi.manager, MPI_COMM_WORLD);
+        json tasks = json::parse(msg);
+        delete[] msg;
+        //
+        // 2. All Workers will receive the tasks and start the process
+        //
+        int num_tasks = tasks.size();
+        // 2.1 A method will tell each worker the range of tasks to process
+        auto [start, end] = part_range_mpi(num_tasks, config_mpi.n_procs, config_mpi.rank);
+        // 2.2 Each worker will process the tasks and return the best scores obtained
+        auto datasets = Datasets(config.discretize, Paths::datasets());
+        json results;
+        for (int i = start; i < end; ++i) {
+            // Process task
+            process_task_mpi(config_mpi, tasks[i], datasets, results);
+        }
+        int size = results.dump().size() + 1;
+        int max_size = 0;
+        //
+        // 3. Manager gather the scores from all the workers and find out the best hyperparameters for each dataset
+        //
+        //3.1 Obtain the maximum size of the results message of all the workers
+        MPI_Allreduce(&size, &max_size, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
+        // Assign the memory to the message and initialize it to 0s
+        char* total = NULL;
+        msg = new char[max_size];
+        strncpy(msg, results.dump().c_str(), size);
+        if (config_mpi.rank == config_mpi.manager) {
+            total = new char[max_size * config_mpi.n_procs];
+        }
+        // 3.2 Gather all the results from the workers into the manager
+        MPI_Gather(msg, max_size, MPI_CHAR, total, max_size, MPI_CHAR, config_mpi.manager, MPI_COMM_WORLD);
+        delete[] msg;
+        if (config_mpi.rank == config_mpi.manager) {
+            std::cout << Colors::RESET() << "|" << std::endl;
+            json total_results;
+            json best_results;
+            // 3.3 Compile the results from all the workers
+            for (int i = 0; i < config_mpi.n_procs; ++i) {
+                json partial_results = json::parse(total + i * max_size);
+                for (auto& [dataset, folds] : partial_results.items()) {
+                    for (auto& [fold, result] : folds.items()) {
+                        total_results[dataset][fold] = result;
+                    }
+                }
+            }
+            delete[] total;
+            // 3.4 Filter the best hyperparameters for each dataset
+            auto grid = GridData(Paths::grid_input(config.model));
+            for (auto& [dataset, folds] : total_results.items()) {
+                double best_score = 0.0;
+                double duration = 0.0;
+                json best_hyper;
+                for (auto& [fold, result] : folds.items()) {
+                    duration += result["duration"].get<double>();
+                    if (result["score"] > best_score) {
+                        best_score = result["score"];
+                        best_hyper = result["hyperparameters"];
+                    }
+                }
+                auto timer = Timer();
+                json result = {
+                    { "score", best_score },
+                    { "hyperparameters", best_hyper },
+                    { "date", get_date() + " " + get_time() },
+                    { "grid", grid.getInputGrid(dataset) },
+                    { "duration", timer.translate2String(duration) }
+                };
+                best_results[dataset] = result;
+            }
+            save(best_results);
+        }
+    }
+    void GridSearch::go()
+    {
+        timer.start();
+        auto grid_type = config.nested == 0 ? "Single" : "Nested";
+        auto datasets = Datasets(config.discretize, Paths::datasets());
+        auto datasets_names = processDatasets(datasets);
+        json results = initializeResults();
+        std::cout << "***************** Starting " << grid_type << " Gridsearch *****************" << std::endl;
+        std::cout << "input file=" << Paths::grid_input(config.model) << std::endl;
+        auto grid = GridData(Paths::grid_input(config.model));
+        Timer timer_dataset;
+        double bestScore = 0;
+        json bestHyperparameters;
+        for (const auto& dataset : datasets_names) {
+            if (!config.quiet)
+                std::cout << "- " << setw(20) << left << dataset << " " << right << flush;
+            auto combinations = grid.getGrid(dataset);
+            timer_dataset.start();
+            if (config.nested == 0)
+                // for dataset // for hyperparameters // for seed // for fold
+                tie(bestScore, bestHyperparameters) = processFileSingle(dataset, datasets, combinations);
+            else
+                // for dataset // for seed // for fold // for hyperparameters // for nested fold
+                tie(bestScore, bestHyperparameters) = processFileNested(dataset, datasets, combinations);
+            if (!config.quiet) {
+                std::cout << "end." << " Score: " << Colors::IBLUE() << setw(9) << setprecision(7) << fixed
+                    << bestScore << Colors::BLUE() << " [" << bestHyperparameters.dump() << "]"
+                    << Colors::RESET() << ::endl;
+            }
+            json result = {
+                { "score", bestScore },
+                { "hyperparameters", bestHyperparameters },
+                { "date", get_date() + " " + get_time() },
+                { "grid", grid.getInputGrid(dataset) },
+                { "duration", timer_dataset.getDurationString() }
+            };
+            results[dataset] = result;
+            // Save partial results
+            save(results);
+        }
+        // Save final results
+        save(results);
+        std::cout << "***************** Ending " << grid_type << " Gridsearch *******************" << std::endl;
+    }
+    pair<double, json> GridSearch::processFileSingle(std::string fileName, Datasets& datasets, vector<json>& combinations)
+    {
+        int num = 0;
+        double bestScore = 0.0;
+        json bestHyperparameters;
+        auto totalComb = combinations.size();
+        for (const auto& hyperparam_line : combinations) {
+            if (!config.quiet)
+                showProgressComb(++num, config.n_folds, totalComb, Colors::CYAN());
+            auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line);
             // Get dataset
             auto [X, y] = datasets.getTensors(fileName);
             auto states = datasets.getStates(fileName);
             auto features = datasets.getFeatures(fileName);
-        auto samples = datasets.getNSamples(fileName);
             auto className = datasets.getClassName(fileName);
             double totalScore = 0.0;
             int numItems = 0;
@@ -55,9 +416,10 @@ namespace platform {
                     fold = new StratifiedKFold(config.n_folds, y, seed);
                 else
                     fold = new KFold(config.n_folds, y.size(0), seed);
-            double bestScore = 0.0;
                 for (int nfold = 0; nfold < config.n_folds; nfold++) {
                     auto clf = Models::instance()->create(config.model);
+                    auto valid = clf->getValidHyperparameters();
+                    hyperparameters.check(valid, fileName);
                     clf->setHyperparameters(hyperparameters.get(fileName));
                     auto [train, test] = fold->getFold(nfold);
                     auto train_t = torch::tensor(train);
@@ -80,51 +442,158 @@ namespace platform {
                 }
                 delete fold;
             }
-        return numItems == 0 ? 0.0 : totalScore / numItems;
+            double score = numItems == 0 ? 0.0 : totalScore / numItems;
-    }
-    void GridSearch::go()
-    {
-        // Load datasets
-        auto datasets = Datasets(config.discretize, Paths::datasets());
-        // Create model
-        std::cout << "***************** Starting Gridsearch *****************" << std::endl;
-        std::cout << "input file=" << config.input_file << std::endl;
-        auto grid = GridData(config.input_file);
-        auto totalComb = grid.getNumCombinations();
-        std::cout << "* Doing " << totalComb << " combinations for each dataset/seed/fold" << std::endl;
-        // Generate hyperparameters grid & run gridsearch
-        // Check each combination of hyperparameters for each dataset and each seed
-        for (const auto& dataset : datasets.getNames()) {
-            if (!config.quiet)
-                std::cout << "- " << setw(20) << left << dataset << " " << right << flush;
-            int num = 0;
-            double bestScore = 0.0;
-            json bestHyperparameters;
-            for (const auto& hyperparam_line : grid.getGrid()) {
-                if (!config.quiet)
-                    showProgressComb(++num, totalComb, Colors::CYAN());
-                auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line);
-                double score = processFile(dataset, datasets, hyperparameters);
             if (score > bestScore) {
                 bestScore = score;
                 bestHyperparameters = hyperparam_line;
             }
         }
-            if (!config.quiet) {
+        return { bestScore, bestHyperparameters };
-                std::cout << "end." << " Score: " << setw(9) << setprecision(7) << fixed
-                    << bestScore << " [" << bestHyperparameters.dump() << "]" << std::endl;
     }
-            results[dataset]["score"] = bestScore;
+    pair<double, json> GridSearch::processFileNested(std::string fileName, Datasets& datasets, vector<json>& combinations)
-            results[dataset]["hyperparameters"] = bestHyperparameters;
-        }
-        // Save results
-        save();
-        std::cout << "***************** Ending Gridsearch *******************" << std::endl;
-    }
-    void GridSearch::save() const
     {
-        std::ofstream file(config.output_file);
+        // Get dataset
-        file << results.dump(4);
+        auto [X, y] = datasets.getTensors(fileName);
-        file.close();
+        auto states = datasets.getStates(fileName);
+        auto features = datasets.getFeatures(fileName);
+        auto className = datasets.getClassName(fileName);
+        int spcs_combinations = int(log(combinations.size()) / log(10)) + 1;
+        double goatScore = 0.0;
+        json goatHyperparameters;
+        // for dataset // for seed // for fold // for hyperparameters // for nested fold
+        for (const auto& seed : config.seeds) {
+            Fold* fold;
+            if (config.stratified)
+                fold = new StratifiedKFold(config.n_folds, y, seed);
+            else
+                fold = new KFold(config.n_folds, y.size(0), seed);
+            double bestScore = 0.0;
+            json bestHyperparameters;
+            std::cout << "(" << seed << ") doing Fold: " << flush;
+            for (int nfold = 0; nfold < config.n_folds; nfold++) {
+                if (!config.quiet)
+                    std::cout << Colors::GREEN() << nfold + 1 << " " << flush;
+                // First level fold
+                auto [train, test] = fold->getFold(nfold);
+                auto train_t = torch::tensor(train);
+                auto test_t = torch::tensor(test);
+                auto X_train = X.index({ "...", train_t });
+                auto y_train = y.index({ train_t });
+                auto X_test = X.index({ "...", test_t });
+                auto y_test = y.index({ test_t });
+                auto num = 0;
+                json result_fold;
+                double hypScore = 0.0;
+                double bestHypScore = 0.0;
+                json bestHypHyperparameters;
+                for (const auto& hyperparam_line : combinations) {
+                    std::cout << "[" << setw(spcs_combinations) << ++num << "/" << setw(spcs_combinations)
+                        << combinations.size() << "] " << std::flush;
+                    Fold* nested_fold;
+                    if (config.stratified)
+                        nested_fold = new StratifiedKFold(config.nested, y_train, seed);
+                    else
+                        nested_fold = new KFold(config.nested, y_train.size(0), seed);
+                    for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) {
+                        // Nested level fold
+                        auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold);
+                        auto train_nested_t = torch::tensor(train_nested);
+                        auto test_nested_t = torch::tensor(test_nested);
+                        auto X_nexted_train = X_train.index({ "...", train_nested_t });
+                        auto y_nested_train = y_train.index({ train_nested_t });
+                        auto X_nested_test = X_train.index({ "...", test_nested_t });
+                        auto y_nested_test = y_train.index({ test_nested_t });
+                        // Build Classifier with selected hyperparameters
+                        auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line);
+                        auto clf = Models::instance()->create(config.model);
+                        auto valid = clf->getValidHyperparameters();
+                        hyperparameters.check(valid, fileName);
+                        clf->setHyperparameters(hyperparameters.get(fileName));
+                        // Train model
+                        if (!config.quiet)
+                            showProgressFold(n_nested_fold + 1, getColor(clf->getStatus()), "a");
+                        clf->fit(X_nexted_train, y_nested_train, features, className, states);
+                        // Test model
+                        if (!config.quiet)
+                            showProgressFold(n_nested_fold + 1, getColor(clf->getStatus()), "b");
+                        hypScore += clf->score(X_nested_test, y_nested_test);
+                        if (!config.quiet)
+                            std::cout << "\b\b\b,  \b" << flush;
+                    }
+                    int magic = 3 * config.nested + 2 * spcs_combinations + 4;
+                    std::cout << string(magic, '\b') << string(magic, ' ') << string(magic, '\b') << flush;
+                    delete nested_fold;
+                    hypScore /= config.nested;
+                    if (hypScore > bestHypScore) {
+                        bestHypScore = hypScore;
+                        bestHypHyperparameters = hyperparam_line;
+                    }
+                }
+                // Build Classifier with selected hyperparameters
+                auto clf = Models::instance()->create(config.model);
+                clf->setHyperparameters(bestHypHyperparameters);
+                // Train model
+                if (!config.quiet)
+                    showProgressFold(nfold + 1, getColor(clf->getStatus()), "a");
+                clf->fit(X_train, y_train, features, className, states);
+                // Test model
+                if (!config.quiet)
+                    showProgressFold(nfold + 1, getColor(clf->getStatus()), "b");
+                double score = clf->score(X_test, y_test);
+                if (!config.quiet)
+                    std::cout << string(2 * config.nested - 1, '\b') << "," << string(2 * config.nested, ' ') << string(2 * config.nested - 1, '\b') << flush;
+                if (score > bestScore) {
+                    bestScore = score;
+                    bestHyperparameters = bestHypHyperparameters;
+                }
+            }
+            if (bestScore > goatScore) {
+                goatScore = bestScore;
+                goatHyperparameters = bestHyperparameters;
+            }
+            delete fold;
+        }
+        return { goatScore, goatHyperparameters };
+    }
+    json GridSearch::initializeResults()
+    {
+        // Load previous results
+        json results;
+        if (config.continue_from != NO_CONTINUE()) {
+            if (!config.quiet)
+                std::cout << "* Loading previous results" << std::endl;
+            try {
+                std::ifstream file(Paths::grid_output(config.model));
+                if (file.is_open()) {
+                    results = json::parse(file);
+                    results = results["results"];
+                }
+            }
+            catch (const std::exception& e) {
+                std::cerr << "* There were no previous results" << std::endl;
+                std::cerr << "* Initizalizing new results" << std::endl;
+                results = json();
+            }
+        }
+        return results;
+    }
+    void GridSearch::save(json& results)
+    {
+        std::ofstream file(Paths::grid_output(config.model));
+        json output = {
+            { "model", config.model },
+            { "score", config.score },
+            { "discretize", config.discretize },
+            { "stratified", config.stratified },
+            { "n_folds", config.n_folds },
+            { "seeds", config.seeds },
+            { "date", get_date() + " " + get_time()},
+            { "nested", config.nested},
+            { "platform", config.platform },
+            { "duration", timer.getDurationString(true)},
+            { "results", results }
+
+        };
+        file << output.dump(4);
     }
 } /* namespace platform */

src/Platform/GridSearch.h

@@ -1,36 +1,54 @@
 #ifndef GRIDSEARCH_H
 #define GRIDSEARCH_H
 #include <string>
-#include <vector>
+#include <map>
+#include <mpi.h>
 #include <nlohmann/json.hpp>
 #include "Datasets.h"
 #include "HyperParameters.h"
 #include "GridData.h"
+#include "Timer.h"
 
 namespace platform {
     using json = nlohmann::json;
     struct ConfigGrid {
         std::string model;
         std::string score;
-        std::string path;
+        std::string continue_from;
-        std::string input_file;
+        std::string platform;
-        std::string output_file;
         bool quiet;
+        bool only; // used with continue_from to only compute that dataset
         bool discretize;
         bool stratified;
+        int nested;
         int n_folds;
+        json excluded;
         std::vector<int> seeds;
     };
+    struct ConfigMPI {
+        int rank;
+        int n_procs;
+        int manager;
+    };
     class GridSearch {
     public:
         explicit GridSearch(struct ConfigGrid& config);
         void go();
-        void save() const;
+        void go_mpi(struct ConfigMPI& config_mpi);
         ~GridSearch() = default;
+        json getResults();
+        static inline std::string NO_CONTINUE() { return "NO_CONTINUE"; }
     private:
-        double processFile(std::string fileName, Datasets& datasets, HyperParameters& hyperparameters);
+        void save(json& results);
-        json results;
+        json initializeResults();
+        vector<std::string> processDatasets(Datasets& datasets);
+        pair<double, json> processFileSingle(std::string fileName, Datasets& datasets, std::vector<json>& combinations);
+        pair<double, json> processFileNested(std::string fileName, Datasets& datasets, std::vector<json>& combinations);
         struct ConfigGrid config;
+        pair<int, int> part_range_mpi(int n_tasks, int nprocs, int rank);
+        json build_tasks_mpi();
+        void process_task_mpi(struct ConfigMPI& config_mpi, json& task, Datasets& datasets, json& results);
+        Timer timer; // used to measure the time of the whole process
     };
 } /* namespace platform */
 #endif /* GRIDSEARCH_H */

src/Platform/HyperParameters.cc

@@ -35,7 +35,7 @@ namespace platform {
                 hyperparameters[dataset] = json({});
                 continue;
             }
-            hyperparameters[dataset] = input_hyperparameters[dataset].get<json>();
+            hyperparameters[dataset] = input_hyperparameters[dataset]["hyperparameters"].get<json>();
         }
     }
     void HyperParameters::check(const std::vector<std::string>& valid, const std::string& fileName)

src/Platform/Paths.h

@@ -26,6 +26,14 @@ namespace platform {
             }
         }
         static std::string excelResults() { return "some_results.xlsx"; }
+        static std::string grid_input(const std::string& model)
+        {
+            return grid() + "grid_" + model + "_input.json";
+        }
+        static std::string grid_output(const std::string& model)
+        {
+            return grid() + "grid_" + model + "_output.json";
+        }
     };
 }
 #endif

src/Platform/Result.h

@@ -32,5 +32,4 @@ namespace platform {
         bool complete;
     };
 };
-
 #endif

src/Platform/Results.h

@@ -7,7 +7,6 @@
 #include "Result.h"
 namespace platform {
     using json = nlohmann::json;
-
     class Results {
     public:
         Results(const std::string& path, const std::string& model, const std::string& score, bool complete, bool partial);
@@ -34,5 +33,4 @@ namespace platform {
         void load(); // Loads the list of results
     };
 };
-
 #endif

src/Platform/Timer.h

@@ -20,13 +20,22 @@ namespace platform {
             std::chrono::duration<double> time_span = std::chrono::duration_cast<std::chrono::duration<double >> (end - begin);
             return time_span.count();
         }
-        std::string getDurationString()
+        double getLapse()
+        {
+            std::chrono::duration<double> time_span = std::chrono::duration_cast<std::chrono::duration<double >> (std::chrono::high_resolution_clock::now() - begin);
+            return time_span.count();
+        }
+        std::string getDurationString(bool lapse = false)
+        {
+            double duration = lapse ? getLapse() : getDuration();
+            return translate2String(duration);
+        }
+        std::string translate2String(double duration)
         {
-            double duration = getDuration();
             double durationShow = duration > 3600 ? duration / 3600 : duration > 60 ? duration / 60 : duration;
             std::string durationUnit = duration > 3600 ? "h" : duration > 60 ? "m" : "s";
             std::stringstream ss;
-            ss << std::setw(7) << std::setprecision(2) << std::fixed << durationShow << " " << durationUnit << " ";
+            ss << std::setprecision(2) << std::fixed << durationShow << " " << durationUnit;
             return ss.str();
         }
     };

src/Platform/b_best.cc

@@ -5,9 +5,8 @@
 #include "Colors.h"
 
 
-argparse::ArgumentParser manageArguments(int argc, char** argv)
+void manageArguments(argparse::ArgumentParser& program, int argc, char** argv)
 {
-    argparse::ArgumentParser program("b_sbest");
     program.add_argument("-m", "--model").default_value("").help("Filter results of the selected model) (any for all models)");
     program.add_argument("-s", "--score").default_value("").help("Filter results of the score name supplied");
     program.add_argument("--build").help("build best score results file").default_value(false).implicit_value(true);
@@ -28,12 +27,12 @@ argparse::ArgumentParser manageArguments(int argc, char** argv)
         catch (...) {
             throw std::runtime_error("Number of folds must be an decimal number");
         }});
-    return program;
 }
 
 int main(int argc, char** argv)
 {
-    auto program = manageArguments(argc, argv);
+    argparse::ArgumentParser program("b_sbest");
+    manageArguments(program, argc, argv);
     std::string model, score;
     bool build, report, friedman, excel;
     double level;

src/Platform/b_grid.cc

@@ -1,17 +1,23 @@
 #include <iostream>
 #include <argparse/argparse.hpp>
+#include <map>
+#include <nlohmann/json.hpp>
+#include <mpi.h>
 #include "DotEnv.h"
 #include "Models.h"
 #include "modelRegister.h"
 #include "GridSearch.h"
 #include "Paths.h"
 #include "Timer.h"
+#include "Colors.h"
 
+using json = nlohmann::json;
+const int MAXL = 133;
 
-argparse::ArgumentParser manageArguments(std::string program_name)
+void manageArguments(argparse::ArgumentParser& program)
 {
     auto env = platform::DotEnv();
-    argparse::ArgumentParser program(program_name);
+    auto& group = program.add_mutually_exclusive_group(true);
     program.add_argument("-m", "--model")
         .help("Model to use " + platform::Models::instance()->tostring())
         .action([](const std::string& value) {
@@ -22,9 +28,17 @@ argparse::ArgumentParser manageArguments(std::string program_name)
         throw std::runtime_error("Model must be one of " + platform::Models::instance()->tostring());
             }
     );
+    group.add_argument("--dump").help("Show the grid combinations").default_value(false).implicit_value(true);
+    group.add_argument("--report").help("Report the computed hyperparameters").default_value(false).implicit_value(true);
+    group.add_argument("--compute").help("Perform computation of the grid output hyperparameters").default_value(false).implicit_value(true);
     program.add_argument("--discretize").help("Discretize input datasets").default_value((bool)stoi(env.get("discretize"))).implicit_value(true);
-    program.add_argument("--quiet").help("Don't display detailed progress").default_value(false).implicit_value(true);
+    program.add_argument("--mpi").help("Use MPI computing grid").default_value(false).implicit_value(true);
     program.add_argument("--stratified").help("If Stratified KFold is to be done").default_value((bool)stoi(env.get("stratified"))).implicit_value(true);
+    program.add_argument("--quiet").help("Don't display detailed progress").default_value(false).implicit_value(true);
+    program.add_argument("--continue").help("Continue computing from that dataset").default_value(platform::GridSearch::NO_CONTINUE());
+    program.add_argument("--only").help("Used with continue to compute that dataset only").default_value(false).implicit_value(true);
+    program.add_argument("--exclude").default_value("[]").help("Datasets to exclude in json format, e.g. [\"dataset1\", \"dataset2\"]");
+    program.add_argument("--nested").help("Do a double/nested cross validation with n folds").default_value(0).scan<'i', int>();
     program.add_argument("--score").help("Score used in gridsearch").default_value("accuracy");
     program.add_argument("-f", "--folds").help("Number of folds").default_value(stoi(env.get("n_folds"))).scan<'i', int>().action([](const std::string& value) {
         try {
@@ -42,13 +56,99 @@ argparse::ArgumentParser manageArguments(std::string program_name)
         }});
     auto seed_values = env.getSeeds();
     program.add_argument("-s", "--seeds").nargs(1, 10).help("Random seeds. Set to -1 to have pseudo random").scan<'i', int>().default_value(seed_values);
-    return program;
 }
 
+void list_dump(std::string& model)
+{
+    auto data = platform::GridData(platform::Paths::grid_input(model));
+    std::cout << Colors::MAGENTA() << "Listing configuration input file (Grid)" << std::endl << std::endl;
+    int index = 0;
+    int max_hyper = 15;
+    int max_dataset = 7;
+    auto combinations = data.getGridFile();
+    for (auto const& item : combinations) {
+        if (item.first.size() > max_dataset) {
+            max_dataset = item.first.size();
+        }
+        if (item.second.dump().size() > max_hyper) {
+            max_hyper = item.second.dump().size();
+        }
+    }
+    std::cout << Colors::GREEN() << left << " #  " << left << setw(max_dataset) << "Dataset" << " #Com. "
+        << setw(max_hyper) << "Hyperparameters" << std::endl;
+    std::cout << "=== " << string(max_dataset, '=') << " ===== " << string(max_hyper, '=') << std::endl;
+    bool odd = true;
+    for (auto const& item : combinations) {
+        auto color = odd ? Colors::CYAN() : Colors::BLUE();
+        std::cout << color;
+        auto num_combinations = data.getNumCombinations(item.first);
+        std::cout << setw(3) << fixed << right << ++index << left << " " << setw(max_dataset) << item.first
+            << " " << setw(5) << right << num_combinations << " " << setw(max_hyper) << item.second.dump() << std::endl;
+        odd = !odd;
+    }
+    std::cout << Colors::RESET() << std::endl;
+}
+std::string headerLine(const std::string& text, int utf = 0)
+{
+    int n = MAXL - text.length() - 3;
+    n = n < 0 ? 0 : n;
+    return "* " + text + std::string(n + utf, ' ') + "*\n";
+}
+void list_results(json& results, std::string& model)
+{
+    std::cout << Colors::MAGENTA() << std::string(MAXL, '*') << std::endl;
+    std::cout << headerLine("Listing computed hyperparameters for model " + model);
+    std::cout << headerLine("Date & time: " + results["date"].get<std::string>() + " Duration: " + results["duration"].get<std::string>());
+    std::cout << headerLine("Score: " + results["score"].get<std::string>());
+    std::cout << headerLine(
+        "Random seeds: " + results["seeds"].dump()
+        + " Discretized: " + (results["discretize"].get<bool>() ? "True" : "False")
+        + " Stratified: " + (results["stratified"].get<bool>() ? "True" : "False")
+        + " #Folds: " + std::to_string(results["n_folds"].get<int>())
+        + " Nested: " + (results["nested"].get<int>() == 0 ? "False" : to_string(results["nested"].get<int>()))
+    );
+    std::cout << std::string(MAXL, '*') << std::endl;
+    int spaces = 0;
+    int hyperparameters_spaces = 0;
+    for (const auto& item : results["results"].items()) {
+        auto key = item.key();
+        auto value = item.value();
+        if (key.size() > spaces) {
+            spaces = key.size();
+        }
+        if (value["hyperparameters"].dump().size() > hyperparameters_spaces) {
+            hyperparameters_spaces = value["hyperparameters"].dump().size();
+        }
+    }
+    std::cout << Colors::GREEN() << " #  " << left << setw(spaces) << "Dataset" << " " << setw(19) << "Date" << " "
+        << "Duration " << setw(8) << "Score" << " " << "Hyperparameters" << std::endl;
+    std::cout << "=== " << string(spaces, '=') << " " << string(19, '=') << " " << string(8, '=') << " "
+        << string(8, '=') << " " << string(hyperparameters_spaces, '=') << std::endl;
+    bool odd = true;
+    int index = 0;
+    for (const auto& item : results["results"].items()) {
+        auto color = odd ? Colors::CYAN() : Colors::BLUE();
+        auto key = item.key();
+        auto value = item.value();
+        std::cout << color;
+        std::cout << std::setw(3) << std::right << index++ << " ";
+        std::cout << left << setw(spaces) << key << " " << value["date"].get<string>()
+            << " " << setw(8) << right << value["duration"].get<string>() << " " << setw(8) << setprecision(6)
+            << fixed << right << value["score"].get<double>() << " " << value["hyperparameters"].dump() << std::endl;
+        odd = !odd;
+    }
+    std::cout << Colors::RESET() << std::endl;
+}
+
+/*
+ * Main
+ */
 int main(int argc, char** argv)
 {
-    auto program = manageArguments("b_grid");
+    argparse::ArgumentParser program("b_grid");
+    manageArguments(program);
     struct platform::ConfigGrid config;
+    bool dump, compute;
     try {
         program.parse_args(argc, argv);
         config.model = program.get<std::string>("model");
@@ -57,7 +157,25 @@ int main(int argc, char** argv)
         config.stratified = program.get<bool>("stratified");
         config.n_folds = program.get<int>("folds");
         config.quiet = program.get<bool>("quiet");
+        config.only = program.get<bool>("only");
         config.seeds = program.get<std::vector<int>>("seeds");
+        config.nested = program.get<int>("nested");
+        config.continue_from = program.get<std::string>("continue");
+        if (config.continue_from == platform::GridSearch::NO_CONTINUE() && config.only) {
+            throw std::runtime_error("Cannot use --only without --continue");
+        }
+        dump = program.get<bool>("dump");
+        compute = program.get<bool>("compute");
+        if (dump && (config.continue_from != platform::GridSearch::NO_CONTINUE() || config.only)) {
+            throw std::runtime_error("Cannot use --dump with --continue or --only");
+        }
+        auto excluded = program.get<std::string>("exclude");
+        config.excluded = json::parse(excluded);
+        if (program.get<bool>("mpi")) {
+            if (!compute || config.nested == 0) {
+                throw std::runtime_error("Cannot use --mpi without --compute or without --nested");
+            }
+        }
     }
     catch (const exception& err) {
         cerr << err.what() << std::endl;
@@ -68,14 +186,42 @@ int main(int argc, char** argv)
      * Begin Processing
      */
     auto env = platform::DotEnv();
+    config.platform = env.get("platform");
     platform::Paths::createPath(platform::Paths::grid());
-    config.path = platform::Paths::grid();
     auto grid_search = platform::GridSearch(config);
     platform::Timer timer;
     timer.start();
+    if (dump) {
+        list_dump(config.model);
+    } else {
+        if (compute) {
+            if (program.get<bool>("mpi")) {
+                struct platform::ConfigMPI mpi_config;
+                mpi_config.manager = 0; // which process is the manager
+                MPI_Init(&argc, &argv);
+                MPI_Comm_rank(MPI_COMM_WORLD, &mpi_config.rank);
+                MPI_Comm_size(MPI_COMM_WORLD, &mpi_config.n_procs);
+                grid_search.go_mpi(mpi_config);
+                if (mpi_config.rank == mpi_config.manager) {
+                    auto results = grid_search.getResults();
+                    list_results(results, config.model);
+                    std::cout << "Process took " << timer.getDurationString() << std::endl;
+                }
+                MPI_Finalize();
+            } else {
                 grid_search.go();
                 std::cout << "Process took " << timer.getDurationString() << std::endl;
-    grid_search.save();
+            }
+        } else {
+            // List results
+            auto results = grid_search.getResults();
+            if (results.empty()) {
+                std::cout << "** No results found" << std::endl;
+            } else {
+                list_results(results, config.model);
+            }
+        }
+    }
     std::cout << "Done!" << std::endl;
     return 0;
 }

src/Platform/b_main.cc

@@ -11,10 +11,9 @@
 
 using json = nlohmann::json;
 
-argparse::ArgumentParser manageArguments(std::string program_name)
+void manageArguments(argparse::ArgumentParser& program)
 {
     auto env = platform::DotEnv();
-    argparse::ArgumentParser program(program_name);
     program.add_argument("-d", "--dataset").default_value("").help("Dataset file name");
     program.add_argument("--hyperparameters").default_value("{}").help("Hyperparameters passed to the model in Experiment");
     program.add_argument("--hyper-file").default_value("").help("Hyperparameters file name." \
@@ -50,18 +49,18 @@ argparse::ArgumentParser manageArguments(std::string program_name)
         }});
     auto seed_values = env.getSeeds();
     program.add_argument("-s", "--seeds").nargs(1, 10).help("Random seeds. Set to -1 to have pseudo random").scan<'i', int>().default_value(seed_values);
-    return program;
 }
 
 int main(int argc, char** argv)
 {
+    argparse::ArgumentParser program("b_main");
+    manageArguments(program);
     std::string file_name, model_name, title, hyperparameters_file;
     json hyperparameters_json;
     bool discretize_dataset, stratified, saveResults, quiet;
     std::vector<int> seeds;
     std::vector<std::string> filesToTest;
     int n_folds;
-    auto program = manageArguments("b_main");
     try {
         program.parse_args(argc, argv);
         file_name = program.get<std::string>("dataset");

src/Platform/b_manage.cc

@@ -3,9 +3,8 @@
 #include "ManageResults.h"
 
 
-argparse::ArgumentParser manageArguments(int argc, char** argv)
+void manageArguments(argparse::ArgumentParser& program, int argc, char** argv)
 {
-    argparse::ArgumentParser program("b_manage");
     program.add_argument("-n", "--number").default_value(0).help("Number of results to show (0 = all)").scan<'i', int>();
     program.add_argument("-m", "--model").default_value("any").help("Filter results of the selected model)");
     program.add_argument("-s", "--score").default_value("any").help("Filter results of the score name supplied");
@@ -29,12 +28,12 @@ argparse::ArgumentParser manageArguments(int argc, char** argv)
         std::cerr << program;
         exit(1);
     }
-    return program;
 }
 
 int main(int argc, char** argv)
 {
-    auto program = manageArguments(argc, argv);
+    auto program = argparse::ArgumentParser("b_manage");
+    manageArguments(program, argc, argv);
     int number = program.get<int>("number");
     std::string model = program.get<std::string>("model");
     std::string score = program.get<std::string>("score");

src/PyClassifiers/ODTE.cc

@@ -5,6 +5,18 @@ namespace pywrap {
     {
         validHyperparameters = { "n_jobs", "n_estimators", "random_state" };
     }
+    int ODTE::getNumberOfNodes() const
+    {
+        return callMethodInt("get_nodes");
+    }
+    int ODTE::getNumberOfEdges() const
+    {
+        return callMethodInt("get_leaves");
+    }
+    int ODTE::getNumberOfStates() const
+    {
+        return callMethodInt("get_depth");
+    }
     std::string ODTE::graph()
     {
         return callMethodString("graph");

src/PyClassifiers/ODTE.h

@@ -8,6 +8,9 @@ namespace pywrap {
     public:
         ODTE();
         ~ODTE() = default;
+        int getNumberOfNodes() const override;
+        int getNumberOfEdges() const override;
+        int getNumberOfStates() const override;
         std::string graph();
     };
 } /* namespace pywrap */

src/PyClassifiers/PyClassifier.cc

@@ -38,6 +38,14 @@ namespace pywrap {
     {
         return pyWrap->callMethodString(id, method);
     }
+    int PyClassifier::callMethodSumOfItems(const std::string& method) const
+    {
+        return pyWrap->callMethodSumOfItems(id, method);
+    }
+    int PyClassifier::callMethodInt(const std::string& method) const
+    {
+        return pyWrap->callMethodInt(id, method);
+    }
     PyClassifier& PyClassifier::fit(torch::Tensor& X, torch::Tensor& y)
     {
         if (!fitted && hyperparameters.size() > 0) {

src/PyClassifiers/PyClassifier.h

@@ -29,8 +29,9 @@ namespace pywrap {
         float score(torch::Tensor& X, torch::Tensor& y) override;
         std::string version();
         std::string callMethodString(const std::string& method);
+        int callMethodSumOfItems(const std::string& method) const;
+        int callMethodInt(const std::string& method) const;
         std::string getVersion() override { return this->version(); };
-        // TODO: Implement these 3 methods
         int getNumberOfNodes() const override { return 0; };
         int getNumberOfEdges() const override { return 0; };
         int getNumberOfStates() const override { return 0; };

src/PyClassifiers/PyWrap.cc

@@ -110,17 +110,80 @@ namespace pywrap {
         Py_XDECREF(result);
         return value;
     }
+    int PyWrap::callMethodInt(const clfId_t id, const std::string& method)
+    {
+        PyObject* instance = getClass(id);
+        PyObject* result;
+        try {
+            if (!(result = PyObject_CallMethod(instance, method.c_str(), NULL)))
+                errorAbort("Couldn't call method " + method);
+        }
+        catch (const std::exception& e) {
+            errorAbort(e.what());
+        }
+        int value = PyLong_AsLong(result);
+        Py_XDECREF(result);
+        return value;
+    }
     std::string PyWrap::sklearnVersion()
     {
-        return "1.0";
+        PyObject* sklearnModule = PyImport_ImportModule("sklearn");
-        // CPyObject data = PyRun_SimpleString("import sklearn;return sklearn.__version__");
+        if (sklearnModule == nullptr) {
-        // std::string result = PyUnicode_AsUTF8(data);
+            errorAbort("Couldn't import sklearn");
-        // return result;
+        }
+        PyObject* versionAttr = PyObject_GetAttrString(sklearnModule, "__version__");
+        if (versionAttr == nullptr || !PyUnicode_Check(versionAttr)) {
+            Py_XDECREF(sklearnModule);
+            errorAbort("Couldn't get sklearn version");
+        }
+        std::string result = PyUnicode_AsUTF8(versionAttr);
+        Py_XDECREF(versionAttr);
+        Py_XDECREF(sklearnModule);
+        return result;
     }
     std::string PyWrap::version(const clfId_t id)
     {
         return callMethodString(id, "version");
     }
+    int PyWrap::callMethodSumOfItems(const clfId_t id, const std::string& method)
+    {
+        // Call method on each estimator and sum the results (made for RandomForest)
+        PyObject* instance = getClass(id);
+        PyObject* estimators = PyObject_GetAttrString(instance, "estimators_");
+        if (estimators == nullptr) {
+            errorAbort("Failed to get attribute: " + method);
+        }
+        int sumOfItems = 0;
+        Py_ssize_t len = PyList_Size(estimators);
+        for (Py_ssize_t i = 0; i < len; i++) {
+            PyObject* estimator = PyList_GetItem(estimators, i);
+            PyObject* result;
+            if (method == "node_count") {
+                PyObject* owner = PyObject_GetAttrString(estimator, "tree_");
+                if (owner == nullptr) {
+                    Py_XDECREF(estimators);
+                    errorAbort("Failed to get attribute tree_ for: " + method);
+                }
+                result = PyObject_GetAttrString(owner, method.c_str());
+                if (result == nullptr) {
+                    Py_XDECREF(estimators);
+                    Py_XDECREF(owner);
+                    errorAbort("Failed to get attribute node_count: " + method);
+                }
+                Py_DECREF(owner);
+            } else {
+                result = PyObject_CallMethod(estimator, method.c_str(), nullptr);
+                if (result == nullptr) {
+                    Py_XDECREF(estimators);
+                    errorAbort("Failed to call method: " + method);
+                }
+            }
+            sumOfItems += PyLong_AsLong(result);
+            Py_DECREF(result);
+        }
+        Py_DECREF(estimators);
+        return sumOfItems;
+    }
     void PyWrap::setHyperparameters(const clfId_t id, const json& hyperparameters)
     {
         // Set hyperparameters as attributes of the class

src/PyClassifiers/PyWrap.h

@@ -24,8 +24,10 @@ namespace pywrap {
         void operator=(const PyWrap&) = delete;
         ~PyWrap() = default;
         std::string callMethodString(const clfId_t id, const std::string& method);
+        int callMethodInt(const clfId_t id, const std::string& method);
         std::string sklearnVersion();
         std::string version(const clfId_t id);
+        int callMethodSumOfItems(const clfId_t id, const std::string& method);
         void setHyperparameters(const clfId_t id, const json& hyperparameters);
         void fit(const clfId_t id, CPyObject& X, CPyObject& y);
         PyObject* predict(const clfId_t id, CPyObject& X);

src/PyClassifiers/RandomForest.cc

@@ -5,4 +5,16 @@ namespace pywrap {
     {
         validHyperparameters = { "n_estimators", "n_jobs", "random_state" };
     }
+    int RandomForest::getNumberOfEdges() const
+    {
+        return callMethodSumOfItems("get_n_leaves");
+    }
+    int RandomForest::getNumberOfStates() const
+    {
+        return callMethodSumOfItems("get_depth");
+    }
+    int RandomForest::getNumberOfNodes() const
+    {
+        return callMethodSumOfItems("node_count");
+    }
 } /* namespace pywrap */

src/PyClassifiers/RandomForest.h

@@ -7,6 +7,9 @@ namespace pywrap {
     public:
         RandomForest();
         ~RandomForest() = default;
+        int getNumberOfEdges() const override;
+        int getNumberOfStates() const override;
+        int getNumberOfNodes() const override;
     };
 } /* namespace pywrap */
 #endif /* RANDOMFOREST_H */

src/PyClassifiers/STree.cc

@@ -5,6 +5,18 @@ namespace pywrap {
     {
         validHyperparameters = { "C", "kernel", "max_iter", "max_depth", "random_state", "multiclass_strategy", "gamma", "max_features", "degree" };
     };
+    int STree::getNumberOfNodes() const
+    {
+        return callMethodInt("get_nodes");
+    }
+    int STree::getNumberOfEdges() const
+    {
+        return callMethodInt("get_leaves");
+    }
+    int STree::getNumberOfStates() const
+    {
+        return callMethodInt("get_depth");
+    }
     std::string STree::graph()
     {
         return callMethodString("graph");

src/PyClassifiers/STree.h

@@ -8,6 +8,9 @@ namespace pywrap {
     public:
         STree();
         ~STree() = default;
+        int getNumberOfNodes() const override;
+        int getNumberOfEdges() const override;
+        int getNumberOfStates() const override;
         std::string graph();
     };
 } /* namespace pywrap */