Check min number of nested folds

Reviewed-on: #14

CMakeLists.txt

@@ -25,12 +25,18 @@ set(CMAKE_CXX_EXTENSIONS                  OFF)
 set(CMAKE_EXPORT_COMPILE_COMMANDS          ON)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")
 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
-
 # Options
 # -------
 option(ENABLE_CLANG_TIDY "Enable to add clang tidy."              OFF)
 option(ENABLE_TESTING "Unit testing build"                        OFF)
 option(CODE_COVERAGE "Collect coverage from test library"         OFF)
+option(MPI_ENABLED "Enable MPI options"                           ON)
+
+if (MPI_ENABLED)
+  find_package(MPI REQUIRED)
+  message("MPI_CXX_LIBRARIES=${MPI_CXX_LIBRARIES}")
+  message("MPI_CXX_INCLUDE_DIRS=${MPI_CXX_INCLUDE_DIRS}")
+endif (MPI_ENABLED)
 
 # Boost Library
 set(Boost_USE_STATIC_LIBS OFF) 

README.md

@@ -8,6 +8,32 @@ Bayesian Network Classifier with libtorch from scratch
 
 Before compiling BayesNet.
 
+### Miniconda
+
+To be able to run Python Classifiers such as STree, ODTE, SVC, etc. it is needed to install Miniconda. To do so, download the installer from [Miniconda](https://docs.conda.io/en/latest/miniconda.html) and run it. It is recommended to install it in the home folder.
+
+In Linux sometimes the library libstdc++ is mistaken from the miniconda installation and produces the next message when running the b_xxxx executables:
+
+```bash
+libstdc++.so.6: version `GLIBCXX_3.4.32' not found (required by b_xxxx)
+```
+
+The solution is to erase the libstdc++ library from the miniconda installation:
+
+### MPI
+
+In Linux just install openmpi & openmpi-devel packages. Only if cmake can't find openmpi installation (like in Oracle Linux) set the following variable:
+
+```bash
+export MPI_HOME="/usr/lib64/openmpi"
+```
+
+In Mac OS X, install mpich with brew and if cmake doesn't find it, edit mpicxx wrapper to remove the ",-commons,use_dylibs" from final_ldflags
+
+```bash
+vi /opt/homebrew/bin/mpicx
+```
+
 ### boost library
 
 [Getting Started](<https://www.boost.org/doc/libs/1_83_0/more/getting_started/index.html>)

src/Platform/CMakeLists.txt

@@ -7,6 +7,7 @@ include_directories(${BayesNet_SOURCE_DIR}/lib/argparse/include)
 include_directories(${BayesNet_SOURCE_DIR}/lib/json/include)
 include_directories(${BayesNet_SOURCE_DIR}/lib/libxlsxwriter/include)
 include_directories(${Python3_INCLUDE_DIRS})
+include_directories(${MPI_CXX_INCLUDE_DIRS})
 
 add_executable(b_best b_best.cc BestResults.cc Result.cc Statistics.cc BestResultsExcel.cc ReportExcel.cc ReportBase.cc Datasets.cc Dataset.cc ExcelFile.cc)
 add_executable(b_grid b_grid.cc GridSearch.cc GridData.cc HyperParameters.cc Folding.cc Datasets.cc Dataset.cc)
@@ -15,7 +16,7 @@ add_executable(b_main b_main.cc Folding.cc Experiment.cc Datasets.cc Dataset.cc
 add_executable(b_manage b_manage.cc Results.cc ManageResults.cc CommandParser.cc Result.cc ReportConsole.cc ReportExcel.cc ReportBase.cc Datasets.cc Dataset.cc ExcelFile.cc)
 
 target_link_libraries(b_best Boost::boost "${XLSXWRITER_LIB}" "${TORCH_LIBRARIES}" ArffFiles mdlp)
-target_link_libraries(b_grid BayesNet  PyWrap)
+target_link_libraries(b_grid BayesNet PyWrap ${MPI_CXX_LIBRARIES})
 target_link_libraries(b_list ArffFiles mdlp "${TORCH_LIBRARIES}")
 target_link_libraries(b_main BayesNet ArffFiles mdlp "${TORCH_LIBRARIES}" PyWrap)
 target_link_libraries(b_manage "${TORCH_LIBRARIES}" "${XLSXWRITER_LIB}" ArffFiles mdlp)

src/Platform/GridSearch.cc

@@ -1,4 +1,5 @@
 #include <iostream>
+#include <cstddef>
 #include <torch/torch.h>
 #include "GridSearch.h"
 #include "Models.h"
@@ -27,10 +28,15 @@ namespace platform {
         oss << std::put_time(timeinfo, "%H:%M:%S");
         return oss.str();
     }
+    std::string get_color_rank(int rank)
+    {
+        auto colors = { Colors::WHITE(), Colors::RED(), Colors::GREEN(),  Colors::BLUE(), Colors::MAGENTA(), Colors::CYAN() };
+        return *(colors.begin() + rank % colors.size());
+    }
     GridSearch::GridSearch(struct ConfigGrid& config) : config(config)
     {
     }
-    json GridSearch::getResults()
+    json GridSearch::loadResults()
     {
         std::ifstream file(Paths::grid_output(config.model));
         if (file.is_open()) {
@@ -38,240 +44,7 @@ namespace platform {
         }
         return json();
     }
-    void showProgressComb(const int num, const int n_folds, const int total, const std::string& color)
-    {
-        int spaces = int(log(total) / log(10)) + 1;
-        int magic = n_folds * 3 + 22 + 2 * spaces;
-        std::string prefix = num == 1 ? "" : string(magic, '\b') + string(magic + 1, ' ') + string(magic + 1, '\b');
-        std::cout << prefix << color << "(" << setw(spaces) << num << "/" << setw(spaces) << total << ") " << Colors::RESET() << flush;
-    }
-    void showProgressFold(int fold, const std::string& color, const std::string& phase)
-    {
-        std::string prefix = phase == "a" ? "" : "\b\b\b\b";
-        std::cout << prefix << color << fold << Colors::RESET() << "(" << color << phase << Colors::RESET() << ")" << flush;
-    }
-    std::string getColor(bayesnet::status_t status)
-    {
-        switch (status) {
-            case bayesnet::NORMAL:
-                return Colors::GREEN();
-            case bayesnet::WARNING:
-                return Colors::YELLOW();
-            case bayesnet::ERROR:
-                return Colors::RED();
-            default:
-                return Colors::RESET();
-        }
-    }
-    void GridSearch::go()
-    {
-        timer.start();
-        auto grid_type = config.nested == 0 ? "Single" : "Nested";
-        auto datasets = Datasets(config.discretize, Paths::datasets());
-        auto datasets_names = processDatasets(datasets);
-        json results = initializeResults();
-        std::cout << "***************** Starting " << grid_type << " Gridsearch *****************" << std::endl;
-        std::cout << "input file=" << Paths::grid_input(config.model) << std::endl;
-        auto grid = GridData(Paths::grid_input(config.model));
-        Timer timer_dataset;
-        double bestScore = 0;
-        json bestHyperparameters;
-        for (const auto& dataset : datasets_names) {
-            if (!config.quiet)
-                std::cout << "- " << setw(20) << left << dataset << " " << right << flush;
-            auto combinations = grid.getGrid(dataset);
-            timer_dataset.start();
-            if (config.nested == 0)
-                // for dataset // for hyperparameters // for seed // for fold
-                tie(bestScore, bestHyperparameters) = processFileSingle(dataset, datasets, combinations);
-            else
-                // for dataset // for seed // for fold // for hyperparameters // for nested fold
-                tie(bestScore, bestHyperparameters) = processFileNested(dataset, datasets, combinations);
-            if (!config.quiet) {
-                std::cout << "end." << " Score: " << Colors::IBLUE() << setw(9) << setprecision(7) << fixed
-                    << bestScore << Colors::BLUE() << " [" << bestHyperparameters.dump() << "]"
-                    << Colors::RESET() << ::endl;
-            }
-            json result = {
-                { "score", bestScore },
-                { "hyperparameters", bestHyperparameters },
-                { "date", get_date() + " " + get_time() },
-                { "grid", grid.getInputGrid(dataset) },
-                { "duration", timer_dataset.getDurationString() }
-            };
-            results[dataset] = result;
-            // Save partial results
-            save(results);
-        }
-        // Save final results
-        save(results);
-        std::cout << "***************** Ending " << grid_type << " Gridsearch *******************" << std::endl;
-    }
-    pair<double, json> GridSearch::processFileSingle(std::string fileName, Datasets& datasets, vector<json>& combinations)
-    {
-        int num = 0;
-        double bestScore = 0.0;
-        json bestHyperparameters;
-        auto totalComb = combinations.size();
-        for (const auto& hyperparam_line : combinations) {
-            if (!config.quiet)
-                showProgressComb(++num, config.n_folds, totalComb, Colors::CYAN());
-            auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line);
-            // Get dataset
-            auto [X, y] = datasets.getTensors(fileName);
-            auto states = datasets.getStates(fileName);
-            auto features = datasets.getFeatures(fileName);
-            auto className = datasets.getClassName(fileName);
-            double totalScore = 0.0;
-            int numItems = 0;
-            for (const auto& seed : config.seeds) {
-                if (!config.quiet)
-                    std::cout << "(" << seed << ") doing Fold: " << flush;
-                Fold* fold;
-                if (config.stratified)
-                    fold = new StratifiedKFold(config.n_folds, y, seed);
-                else
-                    fold = new KFold(config.n_folds, y.size(0), seed);
-                for (int nfold = 0; nfold < config.n_folds; nfold++) {
-                    auto clf = Models::instance()->create(config.model);
-                    auto valid = clf->getValidHyperparameters();
-                    hyperparameters.check(valid, fileName);
-                    clf->setHyperparameters(hyperparameters.get(fileName));
-                    auto [train, test] = fold->getFold(nfold);
-                    auto train_t = torch::tensor(train);
-                    auto test_t = torch::tensor(test);
-                    auto X_train = X.index({ "...", train_t });
-                    auto y_train = y.index({ train_t });
-                    auto X_test = X.index({ "...", test_t });
-                    auto y_test = y.index({ test_t });
-                    // Train model
-                    if (!config.quiet)
-                        showProgressFold(nfold + 1, getColor(clf->getStatus()), "a");
-                    clf->fit(X_train, y_train, features, className, states);
-                    // Test model
-                    if (!config.quiet)
-                        showProgressFold(nfold + 1, getColor(clf->getStatus()), "b");
-                    totalScore += clf->score(X_test, y_test);
-                    numItems++;
-                    if (!config.quiet)
-                        std::cout << "\b\b\b,  \b" << flush;
-                }
-                delete fold;
-            }
-            double score = numItems == 0 ? 0.0 : totalScore / numItems;
-            if (score > bestScore) {
-                bestScore = score;
-                bestHyperparameters = hyperparam_line;
-            }
-        }
-        return { bestScore, bestHyperparameters };
-    }
-    pair<double, json> GridSearch::processFileNested(std::string fileName, Datasets& datasets, vector<json>& combinations)
-    {
-        // Get dataset
-        auto [X, y] = datasets.getTensors(fileName);
-        auto states = datasets.getStates(fileName);
-        auto features = datasets.getFeatures(fileName);
-        auto className = datasets.getClassName(fileName);
-        int spcs_combinations = int(log(combinations.size()) / log(10)) + 1;
-        double goatScore = 0.0;
-        json goatHyperparameters;
-        // for dataset // for seed // for fold // for hyperparameters // for nested fold
-        for (const auto& seed : config.seeds) {
-            Fold* fold;
-            if (config.stratified)
-                fold = new StratifiedKFold(config.n_folds, y, seed);
-            else
-                fold = new KFold(config.n_folds, y.size(0), seed);
-            double bestScore = 0.0;
-            json bestHyperparameters;
-            std::cout << "(" << seed << ") doing Fold: " << flush;
-            for (int nfold = 0; nfold < config.n_folds; nfold++) {
-                if (!config.quiet)
-                    std::cout << Colors::GREEN() << nfold + 1 << " " << flush;
-                // First level fold
-                auto [train, test] = fold->getFold(nfold);
-                auto train_t = torch::tensor(train);
-                auto test_t = torch::tensor(test);
-                auto X_train = X.index({ "...", train_t });
-                auto y_train = y.index({ train_t });
-                auto X_test = X.index({ "...", test_t });
-                auto y_test = y.index({ test_t });
-                auto num = 0;
-                json result_fold;
-                double hypScore = 0.0;
-                double bestHypScore = 0.0;
-                json bestHypHyperparameters;
-                for (const auto& hyperparam_line : combinations) {
-                    std::cout << "[" << setw(spcs_combinations) << ++num << "/" << setw(spcs_combinations)
-                        << combinations.size() << "] " << std::flush;
-                    Fold* nested_fold;
-                    if (config.stratified)
-                        nested_fold = new StratifiedKFold(config.nested, y_train, seed);
-                    else
-                        nested_fold = new KFold(config.nested, y_train.size(0), seed);
-                    for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) {
-                        // Nested level fold
-                        auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold);
-                        auto train_nested_t = torch::tensor(train_nested);
-                        auto test_nested_t = torch::tensor(test_nested);
-                        auto X_nexted_train = X_train.index({ "...", train_nested_t });
-                        auto y_nested_train = y_train.index({ train_nested_t });
-                        auto X_nested_test = X_train.index({ "...", test_nested_t });
-                        auto y_nested_test = y_train.index({ test_nested_t });
-                        // Build Classifier with selected hyperparameters
-                        auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line);
-                        auto clf = Models::instance()->create(config.model);
-                        auto valid = clf->getValidHyperparameters();
-                        hyperparameters.check(valid, fileName);
-                        clf->setHyperparameters(hyperparameters.get(fileName));
-                        // Train model
-                        if (!config.quiet)
-                            showProgressFold(n_nested_fold + 1, getColor(clf->getStatus()), "a");
-                        clf->fit(X_nexted_train, y_nested_train, features, className, states);
-                        // Test model
-                        if (!config.quiet)
-                            showProgressFold(n_nested_fold + 1, getColor(clf->getStatus()), "b");
-                        hypScore += clf->score(X_nested_test, y_nested_test);
-                        if (!config.quiet)
-                            std::cout << "\b\b\b,  \b" << flush;
-                    }
-                    int magic = 3 * config.nested + 2 * spcs_combinations + 4;
-                    std::cout << string(magic, '\b') << string(magic, ' ') << string(magic, '\b') << flush;
-                    delete nested_fold;
-                    hypScore /= config.nested;
-                    if (hypScore > bestHypScore) {
-                        bestHypScore = hypScore;
-                        bestHypHyperparameters = hyperparam_line;
-                    }
-                }
-                // Build Classifier with selected hyperparameters
-                auto clf = Models::instance()->create(config.model);
-                clf->setHyperparameters(bestHypHyperparameters);
-                // Train model
-                if (!config.quiet)
-                    showProgressFold(nfold + 1, getColor(clf->getStatus()), "a");
-                clf->fit(X_train, y_train, features, className, states);
-                // Test model
-                if (!config.quiet)
-                    showProgressFold(nfold + 1, getColor(clf->getStatus()), "b");
-                double score = clf->score(X_test, y_test);
-                if (!config.quiet)
-                    std::cout << string(2 * config.nested - 1, '\b') << "," << string(2 * config.nested, ' ') << string(2 * config.nested - 1, '\b') << flush;
-                if (score > bestScore) {
-                    bestScore = score;
-                    bestHyperparameters = bestHypHyperparameters;
-                }
-            }
-            if (bestScore > goatScore) {
-                goatScore = bestScore;
-                goatHyperparameters = bestHyperparameters;
-            }
-            delete fold;
-        }
-        return { goatScore, goatHyperparameters };
-    }
-    vector<std::string> GridSearch::processDatasets(Datasets& datasets)
+    std::vector<std::string> GridSearch::filterDatasets(Datasets& datasets) const
     {
         // Load datasets
         auto datasets_names = datasets.getNames();
@@ -281,7 +54,7 @@ namespace platform {
                 throw std::invalid_argument("Dataset " + config.continue_from + " not found");
             }
             // Remove datasets already processed
-            vector< string >::iterator it = datasets_names.begin();
+            std::vector<string>::iterator it = datasets_names.begin();
             while (it != datasets_names.end()) {
                 if (*it != config.continue_from) {
                     it = datasets_names.erase(it);
@@ -304,9 +77,329 @@ namespace platform {
         }
         return datasets_names;
     }
+    json GridSearch::build_tasks_mpi(int rank)
+    {
+        auto tasks = json::array();
+        auto grid = GridData(Paths::grid_input(config.model));
+        auto datasets = Datasets(false, Paths::datasets());
+        auto all_datasets = datasets.getNames();
+        auto datasets_names = filterDatasets(datasets);
+        for (int idx_dataset = 0; idx_dataset < datasets_names.size(); ++idx_dataset) {
+            auto dataset = datasets_names[idx_dataset];
+            for (const auto& seed : config.seeds) {
+                auto combinations = grid.getGrid(dataset);
+                for (int n_fold = 0; n_fold < config.n_folds; n_fold++) {
+                    json task = {
+                        { "dataset", dataset },
+                        { "idx_dataset", idx_dataset},
+                        { "seed", seed },
+                        { "fold", n_fold},
+                    };
+                    tasks.push_back(task);
+                }
+            }
+        }
+        // Shuffle the array so heavy datasets are spread across the workers
+        std::mt19937 g{ 271 }; // Use fixed seed to obtain the same shuffle
+        std::shuffle(tasks.begin(), tasks.end(), g);
+        std::cout << get_color_rank(rank) << "* Number of tasks: " << tasks.size() << std::endl;
+        std::cout << "|";
+        for (int i = 0; i < tasks.size(); ++i) {
+            std::cout << (i + 1) % 10;
+        }
+        std::cout << "|" << std::endl << "|" << std::flush;
+        return tasks;
+    }
+    void process_task_mpi_consumer(struct ConfigGrid& config, struct ConfigMPI& config_mpi, json& tasks, int n_task, Datasets& datasets, Task_Result* result)
+    {
+        // initialize
+        Timer timer;
+        timer.start();
+        json task = tasks[n_task];
+        auto model = config.model;
+        auto grid = GridData(Paths::grid_input(model));
+        auto dataset = task["dataset"].get<std::string>();
+        auto idx_dataset = task["idx_dataset"].get<int>();
+        auto seed = task["seed"].get<int>();
+        auto n_fold = task["fold"].get<int>();
+        bool stratified = config.stratified;
+        // Generate the hyperparamters combinations
+        auto combinations = grid.getGrid(dataset);
+        auto [X, y] = datasets.getTensors(dataset);
+        auto states = datasets.getStates(dataset);
+        auto features = datasets.getFeatures(dataset);
+        auto className = datasets.getClassName(dataset);
+        //
+        // Start working on task
+        //
+        Fold* fold;
+        if (stratified)
+            fold = new StratifiedKFold(config.n_folds, y, seed);
+        else
+            fold = new KFold(config.n_folds, y.size(0), seed);
+        auto [train, test] = fold->getFold(n_fold);
+        auto train_t = torch::tensor(train);
+        auto test_t = torch::tensor(test);
+        auto X_train = X.index({ "...", train_t });
+        auto y_train = y.index({ train_t });
+        auto X_test = X.index({ "...", test_t });
+        auto y_test = y.index({ test_t });
+        double best_fold_score = 0.0;
+        int best_idx_combination = -1;
+        json best_fold_hyper;
+        for (int idx_combination = 0; idx_combination < combinations.size(); ++idx_combination) {
+            auto hyperparam_line = combinations[idx_combination];
+            auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line);
+            Fold* nested_fold;
+            if (config.stratified)
+                nested_fold = new StratifiedKFold(config.nested, y_train, seed);
+            else
+                nested_fold = new KFold(config.nested, y_train.size(0), seed);
+            double score = 0.0;
+            for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) {
+                // Nested level fold
+                auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold);
+                auto train_nested_t = torch::tensor(train_nested);
+                auto test_nested_t = torch::tensor(test_nested);
+                auto X_nested_train = X_train.index({ "...", train_nested_t });
+                auto y_nested_train = y_train.index({ train_nested_t });
+                auto X_nested_test = X_train.index({ "...", test_nested_t });
+                auto y_nested_test = y_train.index({ test_nested_t });
+                // Build Classifier with selected hyperparameters
+                auto clf = Models::instance()->create(config.model);
+                auto valid = clf->getValidHyperparameters();
+                hyperparameters.check(valid, dataset);
+                clf->setHyperparameters(hyperparameters.get(dataset));
+                // Train model
+                clf->fit(X_nested_train, y_nested_train, features, className, states);
+                // Test model
+                score += clf->score(X_nested_test, y_nested_test);
+            }
+            delete nested_fold;
+            score /= config.nested;
+            if (score > best_fold_score) {
+                best_fold_score = score;
+                best_idx_combination = idx_combination;
+                best_fold_hyper = hyperparam_line;
+            }
+        }
+        delete fold;
+        // Build Classifier with the best hyperparameters to obtain the best score
+        auto hyperparameters = platform::HyperParameters(datasets.getNames(), best_fold_hyper);
+        auto clf = Models::instance()->create(config.model);
+        auto valid = clf->getValidHyperparameters();
+        hyperparameters.check(valid, dataset);
+        clf->setHyperparameters(best_fold_hyper);
+        clf->fit(X_train, y_train, features, className, states);
+        best_fold_score = clf->score(X_test, y_test);
+        // Return the result
+        result->idx_dataset = task["idx_dataset"].get<int>();
+        result->idx_combination = best_idx_combination;
+        result->score = best_fold_score;
+        result->n_fold = n_fold;
+        result->time = timer.getDuration();
+        // Update progress bar
+        std::cout << get_color_rank(config_mpi.rank) << "*" << std::flush;
+    }
+    json store_result(std::vector<std::string>& names, Task_Result& result, json& results)
+    {
+        json json_result = {
+            { "score", result.score },
+            { "combination", result.idx_combination },
+            { "fold", result.n_fold },
+            { "time", result.time },
+            { "dataset", result.idx_dataset }
+        };
+        auto name = names[result.idx_dataset];
+        if (!results.contains(name)) {
+            results[name] = json::array();
+        }
+        results[name].push_back(json_result);
+        return results;
+    }
+    json producer(std::vector<std::string>& names, json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result)
+    {
+        Task_Result result;
+        json results;
+        int num_tasks = tasks.size();
+
+        //
+        // 2a.1 Producer will loop to send all the tasks to the consumers and receive the results
+        //
+        for (int i = 0; i < num_tasks; ++i) {
+            MPI_Status status;
+            MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
+            if (status.MPI_TAG == TAG_RESULT) {
+                //Store result
+                store_result(names, result, results);
+            }
+            MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD);
+        }
+        //
+        // 2a.2 Producer will send the end message to all the consumers
+        //
+        for (int i = 0; i < config_mpi.n_procs - 1; ++i) {
+            MPI_Status status;
+            MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
+            if (status.MPI_TAG == TAG_RESULT) {
+                //Store result
+                store_result(names, result, results);
+            }
+            MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD);
+        }
+        return results;
+    }
+    void select_best_results_folds(json& results, json& all_results, std::string& model)
+    {
+        Timer timer;
+        auto grid = GridData(Paths::grid_input(model));
+        //
+        // Select the best result of the computed outer folds
+        //
+        for (const auto& result : all_results.items()) {
+            // each result has the results of all the outer folds as each one were a different task
+            double best_score = 0.0;
+            json best;
+            for (const auto& result_fold : result.value()) {
+                double score = result_fold["score"].get<double>();
+                if (score > best_score) {
+                    best_score = score;
+                    best = result_fold;
+                }
+            }
+            auto dataset = result.key();
+            auto combinations = grid.getGrid(dataset);
+            json json_best = {
+                    { "score", best_score },
+                    { "hyperparameters", combinations[best["combination"].get<int>()] },
+                    { "date", get_date() + " " + get_time() },
+                    { "grid", grid.getInputGrid(dataset) },
+                    { "duration", timer.translate2String(best["time"].get<double>()) }
+            };
+            results[dataset] = json_best;
+        }
+    }
+    void consumer(Datasets& datasets, json& tasks, struct ConfigGrid& config, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result)
+    {
+        Task_Result result;
+        //
+        // 2b.1 Consumers announce to the producer that they are ready to receive a task
+        //
+        MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_QUERY, MPI_COMM_WORLD);
+        int task;
+        while (true) {
+            MPI_Status status;
+            //
+            // 2b.2 Consumers receive the task from the producer and process it
+            //
+            MPI_Recv(&task, 1, MPI_INT, config_mpi.manager, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
+            if (status.MPI_TAG == TAG_END) {
+                break;
+            }
+            process_task_mpi_consumer(config, config_mpi, tasks, task, datasets, &result);
+            //
+            // 2b.3 Consumers send the result to the producer
+            //
+            MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_RESULT, MPI_COMM_WORLD);
+        }
+    }
+    void GridSearch::go(struct ConfigMPI& config_mpi)
+    {
+        /*
+        * Each task is a json object with the following structure:
+        * {
+        *   "dataset": "dataset_name",
+        *   "idx_dataset": idx_dataset, // used to identify the dataset in the results
+        *    // this index is relative to the used datasets in the actual run not to the whole datasets
+        *   "seed": # of seed to use,
+        *   "Fold": # of fold to process
+        * }
+        *
+        * The overall process consists in these steps:
+           * 0. Create the MPI result type & tasks
+           * 0.1 Create the MPI result type
+           * 0.2 Manager creates the tasks
+           * 1. Manager will broadcast the tasks to all the processes
+           * 1.1 Broadcast the number of tasks
+           * 1.2 Broadcast the length of the following string
+           * 1.2 Broadcast the tasks as a char* string
+           * 2a. Producer delivers the tasks to the consumers
+           * 2a.1 Producer will loop to send all the tasks to the consumers and receive the results
+           * 2a.2 Producer will send the end message to all the consumers
+           * 2b. Consumers process the tasks and send the results to the producer
+           * 2b.1 Consumers announce to the producer that they are ready to receive a task
+           * 2b.2 Consumers receive the task from the producer and process it
+           * 2b.3 Consumers send the result to the producer
+           * 3. Manager select the bests sccores for each dataset
+           * 3.1 Loop thru all the results obtained from each outer fold (task) and select the best
+           * 3.2 Save the results
+        */
+        //
+        // 0.1 Create the MPI result type
+        //
+        Task_Result result;
+        int tasks_size;
+        MPI_Datatype MPI_Result;
+        MPI_Datatype type[5] = { MPI_UNSIGNED, MPI_UNSIGNED, MPI_INT, MPI_DOUBLE, MPI_DOUBLE };
+        int blocklen[5] = { 1, 1, 1, 1, 1 };
+        MPI_Aint disp[5];
+        disp[0] = offsetof(Task_Result, idx_dataset);
+        disp[1] = offsetof(Task_Result, idx_combination);
+        disp[2] = offsetof(Task_Result, n_fold);
+        disp[3] = offsetof(Task_Result, score);
+        disp[4] = offsetof(Task_Result, time);
+        MPI_Type_create_struct(5, blocklen, disp, type, &MPI_Result);
+        MPI_Type_commit(&MPI_Result);
+        //
+        // 0.2 Manager creates the tasks
+        //
+        char* msg;
+        json tasks;
+        if (config_mpi.rank == config_mpi.manager) {
+            timer.start();
+            tasks = build_tasks_mpi(config_mpi.rank);
+            auto tasks_str = tasks.dump();
+            tasks_size = tasks_str.size();
+            msg = new char[tasks_size + 1];
+            strcpy(msg, tasks_str.c_str());
+        }
+        //
+        // 1. Manager will broadcast the tasks to all the processes
+        //
+        MPI_Bcast(&tasks_size, 1, MPI_INT, config_mpi.manager, MPI_COMM_WORLD);
+        if (config_mpi.rank != config_mpi.manager) {
+            msg = new char[tasks_size + 1];
+        }
+        MPI_Bcast(msg, tasks_size + 1, MPI_CHAR, config_mpi.manager, MPI_COMM_WORLD);
+        tasks = json::parse(msg);
+        delete[] msg;
+        auto datasets = Datasets(config.discretize, Paths::datasets());
+        if (config_mpi.rank == config_mpi.manager) {
+            //
+            // 2a. Producer delivers the tasks to the consumers
+            //
+            auto datasets_names = filterDatasets(datasets);
+            json all_results = producer(datasets_names, tasks, config_mpi, MPI_Result);
+            std::cout << get_color_rank(config_mpi.rank) << "|" << std::endl;
+            //
+            // 3. Manager select the bests sccores for each dataset
+            //
+            auto results = initializeResults();
+            select_best_results_folds(results, all_results, config.model);
+            //
+            // 3.2 Save the results
+            //
+            save(results);
+        } else {
+            //
+            // 2b. Consumers process the tasks and send the results to the producer
+            //
+            consumer(datasets, tasks, config, config_mpi, MPI_Result);
+        }
+    }
     json GridSearch::initializeResults()
     {
-        // Load previous results
+        // Load previous results if continue is set
         json results;
         if (config.continue_from != NO_CONTINUE()) {
             if (!config.quiet)

src/Platform/GridSearch.h

@@ -2,6 +2,7 @@
 #define GRIDSEARCH_H
 #include <string>
 #include <map>
+#include <mpi.h>
 #include <nlohmann/json.hpp>
 #include "Datasets.h"
 #include "HyperParameters.h"
@@ -24,20 +25,35 @@ namespace platform {
         json excluded;
         std::vector<int> seeds;
     };
+    struct ConfigMPI {
+        int rank;
+        int n_procs;
+        int manager;
+    };
+    typedef struct {
+        uint idx_dataset;
+        uint idx_combination;
+        int n_fold;
+        double score;
+        double time;
+    } Task_Result;
+    const int TAG_QUERY = 1;
+    const int TAG_RESULT = 2;
+    const int TAG_TASK = 3;
+    const int TAG_END = 4;
     class GridSearch {
     public:
         explicit GridSearch(struct ConfigGrid& config);
-        void go();
+        void go(struct ConfigMPI& config_mpi);
         ~GridSearch() = default;
-        json getResults();
+        json loadResults();
         static inline std::string NO_CONTINUE() { return "NO_CONTINUE"; }
     private:
         void save(json& results);
         json initializeResults();
-        vector<std::string> processDatasets(Datasets& datasets);
-        pair<double, json> processFileSingle(std::string fileName, Datasets& datasets, std::vector<json>& combinations);
-        pair<double, json> processFileNested(std::string fileName, Datasets& datasets, std::vector<json>& combinations);
+        std::vector<std::string> filterDatasets(Datasets& datasets) const;
         struct ConfigGrid config;
+        json build_tasks_mpi(int rank);
         Timer timer; // used to measure the time of the whole process
     };
 } /* namespace platform */

src/Platform/Timer.h

@@ -28,10 +28,14 @@ namespace platform {
         std::string getDurationString(bool lapse = false)
         {
             double duration = lapse ? getLapse() : getDuration();
+            return translate2String(duration);
+        }
+        std::string translate2String(double duration)
+        {
             double durationShow = duration > 3600 ? duration / 3600 : duration > 60 ? duration / 60 : duration;
             std::string durationUnit = duration > 3600 ? "h" : duration > 60 ? "m" : "s";
             std::stringstream ss;
-            ss << std::setw(7) << std::setprecision(2) << std::fixed << durationShow << " " << durationUnit << " ";
+            ss << std::setprecision(2) << std::fixed << durationShow << " " << durationUnit;
             return ss.str();
         }
     };

src/Platform/b_grid.cc

@@ -2,6 +2,7 @@
 #include <argparse/argparse.hpp>
 #include <map>
 #include <nlohmann/json.hpp>
+#include <mpi.h>
 #include "DotEnv.h"
 #include "Models.h"
 #include "modelRegister.h"
@@ -36,7 +37,20 @@ void manageArguments(argparse::ArgumentParser& program)
     program.add_argument("--continue").help("Continue computing from that dataset").default_value(platform::GridSearch::NO_CONTINUE());
     program.add_argument("--only").help("Used with continue to compute that dataset only").default_value(false).implicit_value(true);
     program.add_argument("--exclude").default_value("[]").help("Datasets to exclude in json format, e.g. [\"dataset1\", \"dataset2\"]");
-    program.add_argument("--nested").help("Do a double/nested cross validation with n folds").default_value(0).scan<'i', int>();
+    program.add_argument("--nested").help("Set the double/nested cross validation number of folds").default_value(5).scan<'i', int>().action([](const std::string& value) {
+        try {
+            auto k = stoi(value);
+            if (k < 2) {
+                throw std::runtime_error("Number of nested folds must be greater than 1");
+            }
+            return k;
+        }
+        catch (const runtime_error& err) {
+            throw std::runtime_error(err.what());
+        }
+        catch (...) {
+            throw std::runtime_error("Number of nested folds must be an integer");
+        }});
     program.add_argument("--score").help("Score used in gridsearch").default_value("accuracy");
     program.add_argument("-f", "--folds").help("Number of folds").default_value(stoi(env.get("n_folds"))).scan<'i', int>().action([](const std::string& value) {
         try {
@@ -106,8 +120,8 @@ void list_results(json& results, std::string& model)
         + " Nested: " + (results["nested"].get<int>() == 0 ? "False" : to_string(results["nested"].get<int>()))
     );
     std::cout << std::string(MAXL, '*') << std::endl;
-    int spaces = 0;
-    int hyperparameters_spaces = 0;
+    int spaces = 7;
+    int hyperparameters_spaces = 15;
     for (const auto& item : results["results"].items()) {
         auto key = item.key();
         auto value = item.value();
@@ -126,19 +140,17 @@ void list_results(json& results, std::string& model)
     int index = 0;
     for (const auto& item : results["results"].items()) {
         auto color = odd ? Colors::CYAN() : Colors::BLUE();
-        auto key = item.key();
         auto value = item.value();
         std::cout << color;
         std::cout << std::setw(3) << std::right << index++ << " ";
-        std::cout << left << setw(spaces) << key << " " << value["date"].get<string>()
-            << " " << setw(8) << value["duration"] << " " << setw(8) << setprecision(6)
+        std::cout << left << setw(spaces) << item.key() << " " << value["date"].get<string>()
+            << " " << setw(8) << right << value["duration"].get<string>() << " " << setw(8) << setprecision(6)
             << fixed << right << value["score"].get<double>() << " " << value["hyperparameters"].dump() << std::endl;
         odd = !odd;
     }
     std::cout << Colors::RESET() << std::endl;
 }
 
-
 /*
  * Main
  */
@@ -189,11 +201,24 @@ int main(int argc, char** argv)
         list_dump(config.model);
     } else {
         if (compute) {
-            grid_search.go();
+            struct platform::ConfigMPI mpi_config;
+            mpi_config.manager = 0; // which process is the manager
+            MPI_Init(&argc, &argv);
+            MPI_Comm_rank(MPI_COMM_WORLD, &mpi_config.rank);
+            MPI_Comm_size(MPI_COMM_WORLD, &mpi_config.n_procs);
+            if (mpi_config.n_procs < 2) {
+                throw std::runtime_error("Cannot use --compute with less than 2 mpi processes, try mpirun -np 2 ...");
+            }
+            grid_search.go(mpi_config);
+            if (mpi_config.rank == mpi_config.manager) {
+                auto results = grid_search.loadResults();
+                list_results(results, config.model);
                 std::cout << "Process took " << timer.getDurationString() << std::endl;
+            }
+            MPI_Finalize();
         } else {
             // List results
-            auto results = grid_search.getResults();
+            auto results = grid_search.loadResults();
             if (results.empty()) {
                 std::cout << "** No results found" << std::endl;
             } else {