From e0b7b2d3168b6c7d312274b4c076cfbb45346738 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana=20G=C3=B3mez?= Date: Fri, 22 Dec 2023 12:47:13 +0100 Subject: [PATCH 01/13] Set structure & protocol of producer-consumer --- src/Platform/GridSearch.cc | 271 ++++++++++++++++++++++++++----------- src/Platform/GridSearch.h | 9 ++ 2 files changed, 202 insertions(+), 78 deletions(-) diff --git a/src/Platform/GridSearch.cc b/src/Platform/GridSearch.cc index 76c4b4c..f8fd2ee 100644 --- a/src/Platform/GridSearch.cc +++ b/src/Platform/GridSearch.cc @@ -149,88 +149,120 @@ namespace platform { auto colors = { Colors::RED(), Colors::GREEN(), Colors::BLUE(), Colors::MAGENTA(), Colors::CYAN() }; return *(colors.begin() + rank % colors.size()); } - void GridSearch::process_task_mpi(struct ConfigMPI& config_mpi, json& task, Datasets& datasets, json& results) + + void GridSearch::go_producer_consumer(struct ConfigMPI& config_mpi) { - // Process the task and store the result in the results json - Timer timer; - timer.start(); - auto grid = GridData(Paths::grid_input(config.model)); - auto dataset = task["dataset"].get(); - auto seed = task["seed"].get(); - auto n_fold = task["fold"].get(); - // Generate the hyperparamters combinations - auto combinations = grid.getGrid(dataset); - auto [X, y] = datasets.getTensors(dataset); - auto states = datasets.getStates(dataset); - auto features = datasets.getFeatures(dataset); - auto className = datasets.getClassName(dataset); + /* + * Each task is a json object with the following structure: + * { + * "dataset": "dataset_name", + * "seed": # of seed to use, + * "model": "model_name", + * "Fold": # of fold to process + * } + * + * The overall process consists in these steps: + * 0. Create the MPI result type & tasks + * 0.1 Create the MPI result type + * 0.2 Manager creates the tasks + * 1. Manager will broadcast the tasks to all the processes + * 1.1 Broadcast the number of tasks + * 1.2 Broadcast the length of the following string + * 1.2 Broadcast the tasks as a char* string + * 2. Workers will receive the tasks and start the process + * 2.1 A method will tell each worker the range of tasks to process + * 2.2 Each worker will process the tasks and generate the best score for each task + * 3. Manager gather the scores from all the workers and find out the best hyperparameters for each dataset + * 3.1 Obtain the maximum size of the results message of all the workers + * 3.2 Gather all the results from the workers into the manager + * 3.3 Compile the results from all the workers + * 3.4 Filter the best hyperparameters for each dataset + */ // - // Start working on task + // 0.1 Create the MPI result type // - Fold* fold; - if (config.stratified) - fold = new StratifiedKFold(config.n_folds, y, seed); - else - fold = new KFold(config.n_folds, y.size(0), seed); - auto [train, test] = fold->getFold(n_fold); - auto train_t = torch::tensor(train); - auto test_t = torch::tensor(test); - auto X_train = X.index({ "...", train_t }); - auto y_train = y.index({ train_t }); - auto X_test = X.index({ "...", test_t }); - auto y_test = y.index({ test_t }); - auto num = 0; - double best_fold_score = 0.0; - json best_fold_hyper; - for (const auto& hyperparam_line : combinations) { - auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); - Fold* nested_fold; - if (config.stratified) - nested_fold = new StratifiedKFold(config.nested, y_train, seed); - else - nested_fold = new KFold(config.nested, y_train.size(0), seed); - double score = 0.0; - for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) { - // Nested level fold - auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold); - auto train_nested_t = torch::tensor(train_nested); - auto test_nested_t = torch::tensor(test_nested); - auto X_nested_train = X_train.index({ "...", train_nested_t }); - auto y_nested_train = y_train.index({ train_nested_t }); - auto X_nested_test = X_train.index({ "...", test_nested_t }); - auto y_nested_test = y_train.index({ test_nested_t }); - // Build Classifier with selected hyperparameters - auto clf = Models::instance()->create(config.model); - auto valid = clf->getValidHyperparameters(); - hyperparameters.check(valid, dataset); - clf->setHyperparameters(hyperparameters.get(dataset)); - // Train model - clf->fit(X_nested_train, y_nested_train, features, className, states); - // Test model - score += clf->score(X_nested_test, y_nested_test); - } - delete nested_fold; - score /= config.nested; - if (score > best_fold_score) { - best_fold_score = score; - best_fold_hyper = hyperparam_line; - } + Task_Result result; + MPI_Datatype MPI_Result; + MPI_Datatype type[3] = { MPI_UNSIGNED, MPI_UNSIGNED, MPI_DOUBLE }; + int blocklen[3] = { 1, 1, 1 }; + MPI_Aint disp[3]; + disp[0] = offsetof(struct MPI_Result, idx_dataset); + disp[1] = offsetof(struct MPI_Result, idx_combination); + disp[2] = offsetof(struct MPI_Result, score); + MPI_Type_create_struct(3, blocklen, disp, type, &MPI_Result); + MPI_Type_commit(&MPI_Result); + // + // 0.2 Manager creates the tasks + // + char* msg; + if (config_mpi.rank == config_mpi.manager) { + timer.start(); + auto tasks = build_tasks_mpi(); + auto tasks_str = tasks.dump(); + tasks_size = tasks_str.size(); + msg = new char[tasks_size + 1]; + strcpy(msg, tasks_str.c_str()); + } + // + // 1. Manager will broadcast the tasks to all the processes + // + MPI_Bcast(&tasks_size, 1, MPI_INT, config_mpi.manager, MPI_COMM_WORLD); + if (config_mpi.rank != config_mpi.manager) { + msg = new char[tasks_size + 1]; + } + MPI_Bcast(msg, tasks_size + 1, MPI_CHAR, config_mpi.manager, MPI_COMM_WORLD); + json tasks = json::parse(msg); + delete[] msg; + // + // 2. All Workers will receive the tasks and start the process + // + if (config_mpi.rank == config_mpi.manager) { + producer(tasks, &MPI_Result); + } else { + consumer(tasks, &MPI_Result); + } + } + void producer(json& tasks, MPI_Datatpe& MPI_Result) + { + Task_Result result; + int num_tasks = tasks.size(); + for (int i = 0; i < num_tasks; ++i) { + MPI_Status status; + MPI_recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); + if (status.MPI_TAG == TAG_RESULT) { + //Store result + + } + MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD); + } + // Send end message to all workers + for (int i = 0; i < config_mpi.n_procs; ++i) { + MPI_Status status; + MPI_recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); + if (status.MPI_TAG == TAG_RESULT) { + //Store result + + } + MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD); + } + } + void consumer(json& tasks, MPI_Datatpe& MPI_Result) + { + Task_Result result; + // Anounce to the producer + MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_QUERY, MPI_COMM_WORLD); + int task; + while (true) { + MPI_Status status; + MPI_recv(&task, 1, MPI_INT, config_mpi.manager, MPI_ANY_TAG, MPI_COMM_WORLD, &status); + if (status.MPI_TAG == TAG_END) { + break; + } + // Process task + process_task_mpi(config_mpi, task, datasets, results); + // Send result to producer + MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_RESULT, MPI_COMM_WORLD); } - delete fold; - // Build Classifier with the best hyperparameters to obtain the best score - auto hyperparameters = platform::HyperParameters(datasets.getNames(), best_fold_hyper); - auto clf = Models::instance()->create(config.model); - auto valid = clf->getValidHyperparameters(); - hyperparameters.check(valid, dataset); - clf->setHyperparameters(best_fold_hyper); - clf->fit(X_train, y_train, features, className, states); - best_fold_score = clf->score(X_test, y_test); - // Save results - results[dataset][std::to_string(n_fold)]["score"] = best_fold_score; - results[dataset][std::to_string(n_fold)]["hyperparameters"] = best_fold_hyper; - results[dataset][std::to_string(n_fold)]["seed"] = seed; - results[dataset][std::to_string(n_fold)]["duration"] = timer.getDuration(); - std::cout << get_color_rank(config_mpi.rank) << "*" << std::flush; } void GridSearch::go_mpi(struct ConfigMPI& config_mpi) { @@ -555,6 +587,89 @@ namespace platform { } return { goatScore, goatHyperparameters }; } + void GridSearch::process_task_mpi(struct ConfigMPI& config_mpi, json& task, Datasets& datasets, json& results) + { + // Process the task and store the result in the results json + Timer timer; + timer.start(); + auto grid = GridData(Paths::grid_input(config.model)); + auto dataset = task["dataset"].get(); + auto seed = task["seed"].get(); + auto n_fold = task["fold"].get(); + // Generate the hyperparamters combinations + auto combinations = grid.getGrid(dataset); + auto [X, y] = datasets.getTensors(dataset); + auto states = datasets.getStates(dataset); + auto features = datasets.getFeatures(dataset); + auto className = datasets.getClassName(dataset); + // + // Start working on task + // + Fold* fold; + if (config.stratified) + fold = new StratifiedKFold(config.n_folds, y, seed); + else + fold = new KFold(config.n_folds, y.size(0), seed); + auto [train, test] = fold->getFold(n_fold); + auto train_t = torch::tensor(train); + auto test_t = torch::tensor(test); + auto X_train = X.index({ "...", train_t }); + auto y_train = y.index({ train_t }); + auto X_test = X.index({ "...", test_t }); + auto y_test = y.index({ test_t }); + auto num = 0; + double best_fold_score = 0.0; + json best_fold_hyper; + for (const auto& hyperparam_line : combinations) { + auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); + Fold* nested_fold; + if (config.stratified) + nested_fold = new StratifiedKFold(config.nested, y_train, seed); + else + nested_fold = new KFold(config.nested, y_train.size(0), seed); + double score = 0.0; + for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) { + // Nested level fold + auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold); + auto train_nested_t = torch::tensor(train_nested); + auto test_nested_t = torch::tensor(test_nested); + auto X_nested_train = X_train.index({ "...", train_nested_t }); + auto y_nested_train = y_train.index({ train_nested_t }); + auto X_nested_test = X_train.index({ "...", test_nested_t }); + auto y_nested_test = y_train.index({ test_nested_t }); + // Build Classifier with selected hyperparameters + auto clf = Models::instance()->create(config.model); + auto valid = clf->getValidHyperparameters(); + hyperparameters.check(valid, dataset); + clf->setHyperparameters(hyperparameters.get(dataset)); + // Train model + clf->fit(X_nested_train, y_nested_train, features, className, states); + // Test model + score += clf->score(X_nested_test, y_nested_test); + } + delete nested_fold; + score /= config.nested; + if (score > best_fold_score) { + best_fold_score = score; + best_fold_hyper = hyperparam_line; + } + } + delete fold; + // Build Classifier with the best hyperparameters to obtain the best score + auto hyperparameters = platform::HyperParameters(datasets.getNames(), best_fold_hyper); + auto clf = Models::instance()->create(config.model); + auto valid = clf->getValidHyperparameters(); + hyperparameters.check(valid, dataset); + clf->setHyperparameters(best_fold_hyper); + clf->fit(X_train, y_train, features, className, states); + best_fold_score = clf->score(X_test, y_test); + // Save results + results[dataset][std::to_string(n_fold)]["score"] = best_fold_score; + results[dataset][std::to_string(n_fold)]["hyperparameters"] = best_fold_hyper; + results[dataset][std::to_string(n_fold)]["seed"] = seed; + results[dataset][std::to_string(n_fold)]["duration"] = timer.getDuration(); + std::cout << get_color_rank(config_mpi.rank) << "*" << std::flush; + } json GridSearch::initializeResults() { // Load previous results diff --git a/src/Platform/GridSearch.h b/src/Platform/GridSearch.h index c00b2ee..1a868a8 100644 --- a/src/Platform/GridSearch.h +++ b/src/Platform/GridSearch.h @@ -30,6 +30,15 @@ namespace platform { int n_procs; int manager; }; + typedef struct { + uint idx_dataset; + uint idx_combination; + double score; + } Task_Result; + const TAG_QUERY = 1; + const TAG_RESULT = 2; + const TAG_TASK = 3; + const TAG_END = 4; class GridSearch { public: explicit GridSearch(struct ConfigGrid& config); From 981bc8f98b91847254611df0963922ed506643cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana=20G=C3=B3mez?= Date: Sat, 23 Dec 2023 01:00:55 +0100 Subject: [PATCH 02/13] Fix install message in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6ddd7c1..a3ebf6a 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Before compiling BayesNet. ### MPI -In Linux just install openmpi & openmpi-devel packages. Only cmake can't find openmpi install (like in Oracle Linux) set the following variable: +In Linux just install openmpi & openmpi-devel packages. Only if cmake can't find openmpi installation (like in Oracle Linux) set the following variable: ```bash export MPI_HOME="/usr/lib64/openmpi" From 702f086706511f69122de31f4cdcd99b837c2c10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana=20G=C3=B3mez?= Date: Sat, 23 Dec 2023 19:54:00 +0100 Subject: [PATCH 03/13] Update miniconda instructions --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index a3ebf6a..a3a4f6a 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,18 @@ Bayesian Network Classifier with libtorch from scratch Before compiling BayesNet. +### Miniconda + +To be able to run Python Classifiers such as STree, ODTE, SVC, etc. it is needed to install Miniconda. To do so, download the installer from [Miniconda](https://docs.conda.io/en/latest/miniconda.html) and run it. It is recommended to install it in the home folder. + +In Linux sometimes the library libstdc++ is mistaken from the miniconda installation and produces the next message when running the b_xxxx executables: + +```bash +libstdc++.so.6: version `GLIBCXX_3.4.32' not found (required by b_xxxx) +``` + +The solution is to erase the libstdc++ library from the miniconda installation: + ### MPI In Linux just install openmpi & openmpi-devel packages. Only if cmake can't find openmpi installation (like in Oracle Linux) set the following variable: From 21c4c6df512de60e3dd80c17ed36ec288b7bd574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana=20G=C3=B3mez?= Date: Mon, 25 Dec 2023 19:33:52 +0100 Subject: [PATCH 04/13] Fix first mistakes in structure --- kk | 0 src/Platform/GridSearch.cc | 105 ++++++++++++++++++++----------------- src/Platform/GridSearch.h | 9 ++-- 3 files changed, 63 insertions(+), 51 deletions(-) create mode 100644 kk diff --git a/kk b/kk new file mode 100644 index 0000000..e69de29 diff --git a/src/Platform/GridSearch.cc b/src/Platform/GridSearch.cc index f8fd2ee..c2ef440 100644 --- a/src/Platform/GridSearch.cc +++ b/src/Platform/GridSearch.cc @@ -1,4 +1,5 @@ #include +#include #include #include "GridSearch.h" #include "Models.h" @@ -101,13 +102,20 @@ namespace platform { auto tasks = json::array(); auto grid = GridData(Paths::grid_input(config.model)); auto datasets = Datasets(false, Paths::datasets()); + auto all_datasets = datasets.getNames(); auto datasets_names = processDatasets(datasets); for (const auto& dataset : datasets_names) { for (const auto& seed : config.seeds) { auto combinations = grid.getGrid(dataset); for (int n_fold = 0; n_fold < config.n_folds; n_fold++) { + auto it = find(all_datasets.begin(), all_datasets.end(), dataset); + if (it == all_datasets.end()) { + throw std::invalid_argument("Dataset " + dataset + " not found"); + } + auto idx_dataset = std::distance(all_datasets.begin(), it); json task = { { "dataset", dataset }, + { "idx_dataset", idx_dataset}, { "seed", seed }, { "fold", n_fold} }; @@ -126,6 +134,9 @@ namespace platform { std::cout << "|" << std::endl << "|" << std::flush; return tasks; } + void process_task_mpi(struct ConfigMPI& config_mpi, int task, Task_Result* result) + { + } std::pair GridSearch::part_range_mpi(int n_tasks, int nprocs, int rank) { int assigned = 0; @@ -149,7 +160,48 @@ namespace platform { auto colors = { Colors::RED(), Colors::GREEN(), Colors::BLUE(), Colors::MAGENTA(), Colors::CYAN() }; return *(colors.begin() + rank % colors.size()); } + void producer(json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) + { + Task_Result result; + int num_tasks = tasks.size(); + for (int i = 0; i < num_tasks; ++i) { + MPI_Status status; + MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); + if (status.MPI_TAG == TAG_RESULT) { + //Store result + } + MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD); + } + // Send end message to all workers + for (int i = 0; i < config_mpi.n_procs; ++i) { + MPI_Status status; + MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); + if (status.MPI_TAG == TAG_RESULT) { + //Store result + + } + MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD); + } + } + void consumer(json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) + { + Task_Result result; + // Anounce to the producer + MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_QUERY, MPI_COMM_WORLD); + int task; + while (true) { + MPI_Status status; + MPI_Recv(&task, 1, MPI_INT, config_mpi.manager, MPI_ANY_TAG, MPI_COMM_WORLD, &status); + if (status.MPI_TAG == TAG_END) { + break; + } + // Process task + process_task_mpi(config_mpi, task, &result); + // Send result to producer + MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_RESULT, MPI_COMM_WORLD); + } + } void GridSearch::go_producer_consumer(struct ConfigMPI& config_mpi) { /* @@ -182,13 +234,14 @@ namespace platform { // 0.1 Create the MPI result type // Task_Result result; + int tasks_size; MPI_Datatype MPI_Result; MPI_Datatype type[3] = { MPI_UNSIGNED, MPI_UNSIGNED, MPI_DOUBLE }; int blocklen[3] = { 1, 1, 1 }; MPI_Aint disp[3]; - disp[0] = offsetof(struct MPI_Result, idx_dataset); - disp[1] = offsetof(struct MPI_Result, idx_combination); - disp[2] = offsetof(struct MPI_Result, score); + disp[0] = offsetof(Task_Result, idx_dataset); + disp[1] = offsetof(Task_Result, idx_combination); + disp[2] = offsetof(Task_Result, score); MPI_Type_create_struct(3, blocklen, disp, type, &MPI_Result); MPI_Type_commit(&MPI_Result); // @@ -217,51 +270,9 @@ namespace platform { // 2. All Workers will receive the tasks and start the process // if (config_mpi.rank == config_mpi.manager) { - producer(tasks, &MPI_Result); + producer(tasks, config_mpi, MPI_Result); } else { - consumer(tasks, &MPI_Result); - } - } - void producer(json& tasks, MPI_Datatpe& MPI_Result) - { - Task_Result result; - int num_tasks = tasks.size(); - for (int i = 0; i < num_tasks; ++i) { - MPI_Status status; - MPI_recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); - if (status.MPI_TAG == TAG_RESULT) { - //Store result - - } - MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD); - } - // Send end message to all workers - for (int i = 0; i < config_mpi.n_procs; ++i) { - MPI_Status status; - MPI_recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); - if (status.MPI_TAG == TAG_RESULT) { - //Store result - - } - MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD); - } - } - void consumer(json& tasks, MPI_Datatpe& MPI_Result) - { - Task_Result result; - // Anounce to the producer - MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_QUERY, MPI_COMM_WORLD); - int task; - while (true) { - MPI_Status status; - MPI_recv(&task, 1, MPI_INT, config_mpi.manager, MPI_ANY_TAG, MPI_COMM_WORLD, &status); - if (status.MPI_TAG == TAG_END) { - break; - } - // Process task - process_task_mpi(config_mpi, task, datasets, results); - // Send result to producer - MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_RESULT, MPI_COMM_WORLD); + consumer(tasks, config_mpi, MPI_Result); } } void GridSearch::go_mpi(struct ConfigMPI& config_mpi) diff --git a/src/Platform/GridSearch.h b/src/Platform/GridSearch.h index 1a868a8..a9e2f6e 100644 --- a/src/Platform/GridSearch.h +++ b/src/Platform/GridSearch.h @@ -35,15 +35,16 @@ namespace platform { uint idx_combination; double score; } Task_Result; - const TAG_QUERY = 1; - const TAG_RESULT = 2; - const TAG_TASK = 3; - const TAG_END = 4; + const int TAG_QUERY = 1; + const int TAG_RESULT = 2; + const int TAG_TASK = 3; + const int TAG_END = 4; class GridSearch { public: explicit GridSearch(struct ConfigGrid& config); void go(); void go_mpi(struct ConfigMPI& config_mpi); + void go_producer_consumer(struct ConfigMPI& config_mpi); ~GridSearch() = default; json getResults(); static inline std::string NO_CONTINUE() { return "NO_CONTINUE"; } From 343269d48c7e73a25867446c1c8fb2352f72c5d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Thu, 28 Dec 2023 23:21:50 +0100 Subject: [PATCH 05/13] Fix syntax errors --- src/Platform/GridSearch.cc | 147 +++++++++++++++++++++++++++++++------ src/Platform/GridSearch.h | 1 + 2 files changed, 124 insertions(+), 24 deletions(-) diff --git a/src/Platform/GridSearch.cc b/src/Platform/GridSearch.cc index c2ef440..4b6741e 100644 --- a/src/Platform/GridSearch.cc +++ b/src/Platform/GridSearch.cc @@ -28,6 +28,11 @@ namespace platform { oss << std::put_time(timeinfo, "%H:%M:%S"); return oss.str(); } + std::string get_color_rank(int rank) + { + auto colors = { Colors::RED(), Colors::GREEN(), Colors::BLUE(), Colors::MAGENTA(), Colors::CYAN() }; + return *(colors.begin() + rank % colors.size()); + } GridSearch::GridSearch(struct ConfigGrid& config) : config(config) { } @@ -104,20 +109,16 @@ namespace platform { auto datasets = Datasets(false, Paths::datasets()); auto all_datasets = datasets.getNames(); auto datasets_names = processDatasets(datasets); - for (const auto& dataset : datasets_names) { + for (int idx_dataset = 0; idx_dataset < all_datasets.size(); ++idx_dataset) { + auto dataset = all_datasets[idx_dataset]; for (const auto& seed : config.seeds) { auto combinations = grid.getGrid(dataset); for (int n_fold = 0; n_fold < config.n_folds; n_fold++) { - auto it = find(all_datasets.begin(), all_datasets.end(), dataset); - if (it == all_datasets.end()) { - throw std::invalid_argument("Dataset " + dataset + " not found"); - } - auto idx_dataset = std::distance(all_datasets.begin(), it); json task = { { "dataset", dataset }, { "idx_dataset", idx_dataset}, { "seed", seed }, - { "fold", n_fold} + { "fold", n_fold}, }; tasks.push_back(task); } @@ -134,8 +135,96 @@ namespace platform { std::cout << "|" << std::endl << "|" << std::flush; return tasks; } - void process_task_mpi(struct ConfigMPI& config_mpi, int task, Task_Result* result) + void process_task_mpi_consumer(struct ConfigGrid& config, struct ConfigMPI& config_mpi, json& tasks, int n_task, Datasets& datasets, Task_Result* result) { + // initialize + Timer timer; + timer.start(); + json task = tasks[n_task]; + auto model = config.model; + auto grid = GridData(Paths::grid_input(model)); + auto dataset = task["dataset"].get(); + auto idx_dataset = task["idx_dataset"].get(); + auto seed = task["seed"].get(); + auto n_fold = task["fold"].get(); + bool stratified = config.stratified; + // Generate the hyperparamters combinations + auto combinations = grid.getGrid(dataset); + auto [X, y] = datasets.getTensors(dataset); + auto states = datasets.getStates(dataset); + auto features = datasets.getFeatures(dataset); + auto className = datasets.getClassName(dataset); + // + // Start working on task + // + Fold* fold; + if (stratified) + fold = new StratifiedKFold(config.n_folds, y, seed); + else + fold = new KFold(config.n_folds, y.size(0), seed); + auto [train, test] = fold->getFold(n_fold); + auto train_t = torch::tensor(train); + auto test_t = torch::tensor(test); + auto X_train = X.index({ "...", train_t }); + auto y_train = y.index({ train_t }); + auto X_test = X.index({ "...", test_t }); + auto y_test = y.index({ test_t }); + auto num = 0; + double best_fold_score = 0.0; + int best_idx_combination = -1; + json best_fold_hyper; + for (int idx_combination = 0; idx_combination < combinations.size(); ++idx_combination) { + auto hyperparam_line = combinations[idx_combination]; + auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); + Fold* nested_fold; + if (config.stratified) + nested_fold = new StratifiedKFold(config.nested, y_train, seed); + else + nested_fold = new KFold(config.nested, y_train.size(0), seed); + double score = 0.0; + for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) { + // Nested level fold + auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold); + auto train_nested_t = torch::tensor(train_nested); + auto test_nested_t = torch::tensor(test_nested); + auto X_nested_train = X_train.index({ "...", train_nested_t }); + auto y_nested_train = y_train.index({ train_nested_t }); + auto X_nested_test = X_train.index({ "...", test_nested_t }); + auto y_nested_test = y_train.index({ test_nested_t }); + // Build Classifier with selected hyperparameters + auto clf = Models::instance()->create(config.model); + auto valid = clf->getValidHyperparameters(); + hyperparameters.check(valid, dataset); + clf->setHyperparameters(hyperparameters.get(dataset)); + // Train model + clf->fit(X_nested_train, y_nested_train, features, className, states); + // Test model + score += clf->score(X_nested_test, y_nested_test); + } + delete nested_fold; + score /= config.nested; + if (score > best_fold_score) { + best_fold_score = score; + best_idx_combination = idx_combination; + best_fold_hyper = hyperparam_line; + } + } + delete fold; + // Build Classifier with the best hyperparameters to obtain the best score + auto hyperparameters = platform::HyperParameters(datasets.getNames(), best_fold_hyper); + auto clf = Models::instance()->create(config.model); + auto valid = clf->getValidHyperparameters(); + hyperparameters.check(valid, dataset); + clf->setHyperparameters(best_fold_hyper); + clf->fit(X_train, y_train, features, className, states); + best_fold_score = clf->score(X_test, y_test); + // Return the result + result->idx_dataset = task["idx_dataset"].get(); + result->idx_combination = best_idx_combination; + result->score = best_fold_score; + result->time = timer.getDuration(); + // Update progress bar + std::cout << get_color_rank(config_mpi.rank) << "*" << std::flush; } std::pair GridSearch::part_range_mpi(int n_tasks, int nprocs, int rank) { @@ -155,14 +244,10 @@ namespace platform { } return { start, end }; } - std::string get_color_rank(int rank) - { - auto colors = { Colors::RED(), Colors::GREEN(), Colors::BLUE(), Colors::MAGENTA(), Colors::CYAN() }; - return *(colors.begin() + rank % colors.size()); - } - void producer(json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) + json producer(json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) { Task_Result result; + json results; int num_tasks = tasks.size(); for (int i = 0; i < num_tasks; ++i) { MPI_Status status; @@ -183,8 +268,17 @@ namespace platform { } MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD); } + return results; } - void consumer(json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) + json select_best_results_folds(json& all_results) + { + json results; + // + // Select the best result of the computed outer folds + // + return results; + } + void consumer(Datasets& datasets, json& tasks, struct ConfigGrid& config, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) { Task_Result result; // Anounce to the producer @@ -197,7 +291,7 @@ namespace platform { break; } // Process task - process_task_mpi(config_mpi, task, &result); + process_task_mpi_consumer(config, config_mpi, tasks, task, datasets, &result); // Send result to producer MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_RESULT, MPI_COMM_WORLD); } @@ -236,21 +330,23 @@ namespace platform { Task_Result result; int tasks_size; MPI_Datatype MPI_Result; - MPI_Datatype type[3] = { MPI_UNSIGNED, MPI_UNSIGNED, MPI_DOUBLE }; - int blocklen[3] = { 1, 1, 1 }; - MPI_Aint disp[3]; + MPI_Datatype type[4] = { MPI_UNSIGNED, MPI_UNSIGNED, MPI_DOUBLE, MPI_DOUBLE }; + int blocklen[4] = { 1, 1, 1, 1 }; + MPI_Aint disp[4]; disp[0] = offsetof(Task_Result, idx_dataset); disp[1] = offsetof(Task_Result, idx_combination); disp[2] = offsetof(Task_Result, score); - MPI_Type_create_struct(3, blocklen, disp, type, &MPI_Result); + disp[3] = offsetof(Task_Result, time); + MPI_Type_create_struct(4, blocklen, disp, type, &MPI_Result); MPI_Type_commit(&MPI_Result); // // 0.2 Manager creates the tasks // char* msg; + json tasks; if (config_mpi.rank == config_mpi.manager) { timer.start(); - auto tasks = build_tasks_mpi(); + tasks = build_tasks_mpi(); auto tasks_str = tasks.dump(); tasks_size = tasks_str.size(); msg = new char[tasks_size + 1]; @@ -264,15 +360,18 @@ namespace platform { msg = new char[tasks_size + 1]; } MPI_Bcast(msg, tasks_size + 1, MPI_CHAR, config_mpi.manager, MPI_COMM_WORLD); - json tasks = json::parse(msg); + tasks = json::parse(msg); delete[] msg; // // 2. All Workers will receive the tasks and start the process // + auto datasets = Datasets(config.discretize, Paths::datasets()); if (config_mpi.rank == config_mpi.manager) { - producer(tasks, config_mpi, MPI_Result); + auto all_results = producer(tasks, config_mpi, MPI_Result); + auto results = select_best_results_folds(all_results); + save(results); } else { - consumer(tasks, config_mpi, MPI_Result); + consumer(datasets, tasks, config, config_mpi, MPI_Result); } } void GridSearch::go_mpi(struct ConfigMPI& config_mpi) diff --git a/src/Platform/GridSearch.h b/src/Platform/GridSearch.h index a9e2f6e..8004eca 100644 --- a/src/Platform/GridSearch.h +++ b/src/Platform/GridSearch.h @@ -34,6 +34,7 @@ namespace platform { uint idx_dataset; uint idx_combination; double score; + double time; } Task_Result; const int TAG_QUERY = 1; const int TAG_RESULT = 2; From b7fef9a99da4b12639fc2484f4e1177d782caa3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Thu, 28 Dec 2023 23:24:59 +0100 Subject: [PATCH 06/13] Remove kk file --- kk | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 kk diff --git a/kk b/kk deleted file mode 100644 index e69de29..0000000 From 652e5f623fcfc499a2bc738b52755ef506a11d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Thu, 28 Dec 2023 23:32:24 +0100 Subject: [PATCH 07/13] Add todo comments --- src/Platform/GridSearch.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Platform/GridSearch.cc b/src/Platform/GridSearch.cc index 4b6741e..4d3c5f0 100644 --- a/src/Platform/GridSearch.cc +++ b/src/Platform/GridSearch.cc @@ -254,7 +254,7 @@ namespace platform { MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); if (status.MPI_TAG == TAG_RESULT) { //Store result - + // TODO } MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD); } @@ -264,7 +264,7 @@ namespace platform { MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); if (status.MPI_TAG == TAG_RESULT) { //Store result - + // TODO } MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD); } @@ -276,6 +276,7 @@ namespace platform { // // Select the best result of the computed outer folds // + // TODO return results; } void consumer(Datasets& datasets, json& tasks, struct ConfigGrid& config, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) From beadb7465fcd29eb78cd18e176829c403d5a38e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana=20G=C3=B3mez?= Date: Sun, 31 Dec 2023 12:02:13 +0100 Subject: [PATCH 08/13] Complete first approach --- src/Platform/GridSearch.cc | 73 +++++++++++++++++++++++++++++--------- src/Platform/GridSearch.h | 3 +- src/Platform/b_grid.cc | 2 +- 3 files changed, 60 insertions(+), 18 deletions(-) diff --git a/src/Platform/GridSearch.cc b/src/Platform/GridSearch.cc index 4d3c5f0..8ace92d 100644 --- a/src/Platform/GridSearch.cc +++ b/src/Platform/GridSearch.cc @@ -44,7 +44,7 @@ namespace platform { } return json(); } - vector GridSearch::processDatasets(Datasets& datasets) + vector GridSearch::processDatasets(Datasets& datasets) const { // Load datasets auto datasets_names = datasets.getNames(); @@ -109,7 +109,7 @@ namespace platform { auto datasets = Datasets(false, Paths::datasets()); auto all_datasets = datasets.getNames(); auto datasets_names = processDatasets(datasets); - for (int idx_dataset = 0; idx_dataset < all_datasets.size(); ++idx_dataset) { + for (int idx_dataset = 0; idx_dataset < datasets_names.size(); ++idx_dataset) { auto dataset = all_datasets[idx_dataset]; for (const auto& seed : config.seeds) { auto combinations = grid.getGrid(dataset); @@ -169,7 +169,6 @@ namespace platform { auto y_train = y.index({ train_t }); auto X_test = X.index({ "...", test_t }); auto y_test = y.index({ test_t }); - auto num = 0; double best_fold_score = 0.0; int best_idx_combination = -1; json best_fold_hyper; @@ -222,6 +221,7 @@ namespace platform { result->idx_dataset = task["idx_dataset"].get(); result->idx_combination = best_idx_combination; result->score = best_fold_score; + result->n_fold = n_fold; result->time = timer.getDuration(); // Update progress bar std::cout << get_color_rank(config_mpi.rank) << "*" << std::flush; @@ -244,17 +244,34 @@ namespace platform { } return { start, end }; } + void store_result(std::vector& names, Task_Result& result, json& results) + { + json json_result = { + { "score", result.score }, + { "combination", result.idx_combination }, + { "fold", result.n_fold }, + { "time", result.time }, + { "dataset", result.idx_dataset } + }; + auto name = names[result.idx_dataset]; + if (!results.contains(name)) { + results[name] = json::array(); + } + results[name].push_back(json_result); + } json producer(json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) { Task_Result result; json results; int num_tasks = tasks.size(); + auto datasets = Datasets(false, Paths::datasets()); + auto names = datasets.getNames(); for (int i = 0; i < num_tasks; ++i) { MPI_Status status; MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); if (status.MPI_TAG == TAG_RESULT) { //Store result - // TODO + store_result(names, result, results); } MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD); } @@ -264,19 +281,42 @@ namespace platform { MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); if (status.MPI_TAG == TAG_RESULT) { //Store result - // TODO + store_result(names, result, results); } MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD); } return results; } - json select_best_results_folds(json& all_results) + json select_best_results_folds(json& all_results, std::string& model) { json results; + Timer timer; + auto grid = GridData(Paths::grid_input(model)); // // Select the best result of the computed outer folds // - // TODO + for (const auto& result : results.items()) { + // each result has the results of all the outer folds as each one were a different task + double best_score = 0.0; + json best; + for (const auto& result_fold : result.value()) { + double score = result_fold["score"].get(); + if (score > best_score) { + best_score = score; + best = result_fold; + } + } + auto dataset = result.key(); + auto combinations = grid.getGrid(dataset); + json json_best = { + { "score", best_score }, + { "hyperparameters", combinations[best["combination"].get()] }, + { "date", get_date() + " " + get_time() }, + { "grid", grid.getInputGrid(dataset) }, + { "duration", timer.translate2String(best["time"].get()) } + }; + results[dataset] = json_best; + } return results; } void consumer(Datasets& datasets, json& tasks, struct ConfigGrid& config, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) @@ -303,8 +343,8 @@ namespace platform { * Each task is a json object with the following structure: * { * "dataset": "dataset_name", + * "idx_dataset": idx_dataset, * "seed": # of seed to use, - * "model": "model_name", * "Fold": # of fold to process * } * @@ -331,14 +371,15 @@ namespace platform { Task_Result result; int tasks_size; MPI_Datatype MPI_Result; - MPI_Datatype type[4] = { MPI_UNSIGNED, MPI_UNSIGNED, MPI_DOUBLE, MPI_DOUBLE }; - int blocklen[4] = { 1, 1, 1, 1 }; - MPI_Aint disp[4]; + MPI_Datatype type[5] = { MPI_UNSIGNED, MPI_UNSIGNED, MPI_INT, MPI_DOUBLE, MPI_DOUBLE }; + int blocklen[5] = { 1, 1, 1, 1, 1 }; + MPI_Aint disp[5]; disp[0] = offsetof(Task_Result, idx_dataset); disp[1] = offsetof(Task_Result, idx_combination); - disp[2] = offsetof(Task_Result, score); - disp[3] = offsetof(Task_Result, time); - MPI_Type_create_struct(4, blocklen, disp, type, &MPI_Result); + disp[2] = offsetof(Task_Result, n_fold); + disp[3] = offsetof(Task_Result, score); + disp[4] = offsetof(Task_Result, time); + MPI_Type_create_struct(5, blocklen, disp, type, &MPI_Result); MPI_Type_commit(&MPI_Result); // // 0.2 Manager creates the tasks @@ -369,7 +410,7 @@ namespace platform { auto datasets = Datasets(config.discretize, Paths::datasets()); if (config_mpi.rank == config_mpi.manager) { auto all_results = producer(tasks, config_mpi, MPI_Result); - auto results = select_best_results_folds(all_results); + auto results = select_best_results_folds(all_results, config.model); save(results); } else { consumer(datasets, tasks, config, config_mpi, MPI_Result); @@ -381,8 +422,8 @@ namespace platform { * Each task is a json object with the following structure: * { * "dataset": "dataset_name", + * "idx_dataset": idx_dataset, * "seed": # of seed to use, - * "model": "model_name", * "Fold": # of fold to process * } * diff --git a/src/Platform/GridSearch.h b/src/Platform/GridSearch.h index 8004eca..08b874d 100644 --- a/src/Platform/GridSearch.h +++ b/src/Platform/GridSearch.h @@ -33,6 +33,7 @@ namespace platform { typedef struct { uint idx_dataset; uint idx_combination; + int n_fold; double score; double time; } Task_Result; @@ -52,7 +53,7 @@ namespace platform { private: void save(json& results); json initializeResults(); - vector processDatasets(Datasets& datasets); + vector processDatasets(Datasets& datasets) const; pair processFileSingle(std::string fileName, Datasets& datasets, std::vector& combinations); pair processFileNested(std::string fileName, Datasets& datasets, std::vector& combinations); struct ConfigGrid config; diff --git a/src/Platform/b_grid.cc b/src/Platform/b_grid.cc index d870353..055da26 100644 --- a/src/Platform/b_grid.cc +++ b/src/Platform/b_grid.cc @@ -201,7 +201,7 @@ int main(int argc, char** argv) MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &mpi_config.rank); MPI_Comm_size(MPI_COMM_WORLD, &mpi_config.n_procs); - grid_search.go_mpi(mpi_config); + grid_search.go_producer_consumer(mpi_config); if (mpi_config.rank == mpi_config.manager) { auto results = grid_search.getResults(); list_results(results, config.model); From 9ab4fc7d76d62a834a192172bf90b2eb42b846fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Wed, 3 Jan 2024 11:53:46 +0100 Subject: [PATCH 09/13] Fix some mistakes in methods --- src/Platform/GridSearch.cc | 885 +++++++++++++++++++------------------ src/Platform/GridSearch.h | 16 +- src/Platform/b_grid.cc | 10 +- 3 files changed, 464 insertions(+), 447 deletions(-) diff --git a/src/Platform/GridSearch.cc b/src/Platform/GridSearch.cc index 8ace92d..7cb36df 100644 --- a/src/Platform/GridSearch.cc +++ b/src/Platform/GridSearch.cc @@ -36,7 +36,7 @@ namespace platform { GridSearch::GridSearch(struct ConfigGrid& config) : config(config) { } - json GridSearch::getResults() + json GridSearch::loadResults() { std::ifstream file(Paths::grid_output(config.model)); if (file.is_open()) { @@ -44,7 +44,7 @@ namespace platform { } return json(); } - vector GridSearch::processDatasets(Datasets& datasets) const + vector GridSearch::filterDatasets(Datasets& datasets) const { // Load datasets auto datasets_names = datasets.getNames(); @@ -108,9 +108,9 @@ namespace platform { auto grid = GridData(Paths::grid_input(config.model)); auto datasets = Datasets(false, Paths::datasets()); auto all_datasets = datasets.getNames(); - auto datasets_names = processDatasets(datasets); + auto datasets_names = filterDatasets(datasets); for (int idx_dataset = 0; idx_dataset < datasets_names.size(); ++idx_dataset) { - auto dataset = all_datasets[idx_dataset]; + auto dataset = datasets_names[idx_dataset]; for (const auto& seed : config.seeds) { auto combinations = grid.getGrid(dataset); for (int n_fold = 0; n_fold < config.n_folds; n_fold++) { @@ -226,25 +226,25 @@ namespace platform { // Update progress bar std::cout << get_color_rank(config_mpi.rank) << "*" << std::flush; } - std::pair GridSearch::part_range_mpi(int n_tasks, int nprocs, int rank) - { - int assigned = 0; - int remainder = n_tasks % nprocs; - int start = 0; - if (rank < remainder) { - assigned = n_tasks / nprocs + 1; - } else { - assigned = n_tasks / nprocs; - start = remainder; - } - start += rank * assigned; - int end = start + assigned; - if (rank == nprocs - 1) { - end = n_tasks; - } - return { start, end }; - } - void store_result(std::vector& names, Task_Result& result, json& results) + // std::pair GridSearch::part_range_mpi(int n_tasks, int nprocs, int rank) + // { + // int assigned = 0; + // int remainder = n_tasks % nprocs; + // int start = 0; + // if (rank < remainder) { + // assigned = n_tasks / nprocs + 1; + // } else { + // assigned = n_tasks / nprocs; + // start = remainder; + // } + // start += rank * assigned; + // int end = start + assigned; + // if (rank == nprocs - 1) { + // end = n_tasks; + // } + // return { start, end }; + // } + json store_result(std::vector& names, Task_Result& result, json& results) { json json_result = { { "score", result.score }, @@ -253,11 +253,16 @@ namespace platform { { "time", result.time }, { "dataset", result.idx_dataset } }; + std::cout << "x Storing result for dataset " << result.idx_dataset << " from " << result.idx_combination << ::endl; + std::cout << json_result.dump() << std::endl; + std::cout << string(80, '-') << std::endl; auto name = names[result.idx_dataset]; if (!results.contains(name)) { results[name] = json::array(); } results[name].push_back(json_result); + std::cout << results.dump() << std::endl; + return results; } json producer(json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) { @@ -268,21 +273,27 @@ namespace platform { auto names = datasets.getNames(); for (int i = 0; i < num_tasks; ++i) { MPI_Status status; + std::cout << "+ Producer waiting for result." << std::endl; MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); if (status.MPI_TAG == TAG_RESULT) { //Store result + std::cout << "+ Producer received result from " << status.MPI_SOURCE << std::endl; store_result(names, result, results); } + std::cout << "+ Producer sending task " << i << " to " << status.MPI_SOURCE << std::endl; MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD); } - // Send end message to all workers - for (int i = 0; i < config_mpi.n_procs; ++i) { + // Send end message to all workers but the manager + for (int i = 0; i < config_mpi.n_procs - 1; ++i) { MPI_Status status; + std::cout << "+ Producer waiting for result (closing)." << std::endl; MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); if (status.MPI_TAG == TAG_RESULT) { //Store result + std::cout << "+ Producer received result from " << status.MPI_SOURCE << " (closing)" << std::endl; store_result(names, result, results); } + std::cout << "+ Producer sending end signal to " << status.MPI_SOURCE << std::endl; MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD); } return results; @@ -295,10 +306,13 @@ namespace platform { // // Select the best result of the computed outer folds // - for (const auto& result : results.items()) { + std::cout << "--- Selecting best results of the outer folds ---" << std::endl; + std::cout << all_results.dump() << std::endl; + for (const auto& result : all_results.items()) { // each result has the results of all the outer folds as each one were a different task double best_score = 0.0; json best; + std::cout << " Processing " << result.key() << std::endl; for (const auto& result_fold : result.value()) { double score = result_fold["score"].get(); if (score > best_score) { @@ -327,14 +341,17 @@ namespace platform { int task; while (true) { MPI_Status status; + std::cout << "- Consumer nº " << config_mpi.rank << " waiting for task." << std::endl; MPI_Recv(&task, 1, MPI_INT, config_mpi.manager, MPI_ANY_TAG, MPI_COMM_WORLD, &status); if (status.MPI_TAG == TAG_END) { break; } // Process task + std::cout << " - Consumer nº " << config_mpi.rank << " processing task " << task << std::endl; process_task_mpi_consumer(config, config_mpi, tasks, task, datasets, &result); // Send result to producer MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_RESULT, MPI_COMM_WORLD); + std::cout << " - Consumer nº " << config_mpi.rank << " sent task " << task << std::endl; } } void GridSearch::go_producer_consumer(struct ConfigMPI& config_mpi) @@ -409,419 +426,419 @@ namespace platform { // auto datasets = Datasets(config.discretize, Paths::datasets()); if (config_mpi.rank == config_mpi.manager) { - auto all_results = producer(tasks, config_mpi, MPI_Result); - auto results = select_best_results_folds(all_results, config.model); + json all_results = producer(tasks, config_mpi, MPI_Result); + json results = select_best_results_folds(all_results, config.model); save(results); } else { consumer(datasets, tasks, config, config_mpi, MPI_Result); } } - void GridSearch::go_mpi(struct ConfigMPI& config_mpi) - { - /* - * Each task is a json object with the following structure: - * { - * "dataset": "dataset_name", - * "idx_dataset": idx_dataset, - * "seed": # of seed to use, - * "Fold": # of fold to process - * } - * - * The overall process consists in these steps: - * 1. Manager will broadcast the tasks to all the processes - * 1.1 Broadcast the number of tasks - * 1.2 Broadcast the length of the following string - * 1.2 Broadcast the tasks as a char* string - * 2. Workers will receive the tasks and start the process - * 2.1 A method will tell each worker the range of tasks to process - * 2.2 Each worker will process the tasks and generate the best score for each task - * 3. Manager gather the scores from all the workers and find out the best hyperparameters for each dataset - * 3.1 Obtain the maximum size of the results message of all the workers - * 3.2 Gather all the results from the workers into the manager - * 3.3 Compile the results from all the workers - * 3.4 Filter the best hyperparameters for each dataset - */ - char* msg; - int tasks_size; - if (config_mpi.rank == config_mpi.manager) { - timer.start(); - auto tasks = build_tasks_mpi(); - auto tasks_str = tasks.dump(); - tasks_size = tasks_str.size(); - msg = new char[tasks_size + 1]; - strcpy(msg, tasks_str.c_str()); - } - // - // 1. Manager will broadcast the tasks to all the processes - // - MPI_Bcast(&tasks_size, 1, MPI_INT, config_mpi.manager, MPI_COMM_WORLD); - if (config_mpi.rank != config_mpi.manager) { - msg = new char[tasks_size + 1]; - } - MPI_Bcast(msg, tasks_size + 1, MPI_CHAR, config_mpi.manager, MPI_COMM_WORLD); - json tasks = json::parse(msg); - delete[] msg; - // - // 2. All Workers will receive the tasks and start the process - // - int num_tasks = tasks.size(); - // 2.1 A method will tell each worker the range of tasks to process - auto [start, end] = part_range_mpi(num_tasks, config_mpi.n_procs, config_mpi.rank); - // 2.2 Each worker will process the tasks and return the best scores obtained - auto datasets = Datasets(config.discretize, Paths::datasets()); - json results; - for (int i = start; i < end; ++i) { - // Process task - process_task_mpi(config_mpi, tasks[i], datasets, results); - } - int size = results.dump().size() + 1; - int max_size = 0; - // - // 3. Manager gather the scores from all the workers and find out the best hyperparameters for each dataset - // - //3.1 Obtain the maximum size of the results message of all the workers - MPI_Allreduce(&size, &max_size, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); - // Assign the memory to the message and initialize it to 0s - char* total = NULL; - msg = new char[max_size]; - strncpy(msg, results.dump().c_str(), size); - if (config_mpi.rank == config_mpi.manager) { - total = new char[max_size * config_mpi.n_procs]; - } - // 3.2 Gather all the results from the workers into the manager - MPI_Gather(msg, max_size, MPI_CHAR, total, max_size, MPI_CHAR, config_mpi.manager, MPI_COMM_WORLD); - delete[] msg; - if (config_mpi.rank == config_mpi.manager) { - std::cout << Colors::RESET() << "|" << std::endl; - json total_results; - json best_results; - // 3.3 Compile the results from all the workers - for (int i = 0; i < config_mpi.n_procs; ++i) { - json partial_results = json::parse(total + i * max_size); - for (auto& [dataset, folds] : partial_results.items()) { - for (auto& [fold, result] : folds.items()) { - total_results[dataset][fold] = result; - } - } - } - delete[] total; - // 3.4 Filter the best hyperparameters for each dataset - auto grid = GridData(Paths::grid_input(config.model)); - for (auto& [dataset, folds] : total_results.items()) { - double best_score = 0.0; - double duration = 0.0; - json best_hyper; - for (auto& [fold, result] : folds.items()) { - duration += result["duration"].get(); - if (result["score"] > best_score) { - best_score = result["score"]; - best_hyper = result["hyperparameters"]; - } - } - auto timer = Timer(); - json result = { - { "score", best_score }, - { "hyperparameters", best_hyper }, - { "date", get_date() + " " + get_time() }, - { "grid", grid.getInputGrid(dataset) }, - { "duration", timer.translate2String(duration) } - }; - best_results[dataset] = result; - } - save(best_results); - } - } - void GridSearch::go() - { - timer.start(); - auto grid_type = config.nested == 0 ? "Single" : "Nested"; - auto datasets = Datasets(config.discretize, Paths::datasets()); - auto datasets_names = processDatasets(datasets); - json results = initializeResults(); - std::cout << "***************** Starting " << grid_type << " Gridsearch *****************" << std::endl; - std::cout << "input file=" << Paths::grid_input(config.model) << std::endl; - auto grid = GridData(Paths::grid_input(config.model)); - Timer timer_dataset; - double bestScore = 0; - json bestHyperparameters; - for (const auto& dataset : datasets_names) { - if (!config.quiet) - std::cout << "- " << setw(20) << left << dataset << " " << right << flush; - auto combinations = grid.getGrid(dataset); - timer_dataset.start(); - if (config.nested == 0) - // for dataset // for hyperparameters // for seed // for fold - tie(bestScore, bestHyperparameters) = processFileSingle(dataset, datasets, combinations); - else - // for dataset // for seed // for fold // for hyperparameters // for nested fold - tie(bestScore, bestHyperparameters) = processFileNested(dataset, datasets, combinations); - if (!config.quiet) { - std::cout << "end." << " Score: " << Colors::IBLUE() << setw(9) << setprecision(7) << fixed - << bestScore << Colors::BLUE() << " [" << bestHyperparameters.dump() << "]" - << Colors::RESET() << ::endl; - } - json result = { - { "score", bestScore }, - { "hyperparameters", bestHyperparameters }, - { "date", get_date() + " " + get_time() }, - { "grid", grid.getInputGrid(dataset) }, - { "duration", timer_dataset.getDurationString() } - }; - results[dataset] = result; - // Save partial results - save(results); - } - // Save final results - save(results); - std::cout << "***************** Ending " << grid_type << " Gridsearch *******************" << std::endl; - } - pair GridSearch::processFileSingle(std::string fileName, Datasets& datasets, vector& combinations) - { - int num = 0; - double bestScore = 0.0; - json bestHyperparameters; - auto totalComb = combinations.size(); - for (const auto& hyperparam_line : combinations) { - if (!config.quiet) - showProgressComb(++num, config.n_folds, totalComb, Colors::CYAN()); - auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); - // Get dataset - auto [X, y] = datasets.getTensors(fileName); - auto states = datasets.getStates(fileName); - auto features = datasets.getFeatures(fileName); - auto className = datasets.getClassName(fileName); - double totalScore = 0.0; - int numItems = 0; - for (const auto& seed : config.seeds) { - if (!config.quiet) - std::cout << "(" << seed << ") doing Fold: " << flush; - Fold* fold; - if (config.stratified) - fold = new StratifiedKFold(config.n_folds, y, seed); - else - fold = new KFold(config.n_folds, y.size(0), seed); - for (int nfold = 0; nfold < config.n_folds; nfold++) { - auto clf = Models::instance()->create(config.model); - auto valid = clf->getValidHyperparameters(); - hyperparameters.check(valid, fileName); - clf->setHyperparameters(hyperparameters.get(fileName)); - auto [train, test] = fold->getFold(nfold); - auto train_t = torch::tensor(train); - auto test_t = torch::tensor(test); - auto X_train = X.index({ "...", train_t }); - auto y_train = y.index({ train_t }); - auto X_test = X.index({ "...", test_t }); - auto y_test = y.index({ test_t }); - // Train model - if (!config.quiet) - showProgressFold(nfold + 1, getColor(clf->getStatus()), "a"); - clf->fit(X_train, y_train, features, className, states); - // Test model - if (!config.quiet) - showProgressFold(nfold + 1, getColor(clf->getStatus()), "b"); - totalScore += clf->score(X_test, y_test); - numItems++; - if (!config.quiet) - std::cout << "\b\b\b, \b" << flush; - } - delete fold; - } - double score = numItems == 0 ? 0.0 : totalScore / numItems; - if (score > bestScore) { - bestScore = score; - bestHyperparameters = hyperparam_line; - } - } - return { bestScore, bestHyperparameters }; - } - pair GridSearch::processFileNested(std::string fileName, Datasets& datasets, vector& combinations) - { - // Get dataset - auto [X, y] = datasets.getTensors(fileName); - auto states = datasets.getStates(fileName); - auto features = datasets.getFeatures(fileName); - auto className = datasets.getClassName(fileName); - int spcs_combinations = int(log(combinations.size()) / log(10)) + 1; - double goatScore = 0.0; - json goatHyperparameters; - // for dataset // for seed // for fold // for hyperparameters // for nested fold - for (const auto& seed : config.seeds) { - Fold* fold; - if (config.stratified) - fold = new StratifiedKFold(config.n_folds, y, seed); - else - fold = new KFold(config.n_folds, y.size(0), seed); - double bestScore = 0.0; - json bestHyperparameters; - std::cout << "(" << seed << ") doing Fold: " << flush; - for (int nfold = 0; nfold < config.n_folds; nfold++) { - if (!config.quiet) - std::cout << Colors::GREEN() << nfold + 1 << " " << flush; - // First level fold - auto [train, test] = fold->getFold(nfold); - auto train_t = torch::tensor(train); - auto test_t = torch::tensor(test); - auto X_train = X.index({ "...", train_t }); - auto y_train = y.index({ train_t }); - auto X_test = X.index({ "...", test_t }); - auto y_test = y.index({ test_t }); - auto num = 0; - json result_fold; - double hypScore = 0.0; - double bestHypScore = 0.0; - json bestHypHyperparameters; - for (const auto& hyperparam_line : combinations) { - std::cout << "[" << setw(spcs_combinations) << ++num << "/" << setw(spcs_combinations) - << combinations.size() << "] " << std::flush; - Fold* nested_fold; - if (config.stratified) - nested_fold = new StratifiedKFold(config.nested, y_train, seed); - else - nested_fold = new KFold(config.nested, y_train.size(0), seed); - for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) { - // Nested level fold - auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold); - auto train_nested_t = torch::tensor(train_nested); - auto test_nested_t = torch::tensor(test_nested); - auto X_nexted_train = X_train.index({ "...", train_nested_t }); - auto y_nested_train = y_train.index({ train_nested_t }); - auto X_nested_test = X_train.index({ "...", test_nested_t }); - auto y_nested_test = y_train.index({ test_nested_t }); - // Build Classifier with selected hyperparameters - auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); - auto clf = Models::instance()->create(config.model); - auto valid = clf->getValidHyperparameters(); - hyperparameters.check(valid, fileName); - clf->setHyperparameters(hyperparameters.get(fileName)); - // Train model - if (!config.quiet) - showProgressFold(n_nested_fold + 1, getColor(clf->getStatus()), "a"); - clf->fit(X_nexted_train, y_nested_train, features, className, states); - // Test model - if (!config.quiet) - showProgressFold(n_nested_fold + 1, getColor(clf->getStatus()), "b"); - hypScore += clf->score(X_nested_test, y_nested_test); - if (!config.quiet) - std::cout << "\b\b\b, \b" << flush; - } - int magic = 3 * config.nested + 2 * spcs_combinations + 4; - std::cout << string(magic, '\b') << string(magic, ' ') << string(magic, '\b') << flush; - delete nested_fold; - hypScore /= config.nested; - if (hypScore > bestHypScore) { - bestHypScore = hypScore; - bestHypHyperparameters = hyperparam_line; - } - } - // Build Classifier with selected hyperparameters - auto clf = Models::instance()->create(config.model); - clf->setHyperparameters(bestHypHyperparameters); - // Train model - if (!config.quiet) - showProgressFold(nfold + 1, getColor(clf->getStatus()), "a"); - clf->fit(X_train, y_train, features, className, states); - // Test model - if (!config.quiet) - showProgressFold(nfold + 1, getColor(clf->getStatus()), "b"); - double score = clf->score(X_test, y_test); - if (!config.quiet) - std::cout << string(2 * config.nested - 1, '\b') << "," << string(2 * config.nested, ' ') << string(2 * config.nested - 1, '\b') << flush; - if (score > bestScore) { - bestScore = score; - bestHyperparameters = bestHypHyperparameters; - } - } - if (bestScore > goatScore) { - goatScore = bestScore; - goatHyperparameters = bestHyperparameters; - } - delete fold; - } - return { goatScore, goatHyperparameters }; - } - void GridSearch::process_task_mpi(struct ConfigMPI& config_mpi, json& task, Datasets& datasets, json& results) - { - // Process the task and store the result in the results json - Timer timer; - timer.start(); - auto grid = GridData(Paths::grid_input(config.model)); - auto dataset = task["dataset"].get(); - auto seed = task["seed"].get(); - auto n_fold = task["fold"].get(); - // Generate the hyperparamters combinations - auto combinations = grid.getGrid(dataset); - auto [X, y] = datasets.getTensors(dataset); - auto states = datasets.getStates(dataset); - auto features = datasets.getFeatures(dataset); - auto className = datasets.getClassName(dataset); - // - // Start working on task - // - Fold* fold; - if (config.stratified) - fold = new StratifiedKFold(config.n_folds, y, seed); - else - fold = new KFold(config.n_folds, y.size(0), seed); - auto [train, test] = fold->getFold(n_fold); - auto train_t = torch::tensor(train); - auto test_t = torch::tensor(test); - auto X_train = X.index({ "...", train_t }); - auto y_train = y.index({ train_t }); - auto X_test = X.index({ "...", test_t }); - auto y_test = y.index({ test_t }); - auto num = 0; - double best_fold_score = 0.0; - json best_fold_hyper; - for (const auto& hyperparam_line : combinations) { - auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); - Fold* nested_fold; - if (config.stratified) - nested_fold = new StratifiedKFold(config.nested, y_train, seed); - else - nested_fold = new KFold(config.nested, y_train.size(0), seed); - double score = 0.0; - for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) { - // Nested level fold - auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold); - auto train_nested_t = torch::tensor(train_nested); - auto test_nested_t = torch::tensor(test_nested); - auto X_nested_train = X_train.index({ "...", train_nested_t }); - auto y_nested_train = y_train.index({ train_nested_t }); - auto X_nested_test = X_train.index({ "...", test_nested_t }); - auto y_nested_test = y_train.index({ test_nested_t }); - // Build Classifier with selected hyperparameters - auto clf = Models::instance()->create(config.model); - auto valid = clf->getValidHyperparameters(); - hyperparameters.check(valid, dataset); - clf->setHyperparameters(hyperparameters.get(dataset)); - // Train model - clf->fit(X_nested_train, y_nested_train, features, className, states); - // Test model - score += clf->score(X_nested_test, y_nested_test); - } - delete nested_fold; - score /= config.nested; - if (score > best_fold_score) { - best_fold_score = score; - best_fold_hyper = hyperparam_line; - } - } - delete fold; - // Build Classifier with the best hyperparameters to obtain the best score - auto hyperparameters = platform::HyperParameters(datasets.getNames(), best_fold_hyper); - auto clf = Models::instance()->create(config.model); - auto valid = clf->getValidHyperparameters(); - hyperparameters.check(valid, dataset); - clf->setHyperparameters(best_fold_hyper); - clf->fit(X_train, y_train, features, className, states); - best_fold_score = clf->score(X_test, y_test); - // Save results - results[dataset][std::to_string(n_fold)]["score"] = best_fold_score; - results[dataset][std::to_string(n_fold)]["hyperparameters"] = best_fold_hyper; - results[dataset][std::to_string(n_fold)]["seed"] = seed; - results[dataset][std::to_string(n_fold)]["duration"] = timer.getDuration(); - std::cout << get_color_rank(config_mpi.rank) << "*" << std::flush; - } + // void GridSearch::go_mpi(struct ConfigMPI& config_mpi) + // { + // /* + // * Each task is a json object with the following structure: + // * { + // * "dataset": "dataset_name", + // * "seed": # of seed to use, + // * "model": "model_name", + // * "Fold": # of fold to process + // * } + // * + // * The overall process consists in these steps: + // * 1. Manager will broadcast the tasks to all the processes + // * 1.1 Broadcast the number of tasks + // * 1.2 Broadcast the length of the following string + // * 1.2 Broadcast the tasks as a char* string + // * 2. Workers will receive the tasks and start the process + // * 2.1 A method will tell each worker the range of tasks to process + // * 2.2 Each worker will process the tasks and generate the best score for each task + // * 3. Manager gather the scores from all the workers and find out the best hyperparameters for each dataset + // * 3.1 Obtain the maximum size of the results message of all the workers + // * 3.2 Gather all the results from the workers into the manager + // * 3.3 Compile the results from all the workers + // * 3.4 Filter the best hyperparameters for each dataset + // */ + // char* msg; + // int tasks_size; + // if (config_mpi.rank == config_mpi.manager) { + // timer.start(); + // auto tasks = build_tasks_mpi(); + // auto tasks_str = tasks.dump(); + // tasks_size = tasks_str.size(); + // msg = new char[tasks_size + 1]; + // strcpy(msg, tasks_str.c_str()); + // } + // // + // // 1. Manager will broadcast the tasks to all the processes + // // + // MPI_Bcast(&tasks_size, 1, MPI_INT, config_mpi.manager, MPI_COMM_WORLD); + // if (config_mpi.rank != config_mpi.manager) { + // msg = new char[tasks_size + 1]; + // } + // MPI_Bcast(msg, tasks_size + 1, MPI_CHAR, config_mpi.manager, MPI_COMM_WORLD); + // json tasks = json::parse(msg); + // delete[] msg; + // // + // // 2. All Workers will receive the tasks and start the process + // // + // int num_tasks = tasks.size(); + // // 2.1 A method will tell each worker the range of tasks to process + // auto [start, end] = part_range_mpi(num_tasks, config_mpi.n_procs, config_mpi.rank); + // // 2.2 Each worker will process the tasks and return the best scores obtained + // auto datasets = Datasets(config.discretize, Paths::datasets()); + // json results; + // for (int i = start; i < end; ++i) { + // // Process task + // process_task_mpi(config_mpi, tasks[i], datasets, results); + // } + // int size = results.dump().size() + 1; + // int max_size = 0; + // // + // // 3. Manager gather the scores from all the workers and find out the best hyperparameters for each dataset + // // + // //3.1 Obtain the maximum size of the results message of all the workers + // MPI_Allreduce(&size, &max_size, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + // // Assign the memory to the message and initialize it to 0s + // char* total = NULL; + // msg = new char[max_size]; + // strncpy(msg, results.dump().c_str(), size); + // if (config_mpi.rank == config_mpi.manager) { + // total = new char[max_size * config_mpi.n_procs]; + // } + // // 3.2 Gather all the results from the workers into the manager + // MPI_Gather(msg, max_size, MPI_CHAR, total, max_size, MPI_CHAR, config_mpi.manager, MPI_COMM_WORLD); + // delete[] msg; + // if (config_mpi.rank == config_mpi.manager) { + // std::cout << Colors::RESET() << "|" << std::endl; + // json total_results; + // json best_results; + // // 3.3 Compile the results from all the workers + // for (int i = 0; i < config_mpi.n_procs; ++i) { + // json partial_results = json::parse(total + i * max_size); + // for (auto& [dataset, folds] : partial_results.items()) { + // for (auto& [fold, result] : folds.items()) { + // total_results[dataset][fold] = result; + // } + // } + // } + // delete[] total; + // // 3.4 Filter the best hyperparameters for each dataset + // auto grid = GridData(Paths::grid_input(config.model)); + // for (auto& [dataset, folds] : total_results.items()) { + // double best_score = 0.0; + // double duration = 0.0; + // json best_hyper; + // for (auto& [fold, result] : folds.items()) { + // duration += result["duration"].get(); + // if (result["score"] > best_score) { + // best_score = result["score"]; + // best_hyper = result["hyperparameters"]; + // } + // } + // auto timer = Timer(); + // json result = { + // { "score", best_score }, + // { "hyperparameters", best_hyper }, + // { "date", get_date() + " " + get_time() }, + // { "grid", grid.getInputGrid(dataset) }, + // { "duration", timer.translate2String(duration) } + // }; + // best_results[dataset] = result; + // } + // save(best_results); + // } + // } + // void GridSearch::go() + // { + // timer.start(); + // auto grid_type = config.nested == 0 ? "Single" : "Nested"; + // auto datasets = Datasets(config.discretize, Paths::datasets()); + // auto datasets_names = processDatasets(datasets); + // json results = initializeResults(); + // std::cout << "***************** Starting " << grid_type << " Gridsearch *****************" << std::endl; + // std::cout << "input file=" << Paths::grid_input(config.model) << std::endl; + // auto grid = GridData(Paths::grid_input(config.model)); + // Timer timer_dataset; + // double bestScore = 0; + // json bestHyperparameters; + // for (const auto& dataset : datasets_names) { + // if (!config.quiet) + // std::cout << "- " << setw(20) << left << dataset << " " << right << flush; + // auto combinations = grid.getGrid(dataset); + // timer_dataset.start(); + // if (config.nested == 0) + // // for dataset // for hyperparameters // for seed // for fold + // tie(bestScore, bestHyperparameters) = processFileSingle(dataset, datasets, combinations); + // else + // // for dataset // for seed // for fold // for hyperparameters // for nested fold + // tie(bestScore, bestHyperparameters) = processFileNested(dataset, datasets, combinations); + // if (!config.quiet) { + // std::cout << "end." << " Score: " << Colors::IBLUE() << setw(9) << setprecision(7) << fixed + // << bestScore << Colors::BLUE() << " [" << bestHyperparameters.dump() << "]" + // << Colors::RESET() << ::endl; + // } + // json result = { + // { "score", bestScore }, + // { "hyperparameters", bestHyperparameters }, + // { "date", get_date() + " " + get_time() }, + // { "grid", grid.getInputGrid(dataset) }, + // { "duration", timer_dataset.getDurationString() } + // }; + // results[dataset] = result; + // // Save partial results + // save(results); + // } + // // Save final results + // save(results); + // std::cout << "***************** Ending " << grid_type << " Gridsearch *******************" << std::endl; + // } + // pair GridSearch::processFileSingle(std::string fileName, Datasets& datasets, vector& combinations) + // { + // int num = 0; + // double bestScore = 0.0; + // json bestHyperparameters; + // auto totalComb = combinations.size(); + // for (const auto& hyperparam_line : combinations) { + // if (!config.quiet) + // showProgressComb(++num, config.n_folds, totalComb, Colors::CYAN()); + // auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); + // // Get dataset + // auto [X, y] = datasets.getTensors(fileName); + // auto states = datasets.getStates(fileName); + // auto features = datasets.getFeatures(fileName); + // auto className = datasets.getClassName(fileName); + // double totalScore = 0.0; + // int numItems = 0; + // for (const auto& seed : config.seeds) { + // if (!config.quiet) + // std::cout << "(" << seed << ") doing Fold: " << flush; + // Fold* fold; + // if (config.stratified) + // fold = new StratifiedKFold(config.n_folds, y, seed); + // else + // fold = new KFold(config.n_folds, y.size(0), seed); + // for (int nfold = 0; nfold < config.n_folds; nfold++) { + // auto clf = Models::instance()->create(config.model); + // auto valid = clf->getValidHyperparameters(); + // hyperparameters.check(valid, fileName); + // clf->setHyperparameters(hyperparameters.get(fileName)); + // auto [train, test] = fold->getFold(nfold); + // auto train_t = torch::tensor(train); + // auto test_t = torch::tensor(test); + // auto X_train = X.index({ "...", train_t }); + // auto y_train = y.index({ train_t }); + // auto X_test = X.index({ "...", test_t }); + // auto y_test = y.index({ test_t }); + // // Train model + // if (!config.quiet) + // showProgressFold(nfold + 1, getColor(clf->getStatus()), "a"); + // clf->fit(X_train, y_train, features, className, states); + // // Test model + // if (!config.quiet) + // showProgressFold(nfold + 1, getColor(clf->getStatus()), "b"); + // totalScore += clf->score(X_test, y_test); + // numItems++; + // if (!config.quiet) + // std::cout << "\b\b\b, \b" << flush; + // } + // delete fold; + // } + // double score = numItems == 0 ? 0.0 : totalScore / numItems; + // if (score > bestScore) { + // bestScore = score; + // bestHyperparameters = hyperparam_line; + // } + // } + // return { bestScore, bestHyperparameters }; + // } + // pair GridSearch::processFileNested(std::string fileName, Datasets& datasets, vector& combinations) + // { + // // Get dataset + // auto [X, y] = datasets.getTensors(fileName); + // auto states = datasets.getStates(fileName); + // auto features = datasets.getFeatures(fileName); + // auto className = datasets.getClassName(fileName); + // int spcs_combinations = int(log(combinations.size()) / log(10)) + 1; + // double goatScore = 0.0; + // json goatHyperparameters; + // // for dataset // for seed // for fold // for hyperparameters // for nested fold + // for (const auto& seed : config.seeds) { + // Fold* fold; + // if (config.stratified) + // fold = new StratifiedKFold(config.n_folds, y, seed); + // else + // fold = new KFold(config.n_folds, y.size(0), seed); + // double bestScore = 0.0; + // json bestHyperparameters; + // std::cout << "(" << seed << ") doing Fold: " << flush; + // for (int nfold = 0; nfold < config.n_folds; nfold++) { + // if (!config.quiet) + // std::cout << Colors::GREEN() << nfold + 1 << " " << flush; + // // First level fold + // auto [train, test] = fold->getFold(nfold); + // auto train_t = torch::tensor(train); + // auto test_t = torch::tensor(test); + // auto X_train = X.index({ "...", train_t }); + // auto y_train = y.index({ train_t }); + // auto X_test = X.index({ "...", test_t }); + // auto y_test = y.index({ test_t }); + // auto num = 0; + // json result_fold; + // double hypScore = 0.0; + // double bestHypScore = 0.0; + // json bestHypHyperparameters; + // for (const auto& hyperparam_line : combinations) { + // std::cout << "[" << setw(spcs_combinations) << ++num << "/" << setw(spcs_combinations) + // << combinations.size() << "] " << std::flush; + // Fold* nested_fold; + // if (config.stratified) + // nested_fold = new StratifiedKFold(config.nested, y_train, seed); + // else + // nested_fold = new KFold(config.nested, y_train.size(0), seed); + // for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) { + // // Nested level fold + // auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold); + // auto train_nested_t = torch::tensor(train_nested); + // auto test_nested_t = torch::tensor(test_nested); + // auto X_nexted_train = X_train.index({ "...", train_nested_t }); + // auto y_nested_train = y_train.index({ train_nested_t }); + // auto X_nested_test = X_train.index({ "...", test_nested_t }); + // auto y_nested_test = y_train.index({ test_nested_t }); + // // Build Classifier with selected hyperparameters + // auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); + // auto clf = Models::instance()->create(config.model); + // auto valid = clf->getValidHyperparameters(); + // hyperparameters.check(valid, fileName); + // clf->setHyperparameters(hyperparameters.get(fileName)); + // // Train model + // if (!config.quiet) + // showProgressFold(n_nested_fold + 1, getColor(clf->getStatus()), "a"); + // clf->fit(X_nexted_train, y_nested_train, features, className, states); + // // Test model + // if (!config.quiet) + // showProgressFold(n_nested_fold + 1, getColor(clf->getStatus()), "b"); + // hypScore += clf->score(X_nested_test, y_nested_test); + // if (!config.quiet) + // std::cout << "\b\b\b, \b" << flush; + // } + // int magic = 3 * config.nested + 2 * spcs_combinations + 4; + // std::cout << string(magic, '\b') << string(magic, ' ') << string(magic, '\b') << flush; + // delete nested_fold; + // hypScore /= config.nested; + // if (hypScore > bestHypScore) { + // bestHypScore = hypScore; + // bestHypHyperparameters = hyperparam_line; + // } + // } + // // Build Classifier with selected hyperparameters + // auto clf = Models::instance()->create(config.model); + // clf->setHyperparameters(bestHypHyperparameters); + // // Train model + // if (!config.quiet) + // showProgressFold(nfold + 1, getColor(clf->getStatus()), "a"); + // clf->fit(X_train, y_train, features, className, states); + // // Test model + // if (!config.quiet) + // showProgressFold(nfold + 1, getColor(clf->getStatus()), "b"); + // double score = clf->score(X_test, y_test); + // if (!config.quiet) + // std::cout << string(2 * config.nested - 1, '\b') << "," << string(2 * config.nested, ' ') << string(2 * config.nested - 1, '\b') << flush; + // if (score > bestScore) { + // bestScore = score; + // bestHyperparameters = bestHypHyperparameters; + // } + // } + // if (bestScore > goatScore) { + // goatScore = bestScore; + // goatHyperparameters = bestHyperparameters; + // } + // delete fold; + // } + // return { goatScore, goatHyperparameters }; + // } + // void GridSearch::process_task_mpi(struct ConfigMPI& config_mpi, json& task, Datasets& datasets, json& results) + // { + // // Process the task and store the result in the results json + // Timer timer; + // timer.start(); + // auto grid = GridData(Paths::grid_input(config.model)); + // auto dataset = task["dataset"].get(); + // auto seed = task["seed"].get(); + // auto n_fold = task["fold"].get(); + // // Generate the hyperparamters combinations + // auto combinations = grid.getGrid(dataset); + // auto [X, y] = datasets.getTensors(dataset); + // auto states = datasets.getStates(dataset); + // auto features = datasets.getFeatures(dataset); + // auto className = datasets.getClassName(dataset); + // // + // // Start working on task + // // + // Fold* fold; + // if (config.stratified) + // fold = new StratifiedKFold(config.n_folds, y, seed); + // else + // fold = new KFold(config.n_folds, y.size(0), seed); + // auto [train, test] = fold->getFold(n_fold); + // auto train_t = torch::tensor(train); + // auto test_t = torch::tensor(test); + // auto X_train = X.index({ "...", train_t }); + // auto y_train = y.index({ train_t }); + // auto X_test = X.index({ "...", test_t }); + // auto y_test = y.index({ test_t }); + // auto num = 0; + // double best_fold_score = 0.0; + // json best_fold_hyper; + // for (const auto& hyperparam_line : combinations) { + // auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); + // Fold* nested_fold; + // if (config.stratified) + // nested_fold = new StratifiedKFold(config.nested, y_train, seed); + // else + // nested_fold = new KFold(config.nested, y_train.size(0), seed); + // double score = 0.0; + // for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) { + // // Nested level fold + // auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold); + // auto train_nested_t = torch::tensor(train_nested); + // auto test_nested_t = torch::tensor(test_nested); + // auto X_nested_train = X_train.index({ "...", train_nested_t }); + // auto y_nested_train = y_train.index({ train_nested_t }); + // auto X_nested_test = X_train.index({ "...", test_nested_t }); + // auto y_nested_test = y_train.index({ test_nested_t }); + // // Build Classifier with selected hyperparameters + // auto clf = Models::instance()->create(config.model); + // auto valid = clf->getValidHyperparameters(); + // hyperparameters.check(valid, dataset); + // clf->setHyperparameters(hyperparameters.get(dataset)); + // // Train model + // clf->fit(X_nested_train, y_nested_train, features, className, states); + // // Test model + // score += clf->score(X_nested_test, y_nested_test); + // } + // delete nested_fold; + // score /= config.nested; + // if (score > best_fold_score) { + // best_fold_score = score; + // best_fold_hyper = hyperparam_line; + // } + // } + // delete fold; + // // Build Classifier with the best hyperparameters to obtain the best score + // auto hyperparameters = platform::HyperParameters(datasets.getNames(), best_fold_hyper); + // auto clf = Models::instance()->create(config.model); + // auto valid = clf->getValidHyperparameters(); + // hyperparameters.check(valid, dataset); + // clf->setHyperparameters(best_fold_hyper); + // clf->fit(X_train, y_train, features, className, states); + // best_fold_score = clf->score(X_test, y_test); + // // Save results + // results[dataset][std::to_string(n_fold)]["score"] = best_fold_score; + // results[dataset][std::to_string(n_fold)]["hyperparameters"] = best_fold_hyper; + // results[dataset][std::to_string(n_fold)]["seed"] = seed; + // results[dataset][std::to_string(n_fold)]["duration"] = timer.getDuration(); + // std::cout << get_color_rank(config_mpi.rank) << "*" << std::flush; + // } json GridSearch::initializeResults() { // Load previous results diff --git a/src/Platform/GridSearch.h b/src/Platform/GridSearch.h index 08b874d..05ece4f 100644 --- a/src/Platform/GridSearch.h +++ b/src/Platform/GridSearch.h @@ -44,22 +44,22 @@ namespace platform { class GridSearch { public: explicit GridSearch(struct ConfigGrid& config); - void go(); - void go_mpi(struct ConfigMPI& config_mpi); + // void go(); + // void go_mpi(struct ConfigMPI& config_mpi); void go_producer_consumer(struct ConfigMPI& config_mpi); ~GridSearch() = default; - json getResults(); + json loadResults(); static inline std::string NO_CONTINUE() { return "NO_CONTINUE"; } private: void save(json& results); json initializeResults(); - vector processDatasets(Datasets& datasets) const; - pair processFileSingle(std::string fileName, Datasets& datasets, std::vector& combinations); - pair processFileNested(std::string fileName, Datasets& datasets, std::vector& combinations); + vector filterDatasets(Datasets& datasets) const; + // pair processFileSingle(std::string fileName, Datasets& datasets, std::vector& combinations); + // pair processFileNested(std::string fileName, Datasets& datasets, std::vector& combinations); struct ConfigGrid config; - pair part_range_mpi(int n_tasks, int nprocs, int rank); + // pair part_range_mpi(int n_tasks, int nprocs, int rank); json build_tasks_mpi(); - void process_task_mpi(struct ConfigMPI& config_mpi, json& task, Datasets& datasets, json& results); + // void process_task_mpi(struct ConfigMPI& config_mpi, json& task, Datasets& datasets, json& results); Timer timer; // used to measure the time of the whole process }; } /* namespace platform */ diff --git a/src/Platform/b_grid.cc b/src/Platform/b_grid.cc index 055da26..1d42543 100644 --- a/src/Platform/b_grid.cc +++ b/src/Platform/b_grid.cc @@ -203,18 +203,18 @@ int main(int argc, char** argv) MPI_Comm_size(MPI_COMM_WORLD, &mpi_config.n_procs); grid_search.go_producer_consumer(mpi_config); if (mpi_config.rank == mpi_config.manager) { - auto results = grid_search.getResults(); + auto results = grid_search.loadResults(); list_results(results, config.model); std::cout << "Process took " << timer.getDurationString() << std::endl; } MPI_Finalize(); - } else { - grid_search.go(); - std::cout << "Process took " << timer.getDurationString() << std::endl; + // } else { + // grid_search.go(); + // std::cout << "Process took " << timer.getDurationString() << std::endl; } } else { // List results - auto results = grid_search.getResults(); + auto results = grid_search.loadResults(); if (results.empty()) { std::cout << "** No results found" << std::endl; } else { From 41a0bd4ddd5456e8ccc7403103353f3b8b2b6e2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana=20G=C3=B3mez?= Date: Wed, 3 Jan 2024 17:15:57 +0100 Subject: [PATCH 10/13] fix dataset name mistakes --- src/Platform/GridSearch.cc | 29 +++++++---------------------- src/Platform/GridSearch.h | 2 +- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/src/Platform/GridSearch.cc b/src/Platform/GridSearch.cc index 7cb36df..b3a8b2f 100644 --- a/src/Platform/GridSearch.cc +++ b/src/Platform/GridSearch.cc @@ -44,7 +44,7 @@ namespace platform { } return json(); } - vector GridSearch::filterDatasets(Datasets& datasets) const + std::vector GridSearch::filterDatasets(Datasets& datasets) const { // Load datasets auto datasets_names = datasets.getNames(); @@ -54,7 +54,7 @@ namespace platform { throw std::invalid_argument("Dataset " + config.continue_from + " not found"); } // Remove datasets already processed - vector< string >::iterator it = datasets_names.begin(); + std::vector::iterator it = datasets_names.begin(); while (it != datasets_names.end()) { if (*it != config.continue_from) { it = datasets_names.erase(it); @@ -253,47 +253,36 @@ namespace platform { { "time", result.time }, { "dataset", result.idx_dataset } }; - std::cout << "x Storing result for dataset " << result.idx_dataset << " from " << result.idx_combination << ::endl; - std::cout << json_result.dump() << std::endl; - std::cout << string(80, '-') << std::endl; auto name = names[result.idx_dataset]; if (!results.contains(name)) { results[name] = json::array(); } results[name].push_back(json_result); - std::cout << results.dump() << std::endl; return results; } - json producer(json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) + json producer(std::vector& names, json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) { Task_Result result; json results; int num_tasks = tasks.size(); - auto datasets = Datasets(false, Paths::datasets()); - auto names = datasets.getNames(); + for (int i = 0; i < num_tasks; ++i) { MPI_Status status; - std::cout << "+ Producer waiting for result." << std::endl; MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); if (status.MPI_TAG == TAG_RESULT) { //Store result - std::cout << "+ Producer received result from " << status.MPI_SOURCE << std::endl; store_result(names, result, results); } - std::cout << "+ Producer sending task " << i << " to " << status.MPI_SOURCE << std::endl; MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD); } // Send end message to all workers but the manager for (int i = 0; i < config_mpi.n_procs - 1; ++i) { MPI_Status status; - std::cout << "+ Producer waiting for result (closing)." << std::endl; MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); if (status.MPI_TAG == TAG_RESULT) { //Store result - std::cout << "+ Producer received result from " << status.MPI_SOURCE << " (closing)" << std::endl; store_result(names, result, results); } - std::cout << "+ Producer sending end signal to " << status.MPI_SOURCE << std::endl; MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD); } return results; @@ -306,13 +295,10 @@ namespace platform { // // Select the best result of the computed outer folds // - std::cout << "--- Selecting best results of the outer folds ---" << std::endl; - std::cout << all_results.dump() << std::endl; for (const auto& result : all_results.items()) { // each result has the results of all the outer folds as each one were a different task double best_score = 0.0; json best; - std::cout << " Processing " << result.key() << std::endl; for (const auto& result_fold : result.value()) { double score = result_fold["score"].get(); if (score > best_score) { @@ -341,17 +327,14 @@ namespace platform { int task; while (true) { MPI_Status status; - std::cout << "- Consumer nº " << config_mpi.rank << " waiting for task." << std::endl; MPI_Recv(&task, 1, MPI_INT, config_mpi.manager, MPI_ANY_TAG, MPI_COMM_WORLD, &status); if (status.MPI_TAG == TAG_END) { break; } // Process task - std::cout << " - Consumer nº " << config_mpi.rank << " processing task " << task << std::endl; process_task_mpi_consumer(config, config_mpi, tasks, task, datasets, &result); // Send result to producer MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_RESULT, MPI_COMM_WORLD); - std::cout << " - Consumer nº " << config_mpi.rank << " sent task " << task << std::endl; } } void GridSearch::go_producer_consumer(struct ConfigMPI& config_mpi) @@ -426,9 +409,11 @@ namespace platform { // auto datasets = Datasets(config.discretize, Paths::datasets()); if (config_mpi.rank == config_mpi.manager) { - json all_results = producer(tasks, config_mpi, MPI_Result); + auto datasets_names = filterDatasets(datasets); + json all_results = producer(datasets_names, tasks, config_mpi, MPI_Result); json results = select_best_results_folds(all_results, config.model); save(results); + std::cout << "|" << std::endl; } else { consumer(datasets, tasks, config, config_mpi, MPI_Result); } diff --git a/src/Platform/GridSearch.h b/src/Platform/GridSearch.h index 05ece4f..36760b6 100644 --- a/src/Platform/GridSearch.h +++ b/src/Platform/GridSearch.h @@ -53,7 +53,7 @@ namespace platform { private: void save(json& results); json initializeResults(); - vector filterDatasets(Datasets& datasets) const; + std::vector filterDatasets(Datasets& datasets) const; // pair processFileSingle(std::string fileName, Datasets& datasets, std::vector& combinations); // pair processFileNested(std::string fileName, Datasets& datasets, std::vector& combinations); struct ConfigGrid config; From b1833a5feb9dde7d3d146aa32b3241afa97bc14d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana=20G=C3=B3mez?= Date: Wed, 3 Jan 2024 22:45:16 +0100 Subject: [PATCH 11/13] Add reset color to final progress bar --- src/Platform/GridSearch.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Platform/GridSearch.cc b/src/Platform/GridSearch.cc index b3a8b2f..94bd617 100644 --- a/src/Platform/GridSearch.cc +++ b/src/Platform/GridSearch.cc @@ -413,7 +413,7 @@ namespace platform { json all_results = producer(datasets_names, tasks, config_mpi, MPI_Result); json results = select_best_results_folds(all_results, config.model); save(results); - std::cout << "|" << std::endl; + std::cout << Colors::RESET() << "|" << std::endl; } else { consumer(datasets, tasks, config, config_mpi, MPI_Result); } From 722da7f7814eb5302417cb47b1da9de3fc15051a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana=20G=C3=B3mez?= Date: Thu, 4 Jan 2024 01:21:56 +0100 Subject: [PATCH 12/13] Keep only mpi b_grid compute --- src/Platform/GridSearch.cc | 527 ++++--------------------------------- src/Platform/GridSearch.h | 10 +- src/Platform/b_grid.cc | 43 ++- 3 files changed, 69 insertions(+), 511 deletions(-) diff --git a/src/Platform/GridSearch.cc b/src/Platform/GridSearch.cc index 94bd617..3e9ae3d 100644 --- a/src/Platform/GridSearch.cc +++ b/src/Platform/GridSearch.cc @@ -30,7 +30,7 @@ namespace platform { } std::string get_color_rank(int rank) { - auto colors = { Colors::RED(), Colors::GREEN(), Colors::BLUE(), Colors::MAGENTA(), Colors::CYAN() }; + auto colors = { Colors::WHITE(), Colors::RED(), Colors::GREEN(), Colors::BLUE(), Colors::MAGENTA(), Colors::CYAN() }; return *(colors.begin() + rank % colors.size()); } GridSearch::GridSearch(struct ConfigGrid& config) : config(config) @@ -77,32 +77,7 @@ namespace platform { } return datasets_names; } - void showProgressComb(const int num, const int n_folds, const int total, const std::string& color) - { - int spaces = int(log(total) / log(10)) + 1; - int magic = n_folds * 3 + 22 + 2 * spaces; - std::string prefix = num == 1 ? "" : string(magic, '\b') + string(magic + 1, ' ') + string(magic + 1, '\b'); - std::cout << prefix << color << "(" << setw(spaces) << num << "/" << setw(spaces) << total << ") " << Colors::RESET() << flush; - } - void showProgressFold(int fold, const std::string& color, const std::string& phase) - { - std::string prefix = phase == "a" ? "" : "\b\b\b\b"; - std::cout << prefix << color << fold << Colors::RESET() << "(" << color << phase << Colors::RESET() << ")" << flush; - } - std::string getColor(bayesnet::status_t status) - { - switch (status) { - case bayesnet::NORMAL: - return Colors::GREEN(); - case bayesnet::WARNING: - return Colors::YELLOW(); - case bayesnet::ERROR: - return Colors::RED(); - default: - return Colors::RESET(); - } - } - json GridSearch::build_tasks_mpi() + json GridSearch::build_tasks_mpi(int rank) { auto tasks = json::array(); auto grid = GridData(Paths::grid_input(config.model)); @@ -124,10 +99,10 @@ namespace platform { } } } - // It's important to shuffle the array so heavy datasets are spread across the Workers + // Shuffle the array so heavy datasets are spread across the workers std::mt19937 g{ 271 }; // Use fixed seed to obtain the same shuffle std::shuffle(tasks.begin(), tasks.end(), g); - std::cout << "Tasks size: " << tasks.size() << std::endl; + std::cout << get_color_rank(rank) << "* Number of tasks: " << tasks.size() << std::endl; std::cout << "|"; for (int i = 0; i < tasks.size(); ++i) { std::cout << (i + 1) % 10; @@ -226,24 +201,6 @@ namespace platform { // Update progress bar std::cout << get_color_rank(config_mpi.rank) << "*" << std::flush; } - // std::pair GridSearch::part_range_mpi(int n_tasks, int nprocs, int rank) - // { - // int assigned = 0; - // int remainder = n_tasks % nprocs; - // int start = 0; - // if (rank < remainder) { - // assigned = n_tasks / nprocs + 1; - // } else { - // assigned = n_tasks / nprocs; - // start = remainder; - // } - // start += rank * assigned; - // int end = start + assigned; - // if (rank == nprocs - 1) { - // end = n_tasks; - // } - // return { start, end }; - // } json store_result(std::vector& names, Task_Result& result, json& results) { json json_result = { @@ -266,6 +223,9 @@ namespace platform { json results; int num_tasks = tasks.size(); + // + // 2a.1 Producer will loop to send all the tasks to the consumers and receive the results + // for (int i = 0; i < num_tasks; ++i) { MPI_Status status; MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); @@ -275,7 +235,9 @@ namespace platform { } MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD); } - // Send end message to all workers but the manager + // + // 2a.2 Producer will send the end message to all the consumers + // for (int i = 0; i < config_mpi.n_procs - 1; ++i) { MPI_Status status; MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); @@ -287,9 +249,8 @@ namespace platform { } return results; } - json select_best_results_folds(json& all_results, std::string& model) + void select_best_results_folds(json& results, json& all_results, std::string& model) { - json results; Timer timer; auto grid = GridData(Paths::grid_input(model)); // @@ -317,33 +278,39 @@ namespace platform { }; results[dataset] = json_best; } - return results; } void consumer(Datasets& datasets, json& tasks, struct ConfigGrid& config, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) { Task_Result result; - // Anounce to the producer + // + // 2b.1 Consumers announce to the producer that they are ready to receive a task + // MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_QUERY, MPI_COMM_WORLD); int task; while (true) { MPI_Status status; + // + // 2b.2 Consumers receive the task from the producer and process it + // MPI_Recv(&task, 1, MPI_INT, config_mpi.manager, MPI_ANY_TAG, MPI_COMM_WORLD, &status); if (status.MPI_TAG == TAG_END) { break; } - // Process task process_task_mpi_consumer(config, config_mpi, tasks, task, datasets, &result); - // Send result to producer + // + // 2b.3 Consumers send the result to the producer + // MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_RESULT, MPI_COMM_WORLD); } } - void GridSearch::go_producer_consumer(struct ConfigMPI& config_mpi) + void GridSearch::go(struct ConfigMPI& config_mpi) { /* * Each task is a json object with the following structure: * { * "dataset": "dataset_name", - * "idx_dataset": idx_dataset, + * "idx_dataset": idx_dataset, // used to identify the dataset in the results + * // this index is relative to the used datasets in the actual run not to the whole datasets * "seed": # of seed to use, * "Fold": # of fold to process * } @@ -356,14 +323,16 @@ namespace platform { * 1.1 Broadcast the number of tasks * 1.2 Broadcast the length of the following string * 1.2 Broadcast the tasks as a char* string - * 2. Workers will receive the tasks and start the process - * 2.1 A method will tell each worker the range of tasks to process - * 2.2 Each worker will process the tasks and generate the best score for each task - * 3. Manager gather the scores from all the workers and find out the best hyperparameters for each dataset - * 3.1 Obtain the maximum size of the results message of all the workers - * 3.2 Gather all the results from the workers into the manager - * 3.3 Compile the results from all the workers - * 3.4 Filter the best hyperparameters for each dataset + * 2a. Producer delivers the tasks to the consumers + * 2a.1 Producer will loop to send all the tasks to the consumers and receive the results + * 2a.2 Producer will send the end message to all the consumers + * 2b. Consumers process the tasks and send the results to the producer + * 2b.1 Consumers announce to the producer that they are ready to receive a task + * 2b.2 Consumers receive the task from the producer and process it + * 2b.3 Consumers send the result to the producer + * 3. Manager select the bests sccores for each dataset + * 3.1 Loop thru all the results obtained from each outer fold (task) and select the best + * 3.2 Save the results */ // // 0.1 Create the MPI result type @@ -388,7 +357,7 @@ namespace platform { json tasks; if (config_mpi.rank == config_mpi.manager) { timer.start(); - tasks = build_tasks_mpi(); + tasks = build_tasks_mpi(config_mpi.rank); auto tasks_str = tasks.dump(); tasks_size = tasks_str.size(); msg = new char[tasks_size + 1]; @@ -404,429 +373,33 @@ namespace platform { MPI_Bcast(msg, tasks_size + 1, MPI_CHAR, config_mpi.manager, MPI_COMM_WORLD); tasks = json::parse(msg); delete[] msg; - // - // 2. All Workers will receive the tasks and start the process - // auto datasets = Datasets(config.discretize, Paths::datasets()); if (config_mpi.rank == config_mpi.manager) { + // + // 2a. Producer delivers the tasks to the consumers + // auto datasets_names = filterDatasets(datasets); json all_results = producer(datasets_names, tasks, config_mpi, MPI_Result); - json results = select_best_results_folds(all_results, config.model); + std::cout << get_color_rank(config_mpi.rank) << "|" << std::endl; + // + // 3. Manager select the bests sccores for each dataset + // + auto results = initializeResults(); + select_best_results_folds(results, all_results, config.model); + // + // 3.2 Save the results + // save(results); - std::cout << Colors::RESET() << "|" << std::endl; } else { + // + // 2b. Consumers process the tasks and send the results to the producer + // consumer(datasets, tasks, config, config_mpi, MPI_Result); } } - // void GridSearch::go_mpi(struct ConfigMPI& config_mpi) - // { - // /* - // * Each task is a json object with the following structure: - // * { - // * "dataset": "dataset_name", - // * "seed": # of seed to use, - // * "model": "model_name", - // * "Fold": # of fold to process - // * } - // * - // * The overall process consists in these steps: - // * 1. Manager will broadcast the tasks to all the processes - // * 1.1 Broadcast the number of tasks - // * 1.2 Broadcast the length of the following string - // * 1.2 Broadcast the tasks as a char* string - // * 2. Workers will receive the tasks and start the process - // * 2.1 A method will tell each worker the range of tasks to process - // * 2.2 Each worker will process the tasks and generate the best score for each task - // * 3. Manager gather the scores from all the workers and find out the best hyperparameters for each dataset - // * 3.1 Obtain the maximum size of the results message of all the workers - // * 3.2 Gather all the results from the workers into the manager - // * 3.3 Compile the results from all the workers - // * 3.4 Filter the best hyperparameters for each dataset - // */ - // char* msg; - // int tasks_size; - // if (config_mpi.rank == config_mpi.manager) { - // timer.start(); - // auto tasks = build_tasks_mpi(); - // auto tasks_str = tasks.dump(); - // tasks_size = tasks_str.size(); - // msg = new char[tasks_size + 1]; - // strcpy(msg, tasks_str.c_str()); - // } - // // - // // 1. Manager will broadcast the tasks to all the processes - // // - // MPI_Bcast(&tasks_size, 1, MPI_INT, config_mpi.manager, MPI_COMM_WORLD); - // if (config_mpi.rank != config_mpi.manager) { - // msg = new char[tasks_size + 1]; - // } - // MPI_Bcast(msg, tasks_size + 1, MPI_CHAR, config_mpi.manager, MPI_COMM_WORLD); - // json tasks = json::parse(msg); - // delete[] msg; - // // - // // 2. All Workers will receive the tasks and start the process - // // - // int num_tasks = tasks.size(); - // // 2.1 A method will tell each worker the range of tasks to process - // auto [start, end] = part_range_mpi(num_tasks, config_mpi.n_procs, config_mpi.rank); - // // 2.2 Each worker will process the tasks and return the best scores obtained - // auto datasets = Datasets(config.discretize, Paths::datasets()); - // json results; - // for (int i = start; i < end; ++i) { - // // Process task - // process_task_mpi(config_mpi, tasks[i], datasets, results); - // } - // int size = results.dump().size() + 1; - // int max_size = 0; - // // - // // 3. Manager gather the scores from all the workers and find out the best hyperparameters for each dataset - // // - // //3.1 Obtain the maximum size of the results message of all the workers - // MPI_Allreduce(&size, &max_size, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); - // // Assign the memory to the message and initialize it to 0s - // char* total = NULL; - // msg = new char[max_size]; - // strncpy(msg, results.dump().c_str(), size); - // if (config_mpi.rank == config_mpi.manager) { - // total = new char[max_size * config_mpi.n_procs]; - // } - // // 3.2 Gather all the results from the workers into the manager - // MPI_Gather(msg, max_size, MPI_CHAR, total, max_size, MPI_CHAR, config_mpi.manager, MPI_COMM_WORLD); - // delete[] msg; - // if (config_mpi.rank == config_mpi.manager) { - // std::cout << Colors::RESET() << "|" << std::endl; - // json total_results; - // json best_results; - // // 3.3 Compile the results from all the workers - // for (int i = 0; i < config_mpi.n_procs; ++i) { - // json partial_results = json::parse(total + i * max_size); - // for (auto& [dataset, folds] : partial_results.items()) { - // for (auto& [fold, result] : folds.items()) { - // total_results[dataset][fold] = result; - // } - // } - // } - // delete[] total; - // // 3.4 Filter the best hyperparameters for each dataset - // auto grid = GridData(Paths::grid_input(config.model)); - // for (auto& [dataset, folds] : total_results.items()) { - // double best_score = 0.0; - // double duration = 0.0; - // json best_hyper; - // for (auto& [fold, result] : folds.items()) { - // duration += result["duration"].get(); - // if (result["score"] > best_score) { - // best_score = result["score"]; - // best_hyper = result["hyperparameters"]; - // } - // } - // auto timer = Timer(); - // json result = { - // { "score", best_score }, - // { "hyperparameters", best_hyper }, - // { "date", get_date() + " " + get_time() }, - // { "grid", grid.getInputGrid(dataset) }, - // { "duration", timer.translate2String(duration) } - // }; - // best_results[dataset] = result; - // } - // save(best_results); - // } - // } - // void GridSearch::go() - // { - // timer.start(); - // auto grid_type = config.nested == 0 ? "Single" : "Nested"; - // auto datasets = Datasets(config.discretize, Paths::datasets()); - // auto datasets_names = processDatasets(datasets); - // json results = initializeResults(); - // std::cout << "***************** Starting " << grid_type << " Gridsearch *****************" << std::endl; - // std::cout << "input file=" << Paths::grid_input(config.model) << std::endl; - // auto grid = GridData(Paths::grid_input(config.model)); - // Timer timer_dataset; - // double bestScore = 0; - // json bestHyperparameters; - // for (const auto& dataset : datasets_names) { - // if (!config.quiet) - // std::cout << "- " << setw(20) << left << dataset << " " << right << flush; - // auto combinations = grid.getGrid(dataset); - // timer_dataset.start(); - // if (config.nested == 0) - // // for dataset // for hyperparameters // for seed // for fold - // tie(bestScore, bestHyperparameters) = processFileSingle(dataset, datasets, combinations); - // else - // // for dataset // for seed // for fold // for hyperparameters // for nested fold - // tie(bestScore, bestHyperparameters) = processFileNested(dataset, datasets, combinations); - // if (!config.quiet) { - // std::cout << "end." << " Score: " << Colors::IBLUE() << setw(9) << setprecision(7) << fixed - // << bestScore << Colors::BLUE() << " [" << bestHyperparameters.dump() << "]" - // << Colors::RESET() << ::endl; - // } - // json result = { - // { "score", bestScore }, - // { "hyperparameters", bestHyperparameters }, - // { "date", get_date() + " " + get_time() }, - // { "grid", grid.getInputGrid(dataset) }, - // { "duration", timer_dataset.getDurationString() } - // }; - // results[dataset] = result; - // // Save partial results - // save(results); - // } - // // Save final results - // save(results); - // std::cout << "***************** Ending " << grid_type << " Gridsearch *******************" << std::endl; - // } - // pair GridSearch::processFileSingle(std::string fileName, Datasets& datasets, vector& combinations) - // { - // int num = 0; - // double bestScore = 0.0; - // json bestHyperparameters; - // auto totalComb = combinations.size(); - // for (const auto& hyperparam_line : combinations) { - // if (!config.quiet) - // showProgressComb(++num, config.n_folds, totalComb, Colors::CYAN()); - // auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); - // // Get dataset - // auto [X, y] = datasets.getTensors(fileName); - // auto states = datasets.getStates(fileName); - // auto features = datasets.getFeatures(fileName); - // auto className = datasets.getClassName(fileName); - // double totalScore = 0.0; - // int numItems = 0; - // for (const auto& seed : config.seeds) { - // if (!config.quiet) - // std::cout << "(" << seed << ") doing Fold: " << flush; - // Fold* fold; - // if (config.stratified) - // fold = new StratifiedKFold(config.n_folds, y, seed); - // else - // fold = new KFold(config.n_folds, y.size(0), seed); - // for (int nfold = 0; nfold < config.n_folds; nfold++) { - // auto clf = Models::instance()->create(config.model); - // auto valid = clf->getValidHyperparameters(); - // hyperparameters.check(valid, fileName); - // clf->setHyperparameters(hyperparameters.get(fileName)); - // auto [train, test] = fold->getFold(nfold); - // auto train_t = torch::tensor(train); - // auto test_t = torch::tensor(test); - // auto X_train = X.index({ "...", train_t }); - // auto y_train = y.index({ train_t }); - // auto X_test = X.index({ "...", test_t }); - // auto y_test = y.index({ test_t }); - // // Train model - // if (!config.quiet) - // showProgressFold(nfold + 1, getColor(clf->getStatus()), "a"); - // clf->fit(X_train, y_train, features, className, states); - // // Test model - // if (!config.quiet) - // showProgressFold(nfold + 1, getColor(clf->getStatus()), "b"); - // totalScore += clf->score(X_test, y_test); - // numItems++; - // if (!config.quiet) - // std::cout << "\b\b\b, \b" << flush; - // } - // delete fold; - // } - // double score = numItems == 0 ? 0.0 : totalScore / numItems; - // if (score > bestScore) { - // bestScore = score; - // bestHyperparameters = hyperparam_line; - // } - // } - // return { bestScore, bestHyperparameters }; - // } - // pair GridSearch::processFileNested(std::string fileName, Datasets& datasets, vector& combinations) - // { - // // Get dataset - // auto [X, y] = datasets.getTensors(fileName); - // auto states = datasets.getStates(fileName); - // auto features = datasets.getFeatures(fileName); - // auto className = datasets.getClassName(fileName); - // int spcs_combinations = int(log(combinations.size()) / log(10)) + 1; - // double goatScore = 0.0; - // json goatHyperparameters; - // // for dataset // for seed // for fold // for hyperparameters // for nested fold - // for (const auto& seed : config.seeds) { - // Fold* fold; - // if (config.stratified) - // fold = new StratifiedKFold(config.n_folds, y, seed); - // else - // fold = new KFold(config.n_folds, y.size(0), seed); - // double bestScore = 0.0; - // json bestHyperparameters; - // std::cout << "(" << seed << ") doing Fold: " << flush; - // for (int nfold = 0; nfold < config.n_folds; nfold++) { - // if (!config.quiet) - // std::cout << Colors::GREEN() << nfold + 1 << " " << flush; - // // First level fold - // auto [train, test] = fold->getFold(nfold); - // auto train_t = torch::tensor(train); - // auto test_t = torch::tensor(test); - // auto X_train = X.index({ "...", train_t }); - // auto y_train = y.index({ train_t }); - // auto X_test = X.index({ "...", test_t }); - // auto y_test = y.index({ test_t }); - // auto num = 0; - // json result_fold; - // double hypScore = 0.0; - // double bestHypScore = 0.0; - // json bestHypHyperparameters; - // for (const auto& hyperparam_line : combinations) { - // std::cout << "[" << setw(spcs_combinations) << ++num << "/" << setw(spcs_combinations) - // << combinations.size() << "] " << std::flush; - // Fold* nested_fold; - // if (config.stratified) - // nested_fold = new StratifiedKFold(config.nested, y_train, seed); - // else - // nested_fold = new KFold(config.nested, y_train.size(0), seed); - // for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) { - // // Nested level fold - // auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold); - // auto train_nested_t = torch::tensor(train_nested); - // auto test_nested_t = torch::tensor(test_nested); - // auto X_nexted_train = X_train.index({ "...", train_nested_t }); - // auto y_nested_train = y_train.index({ train_nested_t }); - // auto X_nested_test = X_train.index({ "...", test_nested_t }); - // auto y_nested_test = y_train.index({ test_nested_t }); - // // Build Classifier with selected hyperparameters - // auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); - // auto clf = Models::instance()->create(config.model); - // auto valid = clf->getValidHyperparameters(); - // hyperparameters.check(valid, fileName); - // clf->setHyperparameters(hyperparameters.get(fileName)); - // // Train model - // if (!config.quiet) - // showProgressFold(n_nested_fold + 1, getColor(clf->getStatus()), "a"); - // clf->fit(X_nexted_train, y_nested_train, features, className, states); - // // Test model - // if (!config.quiet) - // showProgressFold(n_nested_fold + 1, getColor(clf->getStatus()), "b"); - // hypScore += clf->score(X_nested_test, y_nested_test); - // if (!config.quiet) - // std::cout << "\b\b\b, \b" << flush; - // } - // int magic = 3 * config.nested + 2 * spcs_combinations + 4; - // std::cout << string(magic, '\b') << string(magic, ' ') << string(magic, '\b') << flush; - // delete nested_fold; - // hypScore /= config.nested; - // if (hypScore > bestHypScore) { - // bestHypScore = hypScore; - // bestHypHyperparameters = hyperparam_line; - // } - // } - // // Build Classifier with selected hyperparameters - // auto clf = Models::instance()->create(config.model); - // clf->setHyperparameters(bestHypHyperparameters); - // // Train model - // if (!config.quiet) - // showProgressFold(nfold + 1, getColor(clf->getStatus()), "a"); - // clf->fit(X_train, y_train, features, className, states); - // // Test model - // if (!config.quiet) - // showProgressFold(nfold + 1, getColor(clf->getStatus()), "b"); - // double score = clf->score(X_test, y_test); - // if (!config.quiet) - // std::cout << string(2 * config.nested - 1, '\b') << "," << string(2 * config.nested, ' ') << string(2 * config.nested - 1, '\b') << flush; - // if (score > bestScore) { - // bestScore = score; - // bestHyperparameters = bestHypHyperparameters; - // } - // } - // if (bestScore > goatScore) { - // goatScore = bestScore; - // goatHyperparameters = bestHyperparameters; - // } - // delete fold; - // } - // return { goatScore, goatHyperparameters }; - // } - // void GridSearch::process_task_mpi(struct ConfigMPI& config_mpi, json& task, Datasets& datasets, json& results) - // { - // // Process the task and store the result in the results json - // Timer timer; - // timer.start(); - // auto grid = GridData(Paths::grid_input(config.model)); - // auto dataset = task["dataset"].get(); - // auto seed = task["seed"].get(); - // auto n_fold = task["fold"].get(); - // // Generate the hyperparamters combinations - // auto combinations = grid.getGrid(dataset); - // auto [X, y] = datasets.getTensors(dataset); - // auto states = datasets.getStates(dataset); - // auto features = datasets.getFeatures(dataset); - // auto className = datasets.getClassName(dataset); - // // - // // Start working on task - // // - // Fold* fold; - // if (config.stratified) - // fold = new StratifiedKFold(config.n_folds, y, seed); - // else - // fold = new KFold(config.n_folds, y.size(0), seed); - // auto [train, test] = fold->getFold(n_fold); - // auto train_t = torch::tensor(train); - // auto test_t = torch::tensor(test); - // auto X_train = X.index({ "...", train_t }); - // auto y_train = y.index({ train_t }); - // auto X_test = X.index({ "...", test_t }); - // auto y_test = y.index({ test_t }); - // auto num = 0; - // double best_fold_score = 0.0; - // json best_fold_hyper; - // for (const auto& hyperparam_line : combinations) { - // auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); - // Fold* nested_fold; - // if (config.stratified) - // nested_fold = new StratifiedKFold(config.nested, y_train, seed); - // else - // nested_fold = new KFold(config.nested, y_train.size(0), seed); - // double score = 0.0; - // for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) { - // // Nested level fold - // auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold); - // auto train_nested_t = torch::tensor(train_nested); - // auto test_nested_t = torch::tensor(test_nested); - // auto X_nested_train = X_train.index({ "...", train_nested_t }); - // auto y_nested_train = y_train.index({ train_nested_t }); - // auto X_nested_test = X_train.index({ "...", test_nested_t }); - // auto y_nested_test = y_train.index({ test_nested_t }); - // // Build Classifier with selected hyperparameters - // auto clf = Models::instance()->create(config.model); - // auto valid = clf->getValidHyperparameters(); - // hyperparameters.check(valid, dataset); - // clf->setHyperparameters(hyperparameters.get(dataset)); - // // Train model - // clf->fit(X_nested_train, y_nested_train, features, className, states); - // // Test model - // score += clf->score(X_nested_test, y_nested_test); - // } - // delete nested_fold; - // score /= config.nested; - // if (score > best_fold_score) { - // best_fold_score = score; - // best_fold_hyper = hyperparam_line; - // } - // } - // delete fold; - // // Build Classifier with the best hyperparameters to obtain the best score - // auto hyperparameters = platform::HyperParameters(datasets.getNames(), best_fold_hyper); - // auto clf = Models::instance()->create(config.model); - // auto valid = clf->getValidHyperparameters(); - // hyperparameters.check(valid, dataset); - // clf->setHyperparameters(best_fold_hyper); - // clf->fit(X_train, y_train, features, className, states); - // best_fold_score = clf->score(X_test, y_test); - // // Save results - // results[dataset][std::to_string(n_fold)]["score"] = best_fold_score; - // results[dataset][std::to_string(n_fold)]["hyperparameters"] = best_fold_hyper; - // results[dataset][std::to_string(n_fold)]["seed"] = seed; - // results[dataset][std::to_string(n_fold)]["duration"] = timer.getDuration(); - // std::cout << get_color_rank(config_mpi.rank) << "*" << std::flush; - // } json GridSearch::initializeResults() { - // Load previous results + // Load previous results if continue is set json results; if (config.continue_from != NO_CONTINUE()) { if (!config.quiet) diff --git a/src/Platform/GridSearch.h b/src/Platform/GridSearch.h index 36760b6..ec1b3cb 100644 --- a/src/Platform/GridSearch.h +++ b/src/Platform/GridSearch.h @@ -44,9 +44,7 @@ namespace platform { class GridSearch { public: explicit GridSearch(struct ConfigGrid& config); - // void go(); - // void go_mpi(struct ConfigMPI& config_mpi); - void go_producer_consumer(struct ConfigMPI& config_mpi); + void go(struct ConfigMPI& config_mpi); ~GridSearch() = default; json loadResults(); static inline std::string NO_CONTINUE() { return "NO_CONTINUE"; } @@ -54,12 +52,8 @@ namespace platform { void save(json& results); json initializeResults(); std::vector filterDatasets(Datasets& datasets) const; - // pair processFileSingle(std::string fileName, Datasets& datasets, std::vector& combinations); - // pair processFileNested(std::string fileName, Datasets& datasets, std::vector& combinations); struct ConfigGrid config; - // pair part_range_mpi(int n_tasks, int nprocs, int rank); - json build_tasks_mpi(); - // void process_task_mpi(struct ConfigMPI& config_mpi, json& task, Datasets& datasets, json& results); + json build_tasks_mpi(int rank); Timer timer; // used to measure the time of the whole process }; } /* namespace platform */ diff --git a/src/Platform/b_grid.cc b/src/Platform/b_grid.cc index 1d42543..c54a21c 100644 --- a/src/Platform/b_grid.cc +++ b/src/Platform/b_grid.cc @@ -32,7 +32,6 @@ void manageArguments(argparse::ArgumentParser& program) group.add_argument("--report").help("Report the computed hyperparameters").default_value(false).implicit_value(true); group.add_argument("--compute").help("Perform computation of the grid output hyperparameters").default_value(false).implicit_value(true); program.add_argument("--discretize").help("Discretize input datasets").default_value((bool)stoi(env.get("discretize"))).implicit_value(true); - program.add_argument("--mpi").help("Use MPI computing grid").default_value(false).implicit_value(true); program.add_argument("--stratified").help("If Stratified KFold is to be done").default_value((bool)stoi(env.get("stratified"))).implicit_value(true); program.add_argument("--quiet").help("Don't display detailed progress").default_value(false).implicit_value(true); program.add_argument("--continue").help("Continue computing from that dataset").default_value(platform::GridSearch::NO_CONTINUE()); @@ -108,8 +107,8 @@ void list_results(json& results, std::string& model) + " Nested: " + (results["nested"].get() == 0 ? "False" : to_string(results["nested"].get())) ); std::cout << std::string(MAXL, '*') << std::endl; - int spaces = 0; - int hyperparameters_spaces = 0; + int spaces = 7; + int hyperparameters_spaces = 15; for (const auto& item : results["results"].items()) { auto key = item.key(); auto value = item.value(); @@ -128,11 +127,10 @@ void list_results(json& results, std::string& model) int index = 0; for (const auto& item : results["results"].items()) { auto color = odd ? Colors::CYAN() : Colors::BLUE(); - auto key = item.key(); auto value = item.value(); std::cout << color; std::cout << std::setw(3) << std::right << index++ << " "; - std::cout << left << setw(spaces) << key << " " << value["date"].get() + std::cout << left << setw(spaces) << item.key() << " " << value["date"].get() << " " << setw(8) << right << value["duration"].get() << " " << setw(8) << setprecision(6) << fixed << right << value["score"].get() << " " << value["hyperparameters"].dump() << std::endl; odd = !odd; @@ -171,11 +169,6 @@ int main(int argc, char** argv) } auto excluded = program.get("exclude"); config.excluded = json::parse(excluded); - if (program.get("mpi")) { - if (!compute || config.nested == 0) { - throw std::runtime_error("Cannot use --mpi without --compute or without --nested"); - } - } } catch (const exception& err) { cerr << err.what() << std::endl; @@ -195,23 +188,21 @@ int main(int argc, char** argv) list_dump(config.model); } else { if (compute) { - if (program.get("mpi")) { - struct platform::ConfigMPI mpi_config; - mpi_config.manager = 0; // which process is the manager - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &mpi_config.rank); - MPI_Comm_size(MPI_COMM_WORLD, &mpi_config.n_procs); - grid_search.go_producer_consumer(mpi_config); - if (mpi_config.rank == mpi_config.manager) { - auto results = grid_search.loadResults(); - list_results(results, config.model); - std::cout << "Process took " << timer.getDurationString() << std::endl; - } - MPI_Finalize(); - // } else { - // grid_search.go(); - // std::cout << "Process took " << timer.getDurationString() << std::endl; + struct platform::ConfigMPI mpi_config; + mpi_config.manager = 0; // which process is the manager + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_config.rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_config.n_procs); + if (mpi_config.n_procs < 2) { + throw std::runtime_error("Cannot use --compute with less than 2 mpi processes, try mpirun -np 2 ..."); } + grid_search.go(mpi_config); + if (mpi_config.rank == mpi_config.manager) { + auto results = grid_search.loadResults(); + list_results(results, config.model); + std::cout << "Process took " << timer.getDurationString() << std::endl; + } + MPI_Finalize(); } else { // List results auto results = grid_search.loadResults(); From 65a96851efb0148cf3a0a9cf6a91d0769443f896 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Thu, 4 Jan 2024 11:01:59 +0100 Subject: [PATCH 13/13] Check min number of nested folds --- src/Platform/b_grid.cc | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Platform/b_grid.cc b/src/Platform/b_grid.cc index c54a21c..6e9796d 100644 --- a/src/Platform/b_grid.cc +++ b/src/Platform/b_grid.cc @@ -37,7 +37,20 @@ void manageArguments(argparse::ArgumentParser& program) program.add_argument("--continue").help("Continue computing from that dataset").default_value(platform::GridSearch::NO_CONTINUE()); program.add_argument("--only").help("Used with continue to compute that dataset only").default_value(false).implicit_value(true); program.add_argument("--exclude").default_value("[]").help("Datasets to exclude in json format, e.g. [\"dataset1\", \"dataset2\"]"); - program.add_argument("--nested").help("Do a double/nested cross validation with n folds").default_value(0).scan<'i', int>(); + program.add_argument("--nested").help("Set the double/nested cross validation number of folds").default_value(5).scan<'i', int>().action([](const std::string& value) { + try { + auto k = stoi(value); + if (k < 2) { + throw std::runtime_error("Number of nested folds must be greater than 1"); + } + return k; + } + catch (const runtime_error& err) { + throw std::runtime_error(err.what()); + } + catch (...) { + throw std::runtime_error("Number of nested folds must be an integer"); + }}); program.add_argument("--score").help("Score used in gridsearch").default_value("accuracy"); program.add_argument("-f", "--folds").help("Number of folds").default_value(stoi(env.get("n_folds"))).scan<'i', int>().action([](const std::string& value) { try {