From f88944de368b2cab54e7c0a4b4ab07425d214557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= Date: Fri, 20 Dec 2024 18:54:08 +0100 Subject: [PATCH] Add grid base class and static class --- src/grid/GridBase.h | 43 +++++++ src/grid/GridConfig.h | 3 +- src/grid/GridExperiment.cpp | 77 ++---------- src/grid/GridExperiment.h | 242 ++++++++++++++++++++++++++++++++++-- src/grid/GridFunctions.cpp | 226 +-------------------------------- src/grid/GridSearch.cpp | 22 +--- src/grid/GridSearch.h | 234 ++++++++++++++++++++++++++++++++-- 7 files changed, 524 insertions(+), 323 deletions(-) create mode 100644 src/grid/GridBase.h diff --git a/src/grid/GridBase.h b/src/grid/GridBase.h new file mode 100644 index 0000000..0f4484d --- /dev/null +++ b/src/grid/GridBase.h @@ -0,0 +1,43 @@ +#ifndef GRIDBASE_H +#define GRIDBASE_H +#include +#include +#include +#include +#include "common/Datasets.h" +#include "common/Timer.h" +#include "main/HyperParameters.h" +#include "GridData.h" +#include "GridConfig.h" +#include "bayesnet/network/Network.h" + + +namespace platform { + using json = nlohmann::ordered_json; + class GridBase { + public: + explicit GridBase(struct ConfigGrid& config) + { + this->config = config; + if (config.smooth_strategy == "ORIGINAL") + smooth_type = bayesnet::Smoothing_t::ORIGINAL; + else if (config.smooth_strategy == "LAPLACE") + smooth_type = bayesnet::Smoothing_t::LAPLACE; + else if (config.smooth_strategy == "CESTNIK") + smooth_type = bayesnet::Smoothing_t::CESTNIK; + else { + std::cerr << "GridBase: Unknown smoothing strategy: " << config.smooth_strategy << std::endl; + exit(1); + } + }; + ~GridBase() = default; + virtual void go(struct ConfigMPI& config_mpi) = 0; + virtual json build_tasks_mpi() = 0; + protected: + struct ConfigGrid config; + Timer timer; // used to measure the time of the whole process + const std::string separator = "|"; + bayesnet::Smoothing_t smooth_type{ bayesnet::Smoothing_t::NONE }; + }; +} /* namespace platform */ +#endif \ No newline at end of file diff --git a/src/grid/GridConfig.h b/src/grid/GridConfig.h index ccf0b8d..b077678 100644 --- a/src/grid/GridConfig.h +++ b/src/grid/GridConfig.h @@ -8,6 +8,7 @@ #include "common/Timer.h" #include "main/HyperParameters.h" #include "GridData.h" +#include "GridConfig.h" #include "bayesnet/network/Network.h" @@ -69,6 +70,6 @@ namespace platform { void mpi_search_consumer(Datasets& datasets, json& tasks, struct ConfigGrid& config, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result); void select_best_results_folds(json& results, json& all_results, std::string& model); json store_search_result(std::vector& names, Task_Result& result, json& results); - void mpi_experiment_consumer_go(struct ConfigGrid& config, struct ConfigMPI& config_mpi, json& tasks, int n_task, Datasets& datasets, Task_Result* result); + void mpi_search_consumer_go(struct ConfigGrid& config, struct ConfigMPI& config_mpi, json& tasks, int n_task, Datasets& datasets, Task_Result* result); } /* namespace platform */ #endif \ No newline at end of file diff --git a/src/grid/GridExperiment.cpp b/src/grid/GridExperiment.cpp index ccb4016..b9d290f 100644 --- a/src/grid/GridExperiment.cpp +++ b/src/grid/GridExperiment.cpp @@ -10,18 +10,8 @@ namespace platform { - GridExperiment::GridExperiment(struct ConfigGrid& config) : config(config) + GridExperiment::GridExperiment(struct ConfigGrid& config) : GridBase(config) { - if (config.smooth_strategy == "ORIGINAL") - smooth_type = bayesnet::Smoothing_t::ORIGINAL; - else if (config.smooth_strategy == "LAPLACE") - smooth_type = bayesnet::Smoothing_t::LAPLACE; - else if (config.smooth_strategy == "CESTNIK") - smooth_type = bayesnet::Smoothing_t::CESTNIK; - else { - std::cerr << "GridSearch: Unknown smoothing strategy: " << config.smooth_strategy << std::endl; - exit(1); - } } json GridExperiment::loadResults() { @@ -31,46 +21,13 @@ namespace platform { } return json(); } - std::vector GridExperiment::filterDatasets(Datasets& datasets) const - { - // Load datasets - auto datasets_names = datasets.getNames(); - if (config.continue_from != NO_CONTINUE()) { - // Continue previous execution: - if (std::find(datasets_names.begin(), datasets_names.end(), config.continue_from) == datasets_names.end()) { - throw std::invalid_argument("Dataset " + config.continue_from + " not found"); - } - // Remove datasets already processed - std::vector::iterator it = datasets_names.begin(); - while (it != datasets_names.end()) { - if (*it != config.continue_from) { - it = datasets_names.erase(it); - } else { - if (config.only) - ++it; - else - break; - } - } - } - // Exclude datasets - for (const auto& name : config.excluded) { - auto dataset = name.get(); - auto it = std::find(datasets_names.begin(), datasets_names.end(), dataset); - if (it == datasets_names.end()) { - throw std::invalid_argument("Dataset " + dataset + " already excluded or doesn't exist!"); - } - datasets_names.erase(it); - } - return datasets_names; - } - json GridExperiment::build_tasks_mpi(int rank) + json GridExperiment::build_tasks_mpi() { auto tasks = json::array(); auto grid = GridData(Paths::grid_input(config.model)); auto datasets = Datasets(false, Paths::datasets()); auto all_datasets = datasets.getNames(); - auto datasets_names = filterDatasets(datasets); + auto datasets_names = all_datasets; for (int idx_dataset = 0; idx_dataset < datasets_names.size(); ++idx_dataset) { auto dataset = datasets_names[idx_dataset]; for (const auto& seed : config.seeds) { @@ -156,7 +113,7 @@ namespace platform { json tasks; if (config_mpi.rank == config_mpi.manager) { timer.start(); - tasks = build_tasks_mpi(config_mpi.rank); + tasks = build_tasks_mpi(); auto tasks_str = tasks.dump(); tasks_size = tasks_str.size(); msg = new char[tasks_size + 1]; @@ -179,45 +136,29 @@ namespace platform { // // 2a. Producer delivers the tasks to the consumers // - auto datasets_names = filterDatasets(datasets); - json all_results = mpi_search_producer(datasets_names, tasks, config_mpi, MPI_Result); + auto datasets_names = std::vector(); + json all_results = mpi_experiment_producer(datasets_names, tasks, config_mpi, MPI_Result); std::cout << separator << std::endl; // // 3. Manager select the bests sccores for each dataset // auto results = initializeResults(); - select_best_results_folds(results, all_results, config.model); + //select_best_results_folds(results, all_results, config.model); // // 3.2 Save the results // save(results); } else { // - // 2b. Consumers process the tasks and send the results to the producer + // 2b. Consumers prostore_search_resultcess the tasks and send the results to the producer // - mpi_search_consumer(datasets, tasks, config, config_mpi, MPI_Result); + mpi_experiment_consumer(datasets, tasks, config, config_mpi, MPI_Result); } } json GridExperiment::initializeResults() { // Load previous results if continue is set json results; - if (config.continue_from != NO_CONTINUE()) { - if (!config.quiet) - std::cout << "* Loading previous results" << std::endl; - try { - std::ifstream file(Paths::grid_output(config.model)); - if (file.is_open()) { - results = json::parse(file); - results = results["results"]; - } - } - catch (const std::exception& e) { - std::cerr << "* There were no previous results" << std::endl; - std::cerr << "* Initizalizing new results" << std::endl; - results = json(); - } - } return results; } void GridExperiment::save(json& results) diff --git a/src/grid/GridExperiment.h b/src/grid/GridExperiment.h index 6fb109b..13b5ef3 100644 --- a/src/grid/GridExperiment.h +++ b/src/grid/GridExperiment.h @@ -8,28 +8,252 @@ #include "common/Timer.h" #include "main/HyperParameters.h" #include "GridData.h" -#include "GridConfig.h" +#include "GridBase.h" #include "bayesnet/network/Network.h" namespace platform { using json = nlohmann::ordered_json; - class GridExperiment { + class GridExperiment : public GridBase { public: explicit GridExperiment(struct ConfigGrid& config); void go(struct ConfigMPI& config_mpi); ~GridExperiment() = default; json loadResults(); - static inline std::string NO_CONTINUE() { return "NO_CONTINUE"; } private: void save(json& results); json initializeResults(); - std::vector filterDatasets(Datasets& datasets) const; - struct ConfigGrid config; - json build_tasks_mpi(int rank); - Timer timer; // used to measure the time of the whole process - const std::string separator = "|"; - bayesnet::Smoothing_t smooth_type{ bayesnet::Smoothing_t::NONE }; + json build_tasks_mpi(); + }; + /* ************************************************************************************************************* + // + // MPI Search Functions + // + ************************************************************************************************************* */ + class MPI_EXPERIMENT { + public: + static std::string get_color_rank(int rank) + { + auto colors = { Colors::WHITE(), Colors::RED(), Colors::GREEN(), Colors::BLUE(), Colors::MAGENTA(), Colors::CYAN(), Colors::YELLOW(), Colors::BLACK() }; + std::string id = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + auto idx = rank % id.size(); + return *(colors.begin() + rank % colors.size()) + id[idx]; + } + static json producer(std::vector& names, json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) + { + Task_Result result; + json results; + int num_tasks = tasks.size(); + + // + // 2a.1 Producer will loop to send all the tasks to the consumers and receive the results + // + for (int i = 0; i < num_tasks; ++i) { + MPI_Status status; + MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); + if (status.MPI_TAG == TAG_RESULT) { + //Store result + store_search_result(names, result, results); + } + MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD); + } + // + // 2a.2 Producer will send the end message to all the consumers + // + for (int i = 0; i < config_mpi.n_procs - 1; ++i) { + MPI_Status status; + MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); + if (status.MPI_TAG == TAG_RESULT) { + //Store result + store_search_result(names, result, results); + } + MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD); + } + return results; + } + static void consumer(Datasets& datasets, json& tasks, struct ConfigGrid& config, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) + { + Task_Result result; + // + // 2b.1 Consumers announce to the producer that they are ready to receive a task + // + MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_QUERY, MPI_COMM_WORLD); + int task; + while (true) { + MPI_Status status; + // + // 2b.2 Consumers receive the task from the producer and process it + // + MPI_Recv(&task, 1, MPI_INT, config_mpi.manager, MPI_ANY_TAG, MPI_COMM_WORLD, &status); + if (status.MPI_TAG == TAG_END) { + break; + } + mpi_experiment_consumer_go(config, config_mpi, tasks, task, datasets, &result); + // + // 2b.3 Consumers send the result to the producer + // + MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_RESULT, MPI_COMM_WORLD); + } + } + static void select_best_results_folds(json& results, json& all_results, std::string& model) + { + Timer timer; + auto grid = GridData(Paths::grid_input(model)); + // + // Select the best result of the computed outer folds + // + for (const auto& result : all_results.items()) { + // each result has the results of all the outer folds as each one were a different task + double best_score = 0.0; + json best; + for (const auto& result_fold : result.value()) { + double score = result_fold["score"].get(); + if (score > best_score) { + best_score = score; + best = result_fold; + } + } + auto dataset = result.key(); + auto combinations = grid.getGrid(dataset); + json json_best = { + { "score", best_score }, + { "hyperparameters", combinations[best["combination"].get()] }, + { "date", get_date() + " " + get_time() }, + { "grid", grid.getInputGrid(dataset) }, + { "duration", timer.translate2String(best["time"].get()) } + }; + results[dataset] = json_best; + } + } + static json store_search_result(std::vector& names, Task_Result& result, json& results) + { + json json_result = { + { "score", result.score }, + { "combination", result.idx_combination }, + { "fold", result.n_fold }, + { "time", result.time }, + { "dataset", result.idx_dataset } + }; + auto name = names[result.idx_dataset]; + if (!results.contains(name)) { + results[name] = json::array(); + } + results[name].push_back(json_result); + return results; + } + static void consumer_go(struct ConfigGrid& config, struct ConfigMPI& config_mpi, json& tasks, int n_task, Datasets& datasets, Task_Result* result) + { + // + // initialize + // + Timer timer; + timer.start(); + json task = tasks[n_task]; + auto model = config.model; + auto grid = GridData(Paths::grid_input(model)); + auto dataset_name = task["dataset"].get(); + auto idx_dataset = task["idx_dataset"].get(); + auto seed = task["seed"].get(); + auto n_fold = task["fold"].get(); + bool stratified = config.stratified; + bayesnet::Smoothing_t smooth; + if (config.smooth_strategy == "ORIGINAL") + smooth = bayesnet::Smoothing_t::ORIGINAL; + else if (config.smooth_strategy == "LAPLACE") + smooth = bayesnet::Smoothing_t::LAPLACE; + else if (config.smooth_strategy == "CESTNIK") + smooth = bayesnet::Smoothing_t::CESTNIK; + // + // Generate the hyperparameters combinations + // + auto& dataset = datasets.getDataset(dataset_name); + auto combinations = grid.getGrid(dataset_name); + dataset.load(); + auto [X, y] = dataset.getTensors(); + auto features = dataset.getFeatures(); + auto className = dataset.getClassName(); + // + // Start working on task + // + folding::Fold* fold; + if (stratified) + fold = new folding::StratifiedKFold(config.n_folds, y, seed); + else + fold = new folding::KFold(config.n_folds, y.size(0), seed); + auto [train, test] = fold->getFold(n_fold); + auto [X_train, X_test, y_train, y_test] = dataset.getTrainTestTensors(train, test); + auto states = dataset.getStates(); // Get the states of the features Once they are discretized + float best_fold_score = 0.0; + int best_idx_combination = -1; + json best_fold_hyper; + for (int idx_combination = 0; idx_combination < combinations.size(); ++idx_combination) { + auto hyperparam_line = combinations[idx_combination]; + auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); + folding::Fold* nested_fold; + if (config.stratified) + nested_fold = new folding::StratifiedKFold(config.nested, y_train, seed); + else + nested_fold = new folding::KFold(config.nested, y_train.size(0), seed); + double score = 0.0; + for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) { + // + // Nested level fold + // + auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold); + auto train_nested_t = torch::tensor(train_nested); + auto test_nested_t = torch::tensor(test_nested); + auto X_nested_train = X_train.index({ "...", train_nested_t }); + auto y_nested_train = y_train.index({ train_nested_t }); + auto X_nested_test = X_train.index({ "...", test_nested_t }); + auto y_nested_test = y_train.index({ test_nested_t }); + // + // Build Classifier with selected hyperparameters + // + auto clf = Models::instance()->create(config.model); + auto valid = clf->getValidHyperparameters(); + hyperparameters.check(valid, dataset_name); + clf->setHyperparameters(hyperparameters.get(dataset_name)); + // + // Train model + // + clf->fit(X_nested_train, y_nested_train, features, className, states, smooth); + // + // Test model + // + score += clf->score(X_nested_test, y_nested_test); + } + delete nested_fold; + score /= config.nested; + if (score > best_fold_score) { + best_fold_score = score; + best_idx_combination = idx_combination; + best_fold_hyper = hyperparam_line; + } + } + delete fold; + // + // Build Classifier with the best hyperparameters to obtain the best score + // + auto hyperparameters = platform::HyperParameters(datasets.getNames(), best_fold_hyper); + auto clf = Models::instance()->create(config.model); + auto valid = clf->getValidHyperparameters(); + hyperparameters.check(valid, dataset_name); + clf->setHyperparameters(best_fold_hyper); + clf->fit(X_train, y_train, features, className, states, smooth); + best_fold_score = clf->score(X_test, y_test); + // + // Return the result + // + result->idx_dataset = task["idx_dataset"].get(); + result->idx_combination = best_idx_combination; + result->score = best_fold_score; + result->n_fold = n_fold; + result->time = timer.getDuration(); + // + // Update progress bar + // + std::cout << get_color_rank(config_mpi.rank) << std::flush; + } }; } /* namespace platform */ #endif \ No newline at end of file diff --git a/src/grid/GridFunctions.cpp b/src/grid/GridFunctions.cpp index ad0584a..9b81247 100644 --- a/src/grid/GridFunctions.cpp +++ b/src/grid/GridFunctions.cpp @@ -6,6 +6,7 @@ #include "common/Paths.h" #include "common/Colors.h" #include "common/Utils.h" +#include "GridConfig.h" namespace platform { using json = nlohmann::ordered_json; std::string get_color_rank(int rank) @@ -34,7 +35,7 @@ namespace platform { MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); if (status.MPI_TAG == TAG_RESULT) { //Store result - store_search_result(names, result, results); + store_experiment_result(names, result, results); } MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD); } @@ -46,7 +47,7 @@ namespace platform { MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); if (status.MPI_TAG == TAG_RESULT) { //Store result - store_search_result(names, result, results); + store_experiment_result(names, result, results); } MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD); } @@ -69,7 +70,7 @@ namespace platform { if (status.MPI_TAG == TAG_END) { break; } - mpi_search_consumer_go(config, config_mpi, tasks, task, datasets, &result); + mpi_experiment_consumer_go(config, config_mpi, tasks, task, datasets, &result); // // 2b.3 Consumers send the result to the producer // @@ -235,224 +236,5 @@ namespace platform { // std::cout << get_color_rank(config_mpi.rank) << std::flush; } - /* ************************************************************************************************************* - // - // MPI Search Functions - // - ************************************************************************************************************* */ - json mpi_search_producer(std::vector& names, json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) - { - Task_Result result; - json results; - int num_tasks = tasks.size(); - // - // 2a.1 Producer will loop to send all the tasks to the consumers and receive the results - // - for (int i = 0; i < num_tasks; ++i) { - MPI_Status status; - MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); - if (status.MPI_TAG == TAG_RESULT) { - //Store result - store_search_result(names, result, results); - } - MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD); - } - // - // 2a.2 Producer will send the end message to all the consumers - // - for (int i = 0; i < config_mpi.n_procs - 1; ++i) { - MPI_Status status; - MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); - if (status.MPI_TAG == TAG_RESULT) { - //Store result - store_search_result(names, result, results); - } - MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD); - } - return results; - } - void mpi_search_consumer(Datasets& datasets, json& tasks, struct ConfigGrid& config, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) - { - Task_Result result; - // - // 2b.1 Consumers announce to the producer that they are ready to receive a task - // - MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_QUERY, MPI_COMM_WORLD); - int task; - while (true) { - MPI_Status status; - // - // 2b.2 Consumers receive the task from the producer and process it - // - MPI_Recv(&task, 1, MPI_INT, config_mpi.manager, MPI_ANY_TAG, MPI_COMM_WORLD, &status); - if (status.MPI_TAG == TAG_END) { - break; - } - mpi_experiment_consumer_go(config, config_mpi, tasks, task, datasets, &result); - // - // 2b.3 Consumers send the result to the producer - // - MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_RESULT, MPI_COMM_WORLD); - } - } - void select_best_results_folds(json& results, json& all_results, std::string& model) - { - Timer timer; - auto grid = GridData(Paths::grid_input(model)); - // - // Select the best result of the computed outer folds - // - for (const auto& result : all_results.items()) { - // each result has the results of all the outer folds as each one were a different task - double best_score = 0.0; - json best; - for (const auto& result_fold : result.value()) { - double score = result_fold["score"].get(); - if (score > best_score) { - best_score = score; - best = result_fold; - } - } - auto dataset = result.key(); - auto combinations = grid.getGrid(dataset); - json json_best = { - { "score", best_score }, - { "hyperparameters", combinations[best["combination"].get()] }, - { "date", get_date() + " " + get_time() }, - { "grid", grid.getInputGrid(dataset) }, - { "duration", timer.translate2String(best["time"].get()) } - }; - results[dataset] = json_best; - } - } - json store_search_result(std::vector& names, Task_Result& result, json& results) - { - json json_result = { - { "score", result.score }, - { "combination", result.idx_combination }, - { "fold", result.n_fold }, - { "time", result.time }, - { "dataset", result.idx_dataset } - }; - auto name = names[result.idx_dataset]; - if (!results.contains(name)) { - results[name] = json::array(); - } - results[name].push_back(json_result); - return results; - } - void mpi_experiment_consumer_go(struct ConfigGrid& config, struct ConfigMPI& config_mpi, json& tasks, int n_task, Datasets& datasets, Task_Result* result) - { - // - // initialize - // - Timer timer; - timer.start(); - json task = tasks[n_task]; - auto model = config.model; - auto grid = GridData(Paths::grid_input(model)); - auto dataset_name = task["dataset"].get(); - auto idx_dataset = task["idx_dataset"].get(); - auto seed = task["seed"].get(); - auto n_fold = task["fold"].get(); - bool stratified = config.stratified; - bayesnet::Smoothing_t smooth; - if (config.smooth_strategy == "ORIGINAL") - smooth = bayesnet::Smoothing_t::ORIGINAL; - else if (config.smooth_strategy == "LAPLACE") - smooth = bayesnet::Smoothing_t::LAPLACE; - else if (config.smooth_strategy == "CESTNIK") - smooth = bayesnet::Smoothing_t::CESTNIK; - // - // Generate the hyperparameters combinations - // - auto& dataset = datasets.getDataset(dataset_name); - auto combinations = grid.getGrid(dataset_name); - dataset.load(); - auto [X, y] = dataset.getTensors(); - auto features = dataset.getFeatures(); - auto className = dataset.getClassName(); - // - // Start working on task - // - folding::Fold* fold; - if (stratified) - fold = new folding::StratifiedKFold(config.n_folds, y, seed); - else - fold = new folding::KFold(config.n_folds, y.size(0), seed); - auto [train, test] = fold->getFold(n_fold); - auto [X_train, X_test, y_train, y_test] = dataset.getTrainTestTensors(train, test); - auto states = dataset.getStates(); // Get the states of the features Once they are discretized - float best_fold_score = 0.0; - int best_idx_combination = -1; - json best_fold_hyper; - for (int idx_combination = 0; idx_combination < combinations.size(); ++idx_combination) { - auto hyperparam_line = combinations[idx_combination]; - auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); - folding::Fold* nested_fold; - if (config.stratified) - nested_fold = new folding::StratifiedKFold(config.nested, y_train, seed); - else - nested_fold = new folding::KFold(config.nested, y_train.size(0), seed); - double score = 0.0; - for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) { - // - // Nested level fold - // - auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold); - auto train_nested_t = torch::tensor(train_nested); - auto test_nested_t = torch::tensor(test_nested); - auto X_nested_train = X_train.index({ "...", train_nested_t }); - auto y_nested_train = y_train.index({ train_nested_t }); - auto X_nested_test = X_train.index({ "...", test_nested_t }); - auto y_nested_test = y_train.index({ test_nested_t }); - // - // Build Classifier with selected hyperparameters - // - auto clf = Models::instance()->create(config.model); - auto valid = clf->getValidHyperparameters(); - hyperparameters.check(valid, dataset_name); - clf->setHyperparameters(hyperparameters.get(dataset_name)); - // - // Train model - // - clf->fit(X_nested_train, y_nested_train, features, className, states, smooth); - // - // Test model - // - score += clf->score(X_nested_test, y_nested_test); - } - delete nested_fold; - score /= config.nested; - if (score > best_fold_score) { - best_fold_score = score; - best_idx_combination = idx_combination; - best_fold_hyper = hyperparam_line; - } - } - delete fold; - // - // Build Classifier with the best hyperparameters to obtain the best score - // - auto hyperparameters = platform::HyperParameters(datasets.getNames(), best_fold_hyper); - auto clf = Models::instance()->create(config.model); - auto valid = clf->getValidHyperparameters(); - hyperparameters.check(valid, dataset_name); - clf->setHyperparameters(best_fold_hyper); - clf->fit(X_train, y_train, features, className, states, smooth); - best_fold_score = clf->score(X_test, y_test); - // - // Return the result - // - result->idx_dataset = task["idx_dataset"].get(); - result->idx_combination = best_idx_combination; - result->score = best_fold_score; - result->n_fold = n_fold; - result->time = timer.getDuration(); - // - // Update progress bar - // - std::cout << get_color_rank(config_mpi.rank) << std::flush; - } } \ No newline at end of file diff --git a/src/grid/GridSearch.cpp b/src/grid/GridSearch.cpp index 886a11d..51ab895 100644 --- a/src/grid/GridSearch.cpp +++ b/src/grid/GridSearch.cpp @@ -9,18 +9,8 @@ #include "GridSearch.h" namespace platform { - GridSearch::GridSearch(struct ConfigGrid& config) : config(config) + GridSearch::GridSearch(struct ConfigGrid& config) : GridBase(config) { - if (config.smooth_strategy == "ORIGINAL") - smooth_type = bayesnet::Smoothing_t::ORIGINAL; - else if (config.smooth_strategy == "LAPLACE") - smooth_type = bayesnet::Smoothing_t::LAPLACE; - else if (config.smooth_strategy == "CESTNIK") - smooth_type = bayesnet::Smoothing_t::CESTNIK; - else { - std::cerr << "GridSearch: Unknown smoothing strategy: " << config.smooth_strategy << std::endl; - exit(1); - } } json GridSearch::loadResults() { @@ -63,7 +53,7 @@ namespace platform { } return datasets_names; } - json GridSearch::build_tasks_mpi(int rank) + json GridSearch::build_tasks_mpi() { auto tasks = json::array(); auto grid = GridData(Paths::grid_input(config.model)); @@ -155,7 +145,7 @@ namespace platform { json tasks; if (config_mpi.rank == config_mpi.manager) { timer.start(); - tasks = build_tasks_mpi(config_mpi.rank); + tasks = build_tasks_mpi(); auto tasks_str = tasks.dump(); tasks_size = tasks_str.size(); msg = new char[tasks_size + 1]; @@ -179,13 +169,13 @@ namespace platform { // 2a. Producer delivers the tasks to the consumers // auto datasets_names = filterDatasets(datasets); - json all_results = mpi_search_producer(datasets_names, tasks, config_mpi, MPI_Result); + json all_results = MPI_SEARCH::producer(datasets_names, tasks, config_mpi, MPI_Result); std::cout << separator << std::endl; // // 3. Manager select the bests sccores for each dataset // auto results = initializeResults(); - select_best_results_folds(results, all_results, config.model); + MPI_SEARCH::select_best_results_folds(results, all_results, config.model); // // 3.2 Save the results // @@ -194,7 +184,7 @@ namespace platform { // // 2b. Consumers process the tasks and send the results to the producer // - mpi_search_consumer(datasets, tasks, config, config_mpi, MPI_Result); + MPI_SEARCH::consumer(datasets, tasks, config, config_mpi, MPI_Result); } } json GridSearch::initializeResults() diff --git a/src/grid/GridSearch.h b/src/grid/GridSearch.h index 8504f69..3735a5f 100644 --- a/src/grid/GridSearch.h +++ b/src/grid/GridSearch.h @@ -4,17 +4,18 @@ #include #include #include +#include #include "common/Datasets.h" #include "common/Timer.h" #include "main/HyperParameters.h" #include "GridData.h" -#include "GridConfig.h" +#include "GridBase.h" #include "bayesnet/network/Network.h" namespace platform { using json = nlohmann::ordered_json; - class GridSearch { + class GridSearch : public GridBase { public: explicit GridSearch(struct ConfigGrid& config); void go(struct ConfigMPI& config_mpi); @@ -25,11 +26,230 @@ namespace platform { void save(json& results); json initializeResults(); std::vector filterDatasets(Datasets& datasets) const; - struct ConfigGrid config; - json build_tasks_mpi(int rank); - Timer timer; // used to measure the time of the whole process - const std::string separator = "|"; - bayesnet::Smoothing_t smooth_type{ bayesnet::Smoothing_t::NONE }; + json build_tasks_mpi(); + }; + /* ************************************************************************************************************* + // + // MPI Search Functions + // + ************************************************************************************************************* */ + class MPI_SEARCH { + public: + static json producer(std::vector& names, json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) + { + Task_Result result; + json results; + int num_tasks = tasks.size(); + + // + // 2a.1 Producer will loop to send all the tasks to the consumers and receive the results + // + for (int i = 0; i < num_tasks; ++i) { + MPI_Status status; + MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); + if (status.MPI_TAG == TAG_RESULT) { + //Store result + store_search_result(names, result, results); + } + MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD); + } + // + // 2a.2 Producer will send the end message to all the consumers + // + for (int i = 0; i < config_mpi.n_procs - 1; ++i) { + MPI_Status status; + MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); + if (status.MPI_TAG == TAG_RESULT) { + //Store result + store_search_result(names, result, results); + } + MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD); + } + return results; + } + static void consumer(Datasets& datasets, json& tasks, struct ConfigGrid& config, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) + { + Task_Result result; + // + // 2b.1 Consumers announce to the producer that they are ready to receive a task + // + MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_QUERY, MPI_COMM_WORLD); + int task; + while (true) { + MPI_Status status; + // + // 2b.2 Consumers receive the task from the producer and process it + // + MPI_Recv(&task, 1, MPI_INT, config_mpi.manager, MPI_ANY_TAG, MPI_COMM_WORLD, &status); + if (status.MPI_TAG == TAG_END) { + break; + } + mpi_experiment_consumer_go(config, config_mpi, tasks, task, datasets, &result); + // + // 2b.3 Consumers send the result to the producer + // + MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_RESULT, MPI_COMM_WORLD); + } + } + static void select_best_results_folds(json& results, json& all_results, std::string& model) + { + Timer timer; + auto grid = GridData(Paths::grid_input(model)); + // + // Select the best result of the computed outer folds + // + for (const auto& result : all_results.items()) { + // each result has the results of all the outer folds as each one were a different task + double best_score = 0.0; + json best; + for (const auto& result_fold : result.value()) { + double score = result_fold["score"].get(); + if (score > best_score) { + best_score = score; + best = result_fold; + } + } + auto dataset = result.key(); + auto combinations = grid.getGrid(dataset); + json json_best = { + { "score", best_score }, + { "hyperparameters", combinations[best["combination"].get()] }, + { "date", get_date() + " " + get_time() }, + { "grid", grid.getInputGrid(dataset) }, + { "duration", timer.translate2String(best["time"].get()) } + }; + results[dataset] = json_best; + } + } + static json store_search_result(std::vector& names, Task_Result& result, json& results) + { + json json_result = { + { "score", result.score }, + { "combination", result.idx_combination }, + { "fold", result.n_fold }, + { "time", result.time }, + { "dataset", result.idx_dataset } + }; + auto name = names[result.idx_dataset]; + if (!results.contains(name)) { + results[name] = json::array(); + } + results[name].push_back(json_result); + return results; + } + static void consumer_go(struct ConfigGrid& config, struct ConfigMPI& config_mpi, json& tasks, int n_task, Datasets& datasets, Task_Result* result) + { + // + // initialize + // + Timer timer; + timer.start(); + json task = tasks[n_task]; + auto model = config.model; + auto grid = GridData(Paths::grid_input(model)); + auto dataset_name = task["dataset"].get(); + auto idx_dataset = task["idx_dataset"].get(); + auto seed = task["seed"].get(); + auto n_fold = task["fold"].get(); + bool stratified = config.stratified; + bayesnet::Smoothing_t smooth; + if (config.smooth_strategy == "ORIGINAL") + smooth = bayesnet::Smoothing_t::ORIGINAL; + else if (config.smooth_strategy == "LAPLACE") + smooth = bayesnet::Smoothing_t::LAPLACE; + else if (config.smooth_strategy == "CESTNIK") + smooth = bayesnet::Smoothing_t::CESTNIK; + // + // Generate the hyperparameters combinations + // + auto& dataset = datasets.getDataset(dataset_name); + auto combinations = grid.getGrid(dataset_name); + dataset.load(); + auto [X, y] = dataset.getTensors(); + auto features = dataset.getFeatures(); + auto className = dataset.getClassName(); + // + // Start working on task + // + folding::Fold* fold; + if (stratified) + fold = new folding::StratifiedKFold(config.n_folds, y, seed); + else + fold = new folding::KFold(config.n_folds, y.size(0), seed); + auto [train, test] = fold->getFold(n_fold); + auto [X_train, X_test, y_train, y_test] = dataset.getTrainTestTensors(train, test); + auto states = dataset.getStates(); // Get the states of the features Once they are discretized + float best_fold_score = 0.0; + int best_idx_combination = -1; + json best_fold_hyper; + for (int idx_combination = 0; idx_combination < combinations.size(); ++idx_combination) { + auto hyperparam_line = combinations[idx_combination]; + auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); + folding::Fold* nested_fold; + if (config.stratified) + nested_fold = new folding::StratifiedKFold(config.nested, y_train, seed); + else + nested_fold = new folding::KFold(config.nested, y_train.size(0), seed); + double score = 0.0; + for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) { + // + // Nested level fold + // + auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold); + auto train_nested_t = torch::tensor(train_nested); + auto test_nested_t = torch::tensor(test_nested); + auto X_nested_train = X_train.index({ "...", train_nested_t }); + auto y_nested_train = y_train.index({ train_nested_t }); + auto X_nested_test = X_train.index({ "...", test_nested_t }); + auto y_nested_test = y_train.index({ test_nested_t }); + // + // Build Classifier with selected hyperparameters + // + auto clf = Models::instance()->create(config.model); + auto valid = clf->getValidHyperparameters(); + hyperparameters.check(valid, dataset_name); + clf->setHyperparameters(hyperparameters.get(dataset_name)); + // + // Train model + // + clf->fit(X_nested_train, y_nested_train, features, className, states, smooth); + // + // Test model + // + score += clf->score(X_nested_test, y_nested_test); + } + delete nested_fold; + score /= config.nested; + if (score > best_fold_score) { + best_fold_score = score; + best_idx_combination = idx_combination; + best_fold_hyper = hyperparam_line; + } + } + delete fold; + // + // Build Classifier with the best hyperparameters to obtain the best score + // + auto hyperparameters = platform::HyperParameters(datasets.getNames(), best_fold_hyper); + auto clf = Models::instance()->create(config.model); + auto valid = clf->getValidHyperparameters(); + hyperparameters.check(valid, dataset_name); + clf->setHyperparameters(best_fold_hyper); + clf->fit(X_train, y_train, features, className, states, smooth); + best_fold_score = clf->score(X_test, y_test); + // + // Return the result + // + result->idx_dataset = task["idx_dataset"].get(); + result->idx_combination = best_idx_combination; + result->score = best_fold_score; + result->n_fold = n_fold; + result->time = timer.getDuration(); + // + // Update progress bar + // + std::cout << get_color_rank(config_mpi.rank) << std::flush; + } }; } /* namespace platform */ #endif \ No newline at end of file