rmontanana/Platform
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add grid base class and static class

Browse Source
This commit is contained in:
Ricardo Montañana Gómez
2024-12-20 18:54:08 +01:00
parent 1a336a094e
commit f88944de36
7 changed files with 524 additions and 323 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/grid/GridBase.h Normal file
View File

@@ -0,0 +1,43 @@
#ifndef GRIDBASE_H
#define GRIDBASE_H
#include <string>
#include <map>
#include <mpi.h>
#include <nlohmann/json.hpp>
#include "common/Datasets.h"
#include "common/Timer.h"
#include "main/HyperParameters.h"
#include "GridData.h"
#include "GridConfig.h"
#include "bayesnet/network/Network.h"
namespace platform {
using json = nlohmann::ordered_json;
class GridBase {
public:
explicit GridBase(struct ConfigGrid& config)
{
this->config = config;
if (config.smooth_strategy == "ORIGINAL")
smooth_type = bayesnet::Smoothing_t::ORIGINAL;
else if (config.smooth_strategy == "LAPLACE")
smooth_type = bayesnet::Smoothing_t::LAPLACE;
else if (config.smooth_strategy == "CESTNIK")
smooth_type = bayesnet::Smoothing_t::CESTNIK;
else {
std::cerr << "GridBase: Unknown smoothing strategy: " << config.smooth_strategy << std::endl;
exit(1);
}
};
~GridBase() = default;
virtual void go(struct ConfigMPI& config_mpi) = 0;
virtual json build_tasks_mpi() = 0;
protected:
struct ConfigGrid config;
Timer timer; // used to measure the time of the whole process
const std::string separator = "|";
bayesnet::Smoothing_t smooth_type{ bayesnet::Smoothing_t::NONE };
};
} /* namespace platform */
#endif

3
src/grid/GridConfig.h
View File

@@ -8,6 +8,7 @@
#include "common/Timer.h"
#include "main/HyperParameters.h"
#include "GridData.h"
#include "GridConfig.h"
#include "bayesnet/network/Network.h"
@@ -69,6 +70,6 @@ namespace platform {
void mpi_search_consumer(Datasets& datasets, json& tasks, struct ConfigGrid& config, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result);
void select_best_results_folds(json& results, json& all_results, std::string& model);
json store_search_result(std::vector<std::string>& names, Task_Result& result, json& results);
void mpi_experiment_consumer_go(struct ConfigGrid& config, struct ConfigMPI& config_mpi, json& tasks, int n_task, Datasets& datasets, Task_Result* result);
void mpi_search_consumer_go(struct ConfigGrid& config, struct ConfigMPI& config_mpi, json& tasks, int n_task, Datasets& datasets, Task_Result* result);
} /* namespace platform */
#endif

77
src/grid/GridExperiment.cpp
View File

@@ -10,18 +10,8 @@
namespace platform {
GridExperiment::GridExperiment(struct ConfigGrid& config) : config(config)
GridExperiment::GridExperiment(struct ConfigGrid& config) : GridBase(config)
{
if (config.smooth_strategy == "ORIGINAL")
smooth_type = bayesnet::Smoothing_t::ORIGINAL;
else if (config.smooth_strategy == "LAPLACE")
smooth_type = bayesnet::Smoothing_t::LAPLACE;
else if (config.smooth_strategy == "CESTNIK")
smooth_type = bayesnet::Smoothing_t::CESTNIK;
else {
std::cerr << "GridSearch: Unknown smoothing strategy: " << config.smooth_strategy << std::endl;
exit(1);
}
}
json GridExperiment::loadResults()
{
@@ -31,46 +21,13 @@ namespace platform {
}
return json();
}
std::vector<std::string> GridExperiment::filterDatasets(Datasets& datasets) const
{
// Load datasets
auto datasets_names = datasets.getNames();
if (config.continue_from != NO_CONTINUE()) {
// Continue previous execution:
if (std::find(datasets_names.begin(), datasets_names.end(), config.continue_from) == datasets_names.end()) {
throw std::invalid_argument("Dataset " + config.continue_from + " not found");
}
// Remove datasets already processed
std::vector<string>::iterator it = datasets_names.begin();
while (it != datasets_names.end()) {
if (*it != config.continue_from) {
it = datasets_names.erase(it);
} else {
if (config.only)
++it;
else
break;
}
}
}
// Exclude datasets
for (const auto& name : config.excluded) {
auto dataset = name.get<std::string>();
auto it = std::find(datasets_names.begin(), datasets_names.end(), dataset);
if (it == datasets_names.end()) {
throw std::invalid_argument("Dataset " + dataset + " already excluded or doesn't exist!");
}
datasets_names.erase(it);
}
return datasets_names;
}
json GridExperiment::build_tasks_mpi(int rank)
json GridExperiment::build_tasks_mpi()
{
auto tasks = json::array();
auto grid = GridData(Paths::grid_input(config.model));
auto datasets = Datasets(false, Paths::datasets());
auto all_datasets = datasets.getNames();
auto datasets_names = filterDatasets(datasets);
auto datasets_names = all_datasets;
for (int idx_dataset = 0; idx_dataset < datasets_names.size(); ++idx_dataset) {
auto dataset = datasets_names[idx_dataset];
for (const auto& seed : config.seeds) {
@@ -156,7 +113,7 @@ namespace platform {
json tasks;
if (config_mpi.rank == config_mpi.manager) {
timer.start();
tasks = build_tasks_mpi(config_mpi.rank);
tasks = build_tasks_mpi();
auto tasks_str = tasks.dump();
tasks_size = tasks_str.size();
msg = new char[tasks_size + 1];
@@ -179,45 +136,29 @@ namespace platform {
//
// 2a. Producer delivers the tasks to the consumers
//
auto datasets_names = filterDatasets(datasets);
json all_results = mpi_search_producer(datasets_names, tasks, config_mpi, MPI_Result);
auto datasets_names = std::vector<std::string>();
json all_results = mpi_experiment_producer(datasets_names, tasks, config_mpi, MPI_Result);
std::cout << separator << std::endl;
//
// 3. Manager select the bests sccores for each dataset
//
auto results = initializeResults();
select_best_results_folds(results, all_results, config.model);
//select_best_results_folds(results, all_results, config.model);
//
// 3.2 Save the results
//
save(results);
} else {
//
// 2b. Consumers process the tasks and send the results to the producer
// 2b. Consumers prostore_search_resultcess the tasks and send the results to the producer
//
mpi_search_consumer(datasets, tasks, config, config_mpi, MPI_Result);
mpi_experiment_consumer(datasets, tasks, config, config_mpi, MPI_Result);
}
}
json GridExperiment::initializeResults()
{
// Load previous results if continue is set
json results;
if (config.continue_from != NO_CONTINUE()) {
if (!config.quiet)
std::cout << "* Loading previous results" << std::endl;
try {
std::ifstream file(Paths::grid_output(config.model));
if (file.is_open()) {
results = json::parse(file);
results = results["results"];
}
}
catch (const std::exception& e) {
std::cerr << "* There were no previous results" << std::endl;
std::cerr << "* Initizalizing new results" << std::endl;
results = json();
}
}
return results;
}
void GridExperiment::save(json& results)

242
src/grid/GridExperiment.h
View File

@@ -8,28 +8,252 @@
#include "common/Timer.h"
#include "main/HyperParameters.h"
#include "GridData.h"
#include "GridConfig.h"
#include "GridBase.h"
#include "bayesnet/network/Network.h"
namespace platform {
using json = nlohmann::ordered_json;
class GridExperiment {
class GridExperiment : public GridBase {
public:
explicit GridExperiment(struct ConfigGrid& config);
void go(struct ConfigMPI& config_mpi);
~GridExperiment() = default;
json loadResults();
static inline std::string NO_CONTINUE() { return "NO_CONTINUE"; }
private:
void save(json& results);
json initializeResults();
std::vector<std::string> filterDatasets(Datasets& datasets) const;
struct ConfigGrid config;
json build_tasks_mpi(int rank);
Timer timer; // used to measure the time of the whole process
const std::string separator = "|";
bayesnet::Smoothing_t smooth_type{ bayesnet::Smoothing_t::NONE };
json build_tasks_mpi();
};
/* *************************************************************************************************************
//
// MPI Search Functions
//
************************************************************************************************************* */
class MPI_EXPERIMENT {
public:
static std::string get_color_rank(int rank)
{
auto colors = { Colors::WHITE(), Colors::RED(), Colors::GREEN(), Colors::BLUE(), Colors::MAGENTA(), Colors::CYAN(), Colors::YELLOW(), Colors::BLACK() };
std::string id = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
auto idx = rank % id.size();
return *(colors.begin() + rank % colors.size()) + id[idx];
}
static json producer(std::vector<std::string>& names, json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result)
{
Task_Result result;
json results;
int num_tasks = tasks.size();
//
// 2a.1 Producer will loop to send all the tasks to the consumers and receive the results
//
for (int i = 0; i < num_tasks; ++i) {
MPI_Status status;
MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
if (status.MPI_TAG == TAG_RESULT) {
//Store result
store_search_result(names, result, results);
}
MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD);
}
//
// 2a.2 Producer will send the end message to all the consumers
//
for (int i = 0; i < config_mpi.n_procs - 1; ++i) {
MPI_Status status;
MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
if (status.MPI_TAG == TAG_RESULT) {
//Store result
store_search_result(names, result, results);
}
MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD);
}
return results;
}
static void consumer(Datasets& datasets, json& tasks, struct ConfigGrid& config, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result)
{
Task_Result result;
//
// 2b.1 Consumers announce to the producer that they are ready to receive a task
//
MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_QUERY, MPI_COMM_WORLD);
int task;
while (true) {
MPI_Status status;
//
// 2b.2 Consumers receive the task from the producer and process it
//
MPI_Recv(&task, 1, MPI_INT, config_mpi.manager, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
if (status.MPI_TAG == TAG_END) {
break;
}
mpi_experiment_consumer_go(config, config_mpi, tasks, task, datasets, &result);
//
// 2b.3 Consumers send the result to the producer
//
MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_RESULT, MPI_COMM_WORLD);
}
}
static void select_best_results_folds(json& results, json& all_results, std::string& model)
{
Timer timer;
auto grid = GridData(Paths::grid_input(model));
//
// Select the best result of the computed outer folds
//
for (const auto& result : all_results.items()) {
// each result has the results of all the outer folds as each one were a different task
double best_score = 0.0;
json best;
for (const auto& result_fold : result.value()) {
double score = result_fold["score"].get<double>();
if (score > best_score) {
best_score = score;
best = result_fold;
}
}
auto dataset = result.key();
auto combinations = grid.getGrid(dataset);
json json_best = {
{ "score", best_score },
{ "hyperparameters", combinations[best["combination"].get<int>()] },
{ "date", get_date() + " " + get_time() },
{ "grid", grid.getInputGrid(dataset) },
{ "duration", timer.translate2String(best["time"].get<double>()) }
};
results[dataset] = json_best;
}
}
static json store_search_result(std::vector<std::string>& names, Task_Result& result, json& results)
{
json json_result = {
{ "score", result.score },
{ "combination", result.idx_combination },
{ "fold", result.n_fold },
{ "time", result.time },
{ "dataset", result.idx_dataset }
};
auto name = names[result.idx_dataset];
if (!results.contains(name)) {
results[name] = json::array();
}
results[name].push_back(json_result);
return results;
}
static void consumer_go(struct ConfigGrid& config, struct ConfigMPI& config_mpi, json& tasks, int n_task, Datasets& datasets, Task_Result* result)
{
//
// initialize
//
Timer timer;
timer.start();
json task = tasks[n_task];
auto model = config.model;
auto grid = GridData(Paths::grid_input(model));
auto dataset_name = task["dataset"].get<std::string>();
auto idx_dataset = task["idx_dataset"].get<int>();
auto seed = task["seed"].get<int>();
auto n_fold = task["fold"].get<int>();
bool stratified = config.stratified;
bayesnet::Smoothing_t smooth;
if (config.smooth_strategy == "ORIGINAL")
smooth = bayesnet::Smoothing_t::ORIGINAL;
else if (config.smooth_strategy == "LAPLACE")
smooth = bayesnet::Smoothing_t::LAPLACE;
else if (config.smooth_strategy == "CESTNIK")
smooth = bayesnet::Smoothing_t::CESTNIK;
//
// Generate the hyperparameters combinations
//
auto& dataset = datasets.getDataset(dataset_name);
auto combinations = grid.getGrid(dataset_name);
dataset.load();
auto [X, y] = dataset.getTensors();
auto features = dataset.getFeatures();
auto className = dataset.getClassName();
//
// Start working on task
//
folding::Fold* fold;
if (stratified)
fold = new folding::StratifiedKFold(config.n_folds, y, seed);
else
fold = new folding::KFold(config.n_folds, y.size(0), seed);
auto [train, test] = fold->getFold(n_fold);
auto [X_train, X_test, y_train, y_test] = dataset.getTrainTestTensors(train, test);
auto states = dataset.getStates(); // Get the states of the features Once they are discretized
float best_fold_score = 0.0;
int best_idx_combination = -1;
json best_fold_hyper;
for (int idx_combination = 0; idx_combination < combinations.size(); ++idx_combination) {
auto hyperparam_line = combinations[idx_combination];
auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line);
folding::Fold* nested_fold;
if (config.stratified)
nested_fold = new folding::StratifiedKFold(config.nested, y_train, seed);
else
nested_fold = new folding::KFold(config.nested, y_train.size(0), seed);
double score = 0.0;
for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) {
//
// Nested level fold
//
auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold);
auto train_nested_t = torch::tensor(train_nested);
auto test_nested_t = torch::tensor(test_nested);
auto X_nested_train = X_train.index({ "...", train_nested_t });
auto y_nested_train = y_train.index({ train_nested_t });
auto X_nested_test = X_train.index({ "...", test_nested_t });
auto y_nested_test = y_train.index({ test_nested_t });
//
// Build Classifier with selected hyperparameters
//
auto clf = Models::instance()->create(config.model);
auto valid = clf->getValidHyperparameters();
hyperparameters.check(valid, dataset_name);
clf->setHyperparameters(hyperparameters.get(dataset_name));
//
// Train model
//
clf->fit(X_nested_train, y_nested_train, features, className, states, smooth);
//
// Test model
//
score += clf->score(X_nested_test, y_nested_test);
}
delete nested_fold;
score /= config.nested;
if (score > best_fold_score) {
best_fold_score = score;
best_idx_combination = idx_combination;
best_fold_hyper = hyperparam_line;
}
}
delete fold;
//
// Build Classifier with the best hyperparameters to obtain the best score
//
auto hyperparameters = platform::HyperParameters(datasets.getNames(), best_fold_hyper);
auto clf = Models::instance()->create(config.model);
auto valid = clf->getValidHyperparameters();
hyperparameters.check(valid, dataset_name);
clf->setHyperparameters(best_fold_hyper);
clf->fit(X_train, y_train, features, className, states, smooth);
best_fold_score = clf->score(X_test, y_test);
//
// Return the result
//
result->idx_dataset = task["idx_dataset"].get<int>();
result->idx_combination = best_idx_combination;
result->score = best_fold_score;
result->n_fold = n_fold;
result->time = timer.getDuration();
//
// Update progress bar
//
std::cout << get_color_rank(config_mpi.rank) << std::flush;
}
};
} /* namespace platform */
#endif

226
src/grid/GridFunctions.cpp
View File

@@ -6,6 +6,7 @@
#include "common/Paths.h"
#include "common/Colors.h"
#include "common/Utils.h"
#include "GridConfig.h"
namespace platform {
using json = nlohmann::ordered_json;
std::string get_color_rank(int rank)
@@ -34,7 +35,7 @@ namespace platform {
MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
if (status.MPI_TAG == TAG_RESULT) {
//Store result
store_search_result(names, result, results);
store_experiment_result(names, result, results);
}
MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD);
}
@@ -46,7 +47,7 @@ namespace platform {
MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
if (status.MPI_TAG == TAG_RESULT) {
//Store result
store_search_result(names, result, results);
store_experiment_result(names, result, results);
}
MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD);
}
@@ -69,7 +70,7 @@ namespace platform {
if (status.MPI_TAG == TAG_END) {
break;
}
mpi_search_consumer_go(config, config_mpi, tasks, task, datasets, &result);
mpi_experiment_consumer_go(config, config_mpi, tasks, task, datasets, &result);
//
// 2b.3 Consumers send the result to the producer
//
@@ -235,224 +236,5 @@ namespace platform {
//
std::cout << get_color_rank(config_mpi.rank) << std::flush;
}
/* *************************************************************************************************************
//
// MPI Search Functions
//
************************************************************************************************************* */
json mpi_search_producer(std::vector<std::string>& names, json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result)
{
Task_Result result;
json results;
int num_tasks = tasks.size();
//
// 2a.1 Producer will loop to send all the tasks to the consumers and receive the results
//
for (int i = 0; i < num_tasks; ++i) {
MPI_Status status;
MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
if (status.MPI_TAG == TAG_RESULT) {
//Store result
store_search_result(names, result, results);
}
MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD);
}
//
// 2a.2 Producer will send the end message to all the consumers
//
for (int i = 0; i < config_mpi.n_procs - 1; ++i) {
MPI_Status status;
MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
if (status.MPI_TAG == TAG_RESULT) {
//Store result
store_search_result(names, result, results);
}
MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD);
}
return results;
}
void mpi_search_consumer(Datasets& datasets, json& tasks, struct ConfigGrid& config, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result)
{
Task_Result result;
//
// 2b.1 Consumers announce to the producer that they are ready to receive a task
//
MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_QUERY, MPI_COMM_WORLD);
int task;
while (true) {
MPI_Status status;
//
// 2b.2 Consumers receive the task from the producer and process it
//
MPI_Recv(&task, 1, MPI_INT, config_mpi.manager, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
if (status.MPI_TAG == TAG_END) {
break;
}
mpi_experiment_consumer_go(config, config_mpi, tasks, task, datasets, &result);
//
// 2b.3 Consumers send the result to the producer
//
MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_RESULT, MPI_COMM_WORLD);
}
}
void select_best_results_folds(json& results, json& all_results, std::string& model)
{
Timer timer;
auto grid = GridData(Paths::grid_input(model));
//
// Select the best result of the computed outer folds
//
for (const auto& result : all_results.items()) {
// each result has the results of all the outer folds as each one were a different task
double best_score = 0.0;
json best;
for (const auto& result_fold : result.value()) {
double score = result_fold["score"].get<double>();
if (score > best_score) {
best_score = score;
best = result_fold;
}
}
auto dataset = result.key();
auto combinations = grid.getGrid(dataset);
json json_best = {
{ "score", best_score },
{ "hyperparameters", combinations[best["combination"].get<int>()] },
{ "date", get_date() + " " + get_time() },
{ "grid", grid.getInputGrid(dataset) },
{ "duration", timer.translate2String(best["time"].get<double>()) }
};
results[dataset] = json_best;
}
}
json store_search_result(std::vector<std::string>& names, Task_Result& result, json& results)
{
json json_result = {
{ "score", result.score },
{ "combination", result.idx_combination },
{ "fold", result.n_fold },
{ "time", result.time },
{ "dataset", result.idx_dataset }
};
auto name = names[result.idx_dataset];
if (!results.contains(name)) {
results[name] = json::array();
}
results[name].push_back(json_result);
return results;
}
void mpi_experiment_consumer_go(struct ConfigGrid& config, struct ConfigMPI& config_mpi, json& tasks, int n_task, Datasets& datasets, Task_Result* result)
{
//
// initialize
//
Timer timer;
timer.start();
json task = tasks[n_task];
auto model = config.model;
auto grid = GridData(Paths::grid_input(model));
auto dataset_name = task["dataset"].get<std::string>();
auto idx_dataset = task["idx_dataset"].get<int>();
auto seed = task["seed"].get<int>();
auto n_fold = task["fold"].get<int>();
bool stratified = config.stratified;
bayesnet::Smoothing_t smooth;
if (config.smooth_strategy == "ORIGINAL")
smooth = bayesnet::Smoothing_t::ORIGINAL;
else if (config.smooth_strategy == "LAPLACE")
smooth = bayesnet::Smoothing_t::LAPLACE;
else if (config.smooth_strategy == "CESTNIK")
smooth = bayesnet::Smoothing_t::CESTNIK;
//
// Generate the hyperparameters combinations
//
auto& dataset = datasets.getDataset(dataset_name);
auto combinations = grid.getGrid(dataset_name);
dataset.load();
auto [X, y] = dataset.getTensors();
auto features = dataset.getFeatures();
auto className = dataset.getClassName();
//
// Start working on task
//
folding::Fold* fold;
if (stratified)
fold = new folding::StratifiedKFold(config.n_folds, y, seed);
else
fold = new folding::KFold(config.n_folds, y.size(0), seed);
auto [train, test] = fold->getFold(n_fold);
auto [X_train, X_test, y_train, y_test] = dataset.getTrainTestTensors(train, test);
auto states = dataset.getStates(); // Get the states of the features Once they are discretized
float best_fold_score = 0.0;
int best_idx_combination = -1;
json best_fold_hyper;
for (int idx_combination = 0; idx_combination < combinations.size(); ++idx_combination) {
auto hyperparam_line = combinations[idx_combination];
auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line);
folding::Fold* nested_fold;
if (config.stratified)
nested_fold = new folding::StratifiedKFold(config.nested, y_train, seed);
else
nested_fold = new folding::KFold(config.nested, y_train.size(0), seed);
double score = 0.0;
for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) {
//
// Nested level fold
//
auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold);
auto train_nested_t = torch::tensor(train_nested);
auto test_nested_t = torch::tensor(test_nested);
auto X_nested_train = X_train.index({ "...", train_nested_t });
auto y_nested_train = y_train.index({ train_nested_t });
auto X_nested_test = X_train.index({ "...", test_nested_t });
auto y_nested_test = y_train.index({ test_nested_t });
//
// Build Classifier with selected hyperparameters
//
auto clf = Models::instance()->create(config.model);
auto valid = clf->getValidHyperparameters();
hyperparameters.check(valid, dataset_name);
clf->setHyperparameters(hyperparameters.get(dataset_name));
//
// Train model
//
clf->fit(X_nested_train, y_nested_train, features, className, states, smooth);
//
// Test model
//
score += clf->score(X_nested_test, y_nested_test);
}
delete nested_fold;
score /= config.nested;
if (score > best_fold_score) {
best_fold_score = score;
best_idx_combination = idx_combination;
best_fold_hyper = hyperparam_line;
}
}
delete fold;
//
// Build Classifier with the best hyperparameters to obtain the best score
//
auto hyperparameters = platform::HyperParameters(datasets.getNames(), best_fold_hyper);
auto clf = Models::instance()->create(config.model);
auto valid = clf->getValidHyperparameters();
hyperparameters.check(valid, dataset_name);
clf->setHyperparameters(best_fold_hyper);
clf->fit(X_train, y_train, features, className, states, smooth);
best_fold_score = clf->score(X_test, y_test);
//
// Return the result
//
result->idx_dataset = task["idx_dataset"].get<int>();
result->idx_combination = best_idx_combination;
result->score = best_fold_score;
result->n_fold = n_fold;
result->time = timer.getDuration();
//
// Update progress bar
//
std::cout << get_color_rank(config_mpi.rank) << std::flush;
}
}

22
src/grid/GridSearch.cpp
View File

@@ -9,18 +9,8 @@
#include "GridSearch.h"
namespace platform {
GridSearch::GridSearch(struct ConfigGrid& config) : config(config)
GridSearch::GridSearch(struct ConfigGrid& config) : GridBase(config)
{
if (config.smooth_strategy == "ORIGINAL")
smooth_type = bayesnet::Smoothing_t::ORIGINAL;
else if (config.smooth_strategy == "LAPLACE")
smooth_type = bayesnet::Smoothing_t::LAPLACE;
else if (config.smooth_strategy == "CESTNIK")
smooth_type = bayesnet::Smoothing_t::CESTNIK;
else {
std::cerr << "GridSearch: Unknown smoothing strategy: " << config.smooth_strategy << std::endl;
exit(1);
}
}
json GridSearch::loadResults()
{
@@ -63,7 +53,7 @@ namespace platform {
}
return datasets_names;
}
json GridSearch::build_tasks_mpi(int rank)
json GridSearch::build_tasks_mpi()
{
auto tasks = json::array();
auto grid = GridData(Paths::grid_input(config.model));
@@ -155,7 +145,7 @@ namespace platform {
json tasks;
if (config_mpi.rank == config_mpi.manager) {
timer.start();
tasks = build_tasks_mpi(config_mpi.rank);
tasks = build_tasks_mpi();
auto tasks_str = tasks.dump();
tasks_size = tasks_str.size();
msg = new char[tasks_size + 1];
@@ -179,13 +169,13 @@ namespace platform {
// 2a. Producer delivers the tasks to the consumers
//
auto datasets_names = filterDatasets(datasets);
json all_results = mpi_search_producer(datasets_names, tasks, config_mpi, MPI_Result);
json all_results = MPI_SEARCH::producer(datasets_names, tasks, config_mpi, MPI_Result);
std::cout << separator << std::endl;
//
// 3. Manager select the bests sccores for each dataset
//
auto results = initializeResults();
select_best_results_folds(results, all_results, config.model);
MPI_SEARCH::select_best_results_folds(results, all_results, config.model);
//
// 3.2 Save the results
//
@@ -194,7 +184,7 @@ namespace platform {
//
// 2b. Consumers process the tasks and send the results to the producer
//
mpi_search_consumer(datasets, tasks, config, config_mpi, MPI_Result);
MPI_SEARCH::consumer(datasets, tasks, config, config_mpi, MPI_Result);
}
}
json GridSearch::initializeResults()

234
src/grid/GridSearch.h
View File

@@ -4,17 +4,18 @@
#include <map>
#include <mpi.h>
#include <nlohmann/json.hpp>
#include <folding.hpp>
#include "common/Datasets.h"
#include "common/Timer.h"
#include "main/HyperParameters.h"
#include "GridData.h"
#include "GridConfig.h"
#include "GridBase.h"
#include "bayesnet/network/Network.h"
namespace platform {
using json = nlohmann::ordered_json;
class GridSearch {
class GridSearch : public GridBase {
public:
explicit GridSearch(struct ConfigGrid& config);
void go(struct ConfigMPI& config_mpi);
@@ -25,11 +26,230 @@ namespace platform {
void save(json& results);
json initializeResults();
std::vector<std::string> filterDatasets(Datasets& datasets) const;
struct ConfigGrid config;
json build_tasks_mpi(int rank);
Timer timer; // used to measure the time of the whole process
const std::string separator = "|";
bayesnet::Smoothing_t smooth_type{ bayesnet::Smoothing_t::NONE };
json build_tasks_mpi();
};
/* *************************************************************************************************************
//
// MPI Search Functions
//
************************************************************************************************************* */
class MPI_SEARCH {
public:
static json producer(std::vector<std::string>& names, json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result)
{
Task_Result result;
json results;
int num_tasks = tasks.size();
//
// 2a.1 Producer will loop to send all the tasks to the consumers and receive the results
//
for (int i = 0; i < num_tasks; ++i) {
MPI_Status status;
MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
if (status.MPI_TAG == TAG_RESULT) {
//Store result
store_search_result(names, result, results);
}
MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD);
}
//
// 2a.2 Producer will send the end message to all the consumers
//
for (int i = 0; i < config_mpi.n_procs - 1; ++i) {
MPI_Status status;
MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
if (status.MPI_TAG == TAG_RESULT) {
//Store result
store_search_result(names, result, results);
}
MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD);
}
return results;
}
static void consumer(Datasets& datasets, json& tasks, struct ConfigGrid& config, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result)
{
Task_Result result;
//
// 2b.1 Consumers announce to the producer that they are ready to receive a task
//
MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_QUERY, MPI_COMM_WORLD);
int task;
while (true) {
MPI_Status status;
//
// 2b.2 Consumers receive the task from the producer and process it
//
MPI_Recv(&task, 1, MPI_INT, config_mpi.manager, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
if (status.MPI_TAG == TAG_END) {
break;
}
mpi_experiment_consumer_go(config, config_mpi, tasks, task, datasets, &result);
//
// 2b.3 Consumers send the result to the producer
//
MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_RESULT, MPI_COMM_WORLD);
}
}
static void select_best_results_folds(json& results, json& all_results, std::string& model)
{
Timer timer;
auto grid = GridData(Paths::grid_input(model));
//
// Select the best result of the computed outer folds
//
for (const auto& result : all_results.items()) {
// each result has the results of all the outer folds as each one were a different task
double best_score = 0.0;
json best;
for (const auto& result_fold : result.value()) {
double score = result_fold["score"].get<double>();
if (score > best_score) {
best_score = score;
best = result_fold;
}
}
auto dataset = result.key();
auto combinations = grid.getGrid(dataset);
json json_best = {
{ "score", best_score },
{ "hyperparameters", combinations[best["combination"].get<int>()] },
{ "date", get_date() + " " + get_time() },
{ "grid", grid.getInputGrid(dataset) },
{ "duration", timer.translate2String(best["time"].get<double>()) }
};
results[dataset] = json_best;
}
}
static json store_search_result(std::vector<std::string>& names, Task_Result& result, json& results)
{
json json_result = {
{ "score", result.score },
{ "combination", result.idx_combination },
{ "fold", result.n_fold },
{ "time", result.time },
{ "dataset", result.idx_dataset }
};
auto name = names[result.idx_dataset];
if (!results.contains(name)) {
results[name] = json::array();
}
results[name].push_back(json_result);
return results;
}
static void consumer_go(struct ConfigGrid& config, struct ConfigMPI& config_mpi, json& tasks, int n_task, Datasets& datasets, Task_Result* result)
{
//
// initialize
//
Timer timer;
timer.start();
json task = tasks[n_task];
auto model = config.model;
auto grid = GridData(Paths::grid_input(model));
auto dataset_name = task["dataset"].get<std::string>();
auto idx_dataset = task["idx_dataset"].get<int>();
auto seed = task["seed"].get<int>();
auto n_fold = task["fold"].get<int>();
bool stratified = config.stratified;
bayesnet::Smoothing_t smooth;
if (config.smooth_strategy == "ORIGINAL")
smooth = bayesnet::Smoothing_t::ORIGINAL;
else if (config.smooth_strategy == "LAPLACE")
smooth = bayesnet::Smoothing_t::LAPLACE;
else if (config.smooth_strategy == "CESTNIK")
smooth = bayesnet::Smoothing_t::CESTNIK;
//
// Generate the hyperparameters combinations
//
auto& dataset = datasets.getDataset(dataset_name);
auto combinations = grid.getGrid(dataset_name);
dataset.load();
auto [X, y] = dataset.getTensors();
auto features = dataset.getFeatures();
auto className = dataset.getClassName();
//
// Start working on task
//
folding::Fold* fold;
if (stratified)
fold = new folding::StratifiedKFold(config.n_folds, y, seed);
else
fold = new folding::KFold(config.n_folds, y.size(0), seed);
auto [train, test] = fold->getFold(n_fold);
auto [X_train, X_test, y_train, y_test] = dataset.getTrainTestTensors(train, test);
auto states = dataset.getStates(); // Get the states of the features Once they are discretized
float best_fold_score = 0.0;
int best_idx_combination = -1;
json best_fold_hyper;
for (int idx_combination = 0; idx_combination < combinations.size(); ++idx_combination) {
auto hyperparam_line = combinations[idx_combination];
auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line);
folding::Fold* nested_fold;
if (config.stratified)
nested_fold = new folding::StratifiedKFold(config.nested, y_train, seed);
else
nested_fold = new folding::KFold(config.nested, y_train.size(0), seed);
double score = 0.0;
for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) {
//
// Nested level fold
//
auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold);
auto train_nested_t = torch::tensor(train_nested);
auto test_nested_t = torch::tensor(test_nested);
auto X_nested_train = X_train.index({ "...", train_nested_t });
auto y_nested_train = y_train.index({ train_nested_t });
auto X_nested_test = X_train.index({ "...", test_nested_t });
auto y_nested_test = y_train.index({ test_nested_t });
//
// Build Classifier with selected hyperparameters
//
auto clf = Models::instance()->create(config.model);
auto valid = clf->getValidHyperparameters();
hyperparameters.check(valid, dataset_name);
clf->setHyperparameters(hyperparameters.get(dataset_name));
//
// Train model
//
clf->fit(X_nested_train, y_nested_train, features, className, states, smooth);
//
// Test model
//
score += clf->score(X_nested_test, y_nested_test);
}
delete nested_fold;
score /= config.nested;
if (score > best_fold_score) {
best_fold_score = score;
best_idx_combination = idx_combination;
best_fold_hyper = hyperparam_line;
}
}
delete fold;
//
// Build Classifier with the best hyperparameters to obtain the best score
//
auto hyperparameters = platform::HyperParameters(datasets.getNames(), best_fold_hyper);
auto clf = Models::instance()->create(config.model);
auto valid = clf->getValidHyperparameters();
hyperparameters.check(valid, dataset_name);
clf->setHyperparameters(best_fold_hyper);
clf->fit(X_train, y_train, features, className, states, smooth);
best_fold_score = clf->score(X_test, y_test);
//
// Return the result
//
result->idx_dataset = task["idx_dataset"].get<int>();
result->idx_combination = best_idx_combination;
result->score = best_fold_score;
result->n_fold = n_fold;
result->time = timer.getDuration();
//
// Update progress bar
//
std::cout << get_color_rank(config_mpi.rank) << std::flush;
}
};
} /* namespace platform */
#endif