rmontanana/BayesNet
rmontanana/BayesNet
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases 3 Wiki Activity

Fix some mistakes in methods

Browse Source
This commit is contained in:
Ricardo Montañana Gómez 2024-01-03 11:53:46 +01:00
parent beadb7465f
commit 9ab4fc7d76
Signed by: rmontanana
GPG Key ID: 46064262FD9A7ADE
3 changed files with 464 additions and 447 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

885
src/Platform/GridSearch.cc
View File

@ -36,7 +36,7 @@ namespace platform {
GridSearch::GridSearch(struct ConfigGrid& config) : config(config) GridSearch::GridSearch(struct ConfigGrid& config) : config(config)
{ {
} }
json GridSearch::getResults() json GridSearch::loadResults()
{ {
std::ifstream file(Paths::grid_output(config.model)); std::ifstream file(Paths::grid_output(config.model));
if (file.is_open()) { if (file.is_open()) {
@ -44,7 +44,7 @@ namespace platform {
} }
return json(); return json();
} }
vector<std::string> GridSearch::processDatasets(Datasets& datasets) const vector<std::string> GridSearch::filterDatasets(Datasets& datasets) const
{ {
// Load datasets // Load datasets
auto datasets_names = datasets.getNames(); auto datasets_names = datasets.getNames();
@ -108,9 +108,9 @@ namespace platform {
auto grid = GridData(Paths::grid_input(config.model)); auto grid = GridData(Paths::grid_input(config.model));
auto datasets = Datasets(false, Paths::datasets()); auto datasets = Datasets(false, Paths::datasets());
auto all_datasets = datasets.getNames(); auto all_datasets = datasets.getNames();
auto datasets_names = processDatasets(datasets); auto datasets_names = filterDatasets(datasets);
for (int idx_dataset = 0; idx_dataset < datasets_names.size(); ++idx_dataset) { for (int idx_dataset = 0; idx_dataset < datasets_names.size(); ++idx_dataset) {
auto dataset = all_datasets[idx_dataset]; auto dataset = datasets_names[idx_dataset];
for (const auto& seed : config.seeds) { for (const auto& seed : config.seeds) {
auto combinations = grid.getGrid(dataset); auto combinations = grid.getGrid(dataset);
for (int n_fold = 0; n_fold < config.n_folds; n_fold++) { for (int n_fold = 0; n_fold < config.n_folds; n_fold++) {
@ -226,25 +226,25 @@ namespace platform {
// Update progress bar // Update progress bar
std::cout << get_color_rank(config_mpi.rank) << "*" << std::flush; std::cout << get_color_rank(config_mpi.rank) << "*" << std::flush;
} }
std::pair<int, int> GridSearch::part_range_mpi(int n_tasks, int nprocs, int rank) // std::pair<int, int> GridSearch::part_range_mpi(int n_tasks, int nprocs, int rank)
{ // {
int assigned = 0; // int assigned = 0;
int remainder = n_tasks % nprocs; // int remainder = n_tasks % nprocs;
int start = 0; // int start = 0;
if (rank < remainder) { // if (rank < remainder) {
assigned = n_tasks / nprocs + 1; // assigned = n_tasks / nprocs + 1;
} else { // } else {
assigned = n_tasks / nprocs; // assigned = n_tasks / nprocs;
start = remainder; // start = remainder;
} // }
start += rank * assigned; // start += rank * assigned;
int end = start + assigned; // int end = start + assigned;
if (rank == nprocs - 1) { // if (rank == nprocs - 1) {
end = n_tasks; // end = n_tasks;
} // }
return { start, end }; // return { start, end };
} // }
void store_result(std::vector<std::string>& names, Task_Result& result, json& results) json store_result(std::vector<std::string>& names, Task_Result& result, json& results)
{ {
json json_result = { json json_result = {
{ "score", result.score }, { "score", result.score },
@ -253,11 +253,16 @@ namespace platform {
{ "time", result.time }, { "time", result.time },
{ "dataset", result.idx_dataset } { "dataset", result.idx_dataset }
}; };
std::cout << "x Storing result for dataset " << result.idx_dataset << " from " << result.idx_combination << ::endl;
std::cout << json_result.dump() << std::endl;
std::cout << string(80, '-') << std::endl;
auto name = names[result.idx_dataset]; auto name = names[result.idx_dataset];
if (!results.contains(name)) { if (!results.contains(name)) {
results[name] = json::array(); results[name] = json::array();
} }
results[name].push_back(json_result); results[name].push_back(json_result);
std::cout << results.dump() << std::endl;
return results;
} }
json producer(json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result) json producer(json& tasks, struct ConfigMPI& config_mpi, MPI_Datatype& MPI_Result)
{ {
@ -268,21 +273,27 @@ namespace platform {
auto names = datasets.getNames(); auto names = datasets.getNames();
for (int i = 0; i < num_tasks; ++i) { for (int i = 0; i < num_tasks; ++i) {
MPI_Status status; MPI_Status status;
std::cout << "+ Producer waiting for result." << std::endl;
MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
if (status.MPI_TAG == TAG_RESULT) { if (status.MPI_TAG == TAG_RESULT) {
//Store result //Store result
std::cout << "+ Producer received result from " << status.MPI_SOURCE << std::endl;
store_result(names, result, results); store_result(names, result, results);
} }
std::cout << "+ Producer sending task " << i << " to " << status.MPI_SOURCE << std::endl;
MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD); MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_TASK, MPI_COMM_WORLD);
} }
// Send end message to all workers // Send end message to all workers but the manager
for (int i = 0; i < config_mpi.n_procs; ++i) { for (int i = 0; i < config_mpi.n_procs - 1; ++i) {
MPI_Status status; MPI_Status status;
std::cout << "+ Producer waiting for result (closing)." << std::endl;
MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); MPI_Recv(&result, 1, MPI_Result, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
if (status.MPI_TAG == TAG_RESULT) { if (status.MPI_TAG == TAG_RESULT) {
//Store result //Store result
std::cout << "+ Producer received result from " << status.MPI_SOURCE << " (closing)" << std::endl;
store_result(names, result, results); store_result(names, result, results);
} }
std::cout << "+ Producer sending end signal to " << status.MPI_SOURCE << std::endl;
MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD); MPI_Send(&i, 1, MPI_INT, status.MPI_SOURCE, TAG_END, MPI_COMM_WORLD);
} }
return results; return results;
@ -295,10 +306,13 @@ namespace platform {
// //
// Select the best result of the computed outer folds // Select the best result of the computed outer folds
// //
for (const auto& result : results.items()) { std::cout << "--- Selecting best results of the outer folds ---" << std::endl;
std::cout << all_results.dump() << std::endl;
for (const auto& result : all_results.items()) {
// each result has the results of all the outer folds as each one were a different task // each result has the results of all the outer folds as each one were a different task
double best_score = 0.0; double best_score = 0.0;
json best; json best;
std::cout << " Processing " << result.key() << std::endl;
for (const auto& result_fold : result.value()) { for (const auto& result_fold : result.value()) {
double score = result_fold["score"].get<double>(); double score = result_fold["score"].get<double>();
if (score > best_score) { if (score > best_score) {
@ -327,14 +341,17 @@ namespace platform {
int task; int task;
while (true) { while (true) {
MPI_Status status; MPI_Status status;
std::cout << "- Consumer nº " << config_mpi.rank << " waiting for task." << std::endl;
MPI_Recv(&task, 1, MPI_INT, config_mpi.manager, MPI_ANY_TAG, MPI_COMM_WORLD, &status); MPI_Recv(&task, 1, MPI_INT, config_mpi.manager, MPI_ANY_TAG, MPI_COMM_WORLD, &status);
if (status.MPI_TAG == TAG_END) { if (status.MPI_TAG == TAG_END) {
break; break;
} }
// Process task // Process task
std::cout << " - Consumer nº " << config_mpi.rank << " processing task " << task << std::endl;
process_task_mpi_consumer(config, config_mpi, tasks, task, datasets, &result); process_task_mpi_consumer(config, config_mpi, tasks, task, datasets, &result);
// Send result to producer // Send result to producer
MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_RESULT, MPI_COMM_WORLD); MPI_Send(&result, 1, MPI_Result, config_mpi.manager, TAG_RESULT, MPI_COMM_WORLD);
std::cout << " - Consumer nº " << config_mpi.rank << " sent task " << task << std::endl;
} }
} }
void GridSearch::go_producer_consumer(struct ConfigMPI& config_mpi) void GridSearch::go_producer_consumer(struct ConfigMPI& config_mpi)
@ -409,419 +426,419 @@ namespace platform {
// //
auto datasets = Datasets(config.discretize, Paths::datasets()); auto datasets = Datasets(config.discretize, Paths::datasets());
if (config_mpi.rank == config_mpi.manager) { if (config_mpi.rank == config_mpi.manager) {
auto all_results = producer(tasks, config_mpi, MPI_Result); json all_results = producer(tasks, config_mpi, MPI_Result);
auto results = select_best_results_folds(all_results, config.model); json results = select_best_results_folds(all_results, config.model);
save(results); save(results);
} else { } else {
consumer(datasets, tasks, config, config_mpi, MPI_Result); consumer(datasets, tasks, config, config_mpi, MPI_Result);
} }
} }
void GridSearch::go_mpi(struct ConfigMPI& config_mpi) // void GridSearch::go_mpi(struct ConfigMPI& config_mpi)
{ // {
/* // /*
* Each task is a json object with the following structure: // * Each task is a json object with the following structure:
* { // * {
* "dataset": "dataset_name", // * "dataset": "dataset_name",
* "idx_dataset": idx_dataset, // * "seed": # of seed to use,
* "seed": # of seed to use, // * "model": "model_name",
* "Fold": # of fold to process // * "Fold": # of fold to process
* } // * }
* // *
* The overall process consists in these steps: // * The overall process consists in these steps:
* 1. Manager will broadcast the tasks to all the processes // * 1. Manager will broadcast the tasks to all the processes
* 1.1 Broadcast the number of tasks // * 1.1 Broadcast the number of tasks
* 1.2 Broadcast the length of the following string // * 1.2 Broadcast the length of the following string
* 1.2 Broadcast the tasks as a char* string // * 1.2 Broadcast the tasks as a char* string
* 2. Workers will receive the tasks and start the process // * 2. Workers will receive the tasks and start the process
* 2.1 A method will tell each worker the range of tasks to process // * 2.1 A method will tell each worker the range of tasks to process
* 2.2 Each worker will process the tasks and generate the best score for each task // * 2.2 Each worker will process the tasks and generate the best score for each task
* 3. Manager gather the scores from all the workers and find out the best hyperparameters for each dataset // * 3. Manager gather the scores from all the workers and find out the best hyperparameters for each dataset
* 3.1 Obtain the maximum size of the results message of all the workers // * 3.1 Obtain the maximum size of the results message of all the workers
* 3.2 Gather all the results from the workers into the manager // * 3.2 Gather all the results from the workers into the manager
* 3.3 Compile the results from all the workers // * 3.3 Compile the results from all the workers
* 3.4 Filter the best hyperparameters for each dataset // * 3.4 Filter the best hyperparameters for each dataset
*/ // */
char* msg; // char* msg;
int tasks_size; // int tasks_size;
if (config_mpi.rank == config_mpi.manager) { // if (config_mpi.rank == config_mpi.manager) {
timer.start(); // timer.start();
auto tasks = build_tasks_mpi(); // auto tasks = build_tasks_mpi();
auto tasks_str = tasks.dump(); // auto tasks_str = tasks.dump();
tasks_size = tasks_str.size(); // tasks_size = tasks_str.size();
msg = new char[tasks_size + 1]; // msg = new char[tasks_size + 1];
strcpy(msg, tasks_str.c_str()); // strcpy(msg, tasks_str.c_str());
} // }
// // //
// 1. Manager will broadcast the tasks to all the processes // // 1. Manager will broadcast the tasks to all the processes
// // //
MPI_Bcast(&tasks_size, 1, MPI_INT, config_mpi.manager, MPI_COMM_WORLD); // MPI_Bcast(&tasks_size, 1, MPI_INT, config_mpi.manager, MPI_COMM_WORLD);
if (config_mpi.rank != config_mpi.manager) { // if (config_mpi.rank != config_mpi.manager) {
msg = new char[tasks_size + 1]; // msg = new char[tasks_size + 1];
} // }
MPI_Bcast(msg, tasks_size + 1, MPI_CHAR, config_mpi.manager, MPI_COMM_WORLD); // MPI_Bcast(msg, tasks_size + 1, MPI_CHAR, config_mpi.manager, MPI_COMM_WORLD);
json tasks = json::parse(msg); // json tasks = json::parse(msg);
delete[] msg; // delete[] msg;
// // //
// 2. All Workers will receive the tasks and start the process // // 2. All Workers will receive the tasks and start the process
// // //
int num_tasks = tasks.size(); // int num_tasks = tasks.size();
// 2.1 A method will tell each worker the range of tasks to process // // 2.1 A method will tell each worker the range of tasks to process
auto [start, end] = part_range_mpi(num_tasks, config_mpi.n_procs, config_mpi.rank); // auto [start, end] = part_range_mpi(num_tasks, config_mpi.n_procs, config_mpi.rank);
// 2.2 Each worker will process the tasks and return the best scores obtained // // 2.2 Each worker will process the tasks and return the best scores obtained
auto datasets = Datasets(config.discretize, Paths::datasets()); // auto datasets = Datasets(config.discretize, Paths::datasets());
json results; // json results;
for (int i = start; i < end; ++i) { // for (int i = start; i < end; ++i) {
// Process task // // Process task
process_task_mpi(config_mpi, tasks[i], datasets, results); // process_task_mpi(config_mpi, tasks[i], datasets, results);
} // }
int size = results.dump().size() + 1; // int size = results.dump().size() + 1;
int max_size = 0; // int max_size = 0;
// // //
// 3. Manager gather the scores from all the workers and find out the best hyperparameters for each dataset // // 3. Manager gather the scores from all the workers and find out the best hyperparameters for each dataset
// // //
//3.1 Obtain the maximum size of the results message of all the workers // //3.1 Obtain the maximum size of the results message of all the workers
MPI_Allreduce(&size, &max_size, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); // MPI_Allreduce(&size, &max_size, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
// Assign the memory to the message and initialize it to 0s // // Assign the memory to the message and initialize it to 0s
char* total = NULL; // char* total = NULL;
msg = new char[max_size]; // msg = new char[max_size];
strncpy(msg, results.dump().c_str(), size); // strncpy(msg, results.dump().c_str(), size);
if (config_mpi.rank == config_mpi.manager) { // if (config_mpi.rank == config_mpi.manager) {
total = new char[max_size * config_mpi.n_procs]; // total = new char[max_size * config_mpi.n_procs];
} // }
// 3.2 Gather all the results from the workers into the manager // // 3.2 Gather all the results from the workers into the manager
MPI_Gather(msg, max_size, MPI_CHAR, total, max_size, MPI_CHAR, config_mpi.manager, MPI_COMM_WORLD); // MPI_Gather(msg, max_size, MPI_CHAR, total, max_size, MPI_CHAR, config_mpi.manager, MPI_COMM_WORLD);
delete[] msg; // delete[] msg;
if (config_mpi.rank == config_mpi.manager) { // if (config_mpi.rank == config_mpi.manager) {
std::cout << Colors::RESET() << "|" << std::endl; // std::cout << Colors::RESET() << "|" << std::endl;
json total_results; // json total_results;
json best_results; // json best_results;
// 3.3 Compile the results from all the workers // // 3.3 Compile the results from all the workers
for (int i = 0; i < config_mpi.n_procs; ++i) { // for (int i = 0; i < config_mpi.n_procs; ++i) {
json partial_results = json::parse(total + i * max_size); // json partial_results = json::parse(total + i * max_size);
for (auto& [dataset, folds] : partial_results.items()) { // for (auto& [dataset, folds] : partial_results.items()) {
for (auto& [fold, result] : folds.items()) { // for (auto& [fold, result] : folds.items()) {
total_results[dataset][fold] = result; // total_results[dataset][fold] = result;
} // }
} // }
} // }
delete[] total; // delete[] total;
// 3.4 Filter the best hyperparameters for each dataset // // 3.4 Filter the best hyperparameters for each dataset
auto grid = GridData(Paths::grid_input(config.model)); // auto grid = GridData(Paths::grid_input(config.model));
for (auto& [dataset, folds] : total_results.items()) { // for (auto& [dataset, folds] : total_results.items()) {
double best_score = 0.0; // double best_score = 0.0;
double duration = 0.0; // double duration = 0.0;
json best_hyper; // json best_hyper;
for (auto& [fold, result] : folds.items()) { // for (auto& [fold, result] : folds.items()) {
duration += result["duration"].get<double>(); // duration += result["duration"].get<double>();
if (result["score"] > best_score) { // if (result["score"] > best_score) {
best_score = result["score"]; // best_score = result["score"];
best_hyper = result["hyperparameters"]; // best_hyper = result["hyperparameters"];
} // }
} // }
auto timer = Timer(); // auto timer = Timer();
json result = { // json result = {
{ "score", best_score }, // { "score", best_score },
{ "hyperparameters", best_hyper }, // { "hyperparameters", best_hyper },
{ "date", get_date() + " " + get_time() }, // { "date", get_date() + " " + get_time() },
{ "grid", grid.getInputGrid(dataset) }, // { "grid", grid.getInputGrid(dataset) },
{ "duration", timer.translate2String(duration) } // { "duration", timer.translate2String(duration) }
}; // };
best_results[dataset] = result; // best_results[dataset] = result;
} // }
save(best_results); // save(best_results);
} // }
} // }
void GridSearch::go() // void GridSearch::go()
{ // {
timer.start(); // timer.start();
auto grid_type = config.nested == 0 ? "Single" : "Nested"; // auto grid_type = config.nested == 0 ? "Single" : "Nested";
auto datasets = Datasets(config.discretize, Paths::datasets()); // auto datasets = Datasets(config.discretize, Paths::datasets());
auto datasets_names = processDatasets(datasets); // auto datasets_names = processDatasets(datasets);
json results = initializeResults(); // json results = initializeResults();
std::cout << "***************** Starting " << grid_type << " Gridsearch *****************" << std::endl; // std::cout << "***************** Starting " << grid_type << " Gridsearch *****************" << std::endl;
std::cout << "input file=" << Paths::grid_input(config.model) << std::endl; // std::cout << "input file=" << Paths::grid_input(config.model) << std::endl;
auto grid = GridData(Paths::grid_input(config.model)); // auto grid = GridData(Paths::grid_input(config.model));
Timer timer_dataset; // Timer timer_dataset;
double bestScore = 0; // double bestScore = 0;
json bestHyperparameters; // json bestHyperparameters;
for (const auto& dataset : datasets_names) { // for (const auto& dataset : datasets_names) {
if (!config.quiet) // if (!config.quiet)
std::cout << "- " << setw(20) << left << dataset << " " << right << flush; // std::cout << "- " << setw(20) << left << dataset << " " << right << flush;
auto combinations = grid.getGrid(dataset); // auto combinations = grid.getGrid(dataset);
timer_dataset.start(); // timer_dataset.start();
if (config.nested == 0) // if (config.nested == 0)
// for dataset // for hyperparameters // for seed // for fold // // for dataset // for hyperparameters // for seed // for fold
tie(bestScore, bestHyperparameters) = processFileSingle(dataset, datasets, combinations); // tie(bestScore, bestHyperparameters) = processFileSingle(dataset, datasets, combinations);
else // else
// for dataset // for seed // for fold // for hyperparameters // for nested fold // // for dataset // for seed // for fold // for hyperparameters // for nested fold
tie(bestScore, bestHyperparameters) = processFileNested(dataset, datasets, combinations); // tie(bestScore, bestHyperparameters) = processFileNested(dataset, datasets, combinations);
if (!config.quiet) { // if (!config.quiet) {
std::cout << "end." << " Score: " << Colors::IBLUE() << setw(9) << setprecision(7) << fixed // std::cout << "end." << " Score: " << Colors::IBLUE() << setw(9) << setprecision(7) << fixed
<< bestScore << Colors::BLUE() << " [" << bestHyperparameters.dump() << "]" // << bestScore << Colors::BLUE() << " [" << bestHyperparameters.dump() << "]"
<< Colors::RESET() << ::endl; // << Colors::RESET() << ::endl;
} // }
json result = { // json result = {
{ "score", bestScore }, // { "score", bestScore },
{ "hyperparameters", bestHyperparameters }, // { "hyperparameters", bestHyperparameters },
{ "date", get_date() + " " + get_time() }, // { "date", get_date() + " " + get_time() },
{ "grid", grid.getInputGrid(dataset) }, // { "grid", grid.getInputGrid(dataset) },
{ "duration", timer_dataset.getDurationString() } // { "duration", timer_dataset.getDurationString() }
}; // };
results[dataset] = result; // results[dataset] = result;
// Save partial results // // Save partial results
save(results); // save(results);
} // }
// Save final results // // Save final results
save(results); // save(results);
std::cout << "***************** Ending " << grid_type << " Gridsearch *******************" << std::endl; // std::cout << "***************** Ending " << grid_type << " Gridsearch *******************" << std::endl;
} // }
pair<double, json> GridSearch::processFileSingle(std::string fileName, Datasets& datasets, vector<json>& combinations) // pair<double, json> GridSearch::processFileSingle(std::string fileName, Datasets& datasets, vector<json>& combinations)
{ // {
int num = 0; // int num = 0;
double bestScore = 0.0; // double bestScore = 0.0;
json bestHyperparameters; // json bestHyperparameters;
auto totalComb = combinations.size(); // auto totalComb = combinations.size();
for (const auto& hyperparam_line : combinations) { // for (const auto& hyperparam_line : combinations) {
if (!config.quiet) // if (!config.quiet)
showProgressComb(++num, config.n_folds, totalComb, Colors::CYAN()); // showProgressComb(++num, config.n_folds, totalComb, Colors::CYAN());
auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); // auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line);
// Get dataset // // Get dataset
auto [X, y] = datasets.getTensors(fileName); // auto [X, y] = datasets.getTensors(fileName);
auto states = datasets.getStates(fileName); // auto states = datasets.getStates(fileName);
auto features = datasets.getFeatures(fileName); // auto features = datasets.getFeatures(fileName);
auto className = datasets.getClassName(fileName); // auto className = datasets.getClassName(fileName);
double totalScore = 0.0; // double totalScore = 0.0;
int numItems = 0; // int numItems = 0;
for (const auto& seed : config.seeds) { // for (const auto& seed : config.seeds) {
if (!config.quiet) // if (!config.quiet)
std::cout << "(" << seed << ") doing Fold: " << flush; // std::cout << "(" << seed << ") doing Fold: " << flush;
Fold* fold; // Fold* fold;
if (config.stratified) // if (config.stratified)
fold = new StratifiedKFold(config.n_folds, y, seed); // fold = new StratifiedKFold(config.n_folds, y, seed);
else // else
fold = new KFold(config.n_folds, y.size(0), seed); // fold = new KFold(config.n_folds, y.size(0), seed);
for (int nfold = 0; nfold < config.n_folds; nfold++) { // for (int nfold = 0; nfold < config.n_folds; nfold++) {
auto clf = Models::instance()->create(config.model); // auto clf = Models::instance()->create(config.model);
auto valid = clf->getValidHyperparameters(); // auto valid = clf->getValidHyperparameters();
hyperparameters.check(valid, fileName); // hyperparameters.check(valid, fileName);
clf->setHyperparameters(hyperparameters.get(fileName)); // clf->setHyperparameters(hyperparameters.get(fileName));
auto [train, test] = fold->getFold(nfold); // auto [train, test] = fold->getFold(nfold);
auto train_t = torch::tensor(train); // auto train_t = torch::tensor(train);
auto test_t = torch::tensor(test); // auto test_t = torch::tensor(test);
auto X_train = X.index({ "...", train_t }); // auto X_train = X.index({ "...", train_t });
auto y_train = y.index({ train_t }); // auto y_train = y.index({ train_t });
auto X_test = X.index({ "...", test_t }); // auto X_test = X.index({ "...", test_t });
auto y_test = y.index({ test_t }); // auto y_test = y.index({ test_t });
// Train model // // Train model
if (!config.quiet) // if (!config.quiet)
showProgressFold(nfold + 1, getColor(clf->getStatus()), "a"); // showProgressFold(nfold + 1, getColor(clf->getStatus()), "a");
clf->fit(X_train, y_train, features, className, states); // clf->fit(X_train, y_train, features, className, states);
// Test model // // Test model
if (!config.quiet) // if (!config.quiet)
showProgressFold(nfold + 1, getColor(clf->getStatus()), "b"); // showProgressFold(nfold + 1, getColor(clf->getStatus()), "b");
totalScore += clf->score(X_test, y_test); // totalScore += clf->score(X_test, y_test);
numItems++; // numItems++;
if (!config.quiet) // if (!config.quiet)
std::cout << "\b\b\b, \b" << flush; // std::cout << "\b\b\b, \b" << flush;
} // }
delete fold; // delete fold;
} // }
double score = numItems == 0 ? 0.0 : totalScore / numItems; // double score = numItems == 0 ? 0.0 : totalScore / numItems;
if (score > bestScore) { // if (score > bestScore) {
bestScore = score; // bestScore = score;
bestHyperparameters = hyperparam_line; // bestHyperparameters = hyperparam_line;
} // }
} // }
return { bestScore, bestHyperparameters }; // return { bestScore, bestHyperparameters };
} // }
pair<double, json> GridSearch::processFileNested(std::string fileName, Datasets& datasets, vector<json>& combinations) // pair<double, json> GridSearch::processFileNested(std::string fileName, Datasets& datasets, vector<json>& combinations)
{ // {
// Get dataset // // Get dataset
auto [X, y] = datasets.getTensors(fileName); // auto [X, y] = datasets.getTensors(fileName);
auto states = datasets.getStates(fileName); // auto states = datasets.getStates(fileName);
auto features = datasets.getFeatures(fileName); // auto features = datasets.getFeatures(fileName);
auto className = datasets.getClassName(fileName); // auto className = datasets.getClassName(fileName);
int spcs_combinations = int(log(combinations.size()) / log(10)) + 1; // int spcs_combinations = int(log(combinations.size()) / log(10)) + 1;
double goatScore = 0.0; // double goatScore = 0.0;
json goatHyperparameters; // json goatHyperparameters;
// for dataset // for seed // for fold // for hyperparameters // for nested fold // // for dataset // for seed // for fold // for hyperparameters // for nested fold
for (const auto& seed : config.seeds) { // for (const auto& seed : config.seeds) {
Fold* fold; // Fold* fold;
if (config.stratified) // if (config.stratified)
fold = new StratifiedKFold(config.n_folds, y, seed); // fold = new StratifiedKFold(config.n_folds, y, seed);
else // else
fold = new KFold(config.n_folds, y.size(0), seed); // fold = new KFold(config.n_folds, y.size(0), seed);
double bestScore = 0.0; // double bestScore = 0.0;
json bestHyperparameters; // json bestHyperparameters;
std::cout << "(" << seed << ") doing Fold: " << flush; // std::cout << "(" << seed << ") doing Fold: " << flush;
for (int nfold = 0; nfold < config.n_folds; nfold++) { // for (int nfold = 0; nfold < config.n_folds; nfold++) {
if (!config.quiet) // if (!config.quiet)
std::cout << Colors::GREEN() << nfold + 1 << " " << flush; // std::cout << Colors::GREEN() << nfold + 1 << " " << flush;
// First level fold // // First level fold
auto [train, test] = fold->getFold(nfold); // auto [train, test] = fold->getFold(nfold);
auto train_t = torch::tensor(train); // auto train_t = torch::tensor(train);
auto test_t = torch::tensor(test); // auto test_t = torch::tensor(test);
auto X_train = X.index({ "...", train_t }); // auto X_train = X.index({ "...", train_t });
auto y_train = y.index({ train_t }); // auto y_train = y.index({ train_t });
auto X_test = X.index({ "...", test_t }); // auto X_test = X.index({ "...", test_t });
auto y_test = y.index({ test_t }); // auto y_test = y.index({ test_t });
auto num = 0; // auto num = 0;
json result_fold; // json result_fold;
double hypScore = 0.0; // double hypScore = 0.0;
double bestHypScore = 0.0; // double bestHypScore = 0.0;
json bestHypHyperparameters; // json bestHypHyperparameters;
for (const auto& hyperparam_line : combinations) { // for (const auto& hyperparam_line : combinations) {
std::cout << "[" << setw(spcs_combinations) << ++num << "/" << setw(spcs_combinations) // std::cout << "[" << setw(spcs_combinations) << ++num << "/" << setw(spcs_combinations)
<< combinations.size() << "] " << std::flush; // << combinations.size() << "] " << std::flush;
Fold* nested_fold; // Fold* nested_fold;
if (config.stratified) // if (config.stratified)
nested_fold = new StratifiedKFold(config.nested, y_train, seed); // nested_fold = new StratifiedKFold(config.nested, y_train, seed);
else // else
nested_fold = new KFold(config.nested, y_train.size(0), seed); // nested_fold = new KFold(config.nested, y_train.size(0), seed);
for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) { // for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) {
// Nested level fold // // Nested level fold
auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold); // auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold);
auto train_nested_t = torch::tensor(train_nested); // auto train_nested_t = torch::tensor(train_nested);
auto test_nested_t = torch::tensor(test_nested); // auto test_nested_t = torch::tensor(test_nested);
auto X_nexted_train = X_train.index({ "...", train_nested_t }); // auto X_nexted_train = X_train.index({ "...", train_nested_t });
auto y_nested_train = y_train.index({ train_nested_t }); // auto y_nested_train = y_train.index({ train_nested_t });
auto X_nested_test = X_train.index({ "...", test_nested_t }); // auto X_nested_test = X_train.index({ "...", test_nested_t });
auto y_nested_test = y_train.index({ test_nested_t }); // auto y_nested_test = y_train.index({ test_nested_t });
// Build Classifier with selected hyperparameters // // Build Classifier with selected hyperparameters
auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); // auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line);
auto clf = Models::instance()->create(config.model); // auto clf = Models::instance()->create(config.model);
auto valid = clf->getValidHyperparameters(); // auto valid = clf->getValidHyperparameters();
hyperparameters.check(valid, fileName); // hyperparameters.check(valid, fileName);
clf->setHyperparameters(hyperparameters.get(fileName)); // clf->setHyperparameters(hyperparameters.get(fileName));
// Train model // // Train model
if (!config.quiet) // if (!config.quiet)
showProgressFold(n_nested_fold + 1, getColor(clf->getStatus()), "a"); // showProgressFold(n_nested_fold + 1, getColor(clf->getStatus()), "a");
clf->fit(X_nexted_train, y_nested_train, features, className, states); // clf->fit(X_nexted_train, y_nested_train, features, className, states);
// Test model // // Test model
if (!config.quiet) // if (!config.quiet)
showProgressFold(n_nested_fold + 1, getColor(clf->getStatus()), "b"); // showProgressFold(n_nested_fold + 1, getColor(clf->getStatus()), "b");
hypScore += clf->score(X_nested_test, y_nested_test); // hypScore += clf->score(X_nested_test, y_nested_test);
if (!config.quiet) // if (!config.quiet)
std::cout << "\b\b\b, \b" << flush; // std::cout << "\b\b\b, \b" << flush;
} // }
int magic = 3 * config.nested + 2 * spcs_combinations + 4; // int magic = 3 * config.nested + 2 * spcs_combinations + 4;
std::cout << string(magic, '\b') << string(magic, ' ') << string(magic, '\b') << flush; // std::cout << string(magic, '\b') << string(magic, ' ') << string(magic, '\b') << flush;
delete nested_fold; // delete nested_fold;
hypScore /= config.nested; // hypScore /= config.nested;
if (hypScore > bestHypScore) { // if (hypScore > bestHypScore) {
bestHypScore = hypScore; // bestHypScore = hypScore;
bestHypHyperparameters = hyperparam_line; // bestHypHyperparameters = hyperparam_line;
} // }
} // }
// Build Classifier with selected hyperparameters // // Build Classifier with selected hyperparameters
auto clf = Models::instance()->create(config.model); // auto clf = Models::instance()->create(config.model);
clf->setHyperparameters(bestHypHyperparameters); // clf->setHyperparameters(bestHypHyperparameters);
// Train model // // Train model
if (!config.quiet) // if (!config.quiet)
showProgressFold(nfold + 1, getColor(clf->getStatus()), "a"); // showProgressFold(nfold + 1, getColor(clf->getStatus()), "a");
clf->fit(X_train, y_train, features, className, states); // clf->fit(X_train, y_train, features, className, states);
// Test model // // Test model
if (!config.quiet) // if (!config.quiet)
showProgressFold(nfold + 1, getColor(clf->getStatus()), "b"); // showProgressFold(nfold + 1, getColor(clf->getStatus()), "b");
double score = clf->score(X_test, y_test); // double score = clf->score(X_test, y_test);
if (!config.quiet) // if (!config.quiet)
std::cout << string(2 * config.nested - 1, '\b') << "," << string(2 * config.nested, ' ') << string(2 * config.nested - 1, '\b') << flush; // std::cout << string(2 * config.nested - 1, '\b') << "," << string(2 * config.nested, ' ') << string(2 * config.nested - 1, '\b') << flush;
if (score > bestScore) { // if (score > bestScore) {
bestScore = score; // bestScore = score;
bestHyperparameters = bestHypHyperparameters; // bestHyperparameters = bestHypHyperparameters;
} // }
} // }
if (bestScore > goatScore) { // if (bestScore > goatScore) {
goatScore = bestScore; // goatScore = bestScore;
goatHyperparameters = bestHyperparameters; // goatHyperparameters = bestHyperparameters;
} // }
delete fold; // delete fold;
} // }
return { goatScore, goatHyperparameters }; // return { goatScore, goatHyperparameters };
} // }
void GridSearch::process_task_mpi(struct ConfigMPI& config_mpi, json& task, Datasets& datasets, json& results) // void GridSearch::process_task_mpi(struct ConfigMPI& config_mpi, json& task, Datasets& datasets, json& results)
{ // {
// Process the task and store the result in the results json // // Process the task and store the result in the results json
Timer timer; // Timer timer;
timer.start(); // timer.start();
auto grid = GridData(Paths::grid_input(config.model)); // auto grid = GridData(Paths::grid_input(config.model));
auto dataset = task["dataset"].get<std::string>(); // auto dataset = task["dataset"].get<std::string>();
auto seed = task["seed"].get<int>(); // auto seed = task["seed"].get<int>();
auto n_fold = task["fold"].get<int>(); // auto n_fold = task["fold"].get<int>();
// Generate the hyperparamters combinations // // Generate the hyperparamters combinations
auto combinations = grid.getGrid(dataset); // auto combinations = grid.getGrid(dataset);
auto [X, y] = datasets.getTensors(dataset); // auto [X, y] = datasets.getTensors(dataset);
auto states = datasets.getStates(dataset); // auto states = datasets.getStates(dataset);
auto features = datasets.getFeatures(dataset); // auto features = datasets.getFeatures(dataset);
auto className = datasets.getClassName(dataset); // auto className = datasets.getClassName(dataset);
// // //
// Start working on task // // Start working on task
// // //
Fold* fold; // Fold* fold;
if (config.stratified) // if (config.stratified)
fold = new StratifiedKFold(config.n_folds, y, seed); // fold = new StratifiedKFold(config.n_folds, y, seed);
else // else
fold = new KFold(config.n_folds, y.size(0), seed); // fold = new KFold(config.n_folds, y.size(0), seed);
auto [train, test] = fold->getFold(n_fold); // auto [train, test] = fold->getFold(n_fold);
auto train_t = torch::tensor(train); // auto train_t = torch::tensor(train);
auto test_t = torch::tensor(test); // auto test_t = torch::tensor(test);
auto X_train = X.index({ "...", train_t }); // auto X_train = X.index({ "...", train_t });
auto y_train = y.index({ train_t }); // auto y_train = y.index({ train_t });
auto X_test = X.index({ "...", test_t }); // auto X_test = X.index({ "...", test_t });
auto y_test = y.index({ test_t }); // auto y_test = y.index({ test_t });
auto num = 0; // auto num = 0;
double best_fold_score = 0.0; // double best_fold_score = 0.0;
json best_fold_hyper; // json best_fold_hyper;
for (const auto& hyperparam_line : combinations) { // for (const auto& hyperparam_line : combinations) {
auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line); // auto hyperparameters = platform::HyperParameters(datasets.getNames(), hyperparam_line);
Fold* nested_fold; // Fold* nested_fold;
if (config.stratified) // if (config.stratified)
nested_fold = new StratifiedKFold(config.nested, y_train, seed); // nested_fold = new StratifiedKFold(config.nested, y_train, seed);
else // else
nested_fold = new KFold(config.nested, y_train.size(0), seed); // nested_fold = new KFold(config.nested, y_train.size(0), seed);
double score = 0.0; // double score = 0.0;
for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) { // for (int n_nested_fold = 0; n_nested_fold < config.nested; n_nested_fold++) {
// Nested level fold // // Nested level fold
auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold); // auto [train_nested, test_nested] = nested_fold->getFold(n_nested_fold);
auto train_nested_t = torch::tensor(train_nested); // auto train_nested_t = torch::tensor(train_nested);
auto test_nested_t = torch::tensor(test_nested); // auto test_nested_t = torch::tensor(test_nested);
auto X_nested_train = X_train.index({ "...", train_nested_t }); // auto X_nested_train = X_train.index({ "...", train_nested_t });
auto y_nested_train = y_train.index({ train_nested_t }); // auto y_nested_train = y_train.index({ train_nested_t });
auto X_nested_test = X_train.index({ "...", test_nested_t }); // auto X_nested_test = X_train.index({ "...", test_nested_t });
auto y_nested_test = y_train.index({ test_nested_t }); // auto y_nested_test = y_train.index({ test_nested_t });
// Build Classifier with selected hyperparameters // // Build Classifier with selected hyperparameters
auto clf = Models::instance()->create(config.model); // auto clf = Models::instance()->create(config.model);
auto valid = clf->getValidHyperparameters(); // auto valid = clf->getValidHyperparameters();
hyperparameters.check(valid, dataset); // hyperparameters.check(valid, dataset);
clf->setHyperparameters(hyperparameters.get(dataset)); // clf->setHyperparameters(hyperparameters.get(dataset));
// Train model // // Train model
clf->fit(X_nested_train, y_nested_train, features, className, states); // clf->fit(X_nested_train, y_nested_train, features, className, states);
// Test model // // Test model
score += clf->score(X_nested_test, y_nested_test); // score += clf->score(X_nested_test, y_nested_test);
} // }
delete nested_fold; // delete nested_fold;
score /= config.nested; // score /= config.nested;
if (score > best_fold_score) { // if (score > best_fold_score) {
best_fold_score = score; // best_fold_score = score;
best_fold_hyper = hyperparam_line; // best_fold_hyper = hyperparam_line;
} // }
} // }
delete fold; // delete fold;
// Build Classifier with the best hyperparameters to obtain the best score // // Build Classifier with the best hyperparameters to obtain the best score
auto hyperparameters = platform::HyperParameters(datasets.getNames(), best_fold_hyper); // auto hyperparameters = platform::HyperParameters(datasets.getNames(), best_fold_hyper);
auto clf = Models::instance()->create(config.model); // auto clf = Models::instance()->create(config.model);
auto valid = clf->getValidHyperparameters(); // auto valid = clf->getValidHyperparameters();
hyperparameters.check(valid, dataset); // hyperparameters.check(valid, dataset);
clf->setHyperparameters(best_fold_hyper); // clf->setHyperparameters(best_fold_hyper);
clf->fit(X_train, y_train, features, className, states); // clf->fit(X_train, y_train, features, className, states);
best_fold_score = clf->score(X_test, y_test); // best_fold_score = clf->score(X_test, y_test);
// Save results // // Save results
results[dataset][std::to_string(n_fold)]["score"] = best_fold_score; // results[dataset][std::to_string(n_fold)]["score"] = best_fold_score;
results[dataset][std::to_string(n_fold)]["hyperparameters"] = best_fold_hyper; // results[dataset][std::to_string(n_fold)]["hyperparameters"] = best_fold_hyper;
results[dataset][std::to_string(n_fold)]["seed"] = seed; // results[dataset][std::to_string(n_fold)]["seed"] = seed;
results[dataset][std::to_string(n_fold)]["duration"] = timer.getDuration(); // results[dataset][std::to_string(n_fold)]["duration"] = timer.getDuration();
std::cout << get_color_rank(config_mpi.rank) << "*" << std::flush; // std::cout << get_color_rank(config_mpi.rank) << "*" << std::flush;
} // }
json GridSearch::initializeResults() json GridSearch::initializeResults()
{ {
// Load previous results // Load previous results

16
src/Platform/GridSearch.h
View File

@ -44,22 +44,22 @@ namespace platform {
class GridSearch { class GridSearch {
public: public:
explicit GridSearch(struct ConfigGrid& config); explicit GridSearch(struct ConfigGrid& config);
void go(); // void go();
void go_mpi(struct ConfigMPI& config_mpi); // void go_mpi(struct ConfigMPI& config_mpi);
void go_producer_consumer(struct ConfigMPI& config_mpi); void go_producer_consumer(struct ConfigMPI& config_mpi);
~GridSearch() = default; ~GridSearch() = default;
json getResults(); json loadResults();
static inline std::string NO_CONTINUE() { return "NO_CONTINUE"; } static inline std::string NO_CONTINUE() { return "NO_CONTINUE"; }
private: private:
void save(json& results); void save(json& results);
json initializeResults(); json initializeResults();
vector<std::string> processDatasets(Datasets& datasets) const; vector<std::string> filterDatasets(Datasets& datasets) const;
pair<double, json> processFileSingle(std::string fileName, Datasets& datasets, std::vector<json>& combinations); // pair<double, json> processFileSingle(std::string fileName, Datasets& datasets, std::vector<json>& combinations);
pair<double, json> processFileNested(std::string fileName, Datasets& datasets, std::vector<json>& combinations); // pair<double, json> processFileNested(std::string fileName, Datasets& datasets, std::vector<json>& combinations);
struct ConfigGrid config; struct ConfigGrid config;
pair<int, int> part_range_mpi(int n_tasks, int nprocs, int rank); // pair<int, int> part_range_mpi(int n_tasks, int nprocs, int rank);
json build_tasks_mpi(); json build_tasks_mpi();
void process_task_mpi(struct ConfigMPI& config_mpi, json& task, Datasets& datasets, json& results); // void process_task_mpi(struct ConfigMPI& config_mpi, json& task, Datasets& datasets, json& results);
Timer timer; // used to measure the time of the whole process Timer timer; // used to measure the time of the whole process
}; };
} /* namespace platform */ } /* namespace platform */

10
src/Platform/b_grid.cc
View File

@ -203,18 +203,18 @@ int main(int argc, char** argv)
MPI_Comm_size(MPI_COMM_WORLD, &mpi_config.n_procs); MPI_Comm_size(MPI_COMM_WORLD, &mpi_config.n_procs);
grid_search.go_producer_consumer(mpi_config); grid_search.go_producer_consumer(mpi_config);
if (mpi_config.rank == mpi_config.manager) { if (mpi_config.rank == mpi_config.manager) {
auto results = grid_search.getResults(); auto results = grid_search.loadResults();
list_results(results, config.model); list_results(results, config.model);
std::cout << "Process took " << timer.getDurationString() << std::endl; std::cout << "Process took " << timer.getDurationString() << std::endl;
} }
MPI_Finalize(); MPI_Finalize();
} else { // } else {
grid_search.go(); // grid_search.go();
std::cout << "Process took " << timer.getDurationString() << std::endl; // std::cout << "Process took " << timer.getDurationString() << std::endl;
} }
} else { } else {
// List results // List results
auto results = grid_search.getResults(); auto results = grid_search.loadResults();
if (results.empty()) { if (results.empty()) {
std::cout << "** No results found" << std::endl; std::cout << "** No results found" << std::endl;
} else { } else {