diff --git a/src/Platform/BestResults.cc b/src/Platform/BestResults.cc index 3220382..87ad3b3 100644 --- a/src/Platform/BestResults.cc +++ b/src/Platform/BestResults.cc @@ -1,6 +1,7 @@ #include #include #include +#include #include "BestResults.h" #include "Result.h" #include "Colors.h" @@ -45,23 +46,38 @@ namespace platform { string BestResults::bestResultFile() { - return "best_results_" + model + "_" + score + ".json"; + return "best_results_" + score + "_" + model + ".json"; + } + + pair getModelScore(string name) + { + // results_accuracy_BoostAODE_MacBookpro16_2023-09-06_12:27:00_1.json + int i = 0; + auto pos = name.find("_"); + auto pos2 = name.find("_", pos + 1); + string score = name.substr(pos + 1, pos2 - pos - 1); + pos = name.find("_", pos2 + 1); + string model = name.substr(pos2 + 1, pos - pos2 - 1); + return { model, score }; } vector BestResults::loadResultFiles() { vector files; using std::filesystem::directory_iterator; + string fileModel, fileScore; for (const auto& file : directory_iterator(path)) { auto fileName = file.path().filename().string(); - if (fileName.find(".json") != string::npos && fileName.find("results_") == 0 - && fileName.find("_" + score + "_") != string::npos - && (fileName.find("_" + model + "_") != string::npos || model == "any")) { - files.push_back(fileName); + if (fileName.find(".json") != string::npos && fileName.find("results_") == 0) { + tie(fileModel, fileScore) = getModelScore(fileName); + if (score == fileScore && (model == fileModel || model == "any")) { + files.push_back(fileName); + } } } return files; } + json BestResults::loadFile(const string& fileName) { ifstream resultData(fileName); @@ -71,9 +87,48 @@ namespace platform { } throw invalid_argument("Unable to open result file. [" + fileName + "]"); } - void BestResults::reportAll() + set BestResults::getModels() { + set models; + auto files = loadResultFiles(); + if (files.size() == 0) { + cerr << Colors::MAGENTA() << "No result files were found!" << Colors::RESET() << endl; + exit(1); + } + string fileModel, fileScore; + for (const auto& file : files) { + // take the model from the file name and add it to a vector of models + // set model to the name of the first model in the vector + // filter files and build the best results file of this model + // repeat for all models + // another for loop to read the best results file of each model and print al together + // each row is a dataset and each column is a model + // the score is the score of the best result of each model for that dataset + // the rows are datasets the columns are models and the cells are the scores + // the first row is the header with the model names + // the first column is the dataset names + // the last column is the average score of each dataset + // the last row is the average score of each model + // the last cell is the average score of all models + // the last row and column are in bold + // extract the model from the file name + tie(fileModel, fileScore) = getModelScore(file); + // add the model to the vector of models + models.insert(fileModel); + } + return models; + } + + void BestResults::buildAll() + { + auto models = getModels(); + for (const auto& model : models) { + cout << "Building best results for model: " << model << endl; + this->model = model; + build(); + } + model = "any"; } void BestResults::reportSingle() @@ -102,7 +157,118 @@ namespace platform { cout << endl; odd = !odd; } - - + } + json BestResults::buildTableResults(set models) + { + int numberOfDatasets = 0; + bool first = true; + json origin; + json table; + for (const auto& model : models) { + this->model = model; + string bestFileName = path + bestResultFile(); + if (FILE* fileTest = fopen(bestFileName.c_str(), "r")) { + fclose(fileTest); + } else { + cerr << Colors::MAGENTA() << "File " << bestFileName << " doesn't exist." << Colors::RESET() << endl; + exit(1); + } + auto data = loadFile(bestFileName); + if (first) { + // Get the number of datasets of the first file and check that is the same for all the models + first = false; + numberOfDatasets = data.size(); + origin = data; + } else { + if (numberOfDatasets != data.size()) { + cerr << Colors::MAGENTA() << "The number of datasets in the best results files is not the same for all the models." << Colors::RESET() << endl; + exit(1); + } + } + table[model] = data; + } + return table; + } + void BestResults::printTableResults(set models, json table) + { + cout << Colors::GREEN() << "Best results for " << score << endl; + cout << "------------------------------------------" << endl; + cout << Colors::GREEN() << " # Dataset "; + for (const auto& model : models) { + cout << setw(12) << left << model << " "; + } + cout << endl; + cout << "=== ========================= "; + for (const auto& model : models) { + cout << "============ "; + } + cout << endl; + auto i = 0; + bool odd = true; + map totals; + map ranks; + for (const auto& model : models) { + totals[model] = 0.0; + } + json origin = table.begin().value(); + for (auto const& item : origin.items()) { + auto color = odd ? Colors::BLUE() : Colors::CYAN(); + cout << color << setw(3) << fixed << right << i++ << " "; + cout << setw(25) << left << item.key() << " "; + double maxValue = 0; + vector> ranksOrder; + // Find out the max value for this dataset + for (const auto& model : models) { + double value = table[model].at(item.key()).at(0).get(); + if (value > maxValue) { + maxValue = value; + } + ranksOrder.push_back({ model, value }); + } + // sort the ranksOrder vector by value + sort(ranksOrder.begin(), ranksOrder.end(), [](const pair& a, const pair& b) { + return a.second < b.second; + }); + // Assign the ranks + for (int i = 0; i < ranksOrder.size(); i++) { + ranks[ranksOrder[i].first] = i + 1; + } + // Print the row with red colors on max values + for (const auto& model : models) { + string efectiveColor = color; + double value = table[model].at(item.key()).at(0).get(); + if (value == maxValue) { + efectiveColor = Colors::RED(); + } + totals[model] += value; + cout << efectiveColor << setw(12) << setprecision(10) << fixed << value << " "; + } + cout << endl; + odd = !odd; + } + cout << Colors::GREEN() << "=== ========================= "; + for (const auto& model : models) { + cout << "============ "; + } + cout << endl; + cout << Colors::GREEN() << setw(30) << " Totals..................."; + for (const auto& model : models) { + cout << setw(12) << setprecision(9) << fixed << totals[model] << " "; + } + // Output the averaged ranks + cout << endl; + cout << Colors::GREEN() << setw(30) << " Averaged ranks..........."; + for (const auto& model : models) { + cout << setw(12) << setprecision(10) << fixed << (double)ranks[model] / (double)origin.size() << " "; + } + cout << endl; + } + void BestResults::reportAll() + { + auto models = getModels(); + // Build the table of results + json table = buildTableResults(models); + // Print the table of results + printTableResults(models, table); } } \ No newline at end of file diff --git a/src/Platform/BestResults.h b/src/Platform/BestResults.h index 19e10d7..3ba6b9d 100644 --- a/src/Platform/BestResults.h +++ b/src/Platform/BestResults.h @@ -1,6 +1,7 @@ #ifndef BESTRESULTS_H #define BESTRESULTS_H #include +#include #include using namespace std; using json = nlohmann::json; @@ -11,8 +12,12 @@ namespace platform { string build(); void reportSingle(); void reportAll(); + void buildAll(); private: + set getModels(); vector loadResultFiles(); + json buildTableResults(set models); + void printTableResults(set models, json table); string bestResultFile(); json loadFile(const string& fileName); string path; diff --git a/src/Platform/best.cc b/src/Platform/best.cc index 0280a8a..6e6d432 100644 --- a/src/Platform/best.cc +++ b/src/Platform/best.cc @@ -43,15 +43,14 @@ int main(int argc, char** argv) cerr << program; exit(1); } - if (model == "any" && build) { - cerr << "Can't build best results file for all models. \"any\" is only valid for report" << endl; - cerr << program; - exit(1); - } - auto results = platform::BestResults(platform::Paths::results(), model, score); + auto results = platform::BestResults(platform::Paths::results(), score, model); if (build) { - string fileName = results.build(); - cout << Colors::GREEN() << fileName << " created!" << Colors::RESET() << endl; + if (model == "any") { + results.buildAll(); + } else { + string fileName = results.build(); + cout << Colors::GREEN() << fileName << " created!" << Colors::RESET() << endl; + } } if (report) { if (model == "any") {