#include #include #include #include #include #include "BestResults.h" #include "Result.h" #include "Colors.h" namespace fs = std::filesystem; // function ftime_to_string, Code taken from // https://stackoverflow.com/a/58237530/1389271 template std::string ftime_to_string(TP tp) { using namespace std::chrono; auto sctp = time_point_cast(tp - TP::clock::now() + system_clock::now()); auto tt = system_clock::to_time_t(sctp); std::tm* gmt = std::gmtime(&tt); std::stringstream buffer; buffer << std::put_time(gmt, "%Y-%m-%d %H:%M"); return buffer.str(); } namespace platform { string BestResults::build() { auto files = loadResultFiles(); if (files.size() == 0) { cerr << Colors::MAGENTA() << "No result files were found!" << Colors::RESET() << endl; exit(1); } json bests; for (const auto& file : files) { auto result = Result(path, file); auto data = result.load(); for (auto const& item : data.at("results")) { bool update = false; if (bests.contains(item.at("dataset").get())) { if (item.at("score").get() > bests[item.at("dataset").get()].at(0).get()) { update = true; } } else { update = true; } if (update) { bests[item.at("dataset").get()] = { item.at("score").get(), item.at("hyperparameters"), file }; } } } string bestFileName = path + bestResultFile(); if (FILE* fileTest = fopen(bestFileName.c_str(), "r")) { fclose(fileTest); cout << Colors::MAGENTA() << "File " << bestFileName << " already exists and it shall be overwritten." << Colors::RESET() << endl; } ofstream file(bestFileName); file << bests; file.close(); return bestFileName; } string BestResults::bestResultFile() { return "best_results_" + score + "_" + model + ".json"; } pair getModelScore(string name) { // results_accuracy_BoostAODE_MacBookpro16_2023-09-06_12:27:00_1.json int i = 0; auto pos = name.find("_"); auto pos2 = name.find("_", pos + 1); string score = name.substr(pos + 1, pos2 - pos - 1); pos = name.find("_", pos2 + 1); string model = name.substr(pos2 + 1, pos - pos2 - 1); return { model, score }; } vector BestResults::loadResultFiles() { vector files; using std::filesystem::directory_iterator; string fileModel, fileScore; for (const auto& file : directory_iterator(path)) { auto fileName = file.path().filename().string(); if (fileName.find(".json") != string::npos && fileName.find("results_") == 0) { tie(fileModel, fileScore) = getModelScore(fileName); if (score == fileScore && (model == fileModel || model == "any")) { files.push_back(fileName); } } } return files; } json BestResults::loadFile(const string& fileName) { ifstream resultData(fileName); if (resultData.is_open()) { json data = json::parse(resultData); return data; } throw invalid_argument("Unable to open result file. [" + fileName + "]"); } set BestResults::getModels() { set models; auto files = loadResultFiles(); if (files.size() == 0) { cerr << Colors::MAGENTA() << "No result files were found!" << Colors::RESET() << endl; exit(1); } string fileModel, fileScore; for (const auto& file : files) { // extract the model from the file name tie(fileModel, fileScore) = getModelScore(file); // add the model to the vector of models models.insert(fileModel); } return models; } void BestResults::buildAll() { auto models = getModels(); for (const auto& model : models) { cout << "Building best results for model: " << model << endl; this->model = model; build(); } model = "any"; } void BestResults::reportSingle() { string bestFileName = path + bestResultFile(); if (FILE* fileTest = fopen(bestFileName.c_str(), "r")) { fclose(fileTest); } else { cerr << Colors::MAGENTA() << "File " << bestFileName << " doesn't exist." << Colors::RESET() << endl; exit(1); } auto date = ftime_to_string(filesystem::last_write_time(bestFileName)); auto data = loadFile(bestFileName); cout << Colors::GREEN() << "Best results for " << model << " and " << score << " as of " << date << endl; cout << "--------------------------------------------------------" << endl; cout << Colors::GREEN() << " # Dataset Score File Hyperparameters" << endl; cout << "=== ========================= =========== ================================================================== ================================================= " << endl; auto i = 0; bool odd = true; for (auto const& item : data.items()) { auto color = odd ? Colors::BLUE() : Colors::CYAN(); cout << color << setw(3) << fixed << right << i++ << " "; cout << setw(25) << left << item.key() << " "; cout << setw(11) << setprecision(9) << fixed << item.value().at(0).get() << " "; cout << setw(66) << item.value().at(2).get() << " "; cout << item.value().at(1) << " "; cout << endl; odd = !odd; } } json BestResults::buildTableResults(set models) { int numberOfDatasets = 0; bool first = true; json origin; json table; auto maxDate = filesystem::file_time_type::max(); for (const auto& model : models) { this->model = model; string bestFileName = path + bestResultFile(); if (FILE* fileTest = fopen(bestFileName.c_str(), "r")) { fclose(fileTest); } else { cerr << Colors::MAGENTA() << "File " << bestFileName << " doesn't exist." << Colors::RESET() << endl; exit(1); } auto dateWrite = filesystem::last_write_time(bestFileName); if (dateWrite < maxDate) { maxDate = dateWrite; } auto data = loadFile(bestFileName); if (first) { // Get the number of datasets of the first file and check that is the same for all the models first = false; numberOfDatasets = data.size(); origin = data; } else { if (numberOfDatasets != data.size()) { cerr << Colors::MAGENTA() << "The number of datasets in the best results files is not the same for all the models." << Colors::RESET() << endl; exit(1); } } table[model] = data; } table["dateTable"] = ftime_to_string(maxDate); return table; } void BestResults::printTableResults(set models, json table) { cout << Colors::GREEN() << "Best results for " << score << " as of " << table.at("dateTable").get() << endl; cout << "------------------------------------------------" << endl; cout << Colors::GREEN() << " # Dataset "; for (const auto& model : models) { cout << setw(12) << left << model << " "; } cout << endl; cout << "=== ========================= "; for (const auto& model : models) { cout << "============ "; } cout << endl; auto i = 0; bool odd = true; map totals; map ranks; for (const auto& model : models) { totals[model] = 0.0; } json origin = table.begin().value(); for (auto const& item : origin.items()) { auto color = odd ? Colors::BLUE() : Colors::CYAN(); cout << color << setw(3) << fixed << right << i++ << " "; cout << setw(25) << left << item.key() << " "; double maxValue = 0; vector> ranksOrder; // Find out the max value for this dataset for (const auto& model : models) { double value = table[model].at(item.key()).at(0).get(); if (value > maxValue) { maxValue = value; } ranksOrder.push_back({ model, value }); } // sort the ranksOrder vector by value sort(ranksOrder.begin(), ranksOrder.end(), [](const pair& a, const pair& b) { return a.second > b.second; }); // Assign the ranks for (int i = 0; i < ranksOrder.size(); i++) { ranks[ranksOrder[i].first] = i + 1; } // Print the row with red colors on max values for (const auto& model : models) { string efectiveColor = color; double value = table[model].at(item.key()).at(0).get(); if (value == maxValue) { efectiveColor = Colors::RED(); } totals[model] += value; cout << efectiveColor << setw(12) << setprecision(10) << fixed << value << " "; } cout << endl; odd = !odd; } cout << Colors::GREEN() << "=== ========================= "; for (const auto& model : models) { cout << "============ "; } cout << endl; cout << Colors::GREEN() << setw(30) << " Totals..................."; double max = 0.0; for (const auto& total : totals) { if (total.second > max) { max = total.second; } } for (const auto& model : models) { string efectiveColor = Colors::GREEN(); if (totals[model] == max) { efectiveColor = Colors::RED(); } cout << efectiveColor << setw(12) << setprecision(9) << fixed << totals[model] << " "; } // Output the averaged ranks cout << endl; int min = 1; for (const auto& rank : ranks) { if (rank.second < min) { min = rank.second; } } cout << Colors::GREEN() << setw(30) << " Averaged ranks..........."; for (const auto& model : models) { string efectiveColor = Colors::GREEN(); if (ranks[model] == min) { efectiveColor = Colors::RED(); } cout << efectiveColor << setw(12) << setprecision(10) << fixed << (double)ranks[model] / (double)origin.size() << " "; } cout << endl; } void BestResults::reportAll() { auto models = getModels(); // Build the table of results json table = buildTableResults(models); // Print the table of results printTableResults(models, table); } }