rmontanana/BayesNet
rmontanana/BayesNet
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases 3 Wiki Activity

Add report/build all with totals and ranks

Browse Source
This commit is contained in:
Ricardo Montañana Gómez 2023-09-23 01:14:02 +02:00
parent c8597a794e
commit cd0bc02a74
Signed by: rmontanana
GPG Key ID: 46064262FD9A7ADE
3 changed files with 186 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

182
src/Platform/BestResults.cc
View File

@ -1,6 +1,7 @@
#include <filesystem>
#include <fstream>
#include <iostream>
#include <set>
#include "BestResults.h"
#include "Result.h"
#include "Colors.h"
@ -45,23 +46,38 @@ namespace platform {
string BestResults::bestResultFile()
{
return "best_results_" + model + "_" + score + ".json";
return "best_results_" + score + "_" + model + ".json";
}
pair<string, string> getModelScore(string name)
{
// results_accuracy_BoostAODE_MacBookpro16_2023-09-06_12:27:00_1.json
int i = 0;
auto pos = name.find("_");
auto pos2 = name.find("_", pos + 1);
string score = name.substr(pos + 1, pos2 - pos - 1);
pos = name.find("_", pos2 + 1);
string model = name.substr(pos2 + 1, pos - pos2 - 1);
return { model, score };
}
vector<string> BestResults::loadResultFiles()
{
vector<string> files;
using std::filesystem::directory_iterator;
string fileModel, fileScore;
for (const auto& file : directory_iterator(path)) {
auto fileName = file.path().filename().string();
if (fileName.find(".json") != string::npos && fileName.find("results_") == 0
&& fileName.find("_" + score + "_") != string::npos
&& (fileName.find("_" + model + "_") != string::npos || model == "any")) {
files.push_back(fileName);
if (fileName.find(".json") != string::npos && fileName.find("results_") == 0) {
tie(fileModel, fileScore) = getModelScore(fileName);
if (score == fileScore && (model == fileModel || model == "any")) {
files.push_back(fileName);
}
}
}
return files;
}
json BestResults::loadFile(const string& fileName)
{
ifstream resultData(fileName);
@ -71,9 +87,48 @@ namespace platform {
}
throw invalid_argument("Unable to open result file. [" + fileName + "]");
}
void BestResults::reportAll()
set<string> BestResults::getModels()
{
set<string> models;
auto files = loadResultFiles();
if (files.size() == 0) {
cerr << Colors::MAGENTA() << "No result files were found!" << Colors::RESET() << endl;
exit(1);
}
string fileModel, fileScore;
for (const auto& file : files) {
// take the model from the file name and add it to a vector of models
// set model to the name of the first model in the vector
// filter files and build the best results file of this model
// repeat for all models
// another for loop to read the best results file of each model and print al together
// each row is a dataset and each column is a model
// the score is the score of the best result of each model for that dataset
// the rows are datasets the columns are models and the cells are the scores
// the first row is the header with the model names
// the first column is the dataset names
// the last column is the average score of each dataset
// the last row is the average score of each model
// the last cell is the average score of all models
// the last row and column are in bold
// extract the model from the file name
tie(fileModel, fileScore) = getModelScore(file);
// add the model to the vector of models
models.insert(fileModel);
}
return models;
}
void BestResults::buildAll()
{
auto models = getModels();
for (const auto& model : models) {
cout << "Building best results for model: " << model << endl;
this->model = model;
build();
}
model = "any";
}
void BestResults::reportSingle()
@ -102,7 +157,118 @@ namespace platform {
cout << endl;
odd = !odd;
}
}
json BestResults::buildTableResults(set<string> models)
{
int numberOfDatasets = 0;
bool first = true;
json origin;
json table;
for (const auto& model : models) {
this->model = model;
string bestFileName = path + bestResultFile();
if (FILE* fileTest = fopen(bestFileName.c_str(), "r")) {
fclose(fileTest);
} else {
cerr << Colors::MAGENTA() << "File " << bestFileName << " doesn't exist." << Colors::RESET() << endl;
exit(1);
}
auto data = loadFile(bestFileName);
if (first) {
// Get the number of datasets of the first file and check that is the same for all the models
first = false;
numberOfDatasets = data.size();
origin = data;
} else {
if (numberOfDatasets != data.size()) {
cerr << Colors::MAGENTA() << "The number of datasets in the best results files is not the same for all the models." << Colors::RESET() << endl;
exit(1);
}
}
table[model] = data;
}
return table;
}
void BestResults::printTableResults(set<string> models, json table)
{
cout << Colors::GREEN() << "Best results for " << score << endl;
cout << "------------------------------------------" << endl;
cout << Colors::GREEN() << " # Dataset ";
for (const auto& model : models) {
cout << setw(12) << left << model << " ";
}
cout << endl;
cout << "=== ========================= ";
for (const auto& model : models) {
cout << "============ ";
}
cout << endl;
auto i = 0;
bool odd = true;
map<string, double> totals;
map<string, int> ranks;
for (const auto& model : models) {
totals[model] = 0.0;
}
json origin = table.begin().value();
for (auto const& item : origin.items()) {
auto color = odd ? Colors::BLUE() : Colors::CYAN();
cout << color << setw(3) << fixed << right << i++ << " ";
cout << setw(25) << left << item.key() << " ";
double maxValue = 0;
vector<pair<string, double>> ranksOrder;
// Find out the max value for this dataset
for (const auto& model : models) {
double value = table[model].at(item.key()).at(0).get<double>();
if (value > maxValue) {
maxValue = value;
}
ranksOrder.push_back({ model, value });
}
// sort the ranksOrder vector by value
sort(ranksOrder.begin(), ranksOrder.end(), [](const pair<string, double>& a, const pair<string, double>& b) {
return a.second < b.second;
});
// Assign the ranks
for (int i = 0; i < ranksOrder.size(); i++) {
ranks[ranksOrder[i].first] = i + 1;
}
// Print the row with red colors on max values
for (const auto& model : models) {
string efectiveColor = color;
double value = table[model].at(item.key()).at(0).get<double>();
if (value == maxValue) {
efectiveColor = Colors::RED();
}
totals[model] += value;
cout << efectiveColor << setw(12) << setprecision(10) << fixed << value << " ";
}
cout << endl;
odd = !odd;
}
cout << Colors::GREEN() << "=== ========================= ";
for (const auto& model : models) {
cout << "============ ";
}
cout << endl;
cout << Colors::GREEN() << setw(30) << " Totals...................";
for (const auto& model : models) {
cout << setw(12) << setprecision(9) << fixed << totals[model] << " ";
}
// Output the averaged ranks
cout << endl;
cout << Colors::GREEN() << setw(30) << " Averaged ranks...........";
for (const auto& model : models) {
cout << setw(12) << setprecision(10) << fixed << (double)ranks[model] / (double)origin.size() << " ";
}
cout << endl;
}
void BestResults::reportAll()
{
auto models = getModels();
// Build the table of results
json table = buildTableResults(models);
// Print the table of results
printTableResults(models, table);
}
}

5
src/Platform/BestResults.h
View File

@ -1,6 +1,7 @@
#ifndef BESTRESULTS_H
#define BESTRESULTS_H
#include <string>
#include <set>
#include <nlohmann/json.hpp>
using namespace std;
using json = nlohmann::json;
@ -11,8 +12,12 @@ namespace platform {
string build();
void reportSingle();
void reportAll();
void buildAll();
private:
set<string> getModels();
vector<string> loadResultFiles();
json buildTableResults(set<string> models);
void printTableResults(set<string> models, json table);
string bestResultFile();
json loadFile(const string& fileName);
string path;

15
src/Platform/best.cc
View File

@ -43,15 +43,14 @@ int main(int argc, char** argv)
cerr << program;
exit(1);
}
if (model == "any" && build) {
cerr << "Can't build best results file for all models. \"any\" is only valid for report" << endl;
cerr << program;
exit(1);
}
auto results = platform::BestResults(platform::Paths::results(), model, score);
auto results = platform::BestResults(platform::Paths::results(), score, model);
if (build) {
string fileName = results.build();
cout << Colors::GREEN() << fileName << " created!" << Colors::RESET() << endl;
if (model == "any") {
results.buildAll();
} else {
string fileName = results.build();
cout << Colors::GREEN() << fileName << " created!" << Colors::RESET() << endl;
}
}
if (report) {
if (model == "any") {