rmontanana/BayesNet
rmontanana/BayesNet
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases 3 Wiki Activity

Merge pull request 'bestResults' (#9) from bestResults into main

Browse Source 
Reviewed-on: https://gitea.rmontanana.es:3000/rmontanana/BayesNet/pulls/9

Add best results management, build, report, build all & report all
This commit is contained in:
Ricardo Montañana Gómez 2023-09-25 12:02:17 +00:00
commit e3888e1503
16 changed files with 690 additions and 254 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.vscode/launch.json vendored
View File

@ -37,6 +37,20 @@
], ],
"cwd": "/Users/rmontanana/Code/discretizbench", "cwd": "/Users/rmontanana/Code/discretizbench",
}, },
{
"type": "lldb",
"request": "launch",
"name": "best",
"program": "${workspaceFolder}/build/src/Platform/best",
"args": [
"-m",
"BoostAODE",
"-s",
"accuracy",
"--build",
],
"cwd": "/Users/rmontanana/Code/discretizbench",
},
{ {
"type": "lldb", "type": "lldb",
"request": "launch", "request": "launch",

7
Makefile
View File

@ -19,13 +19,14 @@ copy: ## Copy binary files to selected folder
@cp build/src/Platform/main $(dest) @cp build/src/Platform/main $(dest)
@cp build/src/Platform/list $(dest) @cp build/src/Platform/list $(dest)
@cp build/src/Platform/manage $(dest) @cp build/src/Platform/manage $(dest)
@cp build/src/Platform/best $(dest)
@echo ">>> Done" @echo ">>> Done"
dependency: ## Create a dependency graph diagram of the project (build/dependency.png) dependency: ## Create a dependency graph diagram of the project (build/dependency.png)
cd build && cmake .. --graphviz=dependency.dot && dot -Tpng dependency.dot -o dependency.png cd build && cmake .. --graphviz=dependency.dot && dot -Tpng dependency.dot -o dependency.png
build: ## Build the main and BayesNetSample build: ## Build the main and BayesNetSample
cmake --build build -t main -t BayesNetSample -t manage -t list -j 32 cmake --build build -t main -t BayesNetSample -t manage -t list -t best -j 32
clean: ## Clean the debug info clean: ## Clean the debug info
@echo ">>> Cleaning Debug BayesNet ..."; @echo ">>> Cleaning Debug BayesNet ...";
@ -40,7 +41,7 @@ debug: ## Build a debug version of the project
@if [ -d ./build ]; then rm -rf ./build; fi @if [ -d ./build ]; then rm -rf ./build; fi
@mkdir build; @mkdir build;
cmake -S . -B build -D CMAKE_BUILD_TYPE=Debug -D ENABLE_TESTING=ON -D CODE_COVERAGE=ON; \ cmake -S . -B build -D CMAKE_BUILD_TYPE=Debug -D ENABLE_TESTING=ON -D CODE_COVERAGE=ON; \
cmake --build build -t main -t BayesNetSample -t manage -t list unit_tests -j 32; cmake --build build -t main -t BayesNetSample -t manage -t list -t best -t unit_tests -j 32;
@echo ">>> Done"; @echo ">>> Done";
release: ## Build a Release version of the project release: ## Build a Release version of the project
@ -48,7 +49,7 @@ release: ## Build a Release version of the project
@if [ -d ./build ]; then rm -rf ./build; fi @if [ -d ./build ]; then rm -rf ./build; fi
@mkdir build; @mkdir build;
cmake -S . -B build -D CMAKE_BUILD_TYPE=Release; \ cmake -S . -B build -D CMAKE_BUILD_TYPE=Release; \
cmake --build build -t main -t BayesNetSample -t manage -t list -j 32; cmake --build build -t main -t BayesNetSample -t manage -t list -t best -j 32;
@echo ">>> Done"; @echo ">>> Done";
test: ## Run tests test: ## Run tests

344
sample/sample.cc
View File

@ -104,180 +104,180 @@ int main(int argc, char** argv)
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
cout << weights_.index({ i }).item<double>() << endl; cout << weights_.index({ i }).item<double>() << endl;
} }
// map<string, bool> datasets = { map<string, bool> datasets = {
// {"diabetes", true}, {"diabetes", true},
// {"ecoli", true}, {"ecoli", true},
// {"glass", true}, {"glass", true},
// {"iris", true}, {"iris", true},
// {"kdd_JapaneseVowels", false}, {"kdd_JapaneseVowels", false},
// {"letter", true}, {"letter", true},
// {"liver-disorders", true}, {"liver-disorders", true},
// {"mfeat-factors", true}, {"mfeat-factors", true},
// }; };
// auto valid_datasets = vector<string>(); auto valid_datasets = vector<string>();
// transform(datasets.begin(), datasets.end(), back_inserter(valid_datasets), transform(datasets.begin(), datasets.end(), back_inserter(valid_datasets),
// [](const pair<string, bool>& pair) { return pair.first; }); [](const pair<string, bool>& pair) { return pair.first; });
// argparse::ArgumentParser program("BayesNetSample"); argparse::ArgumentParser program("BayesNetSample");
// program.add_argument("-d", "--dataset") program.add_argument("-d", "--dataset")
// .help("Dataset file name") .help("Dataset file name")
// .action([valid_datasets](const std::string& value) { .action([valid_datasets](const std::string& value) {
// if (find(valid_datasets.begin(), valid_datasets.end(), value) != valid_datasets.end()) { if (find(valid_datasets.begin(), valid_datasets.end(), value) != valid_datasets.end()) {
// return value; return value;
// } }
// throw runtime_error("file must be one of {diabetes, ecoli, glass, iris, kdd_JapaneseVowels, letter, liver-disorders, mfeat-factors}"); throw runtime_error("file must be one of {diabetes, ecoli, glass, iris, kdd_JapaneseVowels, letter, liver-disorders, mfeat-factors}");
// } }
// ); );
// program.add_argument("-p", "--path") program.add_argument("-p", "--path")
// .help(" folder where the data files are located, default") .help(" folder where the data files are located, default")
// .default_value(string{ PATH } .default_value(string{ PATH }
// ); );
// program.add_argument("-m", "--model") program.add_argument("-m", "--model")
// .help("Model to use " + platform::Models::instance()->toString()) .help("Model to use " + platform::Models::instance()->toString())
// .action([](const std::string& value) { .action([](const std::string& value) {
// static const vector<string> choices = platform::Models::instance()->getNames(); static const vector<string> choices = platform::Models::instance()->getNames();
// if (find(choices.begin(), choices.end(), value) != choices.end()) { if (find(choices.begin(), choices.end(), value) != choices.end()) {
// return value; return value;
// } }
// throw runtime_error("Model must be one of " + platform::Models::instance()->toString()); throw runtime_error("Model must be one of " + platform::Models::instance()->toString());
// } }
// ); );
// program.add_argument("--discretize").help("Discretize input dataset").default_value(false).implicit_value(true); program.add_argument("--discretize").help("Discretize input dataset").default_value(false).implicit_value(true);
// program.add_argument("--dumpcpt").help("Dump CPT Tables").default_value(false).implicit_value(true); program.add_argument("--dumpcpt").help("Dump CPT Tables").default_value(false).implicit_value(true);
// program.add_argument("--stratified").help("If Stratified KFold is to be done").default_value(false).implicit_value(true); program.add_argument("--stratified").help("If Stratified KFold is to be done").default_value(false).implicit_value(true);
// program.add_argument("--tensors").help("Use tensors to store samples").default_value(false).implicit_value(true); program.add_argument("--tensors").help("Use tensors to store samples").default_value(false).implicit_value(true);
// program.add_argument("-f", "--folds").help("Number of folds").default_value(5).scan<'i', int>().action([](const string& value) { program.add_argument("-f", "--folds").help("Number of folds").default_value(5).scan<'i', int>().action([](const string& value) {
// try { try {
// auto k = stoi(value); auto k = stoi(value);
// if (k < 2) { if (k < 2) {
// throw runtime_error("Number of folds must be greater than 1"); throw runtime_error("Number of folds must be greater than 1");
// } }
// return k; return k;
// } }
// catch (const runtime_error& err) { catch (const runtime_error& err) {
// throw runtime_error(err.what()); throw runtime_error(err.what());
// } }
// catch (...) { catch (...) {
// throw runtime_error("Number of folds must be an integer"); throw runtime_error("Number of folds must be an integer");
// }}); }});
// program.add_argument("-s", "--seed").help("Random seed").default_value(-1).scan<'i', int>(); program.add_argument("-s", "--seed").help("Random seed").default_value(-1).scan<'i', int>();
// bool class_last, stratified, tensors, dump_cpt; bool class_last, stratified, tensors, dump_cpt;
// string model_name, file_name, path, complete_file_name; string model_name, file_name, path, complete_file_name;
// int nFolds, seed; int nFolds, seed;
// try { try {
// program.parse_args(argc, argv); program.parse_args(argc, argv);
// file_name = program.get<string>("dataset"); file_name = program.get<string>("dataset");
// path = program.get<string>("path"); path = program.get<string>("path");
// model_name = program.get<string>("model"); model_name = program.get<string>("model");
// complete_file_name = path + file_name + ".arff"; complete_file_name = path + file_name + ".arff";
// stratified = program.get<bool>("stratified"); stratified = program.get<bool>("stratified");
// tensors = program.get<bool>("tensors"); tensors = program.get<bool>("tensors");
// nFolds = program.get<int>("folds"); nFolds = program.get<int>("folds");
// seed = program.get<int>("seed"); seed = program.get<int>("seed");
// dump_cpt = program.get<bool>("dumpcpt"); dump_cpt = program.get<bool>("dumpcpt");
// class_last = datasets[file_name]; class_last = datasets[file_name];
// if (!file_exists(complete_file_name)) { if (!file_exists(complete_file_name)) {
// throw runtime_error("Data File " + path + file_name + ".arff" + " does not exist"); throw runtime_error("Data File " + path + file_name + ".arff" + " does not exist");
// } }
// } }
// catch (const exception& err) { catch (const exception& err) {
// cerr << err.what() << endl; cerr << err.what() << endl;
// cerr << program; cerr << program;
// exit(1); exit(1);
// } }
/* /*
* Begin Processing * Begin Processing
*/ */
// auto handler = ArffFiles(); auto handler = ArffFiles();
// handler.load(complete_file_name, class_last); handler.load(complete_file_name, class_last);
// // Get Dataset X, y // Get Dataset X, y
// vector<mdlp::samples_t>& X = handler.getX(); vector<mdlp::samples_t>& X = handler.getX();
// mdlp::labels_t& y = handler.getY(); mdlp::labels_t& y = handler.getY();
// // Get className & Features // Get className & Features
// auto className = handler.getClassName(); auto className = handler.getClassName();
// vector<string> features; vector<string> features;
// auto attributes = handler.getAttributes(); auto attributes = handler.getAttributes();
// transform(attributes.begin(), attributes.end(), back_inserter(features), transform(attributes.begin(), attributes.end(), back_inserter(features),
// [](const pair<string, string>& item) { return item.first; }); [](const pair<string, string>& item) { return item.first; });
// // Discretize Dataset // Discretize Dataset
// auto [Xd, maxes] = discretize(X, y, features); auto [Xd, maxes] = discretize(X, y, features);
// maxes[className] = *max_element(y.begin(), y.end()) + 1; maxes[className] = *max_element(y.begin(), y.end()) + 1;
// map<string, vector<int>> states; map<string, vector<int>> states;
// for (auto feature : features) { for (auto feature : features) {
// states[feature] = vector<int>(maxes[feature]); states[feature] = vector<int>(maxes[feature]);
// } }
// states[className] = vector<int>(maxes[className]); states[className] = vector<int>(maxes[className]);
// auto clf = platform::Models::instance()->create(model_name); auto clf = platform::Models::instance()->create(model_name);
// clf->fit(Xd, y, features, className, states); clf->fit(Xd, y, features, className, states);
// if (dump_cpt) { if (dump_cpt) {
// cout << "--- CPT Tables ---" << endl; cout << "--- CPT Tables ---" << endl;
// clf->dump_cpt(); clf->dump_cpt();
// } }
// auto lines = clf->show(); auto lines = clf->show();
// for (auto line : lines) { for (auto line : lines) {
// cout << line << endl; cout << line << endl;
// } }
// cout << "--- Topological Order ---" << endl; cout << "--- Topological Order ---" << endl;
// auto order = clf->topological_order(); auto order = clf->topological_order();
// for (auto name : order) { for (auto name : order) {
// cout << name << ", "; cout << name << ", ";
// } }
// cout << "end." << endl; cout << "end." << endl;
// auto score = clf->score(Xd, y); auto score = clf->score(Xd, y);
// cout << "Score: " << score << endl; cout << "Score: " << score << endl;
// auto graph = clf->graph(); auto graph = clf->graph();
// auto dot_file = model_name + "_" + file_name; auto dot_file = model_name + "_" + file_name;
// ofstream file(dot_file + ".dot"); ofstream file(dot_file + ".dot");
// file << graph; file << graph;
// file.close(); file.close();
// cout << "Graph saved in " << model_name << "_" << file_name << ".dot" << endl; cout << "Graph saved in " << model_name << "_" << file_name << ".dot" << endl;
// cout << "dot -Tpng -o " + dot_file + ".png " + dot_file + ".dot " << endl; cout << "dot -Tpng -o " + dot_file + ".png " + dot_file + ".dot " << endl;
// string stratified_string = stratified ? " Stratified" : ""; string stratified_string = stratified ? " Stratified" : "";
// cout << nFolds << " Folds" << stratified_string << " Cross validation" << endl; cout << nFolds << " Folds" << stratified_string << " Cross validation" << endl;
// cout << "==========================================" << endl; cout << "==========================================" << endl;
// torch::Tensor Xt = torch::zeros({ static_cast<int>(Xd.size()), static_cast<int>(Xd[0].size()) }, torch::kInt32); torch::Tensor Xt = torch::zeros({ static_cast<int>(Xd.size()), static_cast<int>(Xd[0].size()) }, torch::kInt32);
// torch::Tensor yt = torch::tensor(y, torch::kInt32); torch::Tensor yt = torch::tensor(y, torch::kInt32);
// for (int i = 0; i < features.size(); ++i) { for (int i = 0; i < features.size(); ++i) {
// Xt.index_put_({ i, "..." }, torch::tensor(Xd[i], torch::kInt32)); Xt.index_put_({ i, "..." }, torch::tensor(Xd[i], torch::kInt32));
// } }
// float total_score = 0, total_score_train = 0, score_train, score_test; float total_score = 0, total_score_train = 0, score_train, score_test;
// platform::Fold* fold; platform::Fold* fold;
// if (stratified) if (stratified)
// fold = new platform::StratifiedKFold(nFolds, y, seed); fold = new platform::StratifiedKFold(nFolds, y, seed);
// else else
// fold = new platform::KFold(nFolds, y.size(), seed); fold = new platform::KFold(nFolds, y.size(), seed);
// for (auto i = 0; i < nFolds; ++i) { for (auto i = 0; i < nFolds; ++i) {
// auto [train, test] = fold->getFold(i); auto [train, test] = fold->getFold(i);
// cout << "Fold: " << i + 1 << endl; cout << "Fold: " << i + 1 << endl;
// if (tensors) { if (tensors) {
// auto ttrain = torch::tensor(train, torch::kInt64); auto ttrain = torch::tensor(train, torch::kInt64);
// auto ttest = torch::tensor(test, torch::kInt64); auto ttest = torch::tensor(test, torch::kInt64);
// torch::Tensor Xtraint = torch::index_select(Xt, 1, ttrain); torch::Tensor Xtraint = torch::index_select(Xt, 1, ttrain);
// torch::Tensor ytraint = yt.index({ ttrain }); torch::Tensor ytraint = yt.index({ ttrain });
// torch::Tensor Xtestt = torch::index_select(Xt, 1, ttest); torch::Tensor Xtestt = torch::index_select(Xt, 1, ttest);
// torch::Tensor ytestt = yt.index({ ttest }); torch::Tensor ytestt = yt.index({ ttest });
// clf->fit(Xtraint, ytraint, features, className, states); clf->fit(Xtraint, ytraint, features, className, states);
// auto temp = clf->predict(Xtraint); auto temp = clf->predict(Xtraint);
// score_train = clf->score(Xtraint, ytraint); score_train = clf->score(Xtraint, ytraint);
// score_test = clf->score(Xtestt, ytestt); score_test = clf->score(Xtestt, ytestt);
// } else { } else {
// auto [Xtrain, ytrain] = extract_indices(train, Xd, y); auto [Xtrain, ytrain] = extract_indices(train, Xd, y);
// auto [Xtest, ytest] = extract_indices(test, Xd, y); auto [Xtest, ytest] = extract_indices(test, Xd, y);
// clf->fit(Xtrain, ytrain, features, className, states); clf->fit(Xtrain, ytrain, features, className, states);
// score_train = clf->score(Xtrain, ytrain); score_train = clf->score(Xtrain, ytrain);
// score_test = clf->score(Xtest, ytest); score_test = clf->score(Xtest, ytest);
// } }
// if (dump_cpt) { if (dump_cpt) {
// cout << "--- CPT Tables ---" << endl; cout << "--- CPT Tables ---" << endl;
// clf->dump_cpt(); clf->dump_cpt();
// } }
// total_score_train += score_train; total_score_train += score_train;
// total_score += score_test; total_score += score_test;
// cout << "Score Train: " << score_train << endl; cout << "Score Train: " << score_train << endl;
// cout << "Score Test : " << score_test << endl; cout << "Score Test : " << score_test << endl;
// cout << "-------------------------------------------------------------------------------" << endl; cout << "-------------------------------------------------------------------------------" << endl;
// } }
// cout << "**********************************************************************************" << endl; cout << "**********************************************************************************" << endl;
// cout << "Average Score Train: " << total_score_train / nFolds << endl; cout << "Average Score Train: " << total_score_train / nFolds << endl;
// cout << "Average Score Test : " << total_score / nFolds << endl;return 0; cout << "Average Score Test : " << total_score / nFolds << endl;return 0;
} }

305
src/Platform/BestResults.cc Normal file
View File

@ -0,0 +1,305 @@
#include <filesystem>
#include <fstream>
#include <iostream>
#include <sstream>
#include <set>
#include "BestResults.h"
#include "Result.h"
#include "Colors.h"
namespace fs = std::filesystem;
// function ftime_to_string, Code taken from
// https://stackoverflow.com/a/58237530/1389271
template <typename TP>
std::string ftime_to_string(TP tp)
{
using namespace std::chrono;
auto sctp = time_point_cast<system_clock::duration>(tp - TP::clock::now()
+ system_clock::now());
auto tt = system_clock::to_time_t(sctp);
std::tm* gmt = std::gmtime(&tt);
std::stringstream buffer;
buffer << std::put_time(gmt, "%Y-%m-%d %H:%M");
return buffer.str();
}
namespace platform {
string BestResults::build()
{
auto files = loadResultFiles();
if (files.size() == 0) {
cerr << Colors::MAGENTA() << "No result files were found!" << Colors::RESET() << endl;
exit(1);
}
json bests;
for (const auto& file : files) {
auto result = Result(path, file);
auto data = result.load();
for (auto const& item : data.at("results")) {
bool update = false;
if (bests.contains(item.at("dataset").get<string>())) {
if (item.at("score").get<double>() > bests[item.at("dataset").get<string>()].at(0).get<double>()) {
update = true;
}
} else {
update = true;
}
if (update) {
bests[item.at("dataset").get<string>()] = { item.at("score").get<double>(), item.at("hyperparameters"), file };
}
}
}
string bestFileName = path + bestResultFile();
if (FILE* fileTest = fopen(bestFileName.c_str(), "r")) {
fclose(fileTest);
cout << Colors::MAGENTA() << "File " << bestFileName << " already exists and it shall be overwritten." << Colors::RESET() << endl;
}
ofstream file(bestFileName);
file << bests;
file.close();
return bestFileName;
}
string BestResults::bestResultFile()
{
return "best_results_" + score + "_" + model + ".json";
}
pair<string, string> getModelScore(string name)
{
// results_accuracy_BoostAODE_MacBookpro16_2023-09-06_12:27:00_1.json
int i = 0;
auto pos = name.find("_");
auto pos2 = name.find("_", pos + 1);
string score = name.substr(pos + 1, pos2 - pos - 1);
pos = name.find("_", pos2 + 1);
string model = name.substr(pos2 + 1, pos - pos2 - 1);
return { model, score };
}
vector<string> BestResults::loadResultFiles()
{
vector<string> files;
using std::filesystem::directory_iterator;
string fileModel, fileScore;
for (const auto& file : directory_iterator(path)) {
auto fileName = file.path().filename().string();
if (fileName.find(".json") != string::npos && fileName.find("results_") == 0) {
tie(fileModel, fileScore) = getModelScore(fileName);
if (score == fileScore && (model == fileModel || model == "any")) {
files.push_back(fileName);
}
}
}
return files;
}
json BestResults::loadFile(const string& fileName)
{
ifstream resultData(fileName);
if (resultData.is_open()) {
json data = json::parse(resultData);
return data;
}
throw invalid_argument("Unable to open result file. [" + fileName + "]");
}
set<string> BestResults::getModels()
{
set<string> models;
auto files = loadResultFiles();
if (files.size() == 0) {
cerr << Colors::MAGENTA() << "No result files were found!" << Colors::RESET() << endl;
exit(1);
}
string fileModel, fileScore;
for (const auto& file : files) {
// extract the model from the file name
tie(fileModel, fileScore) = getModelScore(file);
// add the model to the vector of models
models.insert(fileModel);
}
return models;
}
void BestResults::buildAll()
{
auto models = getModels();
for (const auto& model : models) {
cout << "Building best results for model: " << model << endl;
this->model = model;
build();
}
model = "any";
}
void BestResults::reportSingle()
{
string bestFileName = path + bestResultFile();
if (FILE* fileTest = fopen(bestFileName.c_str(), "r")) {
fclose(fileTest);
} else {
cerr << Colors::MAGENTA() << "File " << bestFileName << " doesn't exist." << Colors::RESET() << endl;
exit(1);
}
auto date = ftime_to_string(filesystem::last_write_time(bestFileName));
auto data = loadFile(bestFileName);
cout << Colors::GREEN() << "Best results for " << model << " and " << score << " as of " << date << endl;
cout << "--------------------------------------------------------" << endl;
cout << Colors::GREEN() << " # Dataset Score File Hyperparameters" << endl;
cout << "=== ========================= =========== ================================================================== ================================================= " << endl;
auto i = 0;
bool odd = true;
for (auto const& item : data.items()) {
auto color = odd ? Colors::BLUE() : Colors::CYAN();
cout << color << setw(3) << fixed << right << i++ << " ";
cout << setw(25) << left << item.key() << " ";
cout << setw(11) << setprecision(9) << fixed << item.value().at(0).get<double>() << " ";
cout << setw(66) << item.value().at(2).get<string>() << " ";
cout << item.value().at(1) << " ";
cout << endl;
odd = !odd;
}
}
json BestResults::buildTableResults(set<string> models)
{
int numberOfDatasets = 0;
bool first = true;
json origin;
json table;
auto maxDate = filesystem::file_time_type::max();
for (const auto& model : models) {
this->model = model;
string bestFileName = path + bestResultFile();
if (FILE* fileTest = fopen(bestFileName.c_str(), "r")) {
fclose(fileTest);
} else {
cerr << Colors::MAGENTA() << "File " << bestFileName << " doesn't exist." << Colors::RESET() << endl;
exit(1);
}
auto dateWrite = filesystem::last_write_time(bestFileName);
if (dateWrite < maxDate) {
maxDate = dateWrite;
}
auto data = loadFile(bestFileName);
if (first) {
// Get the number of datasets of the first file and check that is the same for all the models
first = false;
numberOfDatasets = data.size();
origin = data;
} else {
if (numberOfDatasets != data.size()) {
cerr << Colors::MAGENTA() << "The number of datasets in the best results files is not the same for all the models." << Colors::RESET() << endl;
exit(1);
}
}
table[model] = data;
}
table["dateTable"] = ftime_to_string(maxDate);
return table;
}
void BestResults::printTableResults(set<string> models, json table)
{
cout << Colors::GREEN() << "Best results for " << score << " as of " << table.at("dateTable").get<string>() << endl;
cout << "------------------------------------------------" << endl;
cout << Colors::GREEN() << " # Dataset ";
for (const auto& model : models) {
cout << setw(12) << left << model << " ";
}
cout << endl;
cout << "=== ========================= ";
for (const auto& model : models) {
cout << "============ ";
}
cout << endl;
auto i = 0;
bool odd = true;
map<string, double> totals;
map<string, int> ranks;
for (const auto& model : models) {
totals[model] = 0.0;
}
json origin = table.begin().value();
for (auto const& item : origin.items()) {
auto color = odd ? Colors::BLUE() : Colors::CYAN();
cout << color << setw(3) << fixed << right << i++ << " ";
cout << setw(25) << left << item.key() << " ";
double maxValue = 0;
vector<pair<string, double>> ranksOrder;
// Find out the max value for this dataset
for (const auto& model : models) {
double value = table[model].at(item.key()).at(0).get<double>();
if (value > maxValue) {
maxValue = value;
}
ranksOrder.push_back({ model, value });
}
// sort the ranksOrder vector by value
sort(ranksOrder.begin(), ranksOrder.end(), [](const pair<string, double>& a, const pair<string, double>& b) {
return a.second > b.second;
});
// Assign the ranks
for (int i = 0; i < ranksOrder.size(); i++) {
ranks[ranksOrder[i].first] = i + 1;
}
// Print the row with red colors on max values
for (const auto& model : models) {
string efectiveColor = color;
double value = table[model].at(item.key()).at(0).get<double>();
if (value == maxValue) {
efectiveColor = Colors::RED();
}
totals[model] += value;
cout << efectiveColor << setw(12) << setprecision(10) << fixed << value << " ";
}
cout << endl;
odd = !odd;
}
cout << Colors::GREEN() << "=== ========================= ";
for (const auto& model : models) {
cout << "============ ";
}
cout << endl;
cout << Colors::GREEN() << setw(30) << " Totals...................";
double max = 0.0;
for (const auto& total : totals) {
if (total.second > max) {
max = total.second;
}
}
for (const auto& model : models) {
string efectiveColor = Colors::GREEN();
if (totals[model] == max) {
efectiveColor = Colors::RED();
}
cout << efectiveColor << setw(12) << setprecision(9) << fixed << totals[model] << " ";
}
// Output the averaged ranks
cout << endl;
int min = 1;
for (const auto& rank : ranks) {
if (rank.second < min) {
min = rank.second;
}
}
cout << Colors::GREEN() << setw(30) << " Averaged ranks...........";
for (const auto& model : models) {
string efectiveColor = Colors::GREEN();
if (ranks[model] == min) {
efectiveColor = Colors::RED();
}
cout << efectiveColor << setw(12) << setprecision(10) << fixed << (double)ranks[model] / (double)origin.size() << " ";
}
cout << endl;
}
void BestResults::reportAll()
{
auto models = getModels();
// Build the table of results
json table = buildTableResults(models);
// Print the table of results
printTableResults(models, table);
}
}

28
src/Platform/BestResults.h Normal file
View File

@ -0,0 +1,28 @@
#ifndef BESTRESULTS_H
#define BESTRESULTS_H
#include <string>
#include <set>
#include <nlohmann/json.hpp>
using namespace std;
using json = nlohmann::json;
namespace platform {
class BestResults {
public:
explicit BestResults(const string& path, const string& score, const string& model) : path(path), score(score), model(model) {}
string build();
void reportSingle();
void reportAll();
void buildAll();
private:
set<string> getModels();
vector<string> loadResultFiles();
json buildTableResults(set<string> models);
void printTableResults(set<string> models, json table);
string bestResultFile();
json loadFile(const string& fileName);
string path;
string score;
string model;
};
}
#endif //BESTRESULTS_H

6
src/Platform/BestResult.h → src/Platform/BestScore.h
View File

@ -1,7 +1,7 @@
#ifndef BESTRESULT_H #ifndef BESTSCORE_H
#define BESTRESULT_H #define BESTSCORE_H
#include <string> #include <string>
class BestResult { class BestScore {
public: public:
static std::string title() { return "STree_default (linear-ovo)"; } static std::string title() { return "STree_default (linear-ovo)"; }
static double score() { return 22.109799; } static double score() { return 22.109799; }

4
src/Platform/CMakeLists.txt
View File

@ -6,11 +6,13 @@ include_directories(${BayesNet_SOURCE_DIR}/lib/argparse/include)
include_directories(${BayesNet_SOURCE_DIR}/lib/json/include) include_directories(${BayesNet_SOURCE_DIR}/lib/json/include)
include_directories(${BayesNet_SOURCE_DIR}/lib/libxlsxwriter/include) include_directories(${BayesNet_SOURCE_DIR}/lib/libxlsxwriter/include)
add_executable(main main.cc Folding.cc platformUtils.cc Experiment.cc Datasets.cc Models.cc ReportConsole.cc ReportBase.cc) add_executable(main main.cc Folding.cc platformUtils.cc Experiment.cc Datasets.cc Models.cc ReportConsole.cc ReportBase.cc)
add_executable(manage manage.cc Results.cc ReportConsole.cc ReportExcel.cc ReportBase.cc Datasets.cc platformUtils.cc) add_executable(manage manage.cc Results.cc Result.cc ReportConsole.cc ReportExcel.cc ReportBase.cc Datasets.cc platformUtils.cc)
add_executable(list list.cc platformUtils Datasets.cc) add_executable(list list.cc platformUtils Datasets.cc)
add_executable(best best.cc BestResults.cc Result.cc)
target_link_libraries(main BayesNet ArffFiles mdlp "${TORCH_LIBRARIES}") target_link_libraries(main BayesNet ArffFiles mdlp "${TORCH_LIBRARIES}")
if (${CMAKE_HOST_SYSTEM_NAME} MATCHES "Linux") if (${CMAKE_HOST_SYSTEM_NAME} MATCHES "Linux")
target_link_libraries(manage "${TORCH_LIBRARIES}" libxlsxwriter.so ArffFiles mdlp stdc++fs) target_link_libraries(manage "${TORCH_LIBRARIES}" libxlsxwriter.so ArffFiles mdlp stdc++fs)
target_link_libraries(best stdc++fs)
else() else()
target_link_libraries(manage "${TORCH_LIBRARIES}" "${XLSXWRITER_LIB}" ArffFiles mdlp) target_link_libraries(manage "${TORCH_LIBRARIES}" "${XLSXWRITER_LIB}" ArffFiles mdlp)
endif() endif()

2
src/Platform/ReportBase.cc
View File

@ -2,7 +2,7 @@
#include <locale> #include <locale>
#include "Datasets.h" #include "Datasets.h"
#include "ReportBase.h" #include "ReportBase.h"
#include "BestResult.h" #include "BestScore.h"
namespace platform { namespace platform {

6
src/Platform/ReportConsole.cc
View File

@ -1,7 +1,7 @@
#include <sstream> #include <sstream>
#include <locale> #include <locale>
#include "ReportConsole.h" #include "ReportConsole.h"
#include "BestResult.h" #include "BestScore.h"
namespace platform { namespace platform {
@ -99,9 +99,9 @@ namespace platform {
cout << Colors::MAGENTA() << string(MAXL, '*') << endl; cout << Colors::MAGENTA() << string(MAXL, '*') << endl;
showSummary(); showSummary();
auto score = data["score_name"].get<string>(); auto score = data["score_name"].get<string>();
if (score == BestResult::scoreName()) { if (score == BestScore::scoreName()) {
stringstream oss; stringstream oss;
oss << score << " compared to " << BestResult::title() << " .: " << totalScore / BestResult::score(); oss << score << " compared to " << BestScore::title() << " .: " << totalScore / BestScore::score();
cout << headerLine(oss.str()); cout << headerLine(oss.str());
} }
if (!getExistBestFile() && compare) { if (!getExistBestFile() && compare) {

8
src/Platform/ReportExcel.cc
View File

@ -1,7 +1,7 @@
#include <sstream> #include <sstream>
#include <locale> #include <locale>
#include "ReportExcel.h" #include "ReportExcel.h"
#include "BestResult.h" #include "BestScore.h"
namespace platform { namespace platform {
@ -322,9 +322,9 @@ namespace platform {
showSummary(); showSummary();
row += 4 + summary.size(); row += 4 + summary.size();
auto score = data["score_name"].get<string>(); auto score = data["score_name"].get<string>();
if (score == BestResult::scoreName()) { if (score == BestScore::scoreName()) {
worksheet_merge_range(worksheet, row, 1, row, 5, (score + " compared to " + BestResult::title() + " .:").c_str(), efectiveStyle("text")); worksheet_merge_range(worksheet, row, 1, row, 5, (score + " compared to " + BestScore::title() + " .:").c_str(), efectiveStyle("text"));
writeDouble(row, 6, totalScore / BestResult::score(), "result"); writeDouble(row, 6, totalScore / BestScore::score(), "result");
} }
if (!getExistBestFile() && compare) { if (!getExistBestFile() && compare) {
worksheet_write_string(worksheet, row + 1, 0, "*** Best Results File not found. Couldn't compare any result!", styles["summaryStyle"]); worksheet_write_string(worksheet, row + 1, 0, "*** Best Results File not found. Couldn't compare any result!", styles["summaryStyle"]);

51
src/Platform/Result.cc Normal file
View File

@ -0,0 +1,51 @@
#include <filesystem>
#include <fstream>
#include <sstream>
#include "Result.h"
#include "Colors.h"
#include "BestScore.h"
namespace platform {
Result::Result(const string& path, const string& filename)
: path(path)
, filename(filename)
{
auto data = load();
date = data["date"];
score = 0;
for (const auto& result : data["results"]) {
score += result["score"].get<double>();
}
scoreName = data["score_name"];
if (scoreName == BestScore::scoreName()) {
score /= BestScore::score();
}
title = data["title"];
duration = data["duration"];
model = data["model"];
complete = data["results"].size() > 1;
}
json Result::load() const
{
ifstream resultData(path + "/" + filename);
if (resultData.is_open()) {
json data = json::parse(resultData);
return data;
}
throw invalid_argument("Unable to open result file. [" + path + "/" + filename + "]");
}
string Result::to_string() const
{
stringstream oss;
oss << date << " ";
oss << setw(12) << left << model << " ";
oss << setw(11) << left << scoreName << " ";
oss << right << setw(11) << setprecision(7) << fixed << score << " ";
auto completeString = isComplete() ? "C" : "P";
oss << setw(1) << " " << completeString << " ";
oss << setw(9) << setprecision(3) << fixed << duration << " ";
oss << setw(50) << left << title << " ";
return oss.str();
}
}

37
src/Platform/Result.h Normal file
View File

@ -0,0 +1,37 @@
#ifndef RESULT_H
#define RESULT_H
#include <map>
#include <vector>
#include <string>
#include <nlohmann/json.hpp>
namespace platform {
using namespace std;
using json = nlohmann::json;
class Result {
public:
Result(const string& path, const string& filename);
json load() const;
string to_string() const;
string getFilename() const { return filename; };
string getDate() const { return date; };
double getScore() const { return score; };
string getTitle() const { return title; };
double getDuration() const { return duration; };
string getModel() const { return model; };
string getScoreName() const { return scoreName; };
bool isComplete() const { return complete; };
private:
string path;
string filename;
string date;
double score;
string title;
double duration;
string model;
string scoreName;
bool complete;
};
};
#endif

43
src/Platform/Results.cc
View File

@ -3,37 +3,9 @@
#include "Results.h" #include "Results.h"
#include "ReportConsole.h" #include "ReportConsole.h"
#include "ReportExcel.h" #include "ReportExcel.h"
#include "BestResult.h" #include "BestScore.h"
#include "Colors.h" #include "Colors.h"
namespace platform { namespace platform {
Result::Result(const string& path, const string& filename)
: path(path)
, filename(filename)
{
auto data = load();
date = data["date"];
score = 0;
for (const auto& result : data["results"]) {
score += result["score"].get<double>();
}
scoreName = data["score_name"];
if (scoreName == BestResult::scoreName()) {
score /= BestResult::score();
}
title = data["title"];
duration = data["duration"];
model = data["model"];
complete = data["results"].size() > 1;
}
json Result::load() const
{
ifstream resultData(path + "/" + filename);
if (resultData.is_open()) {
json data = json::parse(resultData);
return data;
}
throw invalid_argument("Unable to open result file. [" + path + "/" + filename + "]");
}
void Results::load() void Results::load()
{ {
using std::filesystem::directory_iterator; using std::filesystem::directory_iterator;
@ -52,19 +24,6 @@ namespace platform {
max = files.size(); max = files.size();
} }
} }
string Result::to_string() const
{
stringstream oss;
oss << date << " ";
oss << setw(12) << left << model << " ";
oss << setw(11) << left << scoreName << " ";
oss << right << setw(11) << setprecision(7) << fixed << score << " ";
auto completeString = isComplete() ? "C" : "P";
oss << setw(1) << " " << completeString << " ";
oss << setw(9) << setprecision(3) << fixed << duration << " ";
oss << setw(50) << left << title << " ";
return oss.str();
}
void Results::show() const void Results::show() const
{ {
cout << Colors::GREEN() << "Results found: " << files.size() << endl; cout << Colors::GREEN() << "Results found: " << files.size() << endl;

25
src/Platform/Results.h
View File

@ -5,34 +5,11 @@
#include <vector> #include <vector>
#include <string> #include <string>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include "Result.h"
namespace platform { namespace platform {
using namespace std; using namespace std;
using json = nlohmann::json; using json = nlohmann::json;
class Result {
public:
Result(const string& path, const string& filename);
json load() const;
string to_string() const;
string getFilename() const { return filename; };
string getDate() const { return date; };
double getScore() const { return score; };
string getTitle() const { return title; };
double getDuration() const { return duration; };
string getModel() const { return model; };
string getScoreName() const { return scoreName; };
bool isComplete() const { return complete; };
private:
string path;
string filename;
string date;
double score;
string title;
double duration;
string model;
string scoreName;
bool complete;
};
class Results { class Results {
public: public:
Results(const string& path, const int max, const string& model, const string& score, bool complete, bool partial, bool compare) : Results(const string& path, const int max, const string& model, const string& score, bool complete, bool partial, bool compare) :

63
src/Platform/best.cc Normal file
View File

@ -0,0 +1,63 @@
#include <iostream>
#include <argparse/argparse.hpp>
#include "Paths.h"
#include "BestResults.h"
#include "Colors.h"
using namespace std;
argparse::ArgumentParser manageArguments(int argc, char** argv)
{
argparse::ArgumentParser program("best");
program.add_argument("-m", "--model").default_value("").help("Filter results of the selected model) (any for all models)");
program.add_argument("-s", "--score").default_value("").help("Filter results of the score name supplied");
program.add_argument("--build").help("build best score results file").default_value(false).implicit_value(true);
program.add_argument("--report").help("report of best score results file").default_value(false).implicit_value(true);
try {
program.parse_args(argc, argv);
auto model = program.get<string>("model");
auto score = program.get<string>("score");
auto build = program.get<bool>("build");
auto report = program.get<bool>("report");
if (model == "" || score == "") {
throw runtime_error("Model and score name must be supplied");
}
}
catch (const exception& err) {
cerr << err.what() << endl;
cerr << program;
exit(1);
}
return program;
}
int main(int argc, char** argv)
{
auto program = manageArguments(argc, argv);
auto model = program.get<string>("model");
auto score = program.get<string>("score");
auto build = program.get<bool>("build");
auto report = program.get<bool>("report");
if (!report && !build) {
cerr << "Either build, report or both, have to be selected to do anything!" << endl;
cerr << program;
exit(1);
}
auto results = platform::BestResults(platform::Paths::results(), score, model);
if (build) {
if (model == "any") {
results.buildAll();
} else {
string fileName = results.build();
cout << Colors::GREEN() << fileName << " created!" << Colors::RESET() << endl;
}
}
if (report) {
if (model == "any") {
results.reportAll();
} else {
results.reportSingle();
}
}
return 0;
}

1
src/Platform/platformUtils.h
View File

@ -8,7 +8,6 @@
#include "ArffFiles.h" #include "ArffFiles.h"
#include "CPPFImdlp.h" #include "CPPFImdlp.h"
using namespace std; using namespace std;
const string PATH = "../../data/";
bool file_exists(const std::string& name); bool file_exists(const std::string& name);
vector<string> split(const string& text, char delimiter); vector<string> split(const string& text, char delimiter);