rmontanana/Platform
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Begin adding TeX output to b_best -m any command

Browse Source
This commit is contained in:
Ricardo Montañana Gómez
2024-09-02 18:14:53 +02:00
parent 8372987dae
commit 4545f76667
5 changed files with 106 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
src/best/BestResults.cpp
View File

@@ -52,7 +52,7 @@ namespace platform {
} }
} }
if (update) { if (update) {
bests[datasetName] = { item.at("score").get<double>(), item.at("hyperparameters"), file }; bests[datasetName] = { item.at("score").get<double>(), item.at("hyperparameters"), file, item.at("score_std").get<double>() };
} }
} }
} }
@@ -210,15 +210,56 @@ namespace platform {
table["dateTable"] = ftime_to_string(maxDate); table["dateTable"] = ftime_to_string(maxDate);
return table; return table;
} }
void BestResults::printTableResults(std::vector<std::string> models, json table) double compute_std(std::vector<double> values, double mean)
{
// Compute standard devation of the values
double sum = 0.0;
for (const auto& value : values) {
sum += std::pow(value - mean, 2);
}
double variance = sum / values.size();
return std::sqrt(variance);
}
void BestResults::printTableResults(std::vector<std::string> models, json table, bool tex)
{ {
std::stringstream oss; std::stringstream oss;
oss << Colors::GREEN() << "Best results for " << score << " as of " << table.at("dateTable").get<std::string>() << std::endl; oss << Colors::GREEN() << "Best results for " << score << " as of " << table.at("dateTable").get<std::string>() << std::endl;
std::FILE* output_tex;
std::cout << oss.str(); std::cout << oss.str();
std::cout << std::string(oss.str().size() - 8, '-') << std::endl; std::cout << std::string(oss.str().size() - 8, '-') << std::endl;
std::cout << Colors::GREEN() << " # " << std::setw(maxDatasetName + 1) << std::left << std::string("Dataset"); std::cout << Colors::GREEN() << " # " << std::setw(maxDatasetName + 1) << std::left << std::string("Dataset");
if (tex) {
auto file_name = Paths::tex_output();
output_tex = fopen(file_name.c_str(), "w");
if (output_tex == NULL) {
std::cerr << "Error opening file "<< file_name << std::endl;
exit(1);
}
fprintf(output_tex, "%% This file has been generated by the platform program\n");
fprintf(output_tex, "%% Date: %s\n", table.at("dateTable").get<std::string>().c_str());
fprintf(output_tex, "%%\n");
fprintf(output_tex, "%% Table of results\n");
fprintf(output_tex, "%%\n");
fprintf(output_tex, "\\begin{table}[htbp] \n");
fprintf(output_tex, "\\centering \n");
fprintf(output_tex, "\\tiny \n");
fprintf(output_tex, "\\renewcommand{\\arraystretch }{1.2} \n");
fprintf(output_tex, "\\renewcommand{\\tabcolsep }{0.07cm} \n");
fprintf(output_tex, "\\caption{Accuracy results(mean ± std) for all the algorithms and datasets} \n");
fprintf(output_tex, "\\label{tab:results_accuracy}\n");
fprintf(output_tex, "\\begin{tabular} {{r%s}}\n", std::string(models.size(), 'c').c_str());
fprintf(output_tex, "\\hline \n");
fprintf(output_tex, "Id");
}
for (const auto& model : models) { for (const auto& model : models) {
std::cout << std::setw(maxModelName) << std::left << model << " "; std::cout << std::setw(maxModelName) << std::left << model << " ";
if (tex) {
fprintf(output_tex, "& %s ", model.c_str());
}
}
if (tex) {
fprintf(output_tex, "\\\\ \n");
fprintf(output_tex, "\\hline \n");
} }
std::cout << std::endl; std::cout << std::endl;
std::cout << "=== " << std::string(maxDatasetName, '=') << " "; std::cout << "=== " << std::string(maxDatasetName, '=') << " ";
@@ -227,12 +268,10 @@ namespace platform {
} }
std::cout << std::endl; std::cout << std::endl;
auto i = 0; auto i = 0;
std::map<std::string, double> totals; std::map<std::string, std::vector<double>> totals;
int nDatasets = table.begin().value().size(); int nDatasets = table.begin().value().size();
for (const auto& model : models) {
totals[model] = 0.0;
}
auto datasets = getDatasets(table.begin().value()); auto datasets = getDatasets(table.begin().value());
for (auto const& dataset_ : datasets) { for (auto const& dataset_ : datasets) {
auto color = (i % 2) ? Colors::BLUE() : Colors::CYAN(); auto color = (i % 2) ? Colors::BLUE() : Colors::CYAN();
std::cout << color << std::setw(3) << std::fixed << std::right << i++ << " "; std::cout << color << std::setw(3) << std::fixed << std::right << i++ << " ";
@@ -251,6 +290,9 @@ namespace platform {
maxValue = value; maxValue = value;
} }
} }
if (tex) {
fprintf(output_tex, "%d ", i);
}
// Print the row with red colors on max values // Print the row with red colors on max values
for (const auto& model : models) { for (const auto& model : models) {
std::string efectiveColor = color; std::string efectiveColor = color;
@@ -267,30 +309,53 @@ namespace platform {
if (value == -1) { if (value == -1) {
std::cout << Colors::YELLOW() << std::setw(maxModelName) << std::right << "N/A" << " "; std::cout << Colors::YELLOW() << std::setw(maxModelName) << std::right << "N/A" << " ";
} else { } else {
totals[model] += value; totals[model].push_back(value);
std::cout << efectiveColor << std::setw(maxModelName) << std::setprecision(maxModelName - 2) << std::fixed << value << " "; std::cout << efectiveColor << std::setw(maxModelName) << std::setprecision(maxModelName - 2) << std::fixed << value << " ";
} }
if (tex) {
auto std_value = table[model].at(dataset_).at(3).get<double>();
const char* bold = value == maxValue ? "\\bfseries" : "";
fprintf(output_tex, "& %s %0.4f±%0.3f", bold, value, std_value);
}
} }
std::cout << std::endl; std::cout << std::endl;
if (tex) {
fprintf(output_tex, "\\\\\n");
}
} }
std::cout << Colors::GREEN() << "=== " << std::string(maxDatasetName, '=') << " "; std::cout << Colors::GREEN() << "=== " << std::string(maxDatasetName, '=') << " ";
for (const auto& model : models) { for (const auto& model : models) {
std::cout << std::string(maxModelName, '=') << " "; std::cout << std::string(maxModelName, '=') << " ";
} }
std::cout << std::endl; std::cout << std::endl;
std::cout << Colors::GREEN() << " Totals" << std::string(maxDatasetName - 6, '.') << " "; std::cout << Colors::GREEN() << " Average" << std::string(maxDatasetName - 7, '.') << " ";
double max_value = 0.0; double max_value = 0.0;
std::string best_model = "";
for (const auto& total : totals) { for (const auto& total : totals) {
if (total.second > max_value) { auto actual = std::reduce(total.second.begin(), total.second.end());
max_value = total.second; if (actual > max_value) {
max_value = actual;
best_model = total.first;
} }
} }
if (tex) {
fprintf(output_tex, "\\hline \n");
fprintf(output_tex, "Average ");
}
for (const auto& model : models) { for (const auto& model : models) {
std::string efectiveColor = Colors::GREEN(); std::string efectiveColor = model == best_model ? Colors::RED() : Colors::GREEN();
if (totals[model] == max_value) { double value = std::reduce(totals[model].begin(), totals[model].end()) / nDatasets;
efectiveColor = Colors::RED(); double std_value = compute_std(totals[model], value);
std::cout << efectiveColor << std::right << std::setw(maxModelName) << std::setprecision(maxModelName - 4) << std::fixed << value << " ";
if (tex) {
const char* bold = model == best_model ? "\\bfseries" : "";
fprintf(output_tex, "& %s %0.4f±%0.3f", bold, value, std_value);
} }
std::cout << efectiveColor << std::right << std::setw(maxModelName) << std::setprecision(maxModelName - 4) << std::fixed << totals[model] << " "; }
if (tex) {
// Footer for TeX
fprintf(output_tex, "\\ \n\\hline \n\\end{tabular}\n\\end{table}\n");
fclose(output_tex);
} }
std::cout << std::endl; std::cout << std::endl;
} }
@@ -304,17 +369,17 @@ namespace platform {
std::vector<std::string> datasets = getDatasets(table.begin().value()); std::vector<std::string> datasets = getDatasets(table.begin().value());
BestResultsExcel excel_report(score, datasets); BestResultsExcel excel_report(score, datasets);
excel_report.reportSingle(model, path + Paths::bestResultsFile(score, model)); excel_report.reportSingle(model, path + Paths::bestResultsFile(score, model));
messageExcelFile(excel_report.getFileName()); messageOutputFile("Excel", excel_report.getFileName());
} }
} }
void BestResults::reportAll(bool excel) void BestResults::reportAll(bool excel, bool tex)
{ {
auto models = getModels(); auto models = getModels();
// Build the table of results // Build the table of results
json table = buildTableResults(models); json table = buildTableResults(models);
std::vector<std::string> datasets = getDatasets(table.begin().value()); std::vector<std::string> datasets = getDatasets(table.begin().value());
// Print the table of results // Print the table of results
printTableResults(models, table); printTableResults(models, table, tex);
// Compute the Friedman test // Compute the Friedman test
std::map<std::string, std::map<std::string, float>> ranksModels; std::map<std::string, std::map<std::string, float>> ranksModels;
if (friedman) { if (friedman) {
@@ -323,6 +388,9 @@ namespace platform {
stats.postHocHolmTest(result); stats.postHocHolmTest(result);
ranksModels = stats.getRanks(); ranksModels = stats.getRanks();
} }
if (tex) {
messageOutputFile("TeX", Paths::tex_output());
}
if (excel) { if (excel) {
BestResultsExcel excel(score, datasets); BestResultsExcel excel(score, datasets);
excel.reportAll(models, table, ranksModels, friedman, significance); excel.reportAll(models, table, ranksModels, friedman, significance);
@@ -345,11 +413,12 @@ namespace platform {
model = models.at(idx); model = models.at(idx);
excel.reportSingle(model, path + Paths::bestResultsFile(score, model)); excel.reportSingle(model, path + Paths::bestResultsFile(score, model));
} }
messageExcelFile(excel.getFileName()); messageOutputFile("Excel", excel.getFileName());
} }
} }
void BestResults::messageExcelFile(const std::string& fileName) void BestResults::messageOutputFile(const std::string& title, const std::string& fileName)
{ {
std::cout << Colors::YELLOW() << "** Excel file generated: " << fileName << Colors::RESET() << std::endl; std::cout << Colors::YELLOW() << "** " << std::setw(5) << std::left << title
<< " file generated: " << fileName << Colors::RESET() << std::endl;
} }
} }

6
src/best/BestResults.h
View File

@@ -13,15 +13,15 @@ namespace platform {
} }
std::string build(); std::string build();
void reportSingle(bool excel); void reportSingle(bool excel);
void reportAll(bool excel); void reportAll(bool excel, bool tex);
void buildAll(); void buildAll();
private: private:
std::vector<std::string> getModels(); std::vector<std::string> getModels();
std::vector<std::string> getDatasets(json table); std::vector<std::string> getDatasets(json table);
std::vector<std::string> loadResultFiles(); std::vector<std::string> loadResultFiles();
void messageExcelFile(const std::string& fileName); void messageOutputFile(const std::string& title, const std::string& fileName);
json buildTableResults(std::vector<std::string> models); json buildTableResults(std::vector<std::string> models);
void printTableResults(std::vector<std::string> models, json table); void printTableResults(std::vector<std::string> models, json table, bool tex);
json loadFile(const std::string& fileName); json loadFile(const std::string& fileName);
void listFile(); void listFile();
std::string path; std::string path;

2
src/best/BestResultsExcel.cpp
View File

@@ -32,7 +32,7 @@ namespace platform {
} }
BestResultsExcel::BestResultsExcel(const std::string& score, const std::vector<std::string>& datasets) : score(score), datasets(datasets) BestResultsExcel::BestResultsExcel(const std::string& score, const std::vector<std::string>& datasets) : score(score), datasets(datasets)
{ {
file_name = "BestResults.xlsx"; file_name = Paths::bestResultsExcel();
workbook = workbook_new(getFileName().c_str()); workbook = workbook_new(getFileName().c_str());
setProperties("Best Results"); setProperties("Best Results");
int maxDatasetName = (*max_element(datasets.begin(), datasets.end(), [](const std::string& a, const std::string& b) { return a.size() < b.size(); })).size(); int maxDatasetName = (*max_element(datasets.begin(), datasets.end(), [](const std::string& a, const std::string& b) { return a.size() < b.size(); })).size();

6
src/commands/b_best.cpp
View File

@@ -16,6 +16,7 @@ void manageArguments(argparse::ArgumentParser& program)
program.add_argument("-s", "--score").default_value("accuracy").help("Filter results of the score name supplied"); program.add_argument("-s", "--score").default_value("accuracy").help("Filter results of the score name supplied");
program.add_argument("--friedman").help("Friedman test").default_value(false).implicit_value(true); program.add_argument("--friedman").help("Friedman test").default_value(false).implicit_value(true);
program.add_argument("--excel").help("Output to excel").default_value(false).implicit_value(true); program.add_argument("--excel").help("Output to excel").default_value(false).implicit_value(true);
program.add_argument("--tex").help("Output result table to TeX file").default_value(false).implicit_value(true);
program.add_argument("--level").help("significance level").default_value(0.05).scan<'g', double>().action([](const std::string& value) { program.add_argument("--level").help("significance level").default_value(0.05).scan<'g', double>().action([](const std::string& value) {
try { try {
auto k = std::stod(value); auto k = std::stod(value);
@@ -37,7 +38,7 @@ int main(int argc, char** argv)
argparse::ArgumentParser program("b_best", { platform_project_version.begin(), platform_project_version.end() }); argparse::ArgumentParser program("b_best", { platform_project_version.begin(), platform_project_version.end() });
manageArguments(program); manageArguments(program);
std::string model, dataset, score; std::string model, dataset, score;
bool build, report, friedman, excel; bool build, report, friedman, excel, tex;
double level; double level;
try { try {
program.parse_args(argc, argv); program.parse_args(argc, argv);
@@ -46,6 +47,7 @@ int main(int argc, char** argv)
score = program.get<std::string>("score"); score = program.get<std::string>("score");
friedman = program.get<bool>("friedman"); friedman = program.get<bool>("friedman");
excel = program.get<bool>("excel"); excel = program.get<bool>("excel");
tex = program.get<bool>("tex");
level = program.get<double>("level"); level = program.get<double>("level");
if (model == "" || score == "") { if (model == "" || score == "") {
throw std::runtime_error("Model and score name must be supplied"); throw std::runtime_error("Model and score name must be supplied");
@@ -65,7 +67,7 @@ int main(int argc, char** argv)
auto results = platform::BestResults(platform::Paths::results(), score, model, dataset, friedman, level); auto results = platform::BestResults(platform::Paths::results(), score, model, dataset, friedman, level);
if (model == "any") { if (model == "any") {
results.buildAll(); results.buildAll();
results.reportAll(excel); results.reportAll(excel, tex);
} else { } else {
std::string fileName = results.build(); std::string fileName = results.build();
std::cout << Colors::GREEN() << fileName << " created!" << Colors::RESET() << std::endl; std::cout << Colors::GREEN() << fileName << " created!" << Colors::RESET() << std::endl;

9
src/common/Paths.h
View File

@@ -11,6 +11,7 @@ namespace platform {
static std::string excel() { return "excel/"; } static std::string excel() { return "excel/"; }
static std::string grid() { return "grid/"; } static std::string grid() { return "grid/"; }
static std::string graphs() { return "graphs/"; } static std::string graphs() { return "graphs/"; }
static std::string tex() { return "tex/"; }
static std::string datasets() static std::string datasets()
{ {
auto env = platform::DotEnv(); auto env = platform::DotEnv();
@@ -36,6 +37,10 @@ namespace platform {
{ {
return "best_results_" + score + "_" + model + ".json"; return "best_results_" + score + "_" + model + ".json";
} }
static std::string bestResultsExcel()
{
return "BestResults.excel";
}
static std::string excelResults() { return "some_results.xlsx"; } static std::string excelResults() { return "some_results.xlsx"; }
static std::string grid_input(const std::string& model) static std::string grid_input(const std::string& model)
{ {
@@ -45,6 +50,10 @@ namespace platform {
{ {
return grid() + "grid_" + model + "_output.json"; return grid() + "grid_" + model + "_output.json";
} }
static std::string tex_output()
{
return "results.tex";
}
}; };
} }
#endif #endif