Complete Tex output with Holm test

This commit is contained in:
2024-09-03 12:43:50 +02:00
parent 4dbd76df55
commit 6ae2b2182a
8 changed files with 115 additions and 67 deletions

View File

@@ -220,9 +220,9 @@ namespace platform {
std::cout << oss.str();
std::cout << std::string(oss.str().size() - 8, '-') << std::endl;
std::cout << Colors::GREEN() << " # " << std::setw(maxDatasetName + 1) << std::left << std::string("Dataset");
auto bestResultsTex = BestResultsTex(models, table.at("dateTable").get<std::string>());
auto bestResultsTex = BestResultsTex();
if (tex) {
bestResultsTex.results_header();
bestResultsTex.results_header(models, table.at("dateTable").get<std::string>());
}
for (const auto& model : models) {
std::cout << std::setw(maxModelName) << std::left << model << " ";
@@ -333,11 +333,14 @@ namespace platform {
if (friedman) {
Statistics stats(models, datasets, table, significance);
auto result = stats.friedmanTest();
stats.postHocHolmTest(result);
stats.postHocHolmTest(result, tex);
ranksModels = stats.getRanks();
}
if (tex) {
messageOutputFile("TeX", Paths::tex() + Paths::tex_output());
if (friedman) {
messageOutputFile("TeX", Paths::tex() + Paths::tex_post_hoc());
}
}
if (excel) {
BestResultsExcel excel(score, datasets);

View File

@@ -4,37 +4,39 @@
namespace platform {
using json = nlohmann::ordered_json;
BestResultsTex::BestResultsTex(const std::vector<std::string>& models, const std::string& date) : models(models), date(date)
void BestResultsTex::openTexFile(const std::string& name)
{
}
void BestResultsTex::results_header()
{
auto file_name = Paths::tex() + Paths::tex_output();
output_tex = fopen(file_name.c_str(), "w");
if (output_tex == NULL) {
std::cerr << "Error opening file " << file_name << std::endl;
handler.open(name);
if (!handler.is_open()) {
std::cerr << "Error opening file " << name << std::endl;
exit(1);
}
fprintf(output_tex, "%% This file has been generated by the platform program\n");
fprintf(output_tex, "%% Date: %s\n", date.c_str());
fprintf(output_tex, "%%\n");
fprintf(output_tex, "%% Table of results\n");
fprintf(output_tex, "%%\n");
fprintf(output_tex, "\\begin{table}[htbp] \n");
fprintf(output_tex, "\\centering \n");
fprintf(output_tex, "\\tiny \n");
fprintf(output_tex, "\\renewcommand{\\arraystretch }{1.2} \n");
fprintf(output_tex, "\\renewcommand{\\tabcolsep }{0.07cm} \n");
fprintf(output_tex, "\\caption{Accuracy results(mean ± std) for all the algorithms and datasets} \n");
fprintf(output_tex, "\\label{tab:results_accuracy}\n");
fprintf(output_tex, "\\begin{tabular} {{r%s}}\n", std::string(models.size(), 'c').c_str());
fprintf(output_tex, "\\hline \n");
fprintf(output_tex, "Id");
}
void BestResultsTex::results_header(const std::vector<std::string>& models, const std::string& date)
{
this->models = models;
auto file_name = Paths::tex() + Paths::tex_output();
openTexFile(file_name);
handler << "%% This file has been generated by the platform program" << std::endl;
handler << "%% Date: " << date.c_str() << std::endl;
handler << "%%" << std::endl;
handler << "%% Table of results" << std::endl;
handler << "%%" << std::endl;
handler << "\\begin{table}[htbp] " << std::endl;
handler << "\\centering " << std::endl;
handler << "\\tiny " << std::endl;
handler << "\\renewcommand{\\arraystretch }{1.2} " << std::endl;
handler << "\\renewcommand{\\tabcolsep }{0.07cm} " << std::endl;
handler << "\\caption{Accuracy results(mean ± std) for all the algorithms and datasets} " << std::endl;
handler << "\\label{tab:results_accuracy}" << std::endl;
handler << "\\begin{tabular} {{r" << std::string(models.size(), 'c').c_str() << "}}" << std::endl;
handler << "\\hline " << std::endl;
handler << "" << std::endl;
for (const auto& model : models) {
fprintf(output_tex, "& %s ", model.c_str());
handler << "& " << model.c_str();
}
fprintf(output_tex, "\\\\ \n");
fprintf(output_tex, "\\hline \n");
handler << "\\\\" << std::endl;
handler << "\\hline" << std::endl;
}
void BestResultsTex::results_body(const std::vector<std::string>& datasets, json& table)
{
@@ -55,31 +57,61 @@ namespace platform {
max_value = value;
}
}
fprintf(output_tex, "%d ", i);
handler << ++i << " ";
for (const auto& model : models) {
double value = table[model].at(dataset).at(0).get<double>();
double std_value = table[model].at(dataset).at(3).get<double>();
const char* bold = value == max_value ? "\\bfseries" : "";
fprintf(output_tex, "& %s %0.4f±%0.3f", bold, value, std_value);
handler << "& " << bold << std::setprecision(4) << std::fixed << value << "±" << std::setprecision(3) << std_value;
}
fprintf(output_tex, "\\\\\n");
handler << "\\\\" << std::endl;
}
}
void BestResultsTex::results_footer(const std::map<std::string, std::vector<double>>& totals, const std::string& best_model)
{
fprintf(output_tex, "\\hline \n");
fprintf(output_tex, "Average ");
handler << "\\hline" << std::endl;
handler << "Average ";
int nDatasets = totals.begin()->second.size();
for (const auto& model : models) {
double value = std::reduce(totals.at(model).begin(), totals.at(model).end()) / nDatasets;
double std_value = compute_std(totals.at(model), value);
const char* bold = model == best_model ? "\\bfseries" : "";
fprintf(output_tex, "& %s %0.4f±%0.3f", bold, value, std_value);
handler << "& " << bold << std::setprecision(4) << std::fixed << value << "±" << std::setprecision(3) << std::fixed << std_value;
}
fprintf(output_tex, "\\ \n");
fprintf(output_tex, "\\hline \n");
fprintf(output_tex, "\\end{tabular}\n");
fprintf(output_tex, "\\end{table}\n");
fclose(output_tex);
handler << "\\" << std::endl;
handler << "\\hline " << std::endl;
handler << "\\end{tabular}" << std::endl;
handler << "\\end{table}" << std::endl;
handler.close();
}
void BestResultsTex::holm_test(struct HolmResult& holmResult, const std::string& date)
{
auto file_name = Paths::tex() + Paths::tex_post_hoc();
openTexFile(file_name);
handler << "%% This file has been generated by the platform program" << std::endl;
handler << "%% Date: " << date.c_str() << std::endl;
handler << "%%" << std::endl;
handler << "%% Post-hoc handler test" << std::endl;
handler << "%%" << std::endl;
handler << "\\begin{table}[htbp]" << std::endl;
handler << "\\centering" << std::endl;
handler << "\\caption{Results of the post-hoc test for the mean accuracy of the algorithms.}\\label{tab:tests}" << std::endl;
handler << "\\begin{tabular}{lrrrrr}" << std::endl;
handler << "\\hline" << std::endl;
handler << "classifier & pvalue & rank & win & tie & loss\\\\" << std::endl;
handler << "\\hline" << std::endl;
for (auto const& line : holmResult.holmLines) {
auto textStatus = !line.reject ? "\\bf " : " ";
if (line.model == holmResult.model) {
handler << line.model << " & - & " << std::fixed << std::setprecision(2) << line.rank << " & - & - & - \\\\" << std::endl;
} else {
handler << line.model << " & " << std::scientific << std::setprecision(4) << line.pvalue << " & ";
handler << std::fixed << std::setprecision(2) << line.rank << " & " << line.wtl.win << " & " << line.wtl.tie << " & " << line.wtl.loss << "\\\\" << std::endl;
}
}
handler << "\\hline " << std::endl;
handler << "\\end{tabular}" << std::endl;
handler << "\\end{table}" << std::endl;
handler.close();
}
}

View File

@@ -4,19 +4,21 @@
#include <vector>
#include <nlohmann/json.hpp>
#include "common/Paths.h"
#include "Statistics.h"
namespace platform {
using json = nlohmann::ordered_json;
class BestResultsTex {
public:
BestResultsTex(const std::vector<std::string>& models, const std::string& date);
BestResultsTex() = default;
~BestResultsTex() = default;
void results_header();
void results_header(const std::vector<std::string>& models, const std::string& date);
void results_body(const std::vector<std::string>& datasets, json& table);
void results_footer(const std::map<std::string, std::vector<double>>& totals, const std::string& best_model);
void holm_test(struct HolmResult& holmResult, const std::string& date);
private:
std::FILE* output_tex;
void openTexFile(const std::string& name);
std::ofstream handler;
std::vector<std::string> models;
std::string date;
};
}
#endif

View File

@@ -4,6 +4,7 @@
#include "common/Colors.h"
#include "common/Symbols.h"
#include "common/CLocale.h"
#include "BestResultsTex.h"
#include "Statistics.h"
@@ -113,7 +114,7 @@ namespace platform {
}
}
void Statistics::postHocHolmTest(bool friedmanResult)
void Statistics::postHocHolmTest(bool friedmanResult, bool tex)
{
if (!fitted) {
fit();
@@ -195,6 +196,10 @@ namespace platform {
if (output) {
std::cout << oss.str();
}
if (tex) {
BestResultsTex bestResultsTex;
bestResultsTex.holm_test(holmResult, get_date() + " " + get_time());
}
}
bool Statistics::friedmanTest()
{

View File

@@ -34,7 +34,7 @@ namespace platform {
public:
Statistics(const std::vector<std::string>& models, const std::vector<std::string>& datasets, const json& data, double significance = 0.05, bool output = true);
bool friedmanTest();
void postHocHolmTest(bool friedmanResult);
void postHocHolmTest(bool friedmanResult, bool tex=false);
FriedmanResult& getFriedmanResult();
HolmResult& getHolmResult();
std::map<std::string, std::map<std::string, float>>& getRanks();

View File

@@ -61,6 +61,10 @@ namespace platform {
{
return "results.tex";
}
static std::string tex_post_hoc()
{
return "post_hoc.tex";
}
};
}
#endif

View File

@@ -36,7 +36,7 @@ namespace platform {
}
return result;
}
double compute_std(std::vector<double> values, double mean)
inline double compute_std(std::vector<double> values, double mean)
{
// Compute standard devation of the values
double sum = 0.0;
@@ -46,5 +46,25 @@ namespace platform {
double variance = sum / values.size();
return std::sqrt(variance);
}
inline std::string get_date()
{
time_t rawtime;
tm* timeinfo;
time(&rawtime);
timeinfo = std::localtime(&rawtime);
std::ostringstream oss;
oss << std::put_time(timeinfo, "%Y-%m-%d");
return oss.str();
}
inline std::string get_time()
{
time_t rawtime;
tm* timeinfo;
time(&rawtime);
timeinfo = std::localtime(&rawtime);
std::ostringstream oss;
oss << std::put_time(timeinfo, "%H:%M:%S");
return oss.str();
}
}
#endif

View File

@@ -5,29 +5,11 @@
#include "main/Models.h"
#include "common/Paths.h"
#include "common/Colors.h"
#include "common/Utils.h"
#include "GridSearch.h"
namespace platform {
std::string get_date()
{
time_t rawtime;
tm* timeinfo;
time(&rawtime);
timeinfo = std::localtime(&rawtime);
std::ostringstream oss;
oss << std::put_time(timeinfo, "%Y-%m-%d");
return oss.str();
}
std::string get_time()
{
time_t rawtime;
tm* timeinfo;
time(&rawtime);
timeinfo = std::localtime(&rawtime);
std::ostringstream oss;
oss << std::put_time(timeinfo, "%H:%M:%S");
return oss.str();
}
std::string get_color_rank(int rank)
{
auto colors = { Colors::WHITE(), Colors::RED(), Colors::GREEN(), Colors::BLUE(), Colors::MAGENTA(), Colors::CYAN() };