diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index bcece79..b928c5d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -49,5 +49,8 @@ target_link_libraries(b_main PyClassifiers BayesNet ArffFiles mdlp) # b_manage set(manage_sources b_manage.cc ManageResults.cc CommandParser.cc Results.cc) list(TRANSFORM manage_sources PREPEND manage/) -add_executable(b_manage ${manage_sources} main/Result.cc reports/ReportConsole.cc reports/ReportExcel.cc reports/ReportBase.cc reports/ExcelFile.cc common/Datasets.cc common/Dataset.cc) +add_executable(b_manage ${manage_sources} main/Result.cc + reports/ReportConsole.cc reports/ReportExcel.cc reports/ReportExcelCompared.cc reports/ReportBase.cc reports/ExcelFile.cc + common/Datasets.cc common/Dataset.cc +) target_link_libraries(b_manage "${TORCH_LIBRARIES}" "${XLSXWRITER_LIB}" ArffFiles mdlp) diff --git a/src/manage/ManageResults.cc b/src/manage/ManageResults.cc index 078773f..ba4d73a 100644 --- a/src/manage/ManageResults.cc +++ b/src/manage/ManageResults.cc @@ -7,6 +7,7 @@ #include "Paths.h" #include "ReportConsole.h" #include "ReportExcel.h" +#include "ReportExcelCompared.h" namespace platform { @@ -84,6 +85,14 @@ namespace platform { std::cout << "Not done!" << std::endl; return false; } + void ManageResults::report_compared(const int index_A, const int index_B) + { + std::cout << "Comparing " << results.at(index_A).getFilename() << " with " << results.at(index_B).getFilename() << std::endl; + auto data_A = results.at(index_A).getJson(); + auto data_B = results.at(index_B).getJson(); + ReportExcelCompared reporter(data_A, data_B); + reporter.report(); + } void ManageResults::report(const int index, const bool excelReport) { std::cout << Colors::YELLOW() << "Reporting " << results.at(index).getFilename() << std::endl; @@ -140,7 +149,7 @@ namespace platform { void ManageResults::menu() { char option; - int index, subIndex; + int index, subIndex, index_A = -1, index_B = -1; bool finished = false; std::string filename; // tuple @@ -152,7 +161,10 @@ namespace platform { {"sort", 's', false}, {"report", 'r', true}, {"excel", 'e', true}, - {"title", 't', true} + {"title", 't', true}, + {"set A", 'a', true}, + {"set B", 'b', true}, + {"compare A~B", 'c', false} }; // tuple std::vector> listOptions = { @@ -172,9 +184,39 @@ namespace platform { case 'q': finished = true; break; + case 'a': + if (index == index_B) { + std::cout << Colors::RED() << "A and B cannot be the same!" << Colors::RESET() << std::endl; + break; + } + if (!results.at(index).isComplete()) { + std::cout << Colors::RED() << "A must be a complete result!" << Colors::RESET() << std::endl; + break; + } + index_A = index; + break; case 'b': - // back to show the report - report(index, false); + if (indexList) { + if (index == index_A) { + std::cout << Colors::RED() << "A and B cannot be the same!" << Colors::RESET() << std::endl; + break; + } + if (!results.at(index).isComplete()) { + std::cout << Colors::RED() << "B must be a complete result!" << Colors::RESET() << std::endl; + break; + } + index_B = index; + } else { + // back to show the report + report(index, false); + } + break; + case 'c': + if (index_A == -1 || index_B == -1) { + std::cout << Colors::RED() << "Need to set A and B first!" << Colors::RESET() << std::endl; + break; + } + report_compared(index_A, index_B); break; case 'l': list(); diff --git a/src/manage/ManageResults.h b/src/manage/ManageResults.h index 6bd3704..ba14f27 100644 --- a/src/manage/ManageResults.h +++ b/src/manage/ManageResults.h @@ -13,6 +13,7 @@ namespace platform { void list(); bool confirmAction(const std::string& intent, const std::string& fileName) const; void report(const int index, const bool excelReport); + void report_compared(const int index_A, const int index_B); void showIndex(const int index, const int idx); void sortList(); void menu(); @@ -25,7 +26,6 @@ namespace platform { Results results; lxw_workbook* workbook; }; - } #endif /* MANAGE_RESULTS_H */ \ No newline at end of file diff --git a/src/reports/ExcelFile.cc b/src/reports/ExcelFile.cc index e9407b6..611db74 100644 --- a/src/reports/ExcelFile.cc +++ b/src/reports/ExcelFile.cc @@ -121,12 +121,17 @@ namespace platform { format_set_align(style, LXW_ALIGN_VERTICAL_CENTER); format_set_font_size(style, normalSize); format_set_num_format(style, "#,##0.00"); + } else if (name == "percentage") { + format_set_border(style, LXW_BORDER_THIN); + format_set_align(style, LXW_ALIGN_VERTICAL_CENTER); + format_set_font_size(style, normalSize); + format_set_num_format(style, "0.00%"); } } void ExcelFile::createFormats() { - auto styleNames = { "text", "textCentered", "bodyHeader", "result", "time", "ints", "floats" }; + auto styleNames = { "text", "textCentered", "bodyHeader", "result", "time", "ints", "floats", "percentage" }; lxw_format* style; for (std::string name : styleNames) { lxw_format* style = workbook_add_format(workbook); diff --git a/src/reports/ReportExcel.h b/src/reports/ReportExcel.h index 9e7b052..5052378 100644 --- a/src/reports/ReportExcel.h +++ b/src/reports/ReportExcel.h @@ -9,11 +9,11 @@ namespace platform { class ReportExcel : public ReportBase, public ExcelFile { public: explicit ReportExcel(json data_, bool compare, lxw_workbook* workbook, lxw_worksheet* worksheet = NULL); + void closeFile(); private: void formatColumns(); void createFile(); void createWorksheet(); - void closeFile(); void header() override; void body() override; void showSummary() override; diff --git a/src/reports/ReportExcelCompared.cc b/src/reports/ReportExcelCompared.cc new file mode 100644 index 0000000..5d3cbc1 --- /dev/null +++ b/src/reports/ReportExcelCompared.cc @@ -0,0 +1,134 @@ +#include "ReportExcelCompared.h" + +namespace platform { + + ReportExcelCompared::ReportExcelCompared(json& data_A, json& data_B) : data_A(data_A), data_B(data_B), ExcelFile(NULL, NULL) + { + } + ReportExcelCompared::~ReportExcelCompared() + { + if (workbook) + workbook_close(workbook); + } + void ReportExcelCompared::report() + { + // Create a new workbook and add the two worksheets A & B + workbook = workbook_new((Paths::excel() + Paths::excelResults()).c_str()); + worksheet = workbook_add_worksheet(workbook, "A"); + createFormats(); + ReportExcel report(data_A, false, workbook, worksheet); + workbook = report.getWorkbook(); + report.show(); + worksheet = workbook_add_worksheet(workbook, "B"); + report = ReportExcel(data_B, false, workbook, worksheet); + report.show(); + // Add the comparison worksheet + worksheet = workbook_add_worksheet(workbook, "Δ"); + header(); + body(); + } + void ReportExcelCompared::header() + { + worksheet_merge_range(worksheet, 0, 0, 0, 20, "Compare Results A vs B", styles["headerFirst"]); + worksheet_merge_range(worksheet, 1, 0, 1, 20, "Δ = (A - B) / B", styles["headerRest"]); + worksheet_freeze_panes(worksheet, 5, 1); + } + double diff(double a, double b) + { + return (a - b) / b; + } + void ReportExcelCompared::body() + { + // Body Header + auto sizes = std::vector({ 22, 10, 9, 7, 12, 12, 9, 12, 12, 9, 12, 12, 9, 12, 12, 9, 15, 15, 9, 15, 15 }); + auto head_a = std::vector({ "Dataset", "Samples", "Features", "Classes" }); + auto head_b = std::vector({ "Nodes", "Edges", "States", "Score", "Time" }); + int headerRow = 3; + int col = 0; + for (const auto& item : head_a) { + worksheet_merge_range(worksheet, headerRow, col, headerRow + 1, col, item.c_str(), styles["bodyHeader_even"]); + col++; + } + for (const auto& item : head_b) { + worksheet_merge_range(worksheet, headerRow, col, headerRow, col + 2, item.c_str(), styles["bodyHeader_even"]); + writeString(headerRow + 1, col++, "A", "bodyHeader"); + writeString(headerRow + 1, col++, "B", "bodyHeader"); + writeString(headerRow + 1, col++, "Δ", "bodyHeader"); + } + worksheet_merge_range(worksheet, headerRow, col, headerRow, col + 1, "Hyperparameters", styles["bodyHeader_even"]); + int hypCol = col; + writeString(headerRow + 1, hypCol, "A", "bodyHeader"); + writeString(headerRow + 1, hypCol + 1, "B", "bodyHeader"); + col = 0; + for (const auto size : sizes) { + worksheet_set_column(worksheet, col, col, size, NULL); + col++; + } + // Body Data + row = headerRow + 2; + int hypSize_A = 15; + int hypSize_B = 15; + auto compared = std::vector({ "nodes", "leaves", "depth", "score", "time" }); + auto compared_data = std::vector(compared.size(), 0.0); + auto totals_A = std::vector(compared.size(), 0.0); + auto totals_B = std::vector(compared.size(), 0.0); + std::string hyperparameters; + for (int i = 0; i < data_A["results"].size(); i++) { + col = 0; + const auto& r_A = data_A["results"][i]; + const auto& r_B = data_B["results"][i]; + for (int j = 0; j < compared.size(); j++) { + auto key = compared[j]; + compared_data[j] = diff(r_A[key].get(), r_B[key].get()); + totals_A[j] += r_A[key].get(); + totals_B[j] += r_B[key].get(); + } + writeString(row, col++, r_A["dataset"].get(), "text"); + writeInt(row, col++, r_A["samples"].get(), "ints"); + writeInt(row, col++, r_A["features"].get(), "ints"); + writeInt(row, col++, r_A["classes"].get(), "ints"); + writeDouble(row, col++, r_A["nodes"].get(), "floats"); + writeDouble(row, col++, r_B["nodes"].get(), "floats"); + writeDouble(row, col++, compared_data[0], "percentage"); + writeDouble(row, col++, r_A["leaves"].get(), "floats"); + writeDouble(row, col++, r_B["leaves"].get(), "floats"); + writeDouble(row, col++, compared_data[1], "percentage"); + writeDouble(row, col++, r_A["depth"].get(), "floats"); + writeDouble(row, col++, r_B["depth"].get(), "floats"); + writeDouble(row, col++, compared_data[2], "percentage"); + writeDouble(row, col++, r_A["score"].get(), "result"); + writeDouble(row, col++, r_B["score"].get(), "result"); + writeDouble(row, col++, compared_data[3], "percentage"); + writeDouble(row, col++, r_A["time"].get(), "time"); + writeDouble(row, col++, r_B["time"].get(), "time"); + writeDouble(row, col++, compared_data[4], "percentage"); + hyperparameters = r_A["hyperparameters"].dump(); + if (hyperparameters.size() > hypSize_A) { + hypSize_A = hyperparameters.size(); + } + writeString(row, hypCol, hyperparameters, "text"); + hyperparameters = r_B["hyperparameters"].dump(); + if (hyperparameters.size() > hypSize_B) { + hypSize_B = hyperparameters.size(); + } + writeString(row, hypCol + 1, hyperparameters, "text"); + row++; + } + // Set the right column width of hyperparameters with the maximum length + worksheet_set_column(worksheet, hypCol, hypCol, hypSize_A + 5, NULL); + worksheet_set_column(worksheet, hypCol + 1, hypCol + 1, hypSize_B + 5, NULL); + // Show totals if only one dataset is present in the result + footer(totals_A, totals_B, row); + } + void ReportExcelCompared::footer(std::vector& totals_A, std::vector& totals_B, int row) + { + worksheet_merge_range(worksheet, row, 0, row, 3, "Total", styles["bodyHeader_even"]); + auto formats = std::vector({ "floats", "floats", "floats", "result", "result" }); + int col = 4; + for (int i = 0; i < totals_A.size(); i++) { + writeDouble(row, col++, totals_A[i], formats[i]); + writeDouble(row, col++, totals_B[i], formats[i]); + writeDouble(row, col++, diff(totals_A[i], totals_B[i]), "percentage"); + } + } +} \ No newline at end of file diff --git a/src/reports/ReportExcelCompared.h b/src/reports/ReportExcelCompared.h new file mode 100644 index 0000000..fea47cb --- /dev/null +++ b/src/reports/ReportExcelCompared.h @@ -0,0 +1,16 @@ +#pragma once +#include "ReportExcel.h" +namespace platform { + class ReportExcelCompared : public ExcelFile { + public: + explicit ReportExcelCompared(json& data_A, json& data_B); + ~ReportExcelCompared(); + void report(); + private: + void header(); + void body(); + void footer(std::vector& totals_A, std::vector& totals_B, int row); + json& data_A; + json& data_B; + }; +}; \ No newline at end of file