src/CMakeLists.txt

@@ -49,5 +49,8 @@ target_link_libraries(b_main PyClassifiers BayesNet ArffFiles mdlp)
 # b_manage
 set(manage_sources b_manage.cc ManageResults.cc CommandParser.cc Results.cc)
 list(TRANSFORM manage_sources PREPEND manage/)
-add_executable(b_manage ${manage_sources} main/Result.cc reports/ReportConsole.cc reports/ReportExcel.cc reports/ReportBase.cc reports/ExcelFile.cc common/Datasets.cc common/Dataset.cc)
+add_executable(b_manage ${manage_sources} main/Result.cc 
+    reports/ReportConsole.cc reports/ReportExcel.cc reports/ReportExcelCompared.cc reports/ReportBase.cc reports/ExcelFile.cc 
+    common/Datasets.cc common/Dataset.cc
+)
 target_link_libraries(b_manage "${TORCH_LIBRARIES}" "${XLSXWRITER_LIB}" ArffFiles mdlp)

src/manage/ManageResults.cc

@@ -7,6 +7,7 @@
 #include "Paths.h"
 #include "ReportConsole.h"
 #include "ReportExcel.h"
+#include "ReportExcelCompared.h"
 
 namespace platform {
 
@@ -84,6 +85,14 @@ namespace platform {
         std::cout << "Not done!" << std::endl;
         return false;
     }
+    void ManageResults::report_compared(const int index_A, const int index_B)
+    {
+        std::cout << "Comparing " << results.at(index_A).getFilename() << " with " << results.at(index_B).getFilename() << std::endl;
+        auto data_A = results.at(index_A).getJson();
+        auto data_B = results.at(index_B).getJson();
+        ReportExcelCompared reporter(data_A, data_B);
+        reporter.report();
+    }
     void ManageResults::report(const int index, const bool excelReport)
     {
         std::cout << Colors::YELLOW() << "Reporting " << results.at(index).getFilename() << std::endl;
@@ -140,7 +149,7 @@ namespace platform {
     void ManageResults::menu()
     {
         char option;
-        int index, subIndex;
+        int index, subIndex, index_A = -1, index_B = -1;
         bool finished = false;
         std::string filename;
         // tuple<Option, digit, requires value>
@@ -152,7 +161,10 @@ namespace platform {
             {"sort", 's', false},
             {"report", 'r', true},
             {"excel", 'e', true},
-            {"title", 't', true}
+            {"title", 't', true},
+            {"set A", 'a', true},
+            {"set B", 'b', true},
+            {"compare A~B", 'c', false}
         };
         // tuple<Option, digit, requires value>
         std::vector<std::tuple<std::string, char, bool>> listOptions = {
@@ -172,9 +184,39 @@ namespace platform {
                 case 'q':
                     finished = true;
                     break;
+                case 'a':
+                    if (index == index_B) {
+                        std::cout << Colors::RED() << "A and B cannot be the same!" << Colors::RESET() << std::endl;
+                        break;
+                    }
+                    if (!results.at(index).isComplete()) {
+                        std::cout << Colors::RED() << "A must be a complete result!" << Colors::RESET() << std::endl;
+                        break;
+                    }
+                    index_A = index;
+                    break;
                 case 'b':
-                    // back to show the report
-                    report(index, false);
+                    if (indexList) {
+                        if (index == index_A) {
+                            std::cout << Colors::RED() << "A and B cannot be the same!" << Colors::RESET() << std::endl;
+                            break;
+                        }
+                        if (!results.at(index).isComplete()) {
+                            std::cout << Colors::RED() << "B must be a complete result!" << Colors::RESET() << std::endl;
+                            break;
+                        }
+                        index_B = index;
+                    } else {
+                        // back to show the report
+                        report(index, false);
+                    }
+                    break;
+                case 'c':
+                    if (index_A == -1 || index_B == -1) {
+                        std::cout << Colors::RED() << "Need to set A and B first!" << Colors::RESET() << std::endl;
+                        break;
+                    }
+                    report_compared(index_A, index_B);
                     break;
                 case 'l':
                     list();

src/manage/ManageResults.h

@@ -13,6 +13,7 @@ namespace platform {
         void list();
         bool confirmAction(const std::string& intent, const std::string& fileName) const;
         void report(const int index, const bool excelReport);
+        void report_compared(const int index_A, const int index_B);
         void showIndex(const int index, const int idx);
         void sortList();
         void menu();
@@ -25,7 +26,6 @@ namespace platform {
         Results results;
         lxw_workbook* workbook;
     };
-
 }
 
 #endif /* MANAGE_RESULTS_H */

src/reports/ExcelFile.cc

@@ -121,12 +121,17 @@ namespace platform {
             format_set_align(style, LXW_ALIGN_VERTICAL_CENTER);
             format_set_font_size(style, normalSize);
             format_set_num_format(style, "#,##0.00");
+        } else if (name == "percentage") {
+            format_set_border(style, LXW_BORDER_THIN);
+            format_set_align(style, LXW_ALIGN_VERTICAL_CENTER);
+            format_set_font_size(style, normalSize);
+            format_set_num_format(style, "0.00%");
         }
     }
 
     void ExcelFile::createFormats()
     {
-        auto styleNames = { "text", "textCentered", "bodyHeader", "result", "time", "ints", "floats" };
+        auto styleNames = { "text", "textCentered", "bodyHeader", "result", "time", "ints", "floats", "percentage" };
         lxw_format* style;
         for (std::string name : styleNames) {
             lxw_format* style = workbook_add_format(workbook);

src/reports/ReportExcel.h

@@ -9,11 +9,11 @@ namespace platform {
     class ReportExcel : public ReportBase, public ExcelFile {
     public:
         explicit ReportExcel(json data_, bool compare, lxw_workbook* workbook, lxw_worksheet* worksheet = NULL);
+        void closeFile();
     private:
         void formatColumns();
         void createFile();
         void createWorksheet();
-        void closeFile();
         void header() override;
         void body() override;
         void showSummary() override;

src/reports/ReportExcelCompared.cc

@@ -0,0 +1,134 @@
+#include "ReportExcelCompared.h"
+
+namespace platform {
+
+    ReportExcelCompared::ReportExcelCompared(json& data_A, json& data_B) : data_A(data_A), data_B(data_B), ExcelFile(NULL, NULL)
+    {
+    }
+    ReportExcelCompared::~ReportExcelCompared()
+    {
+        if (workbook)
+            workbook_close(workbook);
+    }
+    void ReportExcelCompared::report()
+    {
+        // Create a new workbook and add the two worksheets A & B
+        workbook = workbook_new((Paths::excel() + Paths::excelResults()).c_str());
+        worksheet = workbook_add_worksheet(workbook, "A");
+        createFormats();
+        ReportExcel report(data_A, false, workbook, worksheet);
+        workbook = report.getWorkbook();
+        report.show();
+        worksheet = workbook_add_worksheet(workbook, "B");
+        report = ReportExcel(data_B, false, workbook, worksheet);
+        report.show();
+        // Add the comparison worksheet
+        worksheet = workbook_add_worksheet(workbook, "Δ");
+        header();
+        body();
+    }
+    void ReportExcelCompared::header()
+    {
+        worksheet_merge_range(worksheet, 0, 0, 0, 20, "Compare Results A vs B", styles["headerFirst"]);
+        worksheet_merge_range(worksheet, 1, 0, 1, 20, "Δ = (A - B) / B", styles["headerRest"]);
+        worksheet_freeze_panes(worksheet, 5, 1);
+    }
+    double diff(double a, double b)
+    {
+        return (a - b) / b;
+    }
+    void ReportExcelCompared::body()
+    {
+        // Body Header
+        auto sizes = std::vector<int>({ 22, 10, 9, 7, 12, 12, 9, 12, 12, 9, 12, 12, 9, 12, 12, 9, 15, 15, 9, 15, 15 });
+        auto head_a = std::vector<std::string>({ "Dataset", "Samples", "Features", "Classes" });
+        auto head_b = std::vector<std::string>({ "Nodes", "Edges", "States", "Score", "Time" });
+        int headerRow = 3;
+        int col = 0;
+        for (const auto& item : head_a) {
+            worksheet_merge_range(worksheet, headerRow, col, headerRow + 1, col, item.c_str(), styles["bodyHeader_even"]);
+            col++;
+        }
+        for (const auto& item : head_b) {
+            worksheet_merge_range(worksheet, headerRow, col, headerRow, col + 2, item.c_str(), styles["bodyHeader_even"]);
+            writeString(headerRow + 1, col++, "A", "bodyHeader");
+            writeString(headerRow + 1, col++, "B", "bodyHeader");
+            writeString(headerRow + 1, col++, "Δ", "bodyHeader");
+        }
+        worksheet_merge_range(worksheet, headerRow, col, headerRow, col + 1, "Hyperparameters", styles["bodyHeader_even"]);
+        int hypCol = col;
+        writeString(headerRow + 1, hypCol, "A", "bodyHeader");
+        writeString(headerRow + 1, hypCol + 1, "B", "bodyHeader");
+        col = 0;
+        for (const auto size : sizes) {
+            worksheet_set_column(worksheet, col, col, size, NULL);
+            col++;
+        }
+        // Body Data
+        row = headerRow + 2;
+        int hypSize_A = 15;
+        int hypSize_B = 15;
+        auto compared = std::vector<std::string>({ "nodes", "leaves", "depth", "score", "time" });
+        auto compared_data = std::vector<double>(compared.size(), 0.0);
+        auto totals_A = std::vector<double>(compared.size(), 0.0);
+        auto totals_B = std::vector<double>(compared.size(), 0.0);
+        std::string hyperparameters;
+        for (int i = 0; i < data_A["results"].size(); i++) {
+            col = 0;
+            const auto& r_A = data_A["results"][i];
+            const auto& r_B = data_B["results"][i];
+            for (int j = 0; j < compared.size(); j++) {
+                auto key = compared[j];
+                compared_data[j] = diff(r_A[key].get<double>(), r_B[key].get<double>());
+                totals_A[j] += r_A[key].get<double>();
+                totals_B[j] += r_B[key].get<double>();
+            }
+            writeString(row, col++, r_A["dataset"].get<std::string>(), "text");
+            writeInt(row, col++, r_A["samples"].get<int>(), "ints");
+            writeInt(row, col++, r_A["features"].get<int>(), "ints");
+            writeInt(row, col++, r_A["classes"].get<int>(), "ints");
+            writeDouble(row, col++, r_A["nodes"].get<float>(), "floats");
+            writeDouble(row, col++, r_B["nodes"].get<float>(), "floats");
+            writeDouble(row, col++, compared_data[0], "percentage");
+            writeDouble(row, col++, r_A["leaves"].get<float>(), "floats");
+            writeDouble(row, col++, r_B["leaves"].get<float>(), "floats");
+            writeDouble(row, col++, compared_data[1], "percentage");
+            writeDouble(row, col++, r_A["depth"].get<double>(), "floats");
+            writeDouble(row, col++, r_B["depth"].get<double>(), "floats");
+            writeDouble(row, col++, compared_data[2], "percentage");
+            writeDouble(row, col++, r_A["score"].get<double>(), "result");
+            writeDouble(row, col++, r_B["score"].get<double>(), "result");
+            writeDouble(row, col++, compared_data[3], "percentage");
+            writeDouble(row, col++, r_A["time"].get<double>(), "time");
+            writeDouble(row, col++, r_B["time"].get<double>(), "time");
+            writeDouble(row, col++, compared_data[4], "percentage");
+            hyperparameters = r_A["hyperparameters"].dump();
+            if (hyperparameters.size() > hypSize_A) {
+                hypSize_A = hyperparameters.size();
+            }
+            writeString(row, hypCol, hyperparameters, "text");
+            hyperparameters = r_B["hyperparameters"].dump();
+            if (hyperparameters.size() > hypSize_B) {
+                hypSize_B = hyperparameters.size();
+            }
+            writeString(row, hypCol + 1, hyperparameters, "text");
+            row++;
+        }
+        // Set the right column width of hyperparameters with the maximum length
+        worksheet_set_column(worksheet, hypCol, hypCol, hypSize_A + 5, NULL);
+        worksheet_set_column(worksheet, hypCol + 1, hypCol + 1, hypSize_B + 5, NULL);
+        // Show totals if only one dataset is present in the result
+        footer(totals_A, totals_B, row);
+    }
+    void ReportExcelCompared::footer(std::vector<double>& totals_A, std::vector<double>& totals_B, int row)
+    {
+        worksheet_merge_range(worksheet, row, 0, row, 3, "Total", styles["bodyHeader_even"]);
+        auto formats = std::vector<std::string>({ "floats", "floats", "floats", "result", "result" });
+        int col = 4;
+        for (int i = 0; i < totals_A.size(); i++) {
+            writeDouble(row, col++, totals_A[i], formats[i]);
+            writeDouble(row, col++, totals_B[i], formats[i]);
+            writeDouble(row, col++, diff(totals_A[i], totals_B[i]), "percentage");
+        }
+    }
+}

src/reports/ReportExcelCompared.h

@@ -0,0 +1,16 @@
+#pragma once
+#include "ReportExcel.h"
+namespace platform {
+    class ReportExcelCompared : public ExcelFile {
+    public:
+        explicit ReportExcelCompared(json& data_A, json& data_B);
+        ~ReportExcelCompared();
+        void report();
+    private:
+        void header();
+        void body();
+        void footer(std::vector<double>& totals_A, std::vector<double>& totals_B, int row);
+        json& data_A;
+        json& data_B;
+    };
+};