Add confusion matrix to json results

Add Aggregate method to Scores

src/main/Experiment.cpp

@@ -2,6 +2,7 @@
 #include "reports/ReportConsole.h"
 #include "common/Paths.h"
 #include "Models.h"
+#include "Scores.h"
 #include "Experiment.h"
 namespace platform {
     using json = nlohmann::json;
@@ -96,6 +97,7 @@ namespace platform {
         auto nodes = torch::zeros({ nResults }, torch::kFloat64);
         auto edges = torch::zeros({ nResults }, torch::kFloat64);
         auto num_states = torch::zeros({ nResults }, torch::kFloat64);
+        json confusion_matrices = json::array();
         std::vector<std::string> notes;
         Timer train_timer, test_timer;
         int item = 0;
@@ -150,10 +152,13 @@ namespace platform {
                 if (!quiet)
                     showProgress(nfold + 1, getColor(clf->getStatus()), "c");
                 test_timer.start();
-                auto accuracy_test_value = clf->score(X_test, y_test);
+                auto y_predict = clf->predict(X_test);
+                Scores scores(y_test, y_predict, states[className].size());
+                auto accuracy_test_value = scores.accuracy();
                 test_time[item] = test_timer.getDuration();
                 accuracy_train[item] = accuracy_train_value;
                 accuracy_test[item] = accuracy_test_value;
+                confusion_matrices.push_back(scores.get_confusion_matrix_json());
                 if (!quiet)
                     std::cout << "\b\b\b, " << flush;
                 // Store results and times in std::vector
@@ -173,6 +178,7 @@ namespace platform {
         partial_result.setTestTimeStd(torch::std(test_time).item<double>()).setTrainTimeStd(torch::std(train_time).item<double>());
         partial_result.setNodes(torch::mean(nodes).item<double>()).setLeaves(torch::mean(edges).item<double>()).setDepth(torch::mean(num_states).item<double>());
         partial_result.setDataset(fileName).setNotes(notes);
+        partial_result.setConfusionMatrices(confusion_matrices);
         addResult(partial_result);
     }
 }

src/main/PartialResult.h

@@ -27,6 +27,7 @@ namespace platform {
             data["notes"].insert(data["notes"].end(), notes_.begin(), notes_.end());
             return *this;
         }
+        PartialResult& setConfusionMatrices(const json& confusion_matrices) { data["confusion_matrices"] = confusion_matrices; return *this; }
         PartialResult& setHyperparameters(const json& hyperparameters) { data["hyperparameters"] = hyperparameters; return *this; }
         PartialResult& setSamples(int samples) { data["samples"] = samples; return *this; }
         PartialResult& setFeatures(int features) { data["features"] = features; return *this; }

src/main/Scores.cpp

@@ -25,6 +25,15 @@ namespace platform {
             labels.push_back("Class " + std::to_string(i));
         }
     }
+    void Scores::aggregate(const Scores& a)
+    {
+        if (a.num_classes != num_classes)
+            throw std::invalid_argument("The number of classes must be the same");
+        confusion_matrix += a.confusion_matrix;
+        total += a.total;
+        accuracy_value += a.accuracy_value;
+        accuracy_value /= 2;
+    }
     Scores::Scores(json& confusion_matrix_)
     {
         json values;
@@ -46,7 +55,6 @@ namespace platform {
                 confusion_matrix[i][j] = value_int;
                 total += value_int;
             }
-            std::cout << std::endl;
             i++;
         }
         // Compute accuracy with the confusion matrix

src/main/Scores.h

@@ -19,6 +19,7 @@ namespace platform {
         torch::Tensor get_confusion_matrix() { return confusion_matrix; }
         std::string classification_report();
         json get_confusion_matrix_json(bool labels_as_keys = false);
+        void aggregate(const Scores& a);
     private:
         std::string classification_report_line(std::string label, float precision, float recall, float f1_score, int support);
         void init_confusion_matrix();