From ebaddf1a6c571cac7f5de20304c481b66c58a9fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana=20G=C3=B3mez?= Date: Sun, 12 May 2024 18:23:48 +0200 Subject: [PATCH] Fix json key automatic ordering error when creating Score from json --- src/best/BestResults.h | 2 +- src/best/BestResultsExcel.h | 2 +- src/best/Statistics.h | 2 +- src/commands/b_grid.cpp | 36 +++++++++++++++---------------- src/commands/b_main.cpp | 2 +- src/grid/GridData.h | 2 +- src/grid/GridSearch.h | 2 +- src/main/Experiment.cpp | 2 +- src/main/Experiment.h | 2 +- src/main/HyperParameters.h | 2 +- src/main/PartialResult.h | 2 +- src/main/Scores.cpp | 13 ++++++++--- src/main/Scores.h | 3 ++- src/manage/ResultsManager.h | 2 +- src/reports/DatasetsConsole.h | 2 +- src/reports/DatasetsExcel.h | 2 +- src/reports/ReportBase.h | 2 +- src/reports/ReportsPaged.h | 2 +- src/results/Result.h | 2 +- src/results/ResultsDataset.h | 2 +- src/results/ResultsDatasetExcel.h | 2 +- tests/TestPlatform.cpp | 4 ++-- tests/TestScores.cpp | 32 ++++++++++++++++++++++++++- 23 files changed, 81 insertions(+), 43 deletions(-) diff --git a/src/best/BestResults.h b/src/best/BestResults.h index ce4b0cd..d470948 100644 --- a/src/best/BestResults.h +++ b/src/best/BestResults.h @@ -2,7 +2,7 @@ #include #include -using json = nlohmann::json; +using json = nlohmann::ordered_json; namespace platform { class BestResults { public: diff --git a/src/best/BestResultsExcel.h b/src/best/BestResultsExcel.h index 00a82dc..c0a9d0f 100644 --- a/src/best/BestResultsExcel.h +++ b/src/best/BestResultsExcel.h @@ -5,7 +5,7 @@ #include #include "reports/ExcelFile.h" -using json = nlohmann::json; +using json = nlohmann::ordered_json; namespace platform { diff --git a/src/best/Statistics.h b/src/best/Statistics.h index a37c5b1..a4ac016 100644 --- a/src/best/Statistics.h +++ b/src/best/Statistics.h @@ -5,7 +5,7 @@ #include #include -using json = nlohmann::json; +using json = nlohmann::ordered_json; namespace platform { struct WTL { diff --git a/src/commands/b_grid.cpp b/src/commands/b_grid.cpp index f9fc56d..2fe0367 100644 --- a/src/commands/b_grid.cpp +++ b/src/commands/b_grid.cpp @@ -13,7 +13,7 @@ #include "grid/GridSearch.h" #include "config.h" -using json = nlohmann::json; +using json = nlohmann::ordered_json; const int MAXL = 133; void assignModel(argparse::ArgumentParser& parser) @@ -29,7 +29,7 @@ void assignModel(argparse::ArgumentParser& parser) } throw std::runtime_error("Model must be one of " + models->toString()); } - ); + ); } void add_compute_args(argparse::ArgumentParser& program) { @@ -54,23 +54,23 @@ void add_compute_args(argparse::ArgumentParser& program) catch (...) { throw std::runtime_error("Number of nested folds must be an integer"); }}); - program.add_argument("--score").help("Score used in gridsearch").default_value("accuracy"); - program.add_argument("-f", "--folds").help("Number of folds").default_value(stoi(env.get("n_folds"))).scan<'i', int>().action([](const std::string& value) { - try { - auto k = stoi(value); - if (k < 2) { - throw std::runtime_error("Number of folds must be greater than 1"); + program.add_argument("--score").help("Score used in gridsearch").default_value("accuracy"); + program.add_argument("-f", "--folds").help("Number of folds").default_value(stoi(env.get("n_folds"))).scan<'i', int>().action([](const std::string& value) { + try { + auto k = stoi(value); + if (k < 2) { + throw std::runtime_error("Number of folds must be greater than 1"); + } + return k; } - return k; - } - catch (const runtime_error& err) { - throw std::runtime_error(err.what()); - } - catch (...) { - throw std::runtime_error("Number of folds must be an integer"); - }}); - auto seed_values = env.getSeeds(); - program.add_argument("-s", "--seeds").nargs(1, 10).help("Random seeds. Set to -1 to have pseudo random").scan<'i', int>().default_value(seed_values); + catch (const runtime_error& err) { + throw std::runtime_error(err.what()); + } + catch (...) { + throw std::runtime_error("Number of folds must be an integer"); + }}); + auto seed_values = env.getSeeds(); + program.add_argument("-s", "--seeds").nargs(1, 10).help("Random seeds. Set to -1 to have pseudo random").scan<'i', int>().default_value(seed_values); } std::string headerLine(const std::string& text, int utf = 0) { diff --git a/src/commands/b_main.cpp b/src/commands/b_main.cpp index c8fd606..b11c5ed 100644 --- a/src/commands/b_main.cpp +++ b/src/commands/b_main.cpp @@ -10,7 +10,7 @@ #include "config.h" -using json = nlohmann::json; +using json = nlohmann::ordered_json; void manageArguments(argparse::ArgumentParser& program) { diff --git a/src/grid/GridData.h b/src/grid/GridData.h index e4ca2d0..98abd14 100644 --- a/src/grid/GridData.h +++ b/src/grid/GridData.h @@ -6,7 +6,7 @@ #include namespace platform { - using json = nlohmann::json; + using json = nlohmann::ordered_json; const std::string ALL_DATASETS = "all"; class GridData { public: diff --git a/src/grid/GridSearch.h b/src/grid/GridSearch.h index a7a1e8c..08226be 100644 --- a/src/grid/GridSearch.h +++ b/src/grid/GridSearch.h @@ -10,7 +10,7 @@ #include "GridData.h" namespace platform { - using json = nlohmann::json; + using json = nlohmann::ordered_json; struct ConfigGrid { std::string model; std::string score; diff --git a/src/main/Experiment.cpp b/src/main/Experiment.cpp index dcca794..156cd99 100644 --- a/src/main/Experiment.cpp +++ b/src/main/Experiment.cpp @@ -5,7 +5,7 @@ #include "Scores.h" #include "Experiment.h" namespace platform { - using json = nlohmann::json; + using json = nlohmann::ordered_json; void Experiment::saveResult() { diff --git a/src/main/Experiment.h b/src/main/Experiment.h index 1863aa5..b5868ef 100644 --- a/src/main/Experiment.h +++ b/src/main/Experiment.h @@ -9,7 +9,7 @@ #include "results/Result.h" namespace platform { - using json = nlohmann::json; + using json = nlohmann::ordered_json; class Experiment { public: diff --git a/src/main/HyperParameters.h b/src/main/HyperParameters.h index a038d40..1ef3a3a 100644 --- a/src/main/HyperParameters.h +++ b/src/main/HyperParameters.h @@ -6,7 +6,7 @@ #include namespace platform { - using json = nlohmann::json; + using json = nlohmann::ordered_json; class HyperParameters { public: HyperParameters() = default; diff --git a/src/main/PartialResult.h b/src/main/PartialResult.h index c948b0a..f5f0b2c 100644 --- a/src/main/PartialResult.h +++ b/src/main/PartialResult.h @@ -4,7 +4,7 @@ #include namespace platform { - using json = nlohmann::json; + using json = nlohmann::ordered_json; class PartialResult { public: diff --git a/src/main/Scores.cpp b/src/main/Scores.cpp index 959280b..e1fb9a4 100644 --- a/src/main/Scores.cpp +++ b/src/main/Scores.cpp @@ -37,11 +37,16 @@ namespace platform { } i++; } - // Compute accuracy with the confusion matrix + compute_accuracy_value(); + } + void Scores::compute_accuracy_value() + { + accuracy_value = 0; for (int i = 0; i < num_classes; i++) { accuracy_value += confusion_matrix[i][i].item(); } accuracy_value /= total; + accuracy_value = std::min(accuracy_value, 1.0f); } void Scores::init_confusion_matrix() { @@ -59,8 +64,7 @@ namespace platform { throw std::invalid_argument("The number of classes must be the same"); confusion_matrix += a.confusion_matrix; total += a.total; - accuracy_value += a.accuracy_value; - accuracy_value /= 2; + compute_accuracy_value(); } float Scores::accuracy() { @@ -71,6 +75,7 @@ namespace platform { // Compute f1_score in a one vs rest fashion auto precision_value = precision(num_class); auto recall_value = recall(num_class); + if (precision_value + recall_value == 0) return 0; // Avoid division by zero (0/0 = 0) return 2 * precision_value * recall_value / (precision_value + recall_value); } float Scores::f1_weighted() @@ -94,6 +99,7 @@ namespace platform { int tp = confusion_matrix[num_class][num_class].item(); int fp = confusion_matrix.index({ "...", num_class }).sum().item() - tp; int fn = confusion_matrix[num_class].sum().item() - tp; + if (tp + fp == 0) return 0; // Avoid division by zero (0/0 = 0 return float(tp) / (tp + fp); } float Scores::recall(int num_class) @@ -101,6 +107,7 @@ namespace platform { int tp = confusion_matrix[num_class][num_class].item(); int fp = confusion_matrix.index({ "...", num_class }).sum().item() - tp; int fn = confusion_matrix[num_class].sum().item() - tp; + if (tp + fn == 0) return 0; // Avoid division by zero (0/0 = 0 return float(tp) / (tp + fn); } std::string Scores::classification_report_line(std::string label, float precision, float recall, float f1_score, int support) diff --git a/src/main/Scores.h b/src/main/Scores.h index 9f2990d..9de92a0 100644 --- a/src/main/Scores.h +++ b/src/main/Scores.h @@ -5,7 +5,7 @@ #include #include namespace platform { - using json = nlohmann::json; + using json = nlohmann::ordered_json; class Scores { public: Scores(torch::Tensor& y_test, torch::Tensor& y_pred, int num_classes, std::vector labels = {}); @@ -24,6 +24,7 @@ namespace platform { std::string classification_report_line(std::string label, float precision, float recall, float f1_score, int support); void init_confusion_matrix(); void init_default_labels(); + void compute_accuracy_value(); int num_classes; float accuracy_value; int total; diff --git a/src/manage/ResultsManager.h b/src/manage/ResultsManager.h index 647e00e..9793959 100644 --- a/src/manage/ResultsManager.h +++ b/src/manage/ResultsManager.h @@ -5,7 +5,7 @@ #include #include "results/Result.h" namespace platform { - using json = nlohmann::json; + using json = nlohmann::ordered_json; enum class SortType { ASC = 0, DESC = 1, diff --git a/src/reports/DatasetsConsole.h b/src/reports/DatasetsConsole.h index 1aff5ca..18dfdb9 100644 --- a/src/reports/DatasetsConsole.h +++ b/src/reports/DatasetsConsole.h @@ -6,7 +6,7 @@ #include "ReportsPaged.h" namespace platform { - using json = nlohmann::json; + using json = nlohmann::ordered_json; class DatasetsConsole : public ReportsPaged { diff --git a/src/reports/DatasetsExcel.h b/src/reports/DatasetsExcel.h index dbfbc8f..11e419f 100644 --- a/src/reports/DatasetsExcel.h +++ b/src/reports/DatasetsExcel.h @@ -3,7 +3,7 @@ #include #include "reports/ExcelFile.h" -using json = nlohmann::json; +using json = nlohmann::ordered_json; namespace platform { diff --git a/src/reports/ReportBase.h b/src/reports/ReportBase.h index 52f122c..56ce32a 100644 --- a/src/reports/ReportBase.h +++ b/src/reports/ReportBase.h @@ -6,7 +6,7 @@ #include "common/Paths.h" #include "common/Symbols.h" -using json = nlohmann::json; +using json = nlohmann::ordered_json; namespace platform { class ReportBase { diff --git a/src/reports/ReportsPaged.h b/src/reports/ReportsPaged.h index 4244d36..3e81923 100644 --- a/src/reports/ReportsPaged.h +++ b/src/reports/ReportsPaged.h @@ -5,7 +5,7 @@ #include namespace platform { - using json = nlohmann::json; + using json = nlohmann::ordered_json; class ReportsPaged { public: diff --git a/src/results/Result.h b/src/results/Result.h index ad94745..cafeb69 100644 --- a/src/results/Result.h +++ b/src/results/Result.h @@ -9,7 +9,7 @@ #include "main/PartialResult.h" namespace platform { - using json = nlohmann::json; + using json = nlohmann::ordered_json; class Result { public: diff --git a/src/results/ResultsDataset.h b/src/results/ResultsDataset.h index 0d9bac3..a9d1935 100644 --- a/src/results/ResultsDataset.h +++ b/src/results/ResultsDataset.h @@ -5,7 +5,7 @@ #include #include "results/Result.h" namespace platform { - using json = nlohmann::json; + using json = nlohmann::ordered_json; class ResultsDataset { public: ResultsDataset(const std::string& dataset, const std::string& model, const std::string& score); diff --git a/src/results/ResultsDatasetExcel.h b/src/results/ResultsDatasetExcel.h index 373a4f7..78b4964 100644 --- a/src/results/ResultsDatasetExcel.h +++ b/src/results/ResultsDatasetExcel.h @@ -3,7 +3,7 @@ #include #include "reports/ExcelFile.h" -using json = nlohmann::json; +using json = nlohmann::ordered_json; namespace platform { diff --git a/tests/TestPlatform.cpp b/tests/TestPlatform.cpp index 9a44072..25d810b 100644 --- a/tests/TestPlatform.cpp +++ b/tests/TestPlatform.cpp @@ -19,12 +19,12 @@ TEST_CASE("Test Platform version", "[Platform]") TEST_CASE("Test Folding library version", "[Folding]") { std::string version = folding::KFold(5, 100).version(); - REQUIRE(version == "1.0.1"); + REQUIRE(version == "1.1.0"); } TEST_CASE("Test BayesNet version", "[BayesNet]") { std::string version = bayesnet::TAN().getVersion(); - REQUIRE(version == "1.0.4.1"); + REQUIRE(version == "1.0.5.1"); } TEST_CASE("Test mdlp version", "[mdlp]") { diff --git a/tests/TestScores.cpp b/tests/TestScores.cpp index 965a824..4d481bd 100644 --- a/tests/TestScores.cpp +++ b/tests/TestScores.cpp @@ -10,6 +10,7 @@ #include "main/Scores.h" #include "config.h" +using json = nlohmann::ordered_json; auto epsilon = 1e-4; void make_test_bin(int TP, int TN, int FP, int FN, std::vector& y_test, std::vector& y_pred) @@ -157,7 +158,7 @@ TEST_CASE("JSON constructor", "[Scores]") std::vector y_pred = { 0, 1, 2, 2, 1, 1, 1, 0, 0, 2 }; auto y_test_tensor = torch::tensor(y_test, torch::kInt32); auto y_pred_tensor = torch::tensor(y_pred, torch::kInt32); - std::vector labels = { "Aeroplane", "Boat", "Car" }; + std::vector labels = { "Car", "Boat", "Aeroplane" }; platform::Scores scores(y_test_tensor, y_pred_tensor, 3, labels); auto res_json_int = scores.get_confusion_matrix_json(); platform::Scores scores2(res_json_int); @@ -218,4 +219,33 @@ TEST_CASE("Aggregate", "[Scores]") REQUIRE(scores3.f1_weighted() == scores.f1_weighted()); REQUIRE(scores3.f1_macro() == scores.f1_macro()); REQUIRE(scores3.accuracy() == scores.accuracy()); +} +TEST_CASE("Order of keys", "[Scores]") +{ + std::vector y_test = { 0, 2, 2, 2, 2, 0, 1, 2, 0, 2 }; + std::vector y_pred = { 0, 1, 2, 2, 1, 1, 1, 0, 0, 2 }; + auto y_test_tensor = torch::tensor(y_test, torch::kInt32); + auto y_pred_tensor = torch::tensor(y_pred, torch::kInt32); + std::vector labels = { "Car", "Boat", "Aeroplane" }; + platform::Scores scores(y_test_tensor, y_pred_tensor, 3, labels); + auto res_json_int = scores.get_confusion_matrix_json(true); + // Make a temp file and store the json + std::string filename = "temp.json"; + std::ofstream file(filename); + file << res_json_int; + file.close(); + // Read the json from the file + std::ifstream file2(filename); + json res_json_int2; + file2 >> res_json_int2; + file2.close(); + // Remove the temp file + std::remove(filename.c_str()); + platform::Scores scores2(res_json_int2); + REQUIRE(scores.accuracy() == scores2.accuracy()); + for (int i = 0; i < 2; ++i) { + REQUIRE(scores.f1_score(i) == scores2.f1_score(i)); + REQUIRE(scores.precision(i) == scores2.precision(i)); + REQUIRE(scores.recall(i) == scores2.recall(i)); + } } \ No newline at end of file