From 54496c68f1a4a070e654b429406b61bce8de5ae8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana=20G=C3=B3mez?= Date: Wed, 15 May 2024 19:49:15 +0200 Subject: [PATCH] Create Boost class as Boost classifiers parent --- bayesnet/ensembles/Boost.cc | 214 ++++++++++++++++++++++++++++++++ bayesnet/ensembles/Boost.h | 51 ++++++++ bayesnet/ensembles/BoostA2DE.cc | 87 +++++-------- bayesnet/ensembles/BoostA2DE.h | 24 +--- bayesnet/ensembles/BoostAODE.cc | 213 +------------------------------ bayesnet/ensembles/BoostAODE.h | 23 +--- bayesnet/ensembles/boost.h | 13 -- bayesnet/utils/BayesMetrics.cc | 2 +- tests/CMakeLists.txt | 16 +-- 9 files changed, 318 insertions(+), 325 deletions(-) create mode 100644 bayesnet/ensembles/Boost.cc create mode 100644 bayesnet/ensembles/Boost.h delete mode 100644 bayesnet/ensembles/boost.h diff --git a/bayesnet/ensembles/Boost.cc b/bayesnet/ensembles/Boost.cc new file mode 100644 index 0000000..73d4adb --- /dev/null +++ b/bayesnet/ensembles/Boost.cc @@ -0,0 +1,214 @@ +// *************************************************************** +// SPDX-FileCopyrightText: Copyright 2024 Ricardo Montañana Gómez +// SPDX-FileType: SOURCE +// SPDX-License-Identifier: MIT +// *************************************************************** +#include "bayesnet/feature_selection/CFS.h" +#include "bayesnet/feature_selection/FCBF.h" +#include "bayesnet/feature_selection/IWSS.h" +#include "Boost.h" + +namespace bayesnet { + Boost::Boost(bool predict_voting) : Ensemble(predict_voting) + { + validHyperparameters = { "order", "convergence", "convergence_best", "bisection", "threshold", "maxTolerance", + "predict_voting", "select_features", "block_update" }; + } + void Boost::setHyperparameters(const nlohmann::json& hyperparameters_) + { + auto hyperparameters = hyperparameters_; + if (hyperparameters.contains("order")) { + std::vector algos = { Orders.ASC, Orders.DESC, Orders.RAND }; + order_algorithm = hyperparameters["order"]; + if (std::find(algos.begin(), algos.end(), order_algorithm) == algos.end()) { + throw std::invalid_argument("Invalid order algorithm, valid values [" + Orders.ASC + ", " + Orders.DESC + ", " + Orders.RAND + "]"); + } + hyperparameters.erase("order"); + } + if (hyperparameters.contains("convergence")) { + convergence = hyperparameters["convergence"]; + hyperparameters.erase("convergence"); + } + if (hyperparameters.contains("convergence_best")) { + convergence_best = hyperparameters["convergence_best"]; + hyperparameters.erase("convergence_best"); + } + if (hyperparameters.contains("bisection")) { + bisection = hyperparameters["bisection"]; + hyperparameters.erase("bisection"); + } + if (hyperparameters.contains("threshold")) { + threshold = hyperparameters["threshold"]; + hyperparameters.erase("threshold"); + } + if (hyperparameters.contains("maxTolerance")) { + maxTolerance = hyperparameters["maxTolerance"]; + if (maxTolerance < 1 || maxTolerance > 4) + throw std::invalid_argument("Invalid maxTolerance value, must be greater in [1, 4]"); + hyperparameters.erase("maxTolerance"); + } + if (hyperparameters.contains("predict_voting")) { + predict_voting = hyperparameters["predict_voting"]; + hyperparameters.erase("predict_voting"); + } + if (hyperparameters.contains("select_features")) { + auto selectedAlgorithm = hyperparameters["select_features"]; + std::vector algos = { SelectFeatures.IWSS, SelectFeatures.CFS, SelectFeatures.FCBF }; + selectFeatures = true; + select_features_algorithm = selectedAlgorithm; + if (std::find(algos.begin(), algos.end(), selectedAlgorithm) == algos.end()) { + throw std::invalid_argument("Invalid selectFeatures value, valid values [" + SelectFeatures.IWSS + ", " + SelectFeatures.CFS + ", " + SelectFeatures.FCBF + "]"); + } + hyperparameters.erase("select_features"); + } + if (hyperparameters.contains("block_update")) { + block_update = hyperparameters["block_update"]; + hyperparameters.erase("block_update"); + } + Classifier::setHyperparameters(hyperparameters); + } + std::vector Boost::featureSelection(torch::Tensor& weights_) + { + int maxFeatures = 0; + if (select_features_algorithm == SelectFeatures.CFS) { + featureSelector = new CFS(dataset, features, className, maxFeatures, states.at(className).size(), weights_); + } else if (select_features_algorithm == SelectFeatures.IWSS) { + if (threshold < 0 || threshold >0.5) { + throw std::invalid_argument("Invalid threshold value for " + SelectFeatures.IWSS + " [0, 0.5]"); + } + featureSelector = new IWSS(dataset, features, className, maxFeatures, states.at(className).size(), weights_, threshold); + } else if (select_features_algorithm == SelectFeatures.FCBF) { + if (threshold < 1e-7 || threshold > 1) { + throw std::invalid_argument("Invalid threshold value for " + SelectFeatures.FCBF + " [1e-7, 1]"); + } + featureSelector = new FCBF(dataset, features, className, maxFeatures, states.at(className).size(), weights_, threshold); + } + featureSelector->fit(); + auto featuresUsed = featureSelector->getFeatures(); + delete featureSelector; + return featuresUsed; + } + std::tuple Boost::update_weights(torch::Tensor& ytrain, torch::Tensor& ypred, torch::Tensor& weights) + { + bool terminate = false; + double alpha_t = 0; + auto mask_wrong = ypred != ytrain; + auto mask_right = ypred == ytrain; + auto masked_weights = weights * mask_wrong.to(weights.dtype()); + double epsilon_t = masked_weights.sum().item(); + if (epsilon_t > 0.5) { + // Inverse the weights policy (plot ln(wt)) + // "In each round of AdaBoost, there is a sanity check to ensure that the current base + // learner is better than random guess" (Zhi-Hua Zhou, 2012) + terminate = true; + } else { + double wt = (1 - epsilon_t) / epsilon_t; + alpha_t = epsilon_t == 0 ? 1 : 0.5 * log(wt); + // Step 3.2: Update weights for next classifier + // Step 3.2.1: Update weights of wrong samples + weights += mask_wrong.to(weights.dtype()) * exp(alpha_t) * weights; + // Step 3.2.2: Update weights of right samples + weights += mask_right.to(weights.dtype()) * exp(-alpha_t) * weights; + // Step 3.3: Normalise the weights + double totalWeights = torch::sum(weights).item(); + weights = weights / totalWeights; + } + return { weights, alpha_t, terminate }; + } + std::tuple Boost::update_weights_block(int k, torch::Tensor& ytrain, torch::Tensor& weights) + { + /* Update Block algorithm + k = # of models in block + n_models = # of models in ensemble to make predictions + n_models_bak = # models saved + models = vector of models to make predictions + models_bak = models not used to make predictions + significances_bak = backup of significances vector + + Case list + A) k = 1, n_models = 1 => n = 0 , n_models = n + k + B) k = 1, n_models = n + 1 => n_models = n + k + C) k > 1, n_models = k + 1 => n= 1, n_models = n + k + D) k > 1, n_models = k => n = 0, n_models = n + k + E) k > 1, n_models = k + n => n_models = n + k + + A, D) n=0, k > 0, n_models == k + 1. n_models_bak <- n_models + 2. significances_bak <- significances + 3. significances = vector(k, 1) + 4. Don’t move any classifiers out of models + 5. n_models <- k + 6. Make prediction, compute alpha, update weights + 7. Don’t restore any classifiers to models + 8. significances <- significances_bak + 9. Update last k significances + 10. n_models <- n_models_bak + + B, C, E) n > 0, k > 0, n_models == n + k + 1. n_models_bak <- n_models + 2. significances_bak <- significances + 3. significances = vector(k, 1) + 4. Move first n classifiers to models_bak + 5. n_models <- k + 6. Make prediction, compute alpha, update weights + 7. Insert classifiers in models_bak to be the first n models + 8. significances <- significances_bak + 9. Update last k significances + 10. n_models <- n_models_bak + */ + // + // Make predict with only the last k models + // + std::unique_ptr model; + std::vector> models_bak; + // 1. n_models_bak <- n_models 2. significances_bak <- significances + auto significance_bak = significanceModels; + auto n_models_bak = n_models; + // 3. significances = vector(k, 1) + significanceModels = std::vector(k, 1.0); + // 4. Move first n classifiers to models_bak + // backup the first n_models - k models (if n_models == k, don't backup any) + for (int i = 0; i < n_models - k; ++i) { + model = std::move(models[0]); + models.erase(models.begin()); + models_bak.push_back(std::move(model)); + } + assert(models.size() == k); + // 5. n_models <- k + n_models = k; + // 6. Make prediction, compute alpha, update weights + auto ypred = predict(X_train); + // + // Update weights + // + double alpha_t; + bool terminate; + std::tie(weights, alpha_t, terminate) = update_weights(y_train, ypred, weights); + // + // Restore the models if needed + // + // 7. Insert classifiers in models_bak to be the first n models + // if n_models_bak == k, don't restore any, because none of them were moved + if (k != n_models_bak) { + // Insert in the same order as they were extracted + int bak_size = models_bak.size(); + for (int i = 0; i < bak_size; ++i) { + model = std::move(models_bak[bak_size - 1 - i]); + models_bak.erase(models_bak.end() - 1); + models.insert(models.begin(), std::move(model)); + } + } + // 8. significances <- significances_bak + significanceModels = significance_bak; + // + // Update the significance of the last k models + // + // 9. Update last k significances + for (int i = 0; i < k; ++i) { + significanceModels[n_models_bak - k + i] = alpha_t; + } + // 10. n_models <- n_models_bak + n_models = n_models_bak; + return { weights, alpha_t, terminate }; + } +} \ No newline at end of file diff --git a/bayesnet/ensembles/Boost.h b/bayesnet/ensembles/Boost.h new file mode 100644 index 0000000..d00728b --- /dev/null +++ b/bayesnet/ensembles/Boost.h @@ -0,0 +1,51 @@ +// *************************************************************** +// SPDX-FileCopyrightText: Copyright 2024 Ricardo Montañana Gómez +// SPDX-FileType: SOURCE +// SPDX-License-Identifier: MIT +// *************************************************************** + +#ifndef BOOST_H +#define BOOST_H +#include +#include +#include +#include +#include +#include "Ensemble.h" +#include "bayesnet/feature_selection/FeatureSelect.h" +namespace bayesnet { + const struct { + std::string CFS = "CFS"; + std::string FCBF = "FCBF"; + std::string IWSS = "IWSS"; + }SelectFeatures; + const struct { + std::string ASC = "asc"; + std::string DESC = "desc"; + std::string RAND = "rand"; + }Orders; + class Boost : public Ensemble { + public: + explicit Boost(bool predict_voting = false); + virtual ~Boost() = default; + void setHyperparameters(const nlohmann::json& hyperparameters_) override; + protected: + std::vector featureSelection(torch::Tensor& weights_); + std::tuple update_weights(torch::Tensor& ytrain, torch::Tensor& ypred, torch::Tensor& weights); + std::tuple update_weights_block(int k, torch::Tensor& ytrain, torch::Tensor& weights); + torch::Tensor X_train, y_train, X_test, y_test; + // Hyperparameters + bool bisection = true; // if true, use bisection stratety to add k models at once to the ensemble + int maxTolerance = 3; + std::string order_algorithm; // order to process the KBest features asc, desc, rand + bool convergence = true; //if true, stop when the model does not improve + bool convergence_best = false; // wether to keep the best accuracy to the moment or the last accuracy as prior accuracy + bool selectFeatures = false; // if true, use feature selection + std::string select_features_algorithm = Orders.DESC; // Selected feature selection algorithm + FeatureSelect* featureSelector = nullptr; + double threshold = -1; + bool block_update = false; + + }; +} +#endif \ No newline at end of file diff --git a/bayesnet/ensembles/BoostA2DE.cc b/bayesnet/ensembles/BoostA2DE.cc index aadb4c6..9ad85e9 100644 --- a/bayesnet/ensembles/BoostA2DE.cc +++ b/bayesnet/ensembles/BoostA2DE.cc @@ -16,71 +16,44 @@ namespace bayesnet { - BoostA2DE::BoostA2DE(bool predict_voting) : Ensemble(predict_voting) + BoostA2DE::BoostA2DE(bool predict_voting) : Boost(predict_voting) { - validHyperparameters = { - "maxModels", "bisection", "order", "convergence", "convergence_best", "threshold", - "select_features", "maxTolerance", "predict_voting", "block_update" - }; - } void BoostA2DE::buildModel(const torch::Tensor& weights) { + // Models shall be built in trainModel models.clear(); + significanceModels.clear(); + n_models = 0; + // Prepare the validation dataset + auto y_ = dataset.index({ -1, "..." }); + if (convergence) { + // Prepare train & validation sets from train data + auto fold = folding::StratifiedKFold(5, y_, 271); + auto [train, test] = fold.getFold(0); + auto train_t = torch::tensor(train); + auto test_t = torch::tensor(test); + // Get train and validation sets + X_train = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), train_t }); + y_train = dataset.index({ -1, train_t }); + X_test = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), test_t }); + y_test = dataset.index({ -1, test_t }); + dataset = X_train; + m = X_train.size(1); + auto n_classes = states.at(className).size(); + // Build dataset with train data + buildDataset(y_train); + metrics = Metrics(dataset, features, className, n_classes); + } else { + // Use all data to train + X_train = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), "..." }); + y_train = y_; + } } - void BoostA2DE::setHyperparameters(const nlohmann::json& hyperparameters_) + void BoostA2DE::trainModel(const torch::Tensor& weights) { - auto hyperparameters = hyperparameters_; - if (hyperparameters.contains("order")) { - std::vector algos = { Orders.ASC, Orders.DESC, Orders.RAND }; - order_algorithm = hyperparameters["order"]; - if (std::find(algos.begin(), algos.end(), order_algorithm) == algos.end()) { - throw std::invalid_argument("Invalid order algorithm, valid values [" + Orders.ASC + ", " + Orders.DESC + ", " + Orders.RAND + "]"); - } - hyperparameters.erase("order"); - } - if (hyperparameters.contains("convergence")) { - convergence = hyperparameters["convergence"]; - hyperparameters.erase("convergence"); - } - if (hyperparameters.contains("convergence_best")) { - convergence_best = hyperparameters["convergence_best"]; - hyperparameters.erase("convergence_best"); - } - if (hyperparameters.contains("bisection")) { - bisection = hyperparameters["bisection"]; - hyperparameters.erase("bisection"); - } - if (hyperparameters.contains("threshold")) { - threshold = hyperparameters["threshold"]; - hyperparameters.erase("threshold"); - } - if (hyperparameters.contains("maxTolerance")) { - maxTolerance = hyperparameters["maxTolerance"]; - if (maxTolerance < 1 || maxTolerance > 4) - throw std::invalid_argument("Invalid maxTolerance value, must be greater in [1, 4]"); - hyperparameters.erase("maxTolerance"); - } - if (hyperparameters.contains("predict_voting")) { - predict_voting = hyperparameters["predict_voting"]; - hyperparameters.erase("predict_voting"); - } - if (hyperparameters.contains("select_features")) { - auto selectedAlgorithm = hyperparameters["select_features"]; - std::vector algos = { SelectFeatures.IWSS, SelectFeatures.CFS, SelectFeatures.FCBF }; - selectFeatures = true; - select_features_algorithm = selectedAlgorithm; - if (std::find(algos.begin(), algos.end(), selectedAlgorithm) == algos.end()) { - throw std::invalid_argument("Invalid selectFeatures value, valid values [" + SelectFeatures.IWSS + ", " + SelectFeatures.CFS + ", " + SelectFeatures.FCBF + "]"); - } - hyperparameters.erase("select_features"); - } - if (hyperparameters.contains("block_update")) { - block_update = hyperparameters["block_update"]; - hyperparameters.erase("block_update"); - } - Classifier::setHyperparameters(hyperparameters); + } std::vector BoostA2DE::graph(const std::string& title) const { diff --git a/bayesnet/ensembles/BoostA2DE.h b/bayesnet/ensembles/BoostA2DE.h index b9e9ad7..c81531e 100644 --- a/bayesnet/ensembles/BoostA2DE.h +++ b/bayesnet/ensembles/BoostA2DE.h @@ -6,33 +6,19 @@ #ifndef BOOSTA2DE_H #define BOOSTA2DE_H -#include -#include "boost.h" +#include +#include #include "bayesnet/classifiers/SPnDE.h" -#include "bayesnet/feature_selection/FeatureSelect.h" -#include "Ensemble.h" +#include "Boost.h" namespace bayesnet { - class BoostA2DE : public Ensemble { + class BoostA2DE : public Boost { public: explicit BoostA2DE(bool predict_voting = false); virtual ~BoostA2DE() = default; std::vector graph(const std::string& title = "BoostA2DE") const override; - void setHyperparameters(const nlohmann::json& hyperparameters_) override; protected: void buildModel(const torch::Tensor& weights) override; - private: - torch::Tensor X_train, y_train, X_test, y_test; - // Hyperparameters - bool bisection = true; // if true, use bisection stratety to add k models at once to the ensemble - int maxTolerance = 3; - std::string order_algorithm; // order to process the KBest features asc, desc, rand - bool convergence = true; //if true, stop when the model does not improve - bool convergence_best = false; // wether to keep the best accuracy to the moment or the last accuracy as prior accuracy - bool selectFeatures = false; // if true, use feature selection - std::string select_features_algorithm = Orders.DESC; // Selected feature selection algorithm - FeatureSelect* featureSelector = nullptr; - double threshold = -1; - bool block_update = false; + void trainModel(const torch::Tensor& weights) override; }; } #endif \ No newline at end of file diff --git a/bayesnet/ensembles/BoostAODE.cc b/bayesnet/ensembles/BoostAODE.cc index 1e855c9..a1a748a 100644 --- a/bayesnet/ensembles/BoostAODE.cc +++ b/bayesnet/ensembles/BoostAODE.cc @@ -9,21 +9,13 @@ #include #include #include -#include "bayesnet/feature_selection/CFS.h" -#include "bayesnet/feature_selection/FCBF.h" -#include "bayesnet/feature_selection/IWSS.h" #include "BoostAODE.h" #include "lib/log/loguru.cpp" namespace bayesnet { - BoostAODE::BoostAODE(bool predict_voting) : Ensemble(predict_voting) + BoostAODE::BoostAODE(bool predict_voting) : Boost(predict_voting) { - validHyperparameters = { - "maxModels", "bisection", "order", "convergence", "convergence_best", "threshold", - "select_features", "maxTolerance", "predict_voting", "block_update" - }; - } void BoostAODE::buildModel(const torch::Tensor& weights) { @@ -56,214 +48,19 @@ namespace bayesnet { y_train = y_; } } - void BoostAODE::setHyperparameters(const nlohmann::json& hyperparameters_) - { - auto hyperparameters = hyperparameters_; - if (hyperparameters.contains("order")) { - std::vector algos = { Orders.ASC, Orders.DESC, Orders.RAND }; - order_algorithm = hyperparameters["order"]; - if (std::find(algos.begin(), algos.end(), order_algorithm) == algos.end()) { - throw std::invalid_argument("Invalid order algorithm, valid values [" + Orders.ASC + ", " + Orders.DESC + ", " + Orders.RAND + "]"); - } - hyperparameters.erase("order"); - } - if (hyperparameters.contains("convergence")) { - convergence = hyperparameters["convergence"]; - hyperparameters.erase("convergence"); - } - if (hyperparameters.contains("convergence_best")) { - convergence_best = hyperparameters["convergence_best"]; - hyperparameters.erase("convergence_best"); - } - if (hyperparameters.contains("bisection")) { - bisection = hyperparameters["bisection"]; - hyperparameters.erase("bisection"); - } - if (hyperparameters.contains("threshold")) { - threshold = hyperparameters["threshold"]; - hyperparameters.erase("threshold"); - } - if (hyperparameters.contains("maxTolerance")) { - maxTolerance = hyperparameters["maxTolerance"]; - if (maxTolerance < 1 || maxTolerance > 4) - throw std::invalid_argument("Invalid maxTolerance value, must be greater in [1, 4]"); - hyperparameters.erase("maxTolerance"); - } - if (hyperparameters.contains("predict_voting")) { - predict_voting = hyperparameters["predict_voting"]; - hyperparameters.erase("predict_voting"); - } - if (hyperparameters.contains("select_features")) { - auto selectedAlgorithm = hyperparameters["select_features"]; - std::vector algos = { SelectFeatures.IWSS, SelectFeatures.CFS, SelectFeatures.FCBF }; - selectFeatures = true; - select_features_algorithm = selectedAlgorithm; - if (std::find(algos.begin(), algos.end(), selectedAlgorithm) == algos.end()) { - throw std::invalid_argument("Invalid selectFeatures value, valid values [" + SelectFeatures.IWSS + ", " + SelectFeatures.CFS + ", " + SelectFeatures.FCBF + "]"); - } - hyperparameters.erase("select_features"); - } - if (hyperparameters.contains("block_update")) { - block_update = hyperparameters["block_update"]; - hyperparameters.erase("block_update"); - } - Classifier::setHyperparameters(hyperparameters); - } - std::tuple update_weights(torch::Tensor& ytrain, torch::Tensor& ypred, torch::Tensor& weights) - { - bool terminate = false; - double alpha_t = 0; - auto mask_wrong = ypred != ytrain; - auto mask_right = ypred == ytrain; - auto masked_weights = weights * mask_wrong.to(weights.dtype()); - double epsilon_t = masked_weights.sum().item(); - if (epsilon_t > 0.5) { - // Inverse the weights policy (plot ln(wt)) - // "In each round of AdaBoost, there is a sanity check to ensure that the current base - // learner is better than random guess" (Zhi-Hua Zhou, 2012) - terminate = true; - } else { - double wt = (1 - epsilon_t) / epsilon_t; - alpha_t = epsilon_t == 0 ? 1 : 0.5 * log(wt); - // Step 3.2: Update weights for next classifier - // Step 3.2.1: Update weights of wrong samples - weights += mask_wrong.to(weights.dtype()) * exp(alpha_t) * weights; - // Step 3.2.2: Update weights of right samples - weights += mask_right.to(weights.dtype()) * exp(-alpha_t) * weights; - // Step 3.3: Normalise the weights - double totalWeights = torch::sum(weights).item(); - weights = weights / totalWeights; - } - return { weights, alpha_t, terminate }; - } - std::tuple BoostAODE::update_weights_block(int k, torch::Tensor& ytrain, torch::Tensor& weights) - { - /* Update Block algorithm - k = # of models in block - n_models = # of models in ensemble to make predictions - n_models_bak = # models saved - models = vector of models to make predictions - models_bak = models not used to make predictions - significances_bak = backup of significances vector - - Case list - A) k = 1, n_models = 1 => n = 0 , n_models = n + k - B) k = 1, n_models = n + 1 => n_models = n + k - C) k > 1, n_models = k + 1 => n= 1, n_models = n + k - D) k > 1, n_models = k => n = 0, n_models = n + k - E) k > 1, n_models = k + n => n_models = n + k - - A, D) n=0, k > 0, n_models == k - 1. n_models_bak <- n_models - 2. significances_bak <- significances - 3. significances = vector(k, 1) - 4. Don’t move any classifiers out of models - 5. n_models <- k - 6. Make prediction, compute alpha, update weights - 7. Don’t restore any classifiers to models - 8. significances <- significances_bak - 9. Update last k significances - 10. n_models <- n_models_bak - - B, C, E) n > 0, k > 0, n_models == n + k - 1. n_models_bak <- n_models - 2. significances_bak <- significances - 3. significances = vector(k, 1) - 4. Move first n classifiers to models_bak - 5. n_models <- k - 6. Make prediction, compute alpha, update weights - 7. Insert classifiers in models_bak to be the first n models - 8. significances <- significances_bak - 9. Update last k significances - 10. n_models <- n_models_bak - */ - // - // Make predict with only the last k models - // - std::unique_ptr model; - std::vector> models_bak; - // 1. n_models_bak <- n_models 2. significances_bak <- significances - auto significance_bak = significanceModels; - auto n_models_bak = n_models; - // 3. significances = vector(k, 1) - significanceModels = std::vector(k, 1.0); - // 4. Move first n classifiers to models_bak - // backup the first n_models - k models (if n_models == k, don't backup any) - for (int i = 0; i < n_models - k; ++i) { - model = std::move(models[0]); - models.erase(models.begin()); - models_bak.push_back(std::move(model)); - } - assert(models.size() == k); - // 5. n_models <- k - n_models = k; - // 6. Make prediction, compute alpha, update weights - auto ypred = predict(X_train); - // - // Update weights - // - double alpha_t; - bool terminate; - std::tie(weights, alpha_t, terminate) = update_weights(y_train, ypred, weights); - // - // Restore the models if needed - // - // 7. Insert classifiers in models_bak to be the first n models - // if n_models_bak == k, don't restore any, because none of them were moved - if (k != n_models_bak) { - // Insert in the same order as they were extracted - int bak_size = models_bak.size(); - for (int i = 0; i < bak_size; ++i) { - model = std::move(models_bak[bak_size - 1 - i]); - models_bak.erase(models_bak.end() - 1); - models.insert(models.begin(), std::move(model)); - } - } - // 8. significances <- significances_bak - significanceModels = significance_bak; - // - // Update the significance of the last k models - // - // 9. Update last k significances - for (int i = 0; i < k; ++i) { - significanceModels[n_models_bak - k + i] = alpha_t; - } - // 10. n_models <- n_models_bak - n_models = n_models_bak; - return { weights, alpha_t, terminate }; - } std::vector BoostAODE::initializeModels() { - std::vector featuresUsed; torch::Tensor weights_ = torch::full({ m }, 1.0 / m, torch::kFloat64); - int maxFeatures = 0; - if (select_features_algorithm == SelectFeatures.CFS) { - featureSelector = new CFS(dataset, features, className, maxFeatures, states.at(className).size(), weights_); - } else if (select_features_algorithm == SelectFeatures.IWSS) { - if (threshold < 0 || threshold >0.5) { - throw std::invalid_argument("Invalid threshold value for " + SelectFeatures.IWSS + " [0, 0.5]"); - } - featureSelector = new IWSS(dataset, features, className, maxFeatures, states.at(className).size(), weights_, threshold); - } else if (select_features_algorithm == SelectFeatures.FCBF) { - if (threshold < 1e-7 || threshold > 1) { - throw std::invalid_argument("Invalid threshold value for " + SelectFeatures.FCBF + " [1e-7, 1]"); - } - featureSelector = new FCBF(dataset, features, className, maxFeatures, states.at(className).size(), weights_, threshold); - } - featureSelector->fit(); - auto cfsFeatures = featureSelector->getFeatures(); - auto scores = featureSelector->getScores(); - for (const int& feature : cfsFeatures) { - featuresUsed.push_back(feature); + std::vector featuresSelected = featureSelection(weights_); + for (const int& feature : featuresSelected) { std::unique_ptr model = std::make_unique(feature); model->fit(dataset, features, className, states, weights_); models.push_back(std::move(model)); significanceModels.push_back(1.0); // They will be updated later in trainModel n_models++; } - notes.push_back("Used features in initialization: " + std::to_string(featuresUsed.size()) + " of " + std::to_string(features.size()) + " with " + select_features_algorithm); - delete featureSelector; - return featuresUsed; + notes.push_back("Used features in initialization: " + std::to_string(featuresSelected.size()) + " of " + std::to_string(features.size()) + " with " + select_features_algorithm); + return featuresSelected; } void BoostAODE::trainModel(const torch::Tensor& weights) { diff --git a/bayesnet/ensembles/BoostAODE.h b/bayesnet/ensembles/BoostAODE.h index c4f8844..d6da098 100644 --- a/bayesnet/ensembles/BoostAODE.h +++ b/bayesnet/ensembles/BoostAODE.h @@ -6,36 +6,21 @@ #ifndef BOOSTAODE_H #define BOOSTAODE_H -#include +#include +#include #include "bayesnet/classifiers/SPODE.h" -#include "bayesnet/feature_selection/FeatureSelect.h" -#include "boost.h" -#include "Ensemble.h" +#include "Boost.h" namespace bayesnet { - class BoostAODE : public Ensemble { + class BoostAODE : public Boost { public: explicit BoostAODE(bool predict_voting = false); virtual ~BoostAODE() = default; std::vector graph(const std::string& title = "BoostAODE") const override; - void setHyperparameters(const nlohmann::json& hyperparameters_) override; protected: void buildModel(const torch::Tensor& weights) override; void trainModel(const torch::Tensor& weights) override; private: - std::tuple update_weights_block(int k, torch::Tensor& ytrain, torch::Tensor& weights); std::vector initializeModels(); - torch::Tensor X_train, y_train, X_test, y_test; - // Hyperparameters - bool bisection = true; // if true, use bisection stratety to add k models at once to the ensemble - int maxTolerance = 3; - std::string order_algorithm; // order to process the KBest features asc, desc, rand - bool convergence = true; //if true, stop when the model does not improve - bool convergence_best = false; // wether to keep the best accuracy to the moment or the last accuracy as prior accuracy - bool selectFeatures = false; // if true, use feature selection - std::string select_features_algorithm = Orders.DESC; // Selected feature selection algorithm - FeatureSelect* featureSelector = nullptr; - double threshold = -1; - bool block_update = false; }; } #endif \ No newline at end of file diff --git a/bayesnet/ensembles/boost.h b/bayesnet/ensembles/boost.h deleted file mode 100644 index 54d6ce7..0000000 --- a/bayesnet/ensembles/boost.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef BOOST_H -#define BOOST_H -const struct { - std::string CFS = "CFS"; - std::string FCBF = "FCBF"; - std::string IWSS = "IWSS"; -}SelectFeatures; -const struct { - std::string ASC = "asc"; - std::string DESC = "desc"; - std::string RAND = "rand"; -}Orders; -#endif \ No newline at end of file diff --git a/bayesnet/utils/BayesMetrics.cc b/bayesnet/utils/BayesMetrics.cc index 3e63038..c863704 100644 --- a/bayesnet/utils/BayesMetrics.cc +++ b/bayesnet/utils/BayesMetrics.cc @@ -187,7 +187,7 @@ namespace bayesnet { auto [x, c, y] = keyJoint; auto keyMarginal = std::make_tuple(x, c); - double p_xc = marginalCount[keyMarginal] / totalWeight; + //double p_xc = marginalCount[keyMarginal] / totalWeight; double p_y_given_xc = jointFreq / marginalCount[keyMarginal]; if (p_y_given_xc > 0) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4befe44..99249fc 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -13,15 +13,15 @@ if(ENABLE_TESTING) TestUtils.cc TestBayesEnsemble.cc TestModulesVersions.cc TestBoostA2DE.cc ${BayesNet_SOURCES}) target_link_libraries(TestBayesNet PUBLIC "${TORCH_LIBRARIES}" ArffFiles mdlp PRIVATE Catch2::Catch2WithMain) add_test(NAME BayesNetworkTest COMMAND TestBayesNet) - add_test(NAME Network COMMAND TestBayesNet "[Network]") - add_test(NAME Node COMMAND TestBayesNet "[Node]") - add_test(NAME Metrics COMMAND TestBayesNet "[Metrics]") - add_test(NAME FeatureSelection COMMAND TestBayesNet "[FeatureSelection]") - add_test(NAME Classifier COMMAND TestBayesNet "[Classifier]") - add_test(NAME Ensemble COMMAND TestBayesNet "[Ensemble]") - add_test(NAME Models COMMAND TestBayesNet "[Models]") - add_test(NAME BoostAODE COMMAND TestBayesNet "[BoostAODE]") add_test(NAME A2DE COMMAND TestBayesNet "[A2DE]") add_test(NAME BoostA2DE COMMAND TestBayesNet "[BoostA2DE]") + add_test(NAME BoostAODE COMMAND TestBayesNet "[BoostAODE]") + add_test(NAME Classifier COMMAND TestBayesNet "[Classifier]") + add_test(NAME Ensemble COMMAND TestBayesNet "[Ensemble]") + add_test(NAME FeatureSelection COMMAND TestBayesNet "[FeatureSelection]") + add_test(NAME Metrics COMMAND TestBayesNet "[Metrics]") + add_test(NAME Models COMMAND TestBayesNet "[Models]") add_test(NAME Modules COMMAND TestBayesNet "[Modules]") + add_test(NAME Network COMMAND TestBayesNet "[Network]") + add_test(NAME Node COMMAND TestBayesNet "[Node]") endif(ENABLE_TESTING)