# LCOV - BayesNet Coverage Report - bayesnet/ensembles/BoostAODE.cc

LCOV - code coverage report

Current view: | top level - bayesnet/ensembles - BoostAODE.cc (source / functions) |  |  | Coverage | Total | Hit
Test: | BayesNet Coverage Report |  | Lines: | 98.3 % | 237 | 233
Test Date: | 2024-05-06 17:54:04 |  | Functions: | 100.0 % | 9 | 9
Legend: | Lines: hit not hit |

                Line data    Source code

           1              : // ***************************************************************
           2              : // SPDX-FileCopyrightText: Copyright 2024 Ricardo Montañana Gómez
           3              : // SPDX-FileType: SOURCE
           4              : // SPDX-License-Identifier: MIT
           5              : // ***************************************************************
           6              :
           7              : #include <set>
           8              : #include <functional>
           9              : #include <limits.h>
          10              : #include <tuple>
          11              : #include <folding.hpp>
          12              : #include "bayesnet/feature_selection/CFS.h"
          13              : #include "bayesnet/feature_selection/FCBF.h"
          14              : #include "bayesnet/feature_selection/IWSS.h"
          15              : #include "BoostAODE.h"
          16              : #include "lib/log/loguru.cpp"
          17              :
          18              : namespace bayesnet {
          19              :
          20          168 :     BoostAODE::BoostAODE(bool predict_voting) : Ensemble(predict_voting)
          21              :     {
          22         1848 :         validHyperparameters = {
          23              :             "maxModels", "bisection", "order", "convergence", "convergence_best", "threshold",
          24              :             "select_features", "maxTolerance", "predict_voting", "block_update"
          25         1848 :         };
          26              :
          27          504 :     }
          28           92 :     void BoostAODE::buildModel(const torch::Tensor& weights)
          29              :     {
          30              :         // Models shall be built in trainModel
          31           92 :         models.clear();
          32           92 :         significanceModels.clear();
          33           92 :         n_models = 0;
          34              :         // Prepare the validation dataset
          35          276 :         auto y_ = dataset.index({ -1, "..." });
          36           92 :         if (convergence) {
          37              :             // Prepare train & validation sets from train data
          38           76 :             auto fold = folding::StratifiedKFold(5, y_, 271);
          39           76 :             auto [train, test] = fold.getFold(0);
          40           76 :             auto train_t = torch::tensor(train);
          41           76 :             auto test_t = torch::tensor(test);
          42              :             // Get train and validation sets
          43          380 :             X_train = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), train_t });
          44          228 :             y_train = dataset.index({ -1, train_t });
          45          380 :             X_test = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), test_t });
          46          228 :             y_test = dataset.index({ -1, test_t });
          47           76 :             dataset = X_train;
          48           76 :             m = X_train.size(1);
          49           76 :             auto n_classes = states.at(className).size();
          50              :             // Build dataset with train data
          51           76 :             buildDataset(y_train);
          52           76 :             metrics = Metrics(dataset, features, className, n_classes);
          53           76 :         } else {
          54              :             // Use all data to train
          55           64 :             X_train = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), "..." });
          56           16 :             y_train = y_;
          57              :         }
          58          900 :     }
          59           88 :     void BoostAODE::setHyperparameters(const nlohmann::json& hyperparameters_)
          60              :     {
          61           88 :         auto hyperparameters = hyperparameters_;
          62           88 :         if (hyperparameters.contains("order")) {
          63          100 :             std::vector<std::string> algos = { Orders.ASC, Orders.DESC, Orders.RAND };
          64           20 :             order_algorithm = hyperparameters["order"];
          65           20 :             if (std::find(algos.begin(), algos.end(), order_algorithm) == algos.end()) {
          66            4 :                 throw std::invalid_argument("Invalid order algorithm, valid values [" + Orders.ASC + ", " + Orders.DESC + ", " + Orders.RAND + "]");
          67              :             }
          68           16 :             hyperparameters.erase("order");
          69           20 :         }
          70           84 :         if (hyperparameters.contains("convergence")) {
          71           36 :             convergence = hyperparameters["convergence"];
          72           36 :             hyperparameters.erase("convergence");
          73              :         }
          74           84 :         if (hyperparameters.contains("convergence_best")) {
          75           12 :             convergence_best = hyperparameters["convergence_best"];
          76           12 :             hyperparameters.erase("convergence_best");
          77              :         }
          78           84 :         if (hyperparameters.contains("bisection")) {
          79           32 :             bisection = hyperparameters["bisection"];
          80           32 :             hyperparameters.erase("bisection");
          81              :         }
          82           84 :         if (hyperparameters.contains("threshold")) {
          83           24 :             threshold = hyperparameters["threshold"];
          84           24 :             hyperparameters.erase("threshold");
          85              :         }
          86           84 :         if (hyperparameters.contains("maxTolerance")) {
          87           44 :             maxTolerance = hyperparameters["maxTolerance"];
          88           44 :             if (maxTolerance < 1 || maxTolerance > 4)
          89           12 :                 throw std::invalid_argument("Invalid maxTolerance value, must be greater in [1, 4]");
          90           32 :             hyperparameters.erase("maxTolerance");
          91              :         }
          92           72 :         if (hyperparameters.contains("predict_voting")) {
          93            4 :             predict_voting = hyperparameters["predict_voting"];
          94            4 :             hyperparameters.erase("predict_voting");
          95              :         }
          96           72 :         if (hyperparameters.contains("select_features")) {
          97           36 :             auto selectedAlgorithm = hyperparameters["select_features"];
          98          180 :             std::vector<std::string> algos = { SelectFeatures.IWSS, SelectFeatures.CFS, SelectFeatures.FCBF };
          99           36 :             selectFeatures = true;
         100           36 :             select_features_algorithm = selectedAlgorithm;
         101           36 :             if (std::find(algos.begin(), algos.end(), selectedAlgorithm) == algos.end()) {
         102            4 :                 throw std::invalid_argument("Invalid selectFeatures value, valid values [" + SelectFeatures.IWSS + ", " + SelectFeatures.CFS + ", " + SelectFeatures.FCBF + "]");
         103              :             }
         104           32 :             hyperparameters.erase("select_features");
         105           40 :         }
         106           68 :         if (hyperparameters.contains("block_update")) {
         107            8 :             block_update = hyperparameters["block_update"];
         108            8 :             hyperparameters.erase("block_update");
         109              :         }
         110           68 :         Classifier::setHyperparameters(hyperparameters);
         111          144 :     }
         112          544 :     std::tuple<torch::Tensor&, double, bool> update_weights(torch::Tensor& ytrain, torch::Tensor& ypred, torch::Tensor& weights)
         113              :     {
         114          544 :         bool terminate = false;
         115          544 :         double alpha_t = 0;
         116          544 :         auto mask_wrong = ypred != ytrain;
         117          544 :         auto mask_right = ypred == ytrain;
         118          544 :         auto masked_weights = weights * mask_wrong.to(weights.dtype());
         119          544 :         double epsilon_t = masked_weights.sum().item<double>();
         120          544 :         if (epsilon_t > 0.5) {
         121              :             // Inverse the weights policy (plot ln(wt))
         122              :             // "In each round of AdaBoost, there is a sanity check to ensure that the current base
         123              :             // learner is better than random guess" (Zhi-Hua Zhou, 2012)
         124           16 :             terminate = true;
         125              :         } else {
         126          528 :             double wt = (1 - epsilon_t) / epsilon_t;
         127          528 :             alpha_t = epsilon_t == 0 ? 1 : 0.5 * log(wt);
         128              :             // Step 3.2: Update weights for next classifier
         129              :             // Step 3.2.1: Update weights of wrong samples
         130          528 :             weights += mask_wrong.to(weights.dtype()) * exp(alpha_t) * weights;
         131              :             // Step 3.2.2: Update weights of right samples
         132          528 :             weights += mask_right.to(weights.dtype()) * exp(-alpha_t) * weights;
         133              :             // Step 3.3: Normalise the weights
         134          528 :             double totalWeights = torch::sum(weights).item<double>();
         135          528 :             weights = weights / totalWeights;
         136              :         }
         137         1088 :         return { weights, alpha_t, terminate };
         138          544 :     }
         139           28 :     std::tuple<torch::Tensor&, double, bool> BoostAODE::update_weights_block(int k, torch::Tensor& ytrain, torch::Tensor& weights)
         140              :     {
         141              :         /* Update Block algorithm
         142              :             k = # of models in block
         143              :             n_models = # of models in ensemble to make predictions
         144              :             n_models_bak = # models saved
         145              :             models = vector of models to make predictions
         146              :             models_bak = models not used to make predictions
         147              :             significances_bak = backup of significances vector
         148              :
         149              :             Case list
         150              :             A) k = 1, n_models = 1              => n = 0 , n_models = n + k
         151              :             B) k = 1, n_models = n + 1  => n_models = n + k
         152              :             C) k > 1, n_models = k + 1       => n= 1, n_models = n + k
         153              :             D) k > 1, n_models = k           => n = 0, n_models = n + k
         154              :             E) k > 1, n_models = k + n       => n_models = n + k
         155              :
         156              :             A, D) n=0, k > 0, n_models == k
         157              :             1. n_models_bak <- n_models
         158              :             2. significances_bak <- significances
         159              :             3. significances = vector(k, 1)
         160              :             4. Don’t move any classifiers out of models
         161              :             5. n_models <- k
         162              :             6. Make prediction, compute alpha, update weights
         163              :             7. Don’t restore any classifiers to models
         164              :             8. significances <- significances_bak
         165              :             9. Update last k significances
         166              :             10. n_models <- n_models_bak
         167              :
         168              :             B, C, E) n > 0, k > 0, n_models == n + k
         169              :             1. n_models_bak <- n_models
         170              :             2. significances_bak <- significances
         171              :             3. significances = vector(k, 1)
         172              :             4. Move first n classifiers to models_bak
         173              :             5. n_models <- k
         174              :             6. Make prediction, compute alpha, update weights
         175              :             7. Insert classifiers in models_bak to be the first n models
         176              :             8. significances <- significances_bak
         177              :             9. Update last k significances
         178              :             10. n_models <- n_models_bak
         179              :         */
         180              :         //
         181              :         // Make predict with only the last k models
         182              :         //
         183           28 :         std::unique_ptr<Classifier> model;
         184           28 :         std::vector<std::unique_ptr<Classifier>> models_bak;
         185              :         // 1. n_models_bak <- n_models 2. significances_bak <- significances
         186           28 :         auto significance_bak = significanceModels;
         187           28 :         auto n_models_bak = n_models;
         188              :         // 3. significances = vector(k, 1)
         189           28 :         significanceModels = std::vector<double>(k, 1.0);
         190              :         // 4. Move first n classifiers to models_bak
         191              :         // backup the first n_models - k models (if n_models == k, don't backup any)
         192          148 :         for (int i = 0; i < n_models - k; ++i) {
         193          120 :             model = std::move(models[0]);
         194          120 :             models.erase(models.begin());
         195          120 :             models_bak.push_back(std::move(model));
         196              :         }
         197           28 :         assert(models.size() == k);
         198              :         // 5. n_models <- k
         199           28 :         n_models = k;
         200              :         // 6. Make prediction, compute alpha, update weights
         201           28 :         auto ypred = predict(X_train);
         202              :         //
         203              :         // Update weights
         204              :         //
         205              :         double alpha_t;
         206              :         bool terminate;
         207           28 :         std::tie(weights, alpha_t, terminate) = update_weights(y_train, ypred, weights);
         208              :         //
         209              :         // Restore the models if needed
         210              :         //
         211              :         // 7. Insert classifiers in models_bak to be the first n models
         212              :         // if n_models_bak == k, don't restore any, because none of them were moved
         213           28 :         if (k != n_models_bak) {
         214              :             // Insert in the same order as they were extracted
         215           24 :             int bak_size = models_bak.size();
         216          144 :             for (int i = 0; i < bak_size; ++i) {
         217          120 :                 model = std::move(models_bak[bak_size - 1 - i]);
         218          120 :                 models_bak.erase(models_bak.end() - 1);
         219          120 :                 models.insert(models.begin(), std::move(model));
         220              :             }
         221              :         }
         222              :         // 8. significances <- significances_bak
         223           28 :         significanceModels = significance_bak;
         224              :         //
         225              :         // Update the significance of the last k models
         226              :         //
         227              :         // 9. Update last k significances
         228          104 :         for (int i = 0; i < k; ++i) {
         229           76 :             significanceModels[n_models_bak - k + i] = alpha_t;
         230              :         }
         231              :         // 10. n_models <- n_models_bak
         232           28 :         n_models = n_models_bak;
         233           56 :         return { weights, alpha_t, terminate };
         234           28 :     }
         235           32 :     std::vector<int> BoostAODE::initializeModels()
         236              :     {
         237           32 :         std::vector<int> featuresUsed;
         238           32 :         torch::Tensor weights_ = torch::full({ m }, 1.0 / m, torch::kFloat64);
         239           32 :         int maxFeatures = 0;
         240           32 :         if (select_features_algorithm == SelectFeatures.CFS) {
         241            8 :             featureSelector = new CFS(dataset, features, className, maxFeatures, states.at(className).size(), weights_);
         242           24 :         } else if (select_features_algorithm == SelectFeatures.IWSS) {
         243           12 :             if (threshold < 0 || threshold >0.5) {
         244            8 :                 throw std::invalid_argument("Invalid threshold value for " + SelectFeatures.IWSS + " [0, 0.5]");
         245              :             }
         246            4 :             featureSelector = new IWSS(dataset, features, className, maxFeatures, states.at(className).size(), weights_, threshold);
         247           12 :         } else if (select_features_algorithm == SelectFeatures.FCBF) {
         248           12 :             if (threshold < 1e-7 || threshold > 1) {
         249            8 :                 throw std::invalid_argument("Invalid threshold value for " + SelectFeatures.FCBF + " [1e-7, 1]");
         250              :             }
         251            4 :             featureSelector = new FCBF(dataset, features, className, maxFeatures, states.at(className).size(), weights_, threshold);
         252              :         }
         253           16 :         featureSelector->fit();
         254           16 :         auto cfsFeatures = featureSelector->getFeatures();
         255           16 :         auto scores = featureSelector->getScores();
         256          100 :         for (const int& feature : cfsFeatures) {
         257           84 :             featuresUsed.push_back(feature);
         258           84 :             std::unique_ptr<Classifier> model = std::make_unique<SPODE>(feature);
         259           84 :             model->fit(dataset, features, className, states, weights_);
         260           84 :             models.push_back(std::move(model));
         261           84 :             significanceModels.push_back(1.0); // They will be updated later in trainModel
         262           84 :             n_models++;
         263           84 :         }
         264           16 :         notes.push_back("Used features in initialization: " + std::to_string(featuresUsed.size()) + " of " + std::to_string(features.size()) + " with " + select_features_algorithm);
         265           16 :         delete featureSelector;
         266           32 :         return featuresUsed;
         267           48 :     }
         268           92 :     void BoostAODE::trainModel(const torch::Tensor& weights)
         269              :     {
         270              :         //
         271              :         // Logging setup
         272              :         //
         273           92 :         loguru::set_thread_name("BoostAODE");
         274           92 :         loguru::g_stderr_verbosity = loguru::Verbosity_OFF;
         275           92 :         loguru::add_file("boostAODE.log", loguru::Truncate, loguru::Verbosity_MAX);
         276              :
         277              :         // Algorithm based on the adaboost algorithm for classification
         278              :         // as explained in Ensemble methods (Zhi-Hua Zhou, 2012)
         279           92 :         fitted = true;
         280           92 :         double alpha_t = 0;
         281           92 :         torch::Tensor weights_ = torch::full({ m }, 1.0 / m, torch::kFloat64);
         282           92 :         bool finished = false;
         283           92 :         std::vector<int> featuresUsed;
         284           92 :         if (selectFeatures) {
         285           32 :             featuresUsed = initializeModels();
         286           16 :             auto ypred = predict(X_train);
         287           16 :             std::tie(weights_, alpha_t, finished) = update_weights(y_train, ypred, weights_);
         288              :             // Update significance of the models
         289          100 :             for (int i = 0; i < n_models; ++i) {
         290           84 :                 significanceModels[i] = alpha_t;
         291              :             }
         292           16 :             if (finished) {
         293            0 :                 return;
         294              :             }
         295           16 :         }
         296           76 :         int numItemsPack = 0; // The counter of the models inserted in the current pack
         297              :         // Variables to control the accuracy finish condition
         298           76 :         double priorAccuracy = 0.0;
         299           76 :         double improvement = 1.0;
         300           76 :         double convergence_threshold = 1e-4;
         301           76 :         int tolerance = 0; // number of times the accuracy is lower than the convergence_threshold
         302              :         // Step 0: Set the finish condition
         303              :         // epsilon sub t > 0.5 => inverse the weights policy
         304              :         // validation error is not decreasing
         305              :         // run out of features
         306           76 :         bool ascending = order_algorithm == Orders.ASC;
         307           76 :         std::mt19937 g{ 173 };
         308          504 :         while (!finished) {
         309              :             // Step 1: Build ranking with mutual information
         310          428 :             auto featureSelection = metrics.SelectKBestWeighted(weights_, ascending, n); // Get all the features sorted
         311          428 :             if (order_algorithm == Orders.RAND) {
         312           36 :                 std::shuffle(featureSelection.begin(), featureSelection.end(), g);
         313              :             }
         314              :             // Remove used features
         315          856 :             featureSelection.erase(remove_if(begin(featureSelection), end(featureSelection), [&](auto x)
         316        38800 :                 { return std::find(begin(featuresUsed), end(featuresUsed), x) != end(featuresUsed);}),
         317          428 :                 end(featureSelection)
         318              :             );
         319          428 :             int k = bisection ? pow(2, tolerance) : 1;
         320          428 :             int counter = 0; // The model counter of the current pack
         321          428 :             VLOG_SCOPE_F(1, "counter=%d k=%d featureSelection.size: %zu", counter, k, featureSelection.size());
         322         1004 :             while (counter++ < k && featureSelection.size() > 0) {
         323          576 :                 auto feature = featureSelection[0];
         324          576 :                 featureSelection.erase(featureSelection.begin());
         325          576 :                 std::unique_ptr<Classifier> model;
         326          576 :                 model = std::make_unique<SPODE>(feature);
         327          576 :                 model->fit(dataset, features, className, states, weights_);
         328          576 :                 alpha_t = 0.0;
         329          576 :                 if (!block_update) {
         330          500 :                     auto ypred = model->predict(X_train);
         331              :                     // Step 3.1: Compute the classifier amout of say
         332          500 :                     std::tie(weights_, alpha_t, finished) = update_weights(y_train, ypred, weights_);
         333          500 :                 }
         334              :                 // Step 3.4: Store classifier and its accuracy to weigh its future vote
         335          576 :                 numItemsPack++;
         336          576 :                 featuresUsed.push_back(feature);
         337          576 :                 models.push_back(std::move(model));
         338          576 :                 significanceModels.push_back(alpha_t);
         339          576 :                 n_models++;
         340          576 :                 VLOG_SCOPE_F(2, "numItemsPack: %d n_models: %d featuresUsed: %zu", numItemsPack, n_models, featuresUsed.size());
         341          576 :             }
         342          428 :             if (block_update) {
         343           28 :                 std::tie(weights_, alpha_t, finished) = update_weights_block(k, y_train, weights_);
         344              :             }
         345          428 :             if (convergence && !finished) {
         346          296 :                 auto y_val_predict = predict(X_test);
         347          296 :                 double accuracy = (y_val_predict == y_test).sum().item<double>() / (double)y_test.size(0);
         348          296 :                 if (priorAccuracy == 0) {
         349           60 :                     priorAccuracy = accuracy;
         350              :                 } else {
         351          236 :                     improvement = accuracy - priorAccuracy;
         352              :                 }
         353          296 :                 if (improvement < convergence_threshold) {
         354          176 :                     VLOG_SCOPE_F(3, "  (improvement<threshold) tolerance: %d numItemsPack: %d improvement: %f prior: %f current: %f", tolerance, numItemsPack, improvement, priorAccuracy, accuracy);
         355          176 :                     tolerance++;
         356          176 :                 } else {
         357          120 :                     VLOG_SCOPE_F(3, "* (improvement>=threshold) Reset. tolerance: %d numItemsPack: %d improvement: %f prior: %f current: %f", tolerance, numItemsPack, improvement, priorAccuracy, accuracy);
         358          120 :                     tolerance = 0; // Reset the counter if the model performs better
         359          120 :                     numItemsPack = 0;
         360          120 :                 }
         361          296 :                 if (convergence_best) {
         362              :                     // Keep the best accuracy until now as the prior accuracy
         363           32 :                     priorAccuracy = std::max(accuracy, priorAccuracy);
         364              :                 } else {
         365              :                     // Keep the last accuray obtained as the prior accuracy
         366          264 :                     priorAccuracy = accuracy;
         367              :                 }
         368          296 :             }
         369          428 :             VLOG_SCOPE_F(1, "tolerance: %d featuresUsed.size: %zu features.size: %zu", tolerance, featuresUsed.size(), features.size());
         370          428 :             finished = finished || tolerance > maxTolerance || featuresUsed.size() == features.size();
         371          428 :         }
         372           76 :         if (tolerance > maxTolerance) {
         373            8 :             if (numItemsPack < n_models) {
         374            8 :                 notes.push_back("Convergence threshold reached & " + std::to_string(numItemsPack) + " models eliminated");
         375            8 :                 VLOG_SCOPE_F(4, "Convergence threshold reached & %d models eliminated of %d", numItemsPack, n_models);
         376          104 :                 for (int i = 0; i < numItemsPack; ++i) {
         377           96 :                     significanceModels.pop_back();
         378           96 :                     models.pop_back();
         379           96 :                     n_models--;
         380              :                 }
         381            8 :             } else {
         382            0 :                 notes.push_back("Convergence threshold reached & 0 models eliminated");
         383            0 :                 VLOG_SCOPE_F(4, "Convergence threshold reached & 0 models eliminated n_models=%d numItemsPack=%d", n_models, numItemsPack);
         384            0 :             }
         385              :         }
         386           76 :         if (featuresUsed.size() != features.size()) {
         387            4 :             notes.push_back("Used features in train: " + std::to_string(featuresUsed.size()) + " of " + std::to_string(features.size()));
         388            4 :             status = WARNING;
         389              :         }
         390           76 :         notes.push_back("Number of models: " + std::to_string(n_models));
         391          108 :     }
         392            4 :     std::vector<std::string> BoostAODE::graph(const std::string& title) const
         393              :     {
         394            4 :         return Ensemble::graph(title);
         395              :     }
         396              : }

Generated by: LCOV version 2.0-1