59 KiB
59 KiB
<html lang="en">
<head>
</head>
</html>
| LCOV - code coverage report | ||||||||||||||||||||||
 | ||||||||||||||||||||||
|
||||||||||||||||||||||
|
Line data Source code 1 : // *************************************************************** 2 : // SPDX-FileCopyrightText: Copyright 2024 Ricardo Montañana Gómez 3 : // SPDX-FileType: SOURCE 4 : // SPDX-License-Identifier: MIT 5 : // *************************************************************** 6 : 7 : #include <set> 8 : #include <functional> 9 : #include <limits.h> 10 : #include <tuple> 11 : #include <folding.hpp> 12 : #include "bayesnet/feature_selection/CFS.h" 13 : #include "bayesnet/feature_selection/FCBF.h" 14 : #include "bayesnet/feature_selection/IWSS.h" 15 : #include "BoostAODE.h" 16 : #include "lib/log/loguru.cpp" 17 : 18 : namespace bayesnet { 19 : 20 345 : BoostAODE::BoostAODE(bool predict_voting) : Ensemble(predict_voting) 21 : { 22 3795 : validHyperparameters = { 23 : "maxModels", "bisection", "order", "convergence", "convergence_best", "threshold", 24 : "select_features", "maxTolerance", "predict_voting", "block_update" 25 3795 : }; 26 : 27 1035 : } 28 190 : void BoostAODE::buildModel(const torch::Tensor& weights) 29 : { 30 : // Models shall be built in trainModel 31 190 : models.clear(); 32 190 : significanceModels.clear(); 33 190 : n_models = 0; 34 : // Prepare the validation dataset 35 570 : auto y_ = dataset.index({ -1, "..." }); 36 190 : if (convergence) { 37 : // Prepare train & validation sets from train data 38 155 : auto fold = folding::StratifiedKFold(5, y_, 271); 39 155 : auto [train, test] = fold.getFold(0); 40 155 : auto train_t = torch::tensor(train); 41 155 : auto test_t = torch::tensor(test); 42 : // Get train and validation sets 43 775 : X_train = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), train_t }); 44 465 : y_train = dataset.index({ -1, train_t }); 45 775 : X_test = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), test_t }); 46 465 : y_test = dataset.index({ -1, test_t }); 47 155 : dataset = X_train; 48 155 : m = X_train.size(1); 49 155 : auto n_classes = states.at(className).size(); 50 : // Build dataset with train data 51 155 : buildDataset(y_train); 52 155 : metrics = Metrics(dataset, features, className, n_classes); 53 155 : } else { 54 : // Use all data to train 55 140 : X_train = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), "..." }); 56 35 : y_train = y_; 57 : } 58 1845 : } 59 187 : void BoostAODE::setHyperparameters(const nlohmann::json& hyperparameters_) 60 : { 61 187 : auto hyperparameters = hyperparameters_; 62 187 : if (hyperparameters.contains("order")) { 63 250 : std::vector<std::string> algos = { Orders.ASC, Orders.DESC, Orders.RAND }; 64 50 : order_algorithm = hyperparameters["order"]; 65 50 : if (std::find(algos.begin(), algos.end(), order_algorithm) == algos.end()) { 66 9 : throw std::invalid_argument("Invalid order algorithm, valid values [" + Orders.ASC + ", " + Orders.DESC + ", " + Orders.RAND + "]"); 67 : } 68 41 : hyperparameters.erase("order"); 69 50 : } 70 178 : if (hyperparameters.contains("convergence")) { 71 70 : convergence = hyperparameters["convergence"]; 72 70 : hyperparameters.erase("convergence"); 73 : } 74 178 : if (hyperparameters.contains("convergence_best")) { 75 18 : convergence_best = hyperparameters["convergence_best"]; 76 18 : hyperparameters.erase("convergence_best"); 77 : } 78 178 : if (hyperparameters.contains("bisection")) { 79 56 : bisection = hyperparameters["bisection"]; 80 56 : hyperparameters.erase("bisection"); 81 : } 82 178 : if (hyperparameters.contains("threshold")) { 83 56 : threshold = hyperparameters["threshold"]; 84 56 : hyperparameters.erase("threshold"); 85 : } 86 178 : if (hyperparameters.contains("maxTolerance")) { 87 83 : maxTolerance = hyperparameters["maxTolerance"]; 88 83 : if (maxTolerance < 1 || maxTolerance > 4) 89 27 : throw std::invalid_argument("Invalid maxTolerance value, must be greater in [1, 4]"); 90 56 : hyperparameters.erase("maxTolerance"); 91 : } 92 151 : if (hyperparameters.contains("predict_voting")) { 93 9 : predict_voting = hyperparameters["predict_voting"]; 94 9 : hyperparameters.erase("predict_voting"); 95 : } 96 151 : if (hyperparameters.contains("select_features")) { 97 85 : auto selectedAlgorithm = hyperparameters["select_features"]; 98 425 : std::vector<std::string> algos = { SelectFeatures.IWSS, SelectFeatures.CFS, SelectFeatures.FCBF }; 99 85 : selectFeatures = true; 100 85 : select_features_algorithm = selectedAlgorithm; 101 85 : if (std::find(algos.begin(), algos.end(), selectedAlgorithm) == algos.end()) { 102 9 : throw std::invalid_argument("Invalid selectFeatures value, valid values [" + SelectFeatures.IWSS + ", " + SelectFeatures.CFS + ", " + SelectFeatures.FCBF + "]"); 103 : } 104 76 : hyperparameters.erase("select_features"); 105 94 : } 106 142 : if (hyperparameters.contains("block_update")) { 107 16 : block_update = hyperparameters["block_update"]; 108 16 : hyperparameters.erase("block_update"); 109 : } 110 142 : Classifier::setHyperparameters(hyperparameters); 111 322 : } 112 1025 : std::tuple<torch::Tensor&, double, bool> update_weights(torch::Tensor& ytrain, torch::Tensor& ypred, torch::Tensor& weights) 113 : { 114 1025 : bool terminate = false; 115 1025 : double alpha_t = 0; 116 1025 : auto mask_wrong = ypred != ytrain; 117 1025 : auto mask_right = ypred == ytrain; 118 1025 : auto masked_weights = weights * mask_wrong.to(weights.dtype()); 119 1025 : double epsilon_t = masked_weights.sum().item<double>(); 120 1025 : if (epsilon_t > 0.5) { 121 : // Inverse the weights policy (plot ln(wt)) 122 : // "In each round of AdaBoost, there is a sanity check to ensure that the current base 123 : // learner is better than random guess" (Zhi-Hua Zhou, 2012) 124 34 : terminate = true; 125 : } else { 126 991 : double wt = (1 - epsilon_t) / epsilon_t; 127 991 : alpha_t = epsilon_t == 0 ? 1 : 0.5 * log(wt); 128 : // Step 3.2: Update weights for next classifier 129 : // Step 3.2.1: Update weights of wrong samples 130 991 : weights += mask_wrong.to(weights.dtype()) * exp(alpha_t) * weights; 131 : // Step 3.2.2: Update weights of right samples 132 991 : weights += mask_right.to(weights.dtype()) * exp(-alpha_t) * weights; 133 : // Step 3.3: Normalise the weights 134 991 : double totalWeights = torch::sum(weights).item<double>(); 135 991 : weights = weights / totalWeights; 136 : } 137 2050 : return { weights, alpha_t, terminate }; 138 1025 : } 139 40 : std::tuple<torch::Tensor&, double, bool> BoostAODE::update_weights_block(int k, torch::Tensor& ytrain, torch::Tensor& weights) 140 : { 141 : /* Update Block algorithm 142 : k = # of models in block 143 : n_models = # of models in ensemble to make predictions 144 : n_models_bak = # models saved 145 : models = vector of models to make predictions 146 : models_bak = models not used to make predictions 147 : significances_bak = backup of significances vector 148 : 149 : Case list 150 : A) k = 1, n_models = 1 => n = 0 , n_models = n + k 151 : B) k = 1, n_models = n + 1 => n_models = n + k 152 : C) k > 1, n_models = k + 1 => n= 1, n_models = n + k 153 : D) k > 1, n_models = k => n = 0, n_models = n + k 154 : E) k > 1, n_models = k + n => n_models = n + k 155 : 156 : A, D) n=0, k > 0, n_models == k 157 : 1. n_models_bak <- n_models 158 : 2. significances_bak <- significances 159 : 3. significances = vector(k, 1) 160 : 4. Don’t move any classifiers out of models 161 : 5. n_models <- k 162 : 6. Make prediction, compute alpha, update weights 163 : 7. Don’t restore any classifiers to models 164 : 8. significances <- significances_bak 165 : 9. Update last k significances 166 : 10. n_models <- n_models_bak 167 : 168 : B, C, E) n > 0, k > 0, n_models == n + k 169 : 1. n_models_bak <- n_models 170 : 2. significances_bak <- significances 171 : 3. significances = vector(k, 1) 172 : 4. Move first n classifiers to models_bak 173 : 5. n_models <- k 174 : 6. Make prediction, compute alpha, update weights 175 : 7. Insert classifiers in models_bak to be the first n models 176 : 8. significances <- significances_bak 177 : 9. Update last k significances 178 : 10. n_models <- n_models_bak 179 : */ 180 : // 181 : // Make predict with only the last k models 182 : // 183 40 : std::unique_ptr<Classifier> model; 184 40 : std::vector<std::unique_ptr<Classifier>> models_bak; 185 : // 1. n_models_bak <- n_models 2. significances_bak <- significances 186 40 : auto significance_bak = significanceModels; 187 40 : auto n_models_bak = n_models; 188 : // 3. significances = vector(k, 1) 189 40 : significanceModels = std::vector<double>(k, 1.0); 190 : // 4. Move first n classifiers to models_bak 191 : // backup the first n_models - k models (if n_models == k, don't backup any) 192 160 : for (int i = 0; i < n_models - k; ++i) { 193 120 : model = std::move(models[0]); 194 120 : models.erase(models.begin()); 195 120 : models_bak.push_back(std::move(model)); 196 : } 197 40 : assert(models.size() == k); 198 : // 5. n_models <- k 199 40 : n_models = k; 200 : // 6. Make prediction, compute alpha, update weights 201 40 : auto ypred = predict(X_train); 202 : // 203 : // Update weights 204 : // 205 : double alpha_t; 206 : bool terminate; 207 40 : std::tie(weights, alpha_t, terminate) = update_weights(y_train, ypred, weights); 208 : // 209 : // Restore the models if needed 210 : // 211 : // 7. Insert classifiers in models_bak to be the first n models 212 : // if n_models_bak == k, don't restore any, because none of them were moved 213 40 : if (k != n_models_bak) { 214 : // Insert in the same order as they were extracted 215 32 : int bak_size = models_bak.size(); 216 152 : for (int i = 0; i < bak_size; ++i) { 217 120 : model = std::move(models_bak[bak_size - 1 - i]); 218 120 : models_bak.erase(models_bak.end() - 1); 219 120 : models.insert(models.begin(), std::move(model)); 220 : } 221 : } 222 : // 8. significances <- significances_bak 223 40 : significanceModels = significance_bak; 224 : // 225 : // Update the significance of the last k models 226 : // 227 : // 9. Update last k significances 228 168 : for (int i = 0; i < k; ++i) { 229 128 : significanceModels[n_models_bak - k + i] = alpha_t; 230 : } 231 : // 10. n_models <- n_models_bak 232 40 : n_models = n_models_bak; 233 80 : return { weights, alpha_t, terminate }; 234 40 : } 235 76 : std::vector<int> BoostAODE::initializeModels() 236 : { 237 76 : std::vector<int> featuresUsed; 238 76 : torch::Tensor weights_ = torch::full({ m }, 1.0 / m, torch::kFloat64); 239 76 : int maxFeatures = 0; 240 76 : if (select_features_algorithm == SelectFeatures.CFS) { 241 20 : featureSelector = new CFS(dataset, features, className, maxFeatures, states.at(className).size(), weights_); 242 56 : } else if (select_features_algorithm == SelectFeatures.IWSS) { 243 29 : if (threshold < 0 || threshold >0.5) { 244 18 : throw std::invalid_argument("Invalid threshold value for " + SelectFeatures.IWSS + " [0, 0.5]"); 245 : } 246 11 : featureSelector = new IWSS(dataset, features, className, maxFeatures, states.at(className).size(), weights_, threshold); 247 27 : } else if (select_features_algorithm == SelectFeatures.FCBF) { 248 27 : if (threshold < 1e-7 || threshold > 1) { 249 18 : throw std::invalid_argument("Invalid threshold value for " + SelectFeatures.FCBF + " [1e-7, 1]"); 250 : } 251 9 : featureSelector = new FCBF(dataset, features, className, maxFeatures, states.at(className).size(), weights_, threshold); 252 : } 253 40 : featureSelector->fit(); 254 40 : auto cfsFeatures = featureSelector->getFeatures(); 255 40 : auto scores = featureSelector->getScores(); 256 245 : for (const int& feature : cfsFeatures) { 257 205 : featuresUsed.push_back(feature); 258 205 : std::unique_ptr<Classifier> model = std::make_unique<SPODE>(feature); 259 205 : model->fit(dataset, features, className, states, weights_); 260 205 : models.push_back(std::move(model)); 261 205 : significanceModels.push_back(1.0); // They will be updated later in trainModel 262 205 : n_models++; 263 205 : } 264 40 : notes.push_back("Used features in initialization: " + std::to_string(featuresUsed.size()) + " of " + std::to_string(features.size()) + " with " + select_features_algorithm); 265 40 : delete featureSelector; 266 80 : return featuresUsed; 267 112 : } 268 190 : void BoostAODE::trainModel(const torch::Tensor& weights) 269 : { 270 : // 271 : // Logging setup 272 : // 273 190 : loguru::set_thread_name("BoostAODE"); 274 190 : loguru::g_stderr_verbosity = loguru::Verbosity_OFF; 275 190 : loguru::add_file("boostAODE.log", loguru::Truncate, loguru::Verbosity_MAX); 276 : 277 : // Algorithm based on the adaboost algorithm for classification 278 : // as explained in Ensemble methods (Zhi-Hua Zhou, 2012) 279 190 : fitted = true; 280 190 : double alpha_t = 0; 281 190 : torch::Tensor weights_ = torch::full({ m }, 1.0 / m, torch::kFloat64); 282 190 : bool finished = false; 283 190 : std::vector<int> featuresUsed; 284 190 : if (selectFeatures) { 285 76 : featuresUsed = initializeModels(); 286 40 : auto ypred = predict(X_train); 287 40 : std::tie(weights_, alpha_t, finished) = update_weights(y_train, ypred, weights_); 288 : // Update significance of the models 289 245 : for (int i = 0; i < n_models; ++i) { 290 205 : significanceModels[i] = alpha_t; 291 : } 292 40 : if (finished) { 293 0 : return; 294 : } 295 40 : } 296 154 : int numItemsPack = 0; // The counter of the models inserted in the current pack 297 : // Variables to control the accuracy finish condition 298 154 : double priorAccuracy = 0.0; 299 154 : double improvement = 1.0; 300 154 : double convergence_threshold = 1e-4; 301 154 : int tolerance = 0; // number of times the accuracy is lower than the convergence_threshold 302 : // Step 0: Set the finish condition 303 : // epsilon sub t > 0.5 => inverse the weights policy 304 : // validation error is not decreasing 305 : // run out of features 306 154 : bool ascending = order_algorithm == Orders.ASC; 307 154 : std::mt19937 g{ 173 }; 308 919 : while (!finished) { 309 : // Step 1: Build ranking with mutual information 310 765 : auto featureSelection = metrics.SelectKBestWeighted(weights_, ascending, n); // Get all the features sorted 311 765 : if (order_algorithm == Orders.RAND) { 312 81 : std::shuffle(featureSelection.begin(), featureSelection.end(), g); 313 : } 314 : // Remove used features 315 1530 : featureSelection.erase(remove_if(begin(featureSelection), end(featureSelection), [&](auto x) 316 110548 : { return std::find(begin(featuresUsed), end(featuresUsed), x) != end(featuresUsed);}), 317 765 : end(featureSelection) 318 : ); 319 765 : int k = bisection ? pow(2, tolerance) : 1; 320 765 : int counter = 0; // The model counter of the current pack 321 765 : VLOG_SCOPE_F(1, "counter=%d k=%d featureSelection.size: %zu", counter, k, featureSelection.size()); 322 1838 : while (counter++ < k && featureSelection.size() > 0) { 323 1073 : auto feature = featureSelection[0]; 324 1073 : featureSelection.erase(featureSelection.begin()); 325 1073 : std::unique_ptr<Classifier> model; 326 1073 : model = std::make_unique<SPODE>(feature); 327 1073 : model->fit(dataset, features, className, states, weights_); 328 1073 : alpha_t = 0.0; 329 1073 : if (!block_update) { 330 945 : auto ypred = model->predict(X_train); 331 : // Step 3.1: Compute the classifier amout of say 332 945 : std::tie(weights_, alpha_t, finished) = update_weights(y_train, ypred, weights_); 333 945 : } 334 : // Step 3.4: Store classifier and its accuracy to weigh its future vote 335 1073 : numItemsPack++; 336 1073 : featuresUsed.push_back(feature); 337 1073 : models.push_back(std::move(model)); 338 1073 : significanceModels.push_back(alpha_t); 339 1073 : n_models++; 340 1073 : VLOG_SCOPE_F(2, "numItemsPack: %d n_models: %d featuresUsed: %zu", numItemsPack, n_models, featuresUsed.size()); 341 1073 : } 342 765 : if (block_update) { 343 40 : std::tie(weights_, alpha_t, finished) = update_weights_block(k, y_train, weights_); 344 : } 345 765 : if (convergence && !finished) { 346 474 : auto y_val_predict = predict(X_test); 347 474 : double accuracy = (y_val_predict == y_test).sum().item<double>() / (double)y_test.size(0); 348 474 : if (priorAccuracy == 0) { 349 119 : priorAccuracy = accuracy; 350 : } else { 351 355 : improvement = accuracy - priorAccuracy; 352 : } 353 474 : if (improvement < convergence_threshold) { 354 296 : VLOG_SCOPE_F(3, " (improvement<threshold) tolerance: %d numItemsPack: %d improvement: %f prior: %f current: %f", tolerance, numItemsPack, improvement, priorAccuracy, accuracy); 355 296 : tolerance++; 356 296 : } else { 357 178 : VLOG_SCOPE_F(3, "* (improvement>=threshold) Reset. tolerance: %d numItemsPack: %d improvement: %f prior: %f current: %f", tolerance, numItemsPack, improvement, priorAccuracy, accuracy); 358 178 : tolerance = 0; // Reset the counter if the model performs better 359 178 : numItemsPack = 0; 360 178 : } 361 474 : if (convergence_best) { 362 : // Keep the best accuracy until now as the prior accuracy 363 71 : priorAccuracy = std::max(accuracy, priorAccuracy); 364 : } else { 365 : // Keep the last accuray obtained as the prior accuracy 366 403 : priorAccuracy = accuracy; 367 : } 368 474 : } 369 765 : VLOG_SCOPE_F(1, "tolerance: %d featuresUsed.size: %zu features.size: %zu", tolerance, featuresUsed.size(), features.size()); 370 765 : finished = finished || tolerance > maxTolerance || featuresUsed.size() == features.size(); 371 765 : } 372 154 : if (tolerance > maxTolerance) { 373 21 : if (numItemsPack < n_models) { 374 21 : notes.push_back("Convergence threshold reached & " + std::to_string(numItemsPack) + " models eliminated"); 375 21 : VLOG_SCOPE_F(4, "Convergence threshold reached & %d models eliminated of %d", numItemsPack, n_models); 376 336 : for (int i = 0; i < numItemsPack; ++i) { 377 315 : significanceModels.pop_back(); 378 315 : models.pop_back(); 379 315 : n_models--; 380 : } 381 21 : } else { 382 0 : notes.push_back("Convergence threshold reached & 0 models eliminated"); 383 0 : VLOG_SCOPE_F(4, "Convergence threshold reached & 0 models eliminated n_models=%d numItemsPack=%d", n_models, numItemsPack); 384 0 : } 385 : } 386 154 : if (featuresUsed.size() != features.size()) { 387 21 : notes.push_back("Used features in train: " + std::to_string(featuresUsed.size()) + " of " + std::to_string(features.size())); 388 21 : status = WARNING; 389 : } 390 154 : notes.push_back("Number of models: " + std::to_string(n_models)); 391 226 : } 392 8 : std::vector<std::string> BoostAODE::graph(const std::string& title) const 393 : { 394 8 : return Ensemble::graph(title); 395 : } 396 : } |
|
| Generated by: LCOV version 2.0-1 |
</html>