Line data Source code
1 : // ***************************************************************
2 : // SPDX-FileCopyrightText: Copyright 2024 Ricardo Montañana Gómez
3 : // SPDX-FileType: SOURCE
4 : // SPDX-License-Identifier: MIT
5 : // ***************************************************************
6 :
7 : #include <set>
8 : #include <functional>
9 : #include <limits.h>
10 : #include <tuple>
11 : #include <folding.hpp>
12 : #include "bayesnet/feature_selection/CFS.h"
13 : #include "bayesnet/feature_selection/FCBF.h"
14 : #include "bayesnet/feature_selection/IWSS.h"
15 : #include "BoostAODE.h"
16 : #include "lib/log/loguru.cpp"
17 :
18 : namespace bayesnet {
19 :
20 252 : BoostAODE::BoostAODE(bool predict_voting) : Ensemble(predict_voting)
21 : {
22 2772 : validHyperparameters = {
23 : "maxModels", "bisection", "order", "convergence", "convergence_best", "threshold",
24 : "select_features", "maxTolerance", "predict_voting", "block_update"
25 2772 : };
26 :
27 756 : }
28 138 : void BoostAODE::buildModel(const torch::Tensor& weights)
29 : {
30 : // Models shall be built in trainModel
31 138 : models.clear();
32 138 : significanceModels.clear();
33 138 : n_models = 0;
34 : // Prepare the validation dataset
35 414 : auto y_ = dataset.index({ -1, "..." });
36 138 : if (convergence) {
37 : // Prepare train & validation sets from train data
38 114 : auto fold = folding::StratifiedKFold(5, y_, 271);
39 114 : auto [train, test] = fold.getFold(0);
40 114 : auto train_t = torch::tensor(train);
41 114 : auto test_t = torch::tensor(test);
42 : // Get train and validation sets
43 570 : X_train = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), train_t });
44 342 : y_train = dataset.index({ -1, train_t });
45 570 : X_test = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), test_t });
46 342 : y_test = dataset.index({ -1, test_t });
47 114 : dataset = X_train;
48 114 : m = X_train.size(1);
49 114 : auto n_classes = states.at(className).size();
50 : // Build dataset with train data
51 114 : buildDataset(y_train);
52 114 : metrics = Metrics(dataset, features, className, n_classes);
53 114 : } else {
54 : // Use all data to train
55 96 : X_train = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), "..." });
56 24 : y_train = y_;
57 : }
58 1350 : }
59 132 : void BoostAODE::setHyperparameters(const nlohmann::json& hyperparameters_)
60 : {
61 132 : auto hyperparameters = hyperparameters_;
62 132 : if (hyperparameters.contains("order")) {
63 150 : std::vector<std::string> algos = { Orders.ASC, Orders.DESC, Orders.RAND };
64 30 : order_algorithm = hyperparameters["order"];
65 30 : if (std::find(algos.begin(), algos.end(), order_algorithm) == algos.end()) {
66 6 : throw std::invalid_argument("Invalid order algorithm, valid values [" + Orders.ASC + ", " + Orders.DESC + ", " + Orders.RAND + "]");
67 : }
68 24 : hyperparameters.erase("order");
69 30 : }
70 126 : if (hyperparameters.contains("convergence")) {
71 54 : convergence = hyperparameters["convergence"];
72 54 : hyperparameters.erase("convergence");
73 : }
74 126 : if (hyperparameters.contains("convergence_best")) {
75 18 : convergence_best = hyperparameters["convergence_best"];
76 18 : hyperparameters.erase("convergence_best");
77 : }
78 126 : if (hyperparameters.contains("bisection")) {
79 48 : bisection = hyperparameters["bisection"];
80 48 : hyperparameters.erase("bisection");
81 : }
82 126 : if (hyperparameters.contains("threshold")) {
83 36 : threshold = hyperparameters["threshold"];
84 36 : hyperparameters.erase("threshold");
85 : }
86 126 : if (hyperparameters.contains("maxTolerance")) {
87 66 : maxTolerance = hyperparameters["maxTolerance"];
88 66 : if (maxTolerance < 1 || maxTolerance > 4)
89 18 : throw std::invalid_argument("Invalid maxTolerance value, must be greater in [1, 4]");
90 48 : hyperparameters.erase("maxTolerance");
91 : }
92 108 : if (hyperparameters.contains("predict_voting")) {
93 6 : predict_voting = hyperparameters["predict_voting"];
94 6 : hyperparameters.erase("predict_voting");
95 : }
96 108 : if (hyperparameters.contains("select_features")) {
97 54 : auto selectedAlgorithm = hyperparameters["select_features"];
98 270 : std::vector<std::string> algos = { SelectFeatures.IWSS, SelectFeatures.CFS, SelectFeatures.FCBF };
99 54 : selectFeatures = true;
100 54 : select_features_algorithm = selectedAlgorithm;
101 54 : if (std::find(algos.begin(), algos.end(), selectedAlgorithm) == algos.end()) {
102 6 : throw std::invalid_argument("Invalid selectFeatures value, valid values [" + SelectFeatures.IWSS + ", " + SelectFeatures.CFS + ", " + SelectFeatures.FCBF + "]");
103 : }
104 48 : hyperparameters.erase("select_features");
105 60 : }
106 102 : if (hyperparameters.contains("block_update")) {
107 12 : block_update = hyperparameters["block_update"];
108 12 : hyperparameters.erase("block_update");
109 : }
110 102 : Classifier::setHyperparameters(hyperparameters);
111 216 : }
112 816 : std::tuple<torch::Tensor&, double, bool> update_weights(torch::Tensor& ytrain, torch::Tensor& ypred, torch::Tensor& weights)
113 : {
114 816 : bool terminate = false;
115 816 : double alpha_t = 0;
116 816 : auto mask_wrong = ypred != ytrain;
117 816 : auto mask_right = ypred == ytrain;
118 816 : auto masked_weights = weights * mask_wrong.to(weights.dtype());
119 816 : double epsilon_t = masked_weights.sum().item<double>();
120 816 : if (epsilon_t > 0.5) {
121 : // Inverse the weights policy (plot ln(wt))
122 : // "In each round of AdaBoost, there is a sanity check to ensure that the current base
123 : // learner is better than random guess" (Zhi-Hua Zhou, 2012)
124 24 : terminate = true;
125 : } else {
126 792 : double wt = (1 - epsilon_t) / epsilon_t;
127 792 : alpha_t = epsilon_t == 0 ? 1 : 0.5 * log(wt);
128 : // Step 3.2: Update weights for next classifier
129 : // Step 3.2.1: Update weights of wrong samples
130 792 : weights += mask_wrong.to(weights.dtype()) * exp(alpha_t) * weights;
131 : // Step 3.2.2: Update weights of right samples
132 792 : weights += mask_right.to(weights.dtype()) * exp(-alpha_t) * weights;
133 : // Step 3.3: Normalise the weights
134 792 : double totalWeights = torch::sum(weights).item<double>();
135 792 : weights = weights / totalWeights;
136 : }
137 1632 : return { weights, alpha_t, terminate };
138 816 : }
139 42 : std::tuple<torch::Tensor&, double, bool> BoostAODE::update_weights_block(int k, torch::Tensor& ytrain, torch::Tensor& weights)
140 : {
141 : /* Update Block algorithm
142 : k = # of models in block
143 : n_models = # of models in ensemble to make predictions
144 : n_models_bak = # models saved
145 : models = vector of models to make predictions
146 : models_bak = models not used to make predictions
147 : significances_bak = backup of significances vector
148 :
149 : Case list
150 : A) k = 1, n_models = 1 => n = 0 , n_models = n + k
151 : B) k = 1, n_models = n + 1 => n_models = n + k
152 : C) k > 1, n_models = k + 1 => n= 1, n_models = n + k
153 : D) k > 1, n_models = k => n = 0, n_models = n + k
154 : E) k > 1, n_models = k + n => n_models = n + k
155 :
156 : A, D) n=0, k > 0, n_models == k
157 : 1. n_models_bak <- n_models
158 : 2. significances_bak <- significances
159 : 3. significances = vector(k, 1)
160 : 4. Don’t move any classifiers out of models
161 : 5. n_models <- k
162 : 6. Make prediction, compute alpha, update weights
163 : 7. Don’t restore any classifiers to models
164 : 8. significances <- significances_bak
165 : 9. Update last k significances
166 : 10. n_models <- n_models_bak
167 :
168 : B, C, E) n > 0, k > 0, n_models == n + k
169 : 1. n_models_bak <- n_models
170 : 2. significances_bak <- significances
171 : 3. significances = vector(k, 1)
172 : 4. Move first n classifiers to models_bak
173 : 5. n_models <- k
174 : 6. Make prediction, compute alpha, update weights
175 : 7. Insert classifiers in models_bak to be the first n models
176 : 8. significances <- significances_bak
177 : 9. Update last k significances
178 : 10. n_models <- n_models_bak
179 : */
180 : //
181 : // Make predict with only the last k models
182 : //
183 42 : std::unique_ptr<Classifier> model;
184 42 : std::vector<std::unique_ptr<Classifier>> models_bak;
185 : // 1. n_models_bak <- n_models 2. significances_bak <- significances
186 42 : auto significance_bak = significanceModels;
187 42 : auto n_models_bak = n_models;
188 : // 3. significances = vector(k, 1)
189 42 : significanceModels = std::vector<double>(k, 1.0);
190 : // 4. Move first n classifiers to models_bak
191 : // backup the first n_models - k models (if n_models == k, don't backup any)
192 222 : for (int i = 0; i < n_models - k; ++i) {
193 180 : model = std::move(models[0]);
194 180 : models.erase(models.begin());
195 180 : models_bak.push_back(std::move(model));
196 : }
197 42 : assert(models.size() == k);
198 : // 5. n_models <- k
199 42 : n_models = k;
200 : // 6. Make prediction, compute alpha, update weights
201 42 : auto ypred = predict(X_train);
202 : //
203 : // Update weights
204 : //
205 : double alpha_t;
206 : bool terminate;
207 42 : std::tie(weights, alpha_t, terminate) = update_weights(y_train, ypred, weights);
208 : //
209 : // Restore the models if needed
210 : //
211 : // 7. Insert classifiers in models_bak to be the first n models
212 : // if n_models_bak == k, don't restore any, because none of them were moved
213 42 : if (k != n_models_bak) {
214 : // Insert in the same order as they were extracted
215 36 : int bak_size = models_bak.size();
216 216 : for (int i = 0; i < bak_size; ++i) {
217 180 : model = std::move(models_bak[bak_size - 1 - i]);
218 180 : models_bak.erase(models_bak.end() - 1);
219 180 : models.insert(models.begin(), std::move(model));
220 : }
221 : }
222 : // 8. significances <- significances_bak
223 42 : significanceModels = significance_bak;
224 : //
225 : // Update the significance of the last k models
226 : //
227 : // 9. Update last k significances
228 156 : for (int i = 0; i < k; ++i) {
229 114 : significanceModels[n_models_bak - k + i] = alpha_t;
230 : }
231 : // 10. n_models <- n_models_bak
232 42 : n_models = n_models_bak;
233 84 : return { weights, alpha_t, terminate };
234 42 : }
235 48 : std::vector<int> BoostAODE::initializeModels()
236 : {
237 48 : std::vector<int> featuresUsed;
238 48 : torch::Tensor weights_ = torch::full({ m }, 1.0 / m, torch::kFloat64);
239 48 : int maxFeatures = 0;
240 48 : if (select_features_algorithm == SelectFeatures.CFS) {
241 12 : featureSelector = new CFS(dataset, features, className, maxFeatures, states.at(className).size(), weights_);
242 36 : } else if (select_features_algorithm == SelectFeatures.IWSS) {
243 18 : if (threshold < 0 || threshold >0.5) {
244 12 : throw std::invalid_argument("Invalid threshold value for " + SelectFeatures.IWSS + " [0, 0.5]");
245 : }
246 6 : featureSelector = new IWSS(dataset, features, className, maxFeatures, states.at(className).size(), weights_, threshold);
247 18 : } else if (select_features_algorithm == SelectFeatures.FCBF) {
248 18 : if (threshold < 1e-7 || threshold > 1) {
249 12 : throw std::invalid_argument("Invalid threshold value for " + SelectFeatures.FCBF + " [1e-7, 1]");
250 : }
251 6 : featureSelector = new FCBF(dataset, features, className, maxFeatures, states.at(className).size(), weights_, threshold);
252 : }
253 24 : featureSelector->fit();
254 24 : auto cfsFeatures = featureSelector->getFeatures();
255 24 : auto scores = featureSelector->getScores();
256 150 : for (const int& feature : cfsFeatures) {
257 126 : featuresUsed.push_back(feature);
258 126 : std::unique_ptr<Classifier> model = std::make_unique<SPODE>(feature);
259 126 : model->fit(dataset, features, className, states, weights_);
260 126 : models.push_back(std::move(model));
261 126 : significanceModels.push_back(1.0); // They will be updated later in trainModel
262 126 : n_models++;
263 126 : }
264 24 : notes.push_back("Used features in initialization: " + std::to_string(featuresUsed.size()) + " of " + std::to_string(features.size()) + " with " + select_features_algorithm);
265 24 : delete featureSelector;
266 48 : return featuresUsed;
267 72 : }
268 138 : void BoostAODE::trainModel(const torch::Tensor& weights)
269 : {
270 : //
271 : // Logging setup
272 : //
273 138 : loguru::set_thread_name("BoostAODE");
274 138 : loguru::g_stderr_verbosity = loguru::Verbosity_OFF;
275 138 : loguru::add_file("boostAODE.log", loguru::Truncate, loguru::Verbosity_MAX);
276 :
277 : // Algorithm based on the adaboost algorithm for classification
278 : // as explained in Ensemble methods (Zhi-Hua Zhou, 2012)
279 138 : fitted = true;
280 138 : double alpha_t = 0;
281 138 : torch::Tensor weights_ = torch::full({ m }, 1.0 / m, torch::kFloat64);
282 138 : bool finished = false;
283 138 : std::vector<int> featuresUsed;
284 138 : if (selectFeatures) {
285 48 : featuresUsed = initializeModels();
286 24 : auto ypred = predict(X_train);
287 24 : std::tie(weights_, alpha_t, finished) = update_weights(y_train, ypred, weights_);
288 : // Update significance of the models
289 150 : for (int i = 0; i < n_models; ++i) {
290 126 : significanceModels[i] = alpha_t;
291 : }
292 24 : if (finished) {
293 0 : return;
294 : }
295 24 : }
296 114 : int numItemsPack = 0; // The counter of the models inserted in the current pack
297 : // Variables to control the accuracy finish condition
298 114 : double priorAccuracy = 0.0;
299 114 : double improvement = 1.0;
300 114 : double convergence_threshold = 1e-4;
301 114 : int tolerance = 0; // number of times the accuracy is lower than the convergence_threshold
302 : // Step 0: Set the finish condition
303 : // epsilon sub t > 0.5 => inverse the weights policy
304 : // validation error is not decreasing
305 : // run out of features
306 114 : bool ascending = order_algorithm == Orders.ASC;
307 114 : std::mt19937 g{ 173 };
308 756 : while (!finished) {
309 : // Step 1: Build ranking with mutual information
310 642 : auto featureSelection = metrics.SelectKBestWeighted(weights_, ascending, n); // Get all the features sorted
311 642 : if (order_algorithm == Orders.RAND) {
312 54 : std::shuffle(featureSelection.begin(), featureSelection.end(), g);
313 : }
314 : // Remove used features
315 1284 : featureSelection.erase(remove_if(begin(featureSelection), end(featureSelection), [&](auto x)
316 58200 : { return std::find(begin(featuresUsed), end(featuresUsed), x) != end(featuresUsed);}),
317 642 : end(featureSelection)
318 : );
319 642 : int k = bisection ? pow(2, tolerance) : 1;
320 642 : int counter = 0; // The model counter of the current pack
321 642 : VLOG_SCOPE_F(1, "counter=%d k=%d featureSelection.size: %zu", counter, k, featureSelection.size());
322 1506 : while (counter++ < k && featureSelection.size() > 0) {
323 864 : auto feature = featureSelection[0];
324 864 : featureSelection.erase(featureSelection.begin());
325 864 : std::unique_ptr<Classifier> model;
326 864 : model = std::make_unique<SPODE>(feature);
327 864 : model->fit(dataset, features, className, states, weights_);
328 864 : alpha_t = 0.0;
329 864 : if (!block_update) {
330 750 : auto ypred = model->predict(X_train);
331 : // Step 3.1: Compute the classifier amout of say
332 750 : std::tie(weights_, alpha_t, finished) = update_weights(y_train, ypred, weights_);
333 750 : }
334 : // Step 3.4: Store classifier and its accuracy to weigh its future vote
335 864 : numItemsPack++;
336 864 : featuresUsed.push_back(feature);
337 864 : models.push_back(std::move(model));
338 864 : significanceModels.push_back(alpha_t);
339 864 : n_models++;
340 864 : VLOG_SCOPE_F(2, "numItemsPack: %d n_models: %d featuresUsed: %zu", numItemsPack, n_models, featuresUsed.size());
341 864 : }
342 642 : if (block_update) {
343 42 : std::tie(weights_, alpha_t, finished) = update_weights_block(k, y_train, weights_);
344 : }
345 642 : if (convergence && !finished) {
346 444 : auto y_val_predict = predict(X_test);
347 444 : double accuracy = (y_val_predict == y_test).sum().item<double>() / (double)y_test.size(0);
348 444 : if (priorAccuracy == 0) {
349 90 : priorAccuracy = accuracy;
350 : } else {
351 354 : improvement = accuracy - priorAccuracy;
352 : }
353 444 : if (improvement < convergence_threshold) {
354 264 : VLOG_SCOPE_F(3, " (improvement<threshold) tolerance: %d numItemsPack: %d improvement: %f prior: %f current: %f", tolerance, numItemsPack, improvement, priorAccuracy, accuracy);
355 264 : tolerance++;
356 264 : } else {
357 180 : VLOG_SCOPE_F(3, "* (improvement>=threshold) Reset. tolerance: %d numItemsPack: %d improvement: %f prior: %f current: %f", tolerance, numItemsPack, improvement, priorAccuracy, accuracy);
358 180 : tolerance = 0; // Reset the counter if the model performs better
359 180 : numItemsPack = 0;
360 180 : }
361 444 : if (convergence_best) {
362 : // Keep the best accuracy until now as the prior accuracy
363 48 : priorAccuracy = std::max(accuracy, priorAccuracy);
364 : } else {
365 : // Keep the last accuray obtained as the prior accuracy
366 396 : priorAccuracy = accuracy;
367 : }
368 444 : }
369 642 : VLOG_SCOPE_F(1, "tolerance: %d featuresUsed.size: %zu features.size: %zu", tolerance, featuresUsed.size(), features.size());
370 642 : finished = finished || tolerance > maxTolerance || featuresUsed.size() == features.size();
371 642 : }
372 114 : if (tolerance > maxTolerance) {
373 12 : if (numItemsPack < n_models) {
374 12 : notes.push_back("Convergence threshold reached & " + std::to_string(numItemsPack) + " models eliminated");
375 12 : VLOG_SCOPE_F(4, "Convergence threshold reached & %d models eliminated of %d", numItemsPack, n_models);
376 156 : for (int i = 0; i < numItemsPack; ++i) {
377 144 : significanceModels.pop_back();
378 144 : models.pop_back();
379 144 : n_models--;
380 : }
381 12 : } else {
382 0 : notes.push_back("Convergence threshold reached & 0 models eliminated");
383 0 : VLOG_SCOPE_F(4, "Convergence threshold reached & 0 models eliminated n_models=%d numItemsPack=%d", n_models, numItemsPack);
384 0 : }
385 : }
386 114 : if (featuresUsed.size() != features.size()) {
387 6 : notes.push_back("Used features in train: " + std::to_string(featuresUsed.size()) + " of " + std::to_string(features.size()));
388 6 : status = WARNING;
389 : }
390 114 : notes.push_back("Number of models: " + std::to_string(n_models));
391 162 : }
392 6 : std::vector<std::string> BoostAODE::graph(const std::string& title) const
393 : {
394 6 : return Ensemble::graph(title);
395 : }
396 : }
|
