diff --git a/bayesnet/ensembles/BoostA2DE.cc b/bayesnet/ensembles/BoostA2DE.cc index 7948923..887c1a2 100644 --- a/bayesnet/ensembles/BoostA2DE.cc +++ b/bayesnet/ensembles/BoostA2DE.cc @@ -50,101 +50,101 @@ namespace bayesnet { // loguru::g_stderr_verbosity = loguru::Verbosity_OFF; // loguru::add_file("boostA2DE.log", loguru::Truncate, loguru::Verbosity_MAX); - // // Algorithm based on the adaboost algorithm for classification - // // as explained in Ensemble methods (Zhi-Hua Zhou, 2012) - // fitted = true; - // double alpha_t = 0; - // torch::Tensor weights_ = torch::full({ m }, 1.0 / m, torch::kFloat64); - // bool finished = false; - // std::vector featuresUsed; - // if (selectFeatures) { - // featuresUsed = initializeModels(); - // auto ypred = predict(X_train); - // std::tie(weights_, alpha_t, finished) = update_weights(y_train, ypred, weights_); - // // Update significance of the models - // for (int i = 0; i < n_models; ++i) { - // significanceModels[i] = alpha_t; - // } - // if (finished) { - // return; - // } - // } - // int numItemsPack = 0; // The counter of the models inserted in the current pack - // // Variables to control the accuracy finish condition - // double priorAccuracy = 0.0; - // double improvement = 1.0; - // double convergence_threshold = 1e-4; - // int tolerance = 0; // number of times the accuracy is lower than the convergence_threshold - // // Step 0: Set the finish condition - // // epsilon sub t > 0.5 => inverse the weights policy - // // validation error is not decreasing - // // run out of features - // bool ascending = order_algorithm == Orders.ASC; - // std::mt19937 g{ 173 }; - // while (!finished) { - // // Step 1: Build ranking with mutual information - // auto pairSelection = metrics.SelectKBestWeighted(weights_, ascending, n); // Get all the features sorted - // if (order_algorithm == Orders.RAND) { - // std::shuffle(featureSelection.begin(), featureSelection.end(), g); - // } - // // Remove used features - // featureSelection.erase(remove_if(begin(featureSelection), end(featureSelection), [&](auto x) - // { return std::find(begin(featuresUsed), end(featuresUsed), x) != end(featuresUsed);}), - // end(featureSelection) - // ); - // int k = bisection ? pow(2, tolerance) : 1; - // int counter = 0; // The model counter of the current pack - // VLOG_SCOPE_F(1, "counter=%d k=%d featureSelection.size: %zu", counter, k, featureSelection.size()); - // while (counter++ < k && featureSelection.size() > 0) { - // auto feature = featureSelection[0]; - // featureSelection.erase(featureSelection.begin()); - // std::unique_ptr model; - // model = std::make_unique(feature); - // model->fit(dataset, features, className, states, weights_); - // alpha_t = 0.0; - // if (!block_update) { - // auto ypred = model->predict(X_train); - // // Step 3.1: Compute the classifier amout of say - // std::tie(weights_, alpha_t, finished) = update_weights(y_train, ypred, weights_); - // } - // // Step 3.4: Store classifier and its accuracy to weigh its future vote - // numItemsPack++; - // featuresUsed.push_back(feature); - // models.push_back(std::move(model)); - // significanceModels.push_back(alpha_t); - // n_models++; - // VLOG_SCOPE_F(2, "numItemsPack: %d n_models: %d featuresUsed: %zu", numItemsPack, n_models, featuresUsed.size()); - // } - // if (block_update) { - // std::tie(weights_, alpha_t, finished) = update_weights_block(k, y_train, weights_); - // } - // if (convergence && !finished) { - // auto y_val_predict = predict(X_test); - // double accuracy = (y_val_predict == y_test).sum().item() / (double)y_test.size(0); - // if (priorAccuracy == 0) { - // priorAccuracy = accuracy; - // } else { - // improvement = accuracy - priorAccuracy; - // } - // if (improvement < convergence_threshold) { - // VLOG_SCOPE_F(3, " (improvement=threshold) Reset. tolerance: %d numItemsPack: %d improvement: %f prior: %f current: %f", tolerance, numItemsPack, improvement, priorAccuracy, accuracy); - // tolerance = 0; // Reset the counter if the model performs better - // numItemsPack = 0; - // } - // if (convergence_best) { - // // Keep the best accuracy until now as the prior accuracy - // priorAccuracy = std::max(accuracy, priorAccuracy); - // } else { - // // Keep the last accuray obtained as the prior accuracy - // priorAccuracy = accuracy; - // } - // } - // VLOG_SCOPE_F(1, "tolerance: %d featuresUsed.size: %zu features.size: %zu", tolerance, featuresUsed.size(), features.size()); - // finished = finished || tolerance > maxTolerance || featuresUsed.size() == features.size(); - // } + // Algorithm based on the adaboost algorithm for classification + // as explained in Ensemble methods (Zhi-Hua Zhou, 2012) + fitted = true; + double alpha_t = 0; + torch::Tensor weights_ = torch::full({ m }, 1.0 / m, torch::kFloat64); + bool finished = false; + std::vector featuresUsed; + if (selectFeatures) { + featuresUsed = initializeModels(); + auto ypred = predict(X_train); + std::tie(weights_, alpha_t, finished) = update_weights(y_train, ypred, weights_); + // Update significance of the models + for (int i = 0; i < n_models; ++i) { + significanceModels[i] = alpha_t; + } + if (finished) { + return; + } + } + int numItemsPack = 0; // The counter of the models inserted in the current pack + // Variables to control the accuracy finish condition + double priorAccuracy = 0.0; + double improvement = 1.0; + double convergence_threshold = 1e-4; + int tolerance = 0; // number of times the accuracy is lower than the convergence_threshold + // Step 0: Set the finish condition + // epsilon sub t > 0.5 => inverse the weights policy + // validation error is not decreasing + // run out of features + bool ascending = order_algorithm == Orders.ASC; + std::mt19937 g{ 173 }; + while (!finished) { + // Step 1: Build ranking with mutual information + auto pairSelection = metrics.SelectKPairs(weights_, featuresUsed, ascending, 0); // Get all the pairs sorted + if (order_algorithm == Orders.RAND) { + std::shuffle(pairSelection.begin(), pairSelection.end(), g); + } + // // Remove used features + // featureSelection.erase(remove_if(begin(featureSelection), end(featureSelection), [&](auto x) + // { return std::find(begin(featuresUsed), end(featuresUsed), x) != end(featuresUsed);}), + // end(featureSelection) + // ); + // int k = bisection ? pow(2, tolerance) : 1; + // int counter = 0; // The model counter of the current pack + // VLOG_SCOPE_F(1, "counter=%d k=%d featureSelection.size: %zu", counter, k, featureSelection.size()); + // while (counter++ < k && featureSelection.size() > 0) { + // auto feature = featureSelection[0]; + // featureSelection.erase(featureSelection.begin()); + // std::unique_ptr model; + // model = std::make_unique(feature); + // model->fit(dataset, features, className, states, weights_); + // alpha_t = 0.0; + // if (!block_update) { + // auto ypred = model->predict(X_train); + // // Step 3.1: Compute the classifier amout of say + // std::tie(weights_, alpha_t, finished) = update_weights(y_train, ypred, weights_); + // } + // // Step 3.4: Store classifier and its accuracy to weigh its future vote + // numItemsPack++; + // featuresUsed.push_back(feature); + // models.push_back(std::move(model)); + // significanceModels.push_back(alpha_t); + // n_models++; + // VLOG_SCOPE_F(2, "numItemsPack: %d n_models: %d featuresUsed: %zu", numItemsPack, n_models, featuresUsed.size()); + // } + // if (block_update) { + // std::tie(weights_, alpha_t, finished) = update_weights_block(k, y_train, weights_); + // } + // if (convergence && !finished) { + // auto y_val_predict = predict(X_test); + // double accuracy = (y_val_predict == y_test).sum().item() / (double)y_test.size(0); + // if (priorAccuracy == 0) { + // priorAccuracy = accuracy; + // } else { + // improvement = accuracy - priorAccuracy; + // } + // if (improvement < convergence_threshold) { + // VLOG_SCOPE_F(3, " (improvement=threshold) Reset. tolerance: %d numItemsPack: %d improvement: %f prior: %f current: %f", tolerance, numItemsPack, improvement, priorAccuracy, accuracy); + // tolerance = 0; // Reset the counter if the model performs better + // numItemsPack = 0; + // } + // if (convergence_best) { + // // Keep the best accuracy until now as the prior accuracy + // priorAccuracy = std::max(accuracy, priorAccuracy); + // } else { + // // Keep the last accuray obtained as the prior accuracy + // priorAccuracy = accuracy; + // } + // } + // VLOG_SCOPE_F(1, "tolerance: %d featuresUsed.size: %zu features.size: %zu", tolerance, featuresUsed.size(), features.size()); + // finished = finished || tolerance > maxTolerance || featuresUsed.size() == features.size(); + } // if (tolerance > maxTolerance) { // if (numItemsPack < n_models) { // notes.push_back("Convergence threshold reached & " + std::to_string(numItemsPack) + " models eliminated"); diff --git a/bayesnet/utils/BayesMetrics.cc b/bayesnet/utils/BayesMetrics.cc index 6083662..85645c7 100644 --- a/bayesnet/utils/BayesMetrics.cc +++ b/bayesnet/utils/BayesMetrics.cc @@ -30,7 +30,7 @@ namespace bayesnet { } samples.index_put_({ -1, "..." }, torch::tensor(labels, torch::kInt32)); } - std::vector> Metrics::SelectKPairs(const torch::Tensor& weights, bool ascending, unsigned k) + std::vector> Metrics::SelectKPairs(const torch::Tensor& weights, std::vector& featuresExcluded, bool ascending, unsigned k) { // Return the K Best features auto n = features.size(); @@ -39,7 +39,13 @@ namespace bayesnet { pairsKBest.clear(); auto labels = samples.index({ -1, "..." }); for (int i = 0; i < n - 1; ++i) { + if (std::find(featuresExcluded.begin(), featuresExcluded.end(), i) != featuresExcluded.end()) { + continue; + } for (int j = i + 1; j < n; ++j) { + if (std::find(featuresExcluded.begin(), featuresExcluded.end(), j) != featuresExcluded.end()) { + continue; + } auto key = std::make_pair(i, j); auto value = conditionalMutualInformation(samples.index({ i, "..." }), samples.index({ j, "..." }), labels, weights); scoresKPairs.push_back({ key, value }); @@ -57,9 +63,10 @@ namespace bayesnet { for (auto& [pairs, score] : scoresKPairs) { pairsKBest.push_back(pairs); } - if (k != 0) { + if (k != 0 && k < pairsKBest.size()) { if (ascending) { - for (int i = 0; i < n - k; ++i) { + int limit = pairsKBest.size() - k; + for (int i = 0; i < limit; i++) { pairsKBest.erase(pairsKBest.begin()); scoresKPairs.erase(scoresKPairs.begin()); } diff --git a/bayesnet/utils/BayesMetrics.h b/bayesnet/utils/BayesMetrics.h index 0e58b82..5b62a72 100644 --- a/bayesnet/utils/BayesMetrics.h +++ b/bayesnet/utils/BayesMetrics.h @@ -16,7 +16,7 @@ namespace bayesnet { Metrics(const torch::Tensor& samples, const std::vector& features, const std::string& className, const int classNumStates); Metrics(const std::vector>& vsamples, const std::vector& labels, const std::vector& features, const std::string& className, const int classNumStates); std::vector SelectKBestWeighted(const torch::Tensor& weights, bool ascending = false, unsigned k = 0); - std::vector> SelectKPairs(const torch::Tensor& weights, bool ascending = false, unsigned k = 0); + std::vector> SelectKPairs(const torch::Tensor& weights, std::vector& featuresExcluded, bool ascending = false, unsigned k = 0); std::vector getScoresKBest() const; std::vector, double>> getScoresKPairs() const; double mutualInformation(const torch::Tensor& firstFeature, const torch::Tensor& secondFeature, const torch::Tensor& weights); diff --git a/tests/TestBayesMetrics.cc b/tests/TestBayesMetrics.cc index b40e3f2..755a4a0 100644 --- a/tests/TestBayesMetrics.cc +++ b/tests/TestBayesMetrics.cc @@ -141,7 +141,8 @@ TEST_CASE("Select K Pairs descending", "[Metrics]") { auto raw = RawDatasets("iris", true); bayesnet::Metrics metrics(raw.dataset, raw.features, raw.className, raw.classNumStates); - auto results = metrics.SelectKPairs(raw.weights, false); + std::vector empty; + auto results = metrics.SelectKPairs(raw.weights, empty, false); auto expected = std::vector, double>>{ { { 1, 3 }, 1.31852 }, { { 1, 2 }, 1.17112 }, @@ -168,7 +169,8 @@ TEST_CASE("Select K Pairs ascending", "[Metrics]") { auto raw = RawDatasets("iris", true); bayesnet::Metrics metrics(raw.dataset, raw.features, raw.className, raw.classNumStates); - auto results = metrics.SelectKPairs(raw.weights, true); + std::vector empty; + auto results = metrics.SelectKPairs(raw.weights, empty, true); auto expected = std::vector, double>>{ { { 0, 1 }, 0.0 }, { { 2, 3 }, 0.210068 }, @@ -190,4 +192,77 @@ TEST_CASE("Select K Pairs ascending", "[Metrics]") } REQUIRE(results.size() == 6); REQUIRE(scores.size() == 6); +} +TEST_CASE("Select K Pairs with features excluded", "[Metrics]") +{ + auto raw = RawDatasets("iris", true); + bayesnet::Metrics metrics(raw.dataset, raw.features, raw.className, raw.classNumStates); + std::vector excluded = { 0, 3 }; + auto results = metrics.SelectKPairs(raw.weights, excluded, true); + auto expected = std::vector, double>>{ + { { 1, 2 }, 1.17112 }, + }; + auto scores = metrics.getScoresKPairs(); + for (int i = 0; i < results.size(); ++i) { + auto result = results[i]; + auto expect = expected[i]; + auto score = scores[i]; + REQUIRE(result.first == expect.first.first); + REQUIRE(result.second == expect.first.second); + REQUIRE(score.first.first == expect.first.first); + REQUIRE(score.first.second == expect.first.second); + REQUIRE(score.second == Catch::Approx(expect.second).epsilon(raw.epsilon)); + } + REQUIRE(results.size() == 1); + REQUIRE(scores.size() == 1); +} +TEST_CASE("Select K Pairs with number of pairs descending", "[Metrics]") +{ + auto raw = RawDatasets("iris", true); + bayesnet::Metrics metrics(raw.dataset, raw.features, raw.className, raw.classNumStates); + std::vector empty; + auto results = metrics.SelectKPairs(raw.weights, empty, false, 3); + auto expected = std::vector, double>>{ + { { 1, 3 }, 1.31852 }, + { { 1, 2 }, 1.17112 }, + { { 0, 3 }, 0.403749 } + }; + auto scores = metrics.getScoresKPairs(); + REQUIRE(results.size() == 3); + REQUIRE(scores.size() == 3); + for (int i = 0; i < results.size(); ++i) { + auto result = results[i]; + auto expect = expected[i]; + auto score = scores[i]; + REQUIRE(result.first == expect.first.first); + REQUIRE(result.second == expect.first.second); + REQUIRE(score.first.first == expect.first.first); + REQUIRE(score.first.second == expect.first.second); + REQUIRE(score.second == Catch::Approx(expect.second).epsilon(raw.epsilon)); + } +} +TEST_CASE("Select K Pairs with number of pairs ascending", "[Metrics]") +{ + auto raw = RawDatasets("iris", true); + bayesnet::Metrics metrics(raw.dataset, raw.features, raw.className, raw.classNumStates); + std::vector empty; + auto results = metrics.SelectKPairs(raw.weights, empty, true, 3); + auto expected = std::vector, double>>{ + { { 0, 3 }, 0.403749 }, + { { 1, 2 }, 1.17112 }, + { { 1, 3 }, 1.31852 } + }; + auto scores = metrics.getScoresKPairs(); + REQUIRE(results.size() == 3); + REQUIRE(scores.size() == 3); + for (int i = 0; i < results.size(); ++i) { + auto result = results[i]; + auto expect = expected[i]; + auto score = scores[i]; + REQUIRE(result.first == expect.first.first); + REQUIRE(result.second == expect.first.second); + REQUIRE(score.first.first == expect.first.first); + REQUIRE(score.first.second == expect.first.second); + REQUIRE(score.second == Catch::Approx(expect.second).epsilon(raw.epsilon)); + } } \ No newline at end of file