rmontanana/BayesNet
rmontanana/BayesNet
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases 3 Wiki Activity

Add features used to selectKPairs

Browse Source
This commit is contained in:
Ricardo Montañana Gómez 2024-05-16 14:18:45 +02:00
parent cccaa6e0af
commit 677ec5613d
Signed by: rmontanana
GPG Key ID: 46064262FD9A7ADE
4 changed files with 183 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

190
bayesnet/ensembles/BoostA2DE.cc
View File

@ -50,101 +50,101 @@ namespace bayesnet {
// loguru::g_stderr_verbosity = loguru::Verbosity_OFF; // loguru::g_stderr_verbosity = loguru::Verbosity_OFF;
// loguru::add_file("boostA2DE.log", loguru::Truncate, loguru::Verbosity_MAX); // loguru::add_file("boostA2DE.log", loguru::Truncate, loguru::Verbosity_MAX);
// // Algorithm based on the adaboost algorithm for classification // Algorithm based on the adaboost algorithm for classification
// // as explained in Ensemble methods (Zhi-Hua Zhou, 2012) // as explained in Ensemble methods (Zhi-Hua Zhou, 2012)
// fitted = true; fitted = true;
// double alpha_t = 0; double alpha_t = 0;
// torch::Tensor weights_ = torch::full({ m }, 1.0 / m, torch::kFloat64); torch::Tensor weights_ = torch::full({ m }, 1.0 / m, torch::kFloat64);
// bool finished = false; bool finished = false;
// std::vector<int> featuresUsed; std::vector<int> featuresUsed;
// if (selectFeatures) { if (selectFeatures) {
// featuresUsed = initializeModels(); featuresUsed = initializeModels();
// auto ypred = predict(X_train); auto ypred = predict(X_train);
// std::tie(weights_, alpha_t, finished) = update_weights(y_train, ypred, weights_); std::tie(weights_, alpha_t, finished) = update_weights(y_train, ypred, weights_);
// // Update significance of the models // Update significance of the models
// for (int i = 0; i < n_models; ++i) { for (int i = 0; i < n_models; ++i) {
// significanceModels[i] = alpha_t; significanceModels[i] = alpha_t;
// } }
// if (finished) { if (finished) {
// return; return;
// } }
// } }
// int numItemsPack = 0; // The counter of the models inserted in the current pack int numItemsPack = 0; // The counter of the models inserted in the current pack
// // Variables to control the accuracy finish condition // Variables to control the accuracy finish condition
// double priorAccuracy = 0.0; double priorAccuracy = 0.0;
// double improvement = 1.0; double improvement = 1.0;
// double convergence_threshold = 1e-4; double convergence_threshold = 1e-4;
// int tolerance = 0; // number of times the accuracy is lower than the convergence_threshold int tolerance = 0; // number of times the accuracy is lower than the convergence_threshold
// // Step 0: Set the finish condition // Step 0: Set the finish condition
// // epsilon sub t > 0.5 => inverse the weights policy // epsilon sub t > 0.5 => inverse the weights policy
// // validation error is not decreasing // validation error is not decreasing
// // run out of features // run out of features
// bool ascending = order_algorithm == Orders.ASC; bool ascending = order_algorithm == Orders.ASC;
// std::mt19937 g{ 173 }; std::mt19937 g{ 173 };
// while (!finished) { while (!finished) {
// // Step 1: Build ranking with mutual information // Step 1: Build ranking with mutual information
// auto pairSelection = metrics.SelectKBestWeighted(weights_, ascending, n); // Get all the features sorted auto pairSelection = metrics.SelectKPairs(weights_, featuresUsed, ascending, 0); // Get all the pairs sorted
// if (order_algorithm == Orders.RAND) { if (order_algorithm == Orders.RAND) {
// std::shuffle(featureSelection.begin(), featureSelection.end(), g); std::shuffle(pairSelection.begin(), pairSelection.end(), g);
// } }
// // Remove used features // // Remove used features
// featureSelection.erase(remove_if(begin(featureSelection), end(featureSelection), [&](auto x) // featureSelection.erase(remove_if(begin(featureSelection), end(featureSelection), [&](auto x)
// { return std::find(begin(featuresUsed), end(featuresUsed), x) != end(featuresUsed);}), // { return std::find(begin(featuresUsed), end(featuresUsed), x) != end(featuresUsed);}),
// end(featureSelection) // end(featureSelection)
// ); // );
// int k = bisection ? pow(2, tolerance) : 1; // int k = bisection ? pow(2, tolerance) : 1;
// int counter = 0; // The model counter of the current pack // int counter = 0; // The model counter of the current pack
// VLOG_SCOPE_F(1, "counter=%d k=%d featureSelection.size: %zu", counter, k, featureSelection.size()); // VLOG_SCOPE_F(1, "counter=%d k=%d featureSelection.size: %zu", counter, k, featureSelection.size());
// while (counter++ < k && featureSelection.size() > 0) { // while (counter++ < k && featureSelection.size() > 0) {
// auto feature = featureSelection[0]; // auto feature = featureSelection[0];
// featureSelection.erase(featureSelection.begin()); // featureSelection.erase(featureSelection.begin());
// std::unique_ptr<Classifier> model; // std::unique_ptr<Classifier> model;
// model = std::make_unique<SPODE>(feature); // model = std::make_unique<SPODE>(feature);
// model->fit(dataset, features, className, states, weights_); // model->fit(dataset, features, className, states, weights_);
// alpha_t = 0.0; // alpha_t = 0.0;
// if (!block_update) { // if (!block_update) {
// auto ypred = model->predict(X_train); // auto ypred = model->predict(X_train);
// // Step 3.1: Compute the classifier amout of say // // Step 3.1: Compute the classifier amout of say
// std::tie(weights_, alpha_t, finished) = update_weights(y_train, ypred, weights_); // std::tie(weights_, alpha_t, finished) = update_weights(y_train, ypred, weights_);
// } // }
// // Step 3.4: Store classifier and its accuracy to weigh its future vote // // Step 3.4: Store classifier and its accuracy to weigh its future vote
// numItemsPack++; // numItemsPack++;
// featuresUsed.push_back(feature); // featuresUsed.push_back(feature);
// models.push_back(std::move(model)); // models.push_back(std::move(model));
// significanceModels.push_back(alpha_t); // significanceModels.push_back(alpha_t);
// n_models++; // n_models++;
// VLOG_SCOPE_F(2, "numItemsPack: %d n_models: %d featuresUsed: %zu", numItemsPack, n_models, featuresUsed.size()); // VLOG_SCOPE_F(2, "numItemsPack: %d n_models: %d featuresUsed: %zu", numItemsPack, n_models, featuresUsed.size());
// } // }
// if (block_update) { // if (block_update) {
// std::tie(weights_, alpha_t, finished) = update_weights_block(k, y_train, weights_); // std::tie(weights_, alpha_t, finished) = update_weights_block(k, y_train, weights_);
// } // }
// if (convergence && !finished) { // if (convergence && !finished) {
// auto y_val_predict = predict(X_test); // auto y_val_predict = predict(X_test);
// double accuracy = (y_val_predict == y_test).sum().item<double>() / (double)y_test.size(0); // double accuracy = (y_val_predict == y_test).sum().item<double>() / (double)y_test.size(0);
// if (priorAccuracy == 0) { // if (priorAccuracy == 0) {
// priorAccuracy = accuracy; // priorAccuracy = accuracy;
// } else { // } else {
// improvement = accuracy - priorAccuracy; // improvement = accuracy - priorAccuracy;
// } // }
// if (improvement < convergence_threshold) { // if (improvement < convergence_threshold) {
// VLOG_SCOPE_F(3, " (improvement<threshold) tolerance: %d numItemsPack: %d improvement: %f prior: %f current: %f", tolerance, numItemsPack, improvement, priorAccuracy, accuracy); // VLOG_SCOPE_F(3, " (improvement<threshold) tolerance: %d numItemsPack: %d improvement: %f prior: %f current: %f", tolerance, numItemsPack, improvement, priorAccuracy, accuracy);
// tolerance++; // tolerance++;
// } else { // } else {
// VLOG_SCOPE_F(3, "* (improvement>=threshold) Reset. tolerance: %d numItemsPack: %d improvement: %f prior: %f current: %f", tolerance, numItemsPack, improvement, priorAccuracy, accuracy); // VLOG_SCOPE_F(3, "* (improvement>=threshold) Reset. tolerance: %d numItemsPack: %d improvement: %f prior: %f current: %f", tolerance, numItemsPack, improvement, priorAccuracy, accuracy);
// tolerance = 0; // Reset the counter if the model performs better // tolerance = 0; // Reset the counter if the model performs better
// numItemsPack = 0; // numItemsPack = 0;
// } // }
// if (convergence_best) { // if (convergence_best) {
// // Keep the best accuracy until now as the prior accuracy // // Keep the best accuracy until now as the prior accuracy
// priorAccuracy = std::max(accuracy, priorAccuracy); // priorAccuracy = std::max(accuracy, priorAccuracy);
// } else { // } else {
// // Keep the last accuray obtained as the prior accuracy // // Keep the last accuray obtained as the prior accuracy
// priorAccuracy = accuracy; // priorAccuracy = accuracy;
// } // }
// } // }
// VLOG_SCOPE_F(1, "tolerance: %d featuresUsed.size: %zu features.size: %zu", tolerance, featuresUsed.size(), features.size()); // VLOG_SCOPE_F(1, "tolerance: %d featuresUsed.size: %zu features.size: %zu", tolerance, featuresUsed.size(), features.size());
// finished = finished || tolerance > maxTolerance || featuresUsed.size() == features.size(); // finished = finished || tolerance > maxTolerance || featuresUsed.size() == features.size();
// } }
// if (tolerance > maxTolerance) { // if (tolerance > maxTolerance) {
// if (numItemsPack < n_models) { // if (numItemsPack < n_models) {
// notes.push_back("Convergence threshold reached & " + std::to_string(numItemsPack) + " models eliminated"); // notes.push_back("Convergence threshold reached & " + std::to_string(numItemsPack) + " models eliminated");

13
bayesnet/utils/BayesMetrics.cc
View File

@ -30,7 +30,7 @@ namespace bayesnet {
} }
samples.index_put_({ -1, "..." }, torch::tensor(labels, torch::kInt32)); samples.index_put_({ -1, "..." }, torch::tensor(labels, torch::kInt32));
} }
std::vector<std::pair<int, int>> Metrics::SelectKPairs(const torch::Tensor& weights, bool ascending, unsigned k) std::vector<std::pair<int, int>> Metrics::SelectKPairs(const torch::Tensor& weights, std::vector<int>& featuresExcluded, bool ascending, unsigned k)
{ {
// Return the K Best features // Return the K Best features
auto n = features.size(); auto n = features.size();
@ -39,7 +39,13 @@ namespace bayesnet {
pairsKBest.clear(); pairsKBest.clear();
auto labels = samples.index({ -1, "..." }); auto labels = samples.index({ -1, "..." });
for (int i = 0; i < n - 1; ++i) { for (int i = 0; i < n - 1; ++i) {
if (std::find(featuresExcluded.begin(), featuresExcluded.end(), i) != featuresExcluded.end()) {
continue;
}
for (int j = i + 1; j < n; ++j) { for (int j = i + 1; j < n; ++j) {
if (std::find(featuresExcluded.begin(), featuresExcluded.end(), j) != featuresExcluded.end()) {
continue;
}
auto key = std::make_pair(i, j); auto key = std::make_pair(i, j);
auto value = conditionalMutualInformation(samples.index({ i, "..." }), samples.index({ j, "..." }), labels, weights); auto value = conditionalMutualInformation(samples.index({ i, "..." }), samples.index({ j, "..." }), labels, weights);
scoresKPairs.push_back({ key, value }); scoresKPairs.push_back({ key, value });
@ -57,9 +63,10 @@ namespace bayesnet {
for (auto& [pairs, score] : scoresKPairs) { for (auto& [pairs, score] : scoresKPairs) {
pairsKBest.push_back(pairs); pairsKBest.push_back(pairs);
} }
if (k != 0) { if (k != 0 && k < pairsKBest.size()) {
if (ascending) { if (ascending) {
for (int i = 0; i < n - k; ++i) { int limit = pairsKBest.size() - k;
for (int i = 0; i < limit; i++) {
pairsKBest.erase(pairsKBest.begin()); pairsKBest.erase(pairsKBest.begin());
scoresKPairs.erase(scoresKPairs.begin()); scoresKPairs.erase(scoresKPairs.begin());
} }

2
bayesnet/utils/BayesMetrics.h
View File

@ -16,7 +16,7 @@ namespace bayesnet {
Metrics(const torch::Tensor& samples, const std::vector<std::string>& features, const std::string& className, const int classNumStates); Metrics(const torch::Tensor& samples, const std::vector<std::string>& features, const std::string& className, const int classNumStates);
Metrics(const std::vector<std::vector<int>>& vsamples, const std::vector<int>& labels, const std::vector<std::string>& features, const std::string& className, const int classNumStates); Metrics(const std::vector<std::vector<int>>& vsamples, const std::vector<int>& labels, const std::vector<std::string>& features, const std::string& className, const int classNumStates);
std::vector<int> SelectKBestWeighted(const torch::Tensor& weights, bool ascending = false, unsigned k = 0); std::vector<int> SelectKBestWeighted(const torch::Tensor& weights, bool ascending = false, unsigned k = 0);
std::vector<std::pair<int, int>> SelectKPairs(const torch::Tensor& weights, bool ascending = false, unsigned k = 0); std::vector<std::pair<int, int>> SelectKPairs(const torch::Tensor& weights, std::vector<int>& featuresExcluded, bool ascending = false, unsigned k = 0);
std::vector<double> getScoresKBest() const; std::vector<double> getScoresKBest() const;
std::vector<std::pair<std::pair<int, int>, double>> getScoresKPairs() const; std::vector<std::pair<std::pair<int, int>, double>> getScoresKPairs() const;
double mutualInformation(const torch::Tensor& firstFeature, const torch::Tensor& secondFeature, const torch::Tensor& weights); double mutualInformation(const torch::Tensor& firstFeature, const torch::Tensor& secondFeature, const torch::Tensor& weights);

79
tests/TestBayesMetrics.cc
View File

@ -141,7 +141,8 @@ TEST_CASE("Select K Pairs descending", "[Metrics]")
{ {
auto raw = RawDatasets("iris", true); auto raw = RawDatasets("iris", true);
bayesnet::Metrics metrics(raw.dataset, raw.features, raw.className, raw.classNumStates); bayesnet::Metrics metrics(raw.dataset, raw.features, raw.className, raw.classNumStates);
auto results = metrics.SelectKPairs(raw.weights, false); std::vector<int> empty;
auto results = metrics.SelectKPairs(raw.weights, empty, false);
auto expected = std::vector<std::pair<std::pair<int, int>, double>>{ auto expected = std::vector<std::pair<std::pair<int, int>, double>>{
{ { 1, 3 }, 1.31852 }, { { 1, 3 }, 1.31852 },
{ { 1, 2 }, 1.17112 }, { { 1, 2 }, 1.17112 },
@ -168,7 +169,8 @@ TEST_CASE("Select K Pairs ascending", "[Metrics]")
{ {
auto raw = RawDatasets("iris", true); auto raw = RawDatasets("iris", true);
bayesnet::Metrics metrics(raw.dataset, raw.features, raw.className, raw.classNumStates); bayesnet::Metrics metrics(raw.dataset, raw.features, raw.className, raw.classNumStates);
auto results = metrics.SelectKPairs(raw.weights, true); std::vector<int> empty;
auto results = metrics.SelectKPairs(raw.weights, empty, true);
auto expected = std::vector<std::pair<std::pair<int, int>, double>>{ auto expected = std::vector<std::pair<std::pair<int, int>, double>>{
{ { 0, 1 }, 0.0 }, { { 0, 1 }, 0.0 },
{ { 2, 3 }, 0.210068 }, { { 2, 3 }, 0.210068 },
@ -190,4 +192,77 @@ TEST_CASE("Select K Pairs ascending", "[Metrics]")
} }
REQUIRE(results.size() == 6); REQUIRE(results.size() == 6);
REQUIRE(scores.size() == 6); REQUIRE(scores.size() == 6);
}
TEST_CASE("Select K Pairs with features excluded", "[Metrics]")
{
auto raw = RawDatasets("iris", true);
bayesnet::Metrics metrics(raw.dataset, raw.features, raw.className, raw.classNumStates);
std::vector<int> excluded = { 0, 3 };
auto results = metrics.SelectKPairs(raw.weights, excluded, true);
auto expected = std::vector<std::pair<std::pair<int, int>, double>>{
{ { 1, 2 }, 1.17112 },
};
auto scores = metrics.getScoresKPairs();
for (int i = 0; i < results.size(); ++i) {
auto result = results[i];
auto expect = expected[i];
auto score = scores[i];
REQUIRE(result.first == expect.first.first);
REQUIRE(result.second == expect.first.second);
REQUIRE(score.first.first == expect.first.first);
REQUIRE(score.first.second == expect.first.second);
REQUIRE(score.second == Catch::Approx(expect.second).epsilon(raw.epsilon));
}
REQUIRE(results.size() == 1);
REQUIRE(scores.size() == 1);
}
TEST_CASE("Select K Pairs with number of pairs descending", "[Metrics]")
{
auto raw = RawDatasets("iris", true);
bayesnet::Metrics metrics(raw.dataset, raw.features, raw.className, raw.classNumStates);
std::vector<int> empty;
auto results = metrics.SelectKPairs(raw.weights, empty, false, 3);
auto expected = std::vector<std::pair<std::pair<int, int>, double>>{
{ { 1, 3 }, 1.31852 },
{ { 1, 2 }, 1.17112 },
{ { 0, 3 }, 0.403749 }
};
auto scores = metrics.getScoresKPairs();
REQUIRE(results.size() == 3);
REQUIRE(scores.size() == 3);
for (int i = 0; i < results.size(); ++i) {
auto result = results[i];
auto expect = expected[i];
auto score = scores[i];
REQUIRE(result.first == expect.first.first);
REQUIRE(result.second == expect.first.second);
REQUIRE(score.first.first == expect.first.first);
REQUIRE(score.first.second == expect.first.second);
REQUIRE(score.second == Catch::Approx(expect.second).epsilon(raw.epsilon));
}
}
TEST_CASE("Select K Pairs with number of pairs ascending", "[Metrics]")
{
auto raw = RawDatasets("iris", true);
bayesnet::Metrics metrics(raw.dataset, raw.features, raw.className, raw.classNumStates);
std::vector<int> empty;
auto results = metrics.SelectKPairs(raw.weights, empty, true, 3);
auto expected = std::vector<std::pair<std::pair<int, int>, double>>{
{ { 0, 3 }, 0.403749 },
{ { 1, 2 }, 1.17112 },
{ { 1, 3 }, 1.31852 }
};
auto scores = metrics.getScoresKPairs();
REQUIRE(results.size() == 3);
REQUIRE(scores.size() == 3);
for (int i = 0; i < results.size(); ++i) {
auto result = results[i];
auto expect = expected[i];
auto score = scores[i];
REQUIRE(result.first == expect.first.first);
REQUIRE(result.second == expect.first.second);
REQUIRE(score.first.first == expect.first.first);
REQUIRE(score.first.second == expect.first.second);
REQUIRE(score.second == Catch::Approx(expect.second).epsilon(raw.epsilon));
}
} }