From 8c413a1eb0a0602811e12456fd6c03a0bafed8a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana=20G=C3=B3mez?= Date: Mon, 16 Jun 2025 00:11:51 +0200 Subject: [PATCH] Begin to add AdaBoost implementation --- src/CMakeLists.txt | 2 + src/experimental_clfs/AdaBoost.cpp | 214 +++++++++++++++++++++++++++++ src/experimental_clfs/AdaBoost.h | 58 ++++++++ src/main/Models.h | 1 + src/main/modelRegister.h | 4 +- 5 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 src/experimental_clfs/AdaBoost.cpp create mode 100644 src/experimental_clfs/AdaBoost.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b89cebc..1f719c1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -33,6 +33,7 @@ add_executable(b_grid commands/b_grid.cpp ${grid_sources} results/Result.cpp experimental_clfs/XA1DE.cpp experimental_clfs/ExpClf.cpp + experimental_clfs/AdaBoost.cpp ) target_link_libraries(b_grid ${MPI_CXX_LIBRARIES} "${PyClassifiers}" bayesnet::bayesnet fimdlp ${Python3_LIBRARIES} "${TORCH_LIBRARIES}" Boost::python Boost::numpy) @@ -56,6 +57,7 @@ add_executable(b_main commands/b_main.cpp ${main_sources} results/Result.cpp experimental_clfs/XA1DE.cpp experimental_clfs/ExpClf.cpp + experimental_clfs/ExpClf.cpp ) target_link_libraries(b_main PRIVATE nlohmann_json::nlohmann_json "${PyClassifiers}" bayesnet::bayesnet fimdlp ${Python3_LIBRARIES} "${TORCH_LIBRARIES}" Boost::python Boost::numpy) diff --git a/src/experimental_clfs/AdaBoost.cpp b/src/experimental_clfs/AdaBoost.cpp new file mode 100644 index 0000000..63523d7 --- /dev/null +++ b/src/experimental_clfs/AdaBoost.cpp @@ -0,0 +1,214 @@ +// *************************************************************** +// SPDX-FileCopyrightText: Copyright 2024 Ricardo Montañana Gómez +// SPDX-FileType: SOURCE +// SPDX-License-Identifier: MIT +// *************************************************************** + +#include "AdaBoost.h" +#include +#include +#include +#include +#include + +namespace platform { + + AdaBoost::AdaBoost(int n_estimators) + : Ensemble(true), n_estimators(n_estimators) + { + validHyperparameters = { "n_estimators" }; + } + + void AdaBoost::buildModel(const torch::Tensor& weights) + { + // Initialize variables + models.clear(); + alphas.clear(); + training_errors.clear(); + + // Initialize sample weights uniformly + int n_samples = dataset.size(1); + sample_weights = torch::ones({ n_samples }) / n_samples; + + // If initial weights are provided, incorporate them + if (weights.defined() && weights.numel() > 0) { + sample_weights *= weights; + normalizeWeights(); + } + + // Main AdaBoost training loop (SAMME algorithm) + for (int iter = 0; iter < n_estimators; ++iter) { + // Train base estimator with current sample weights + auto estimator = trainBaseEstimator(sample_weights); + + // Calculate weighted error + double weighted_error = calculateWeightedError(estimator.get(), sample_weights); + training_errors.push_back(weighted_error); + + // Check if error is too high (worse than random guessing) + double random_guess_error = 1.0 - (1.0 / getClassNumStates()); + if (weighted_error >= random_guess_error) { + // If only one estimator and it's worse than random, keep it with zero weight + if (models.empty()) { + models.push_back(std::move(estimator)); + alphas.push_back(0.0); + } + break; // Stop boosting + } + + // Calculate alpha (estimator weight) using SAMME formula + // alpha = log((1 - err) / err) + log(K - 1) + double alpha = std::log((1.0 - weighted_error) / weighted_error) + + std::log(static_cast(getClassNumStates() - 1)); + + // Store the estimator and its weight + models.push_back(std::move(estimator)); + alphas.push_back(alpha); + + // Update sample weights + updateSampleWeights(models.back().get(), alpha); + + // Normalize weights + normalizeWeights(); + + // Check for perfect classification + if (weighted_error < 1e-10) { + break; + } + } + + // Set the number of models actually trained + n_models = models.size(); + } + + void AdaBoost::trainModel(const torch::Tensor& weights, const Smoothing_t smoothing) + { + // AdaBoost handles its own weight management, so we just build the model + buildModel(weights); + } + + std::unique_ptr AdaBoost::trainBaseEstimator(const torch::Tensor& weights) + { + // Create a new classifier instance + // You need to implement this based on your specific base classifier + // For example, if using Decision Trees: + // auto classifier = std::make_unique(); + + // Or if using a factory method: + // auto classifier = ClassifierFactory::create("DecisionTree"); + + // Placeholder - replace with actual classifier creation + throw std::runtime_error("AdaBoost::trainBaseEstimator - You need to implement base classifier creation"); + + // Once you have the classifier creation implemented, uncomment: + // classifier->fit(dataset, features, className, states, weights, Smoothing_t::NONE); + // return classifier; + } + + double AdaBoost::calculateWeightedError(Classifier* estimator, const torch::Tensor& weights) + { + // Get predictions from the estimator + auto X = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), torch::indexing::Slice() }); + auto y_true = dataset.index({ -1, torch::indexing::Slice() }); + auto y_pred = estimator->predict(X.t()); + + // Calculate weighted error + auto incorrect = (y_pred != y_true).to(torch::kFloat); + double weighted_error = torch::sum(incorrect * weights).item(); + + return weighted_error; + } + + void AdaBoost::updateSampleWeights(Classifier* estimator, double alpha) + { + // Get predictions from the estimator + auto X = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), torch::indexing::Slice() }); + auto y_true = dataset.index({ -1, torch::indexing::Slice() }); + auto y_pred = estimator->predict(X.t()); + + // Update weights according to SAMME algorithm + // w_i = w_i * exp(alpha * I(y_i != y_pred_i)) + auto incorrect = (y_pred != y_true).to(torch::kFloat); + sample_weights *= torch::exp(alpha * incorrect); + } + + void AdaBoost::normalizeWeights() + { + // Normalize weights to sum to 1 + double sum_weights = torch::sum(sample_weights).item(); + if (sum_weights > 0) { + sample_weights /= sum_weights; + } + } + + std::vector AdaBoost::graph(const std::string& title) const + { + // Create a graph representation of the AdaBoost ensemble + std::vector graph_lines; + + // Header + graph_lines.push_back("digraph AdaBoost {"); + graph_lines.push_back(" rankdir=TB;"); + graph_lines.push_back(" node [shape=box];"); + + if (!title.empty()) { + graph_lines.push_back(" label=\"" + title + "\";"); + graph_lines.push_back(" labelloc=t;"); + } + + // Add input node + graph_lines.push_back(" Input [shape=ellipse, label=\"Input Features\"];"); + + // Add base estimators + for (size_t i = 0; i < models.size(); ++i) { + std::stringstream ss; + ss << " Estimator" << i << " [label=\"Base Estimator " << i + 1 + << "\\nα = " << std::fixed << std::setprecision(3) << alphas[i] << "\"];"; + graph_lines.push_back(ss.str()); + + // Connect input to estimator + ss.str(""); + ss << " Input -> Estimator" << i << ";"; + graph_lines.push_back(ss.str()); + } + + // Add combination node + graph_lines.push_back(" Combination [shape=diamond, label=\"Weighted Vote\"];"); + + // Connect estimators to combination + for (size_t i = 0; i < models.size(); ++i) { + std::stringstream ss; + ss << " Estimator" << i << " -> Combination;"; + graph_lines.push_back(ss.str()); + } + + // Add output node + graph_lines.push_back(" Output [shape=ellipse, label=\"Final Prediction\"];"); + graph_lines.push_back(" Combination -> Output;"); + + // Close graph + graph_lines.push_back("}"); + + return graph_lines; + } + + void AdaBoost::setHyperparameters(const nlohmann::json& hyperparameters) + { + // Set hyperparameters from JSON + auto it = hyperparameters.find("n_estimators"); + if (it != hyperparameters.end()) { + n_estimators = it->get(); + if (n_estimators <= 0) { + throw std::invalid_argument("n_estimators must be positive"); + } + } + + // Check for invalid hyperparameters + for (auto& [key, value] : hyperparameters.items()) { + if (std::find(validHyperparameters.begin(), validHyperparameters.end(), key) == validHyperparameters.end()) { + throw std::invalid_argument("Invalid hyperparameter: " + key); + } + } + } + +} // namespace bayesnet \ No newline at end of file diff --git a/src/experimental_clfs/AdaBoost.h b/src/experimental_clfs/AdaBoost.h new file mode 100644 index 0000000..59ac241 --- /dev/null +++ b/src/experimental_clfs/AdaBoost.h @@ -0,0 +1,58 @@ +// *************************************************************** +// SPDX-FileCopyrightText: Copyright 2024 Ricardo Montañana Gómez +// SPDX-FileType: SOURCE +// SPDX-License-Identifier: MIT +// *************************************************************** + +#ifndef ADABOOST_H +#define ADABOOST_H + +#include +#include +#include +#include + +namespace platform { + class AdaBoost : public bayesnet::Ensemble { + public: + explicit AdaBoost(int n_estimators = 100); + virtual ~AdaBoost() = default; + + // Override base class methods + std::vector graph(const std::string& title = "") const override; + + // AdaBoost specific methods + void setNEstimators(int n_estimators) { this->n_estimators = n_estimators; } + int getNEstimators() const { return n_estimators; } + + // Get the weight of each base estimator + std::vector getEstimatorWeights() const { return alphas; } + + // Override setHyperparameters from BaseClassifier + void setHyperparameters(const nlohmann::json& hyperparameters) override; + + protected: + void buildModel(const torch::Tensor& weights) override; + void trainModel(const torch::Tensor& weights, const Smoothing_t smoothing) override; + + private: + int n_estimators; + std::vector alphas; // Weight of each base estimator + std::vector training_errors; // Training error at each iteration + torch::Tensor sample_weights; // Current sample weights + + // Train a single base estimator + std::unique_ptr trainBaseEstimator(const torch::Tensor& weights); + + // Calculate weighted error + double calculateWeightedError(Classifier* estimator, const torch::Tensor& weights); + + // Update sample weights based on predictions + void updateSampleWeights(Classifier* estimator, double alpha); + + // Normalize weights to sum to 1 + void normalizeWeights(); + }; +} + +#endif // ADABOOST_H \ No newline at end of file diff --git a/src/main/Models.h b/src/main/Models.h index 076a926..d6cf449 100644 --- a/src/main/Models.h +++ b/src/main/Models.h @@ -26,6 +26,7 @@ #include #include #include "../experimental_clfs/XA1DE.h" +#include "../experimental_clfs/AdaBoost.h" namespace platform { class Models { diff --git a/src/main/modelRegister.h b/src/main/modelRegister.h index 22a800c..11ebc84 100644 --- a/src/main/modelRegister.h +++ b/src/main/modelRegister.h @@ -35,8 +35,10 @@ namespace platform { [](void) -> bayesnet::BaseClassifier* { return new pywrap::RandomForest();}); static Registrar registrarXGB("XGBoost", [](void) -> bayesnet::BaseClassifier* { return new pywrap::XGBoost();}); - static Registrar registrarAda("AdaBoost", + static Registrar registrarAda("AdaBoostPy", [](void) -> bayesnet::BaseClassifier* { return new pywrap::AdaBoost();}); + // static Registrar registrarAda2("AdaBoost", + // [](void) -> bayesnet::BaseClassifier* { return new platform::AdaBoost();}); static Registrar registrarXSPODE("XSPODE", [](void) -> bayesnet::BaseClassifier* { return new bayesnet::XSpode(0);}); static Registrar registrarXSP2DE("XSP2DE",