Implement classifier.predict_proba & test

CHANGELOG.md

@@ -5,7 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## =[Unreleased]
+
+### Added
+
+- predict_proba method in Classifier
+- predict_proba method in BoostAODE
+- predict_voting parameter in BoostAODE constructor to use voting or probability to predict (default is voting)
 
 ## [1.0.2] - 2024-02-20
 

src/BayesNet/BaseClassifier.h

@@ -16,12 +16,15 @@ namespace bayesnet {
         virtual ~BaseClassifier() = default;
         torch::Tensor virtual predict(torch::Tensor& X) = 0;
         std::vector<int> virtual predict(std::vector<std::vector<int >>& X) = 0;
+        torch::Tensor virtual predict_proba(torch::Tensor& X) = 0;
+        std::vector<std::vector<double>> virtual predict_proba(std::vector<std::vector<int >>& X) = 0;
         status_t virtual getStatus() const = 0;
         float virtual score(std::vector<std::vector<int>>& X, std::vector<int>& y) = 0;
         float virtual score(torch::Tensor& X, torch::Tensor& y) = 0;
         int virtual getNumberOfNodes()const = 0;
         int virtual getNumberOfEdges()const = 0;
         int virtual getNumberOfStates() const = 0;
+        int virtual getClassNumStates() const = 0;
         std::vector<std::string> virtual show() const = 0;
         std::vector<std::string> virtual graph(const std::string& title = "") const = 0;
         virtual std::string getVersion() = 0;

src/BayesNet/BoostAODE.cc

@@ -8,7 +8,7 @@
 #include "folding.hpp"
 
 namespace bayesnet {
-    BoostAODE::BoostAODE() : Ensemble()
+    BoostAODE::BoostAODE() : Ensemble(false)
     {
         validHyperparameters = { "repeatSparent", "maxModels", "ascending", "convergence", "threshold", "select_features", "tolerance" };
 

src/BayesNet/Classifier.cc

@@ -3,6 +3,7 @@
 
 namespace bayesnet {
     Classifier::Classifier(Network model) : model(model), m(0), n(0), metrics(Metrics()), fitted(false) {}
+    const std::string CLASSIFIER_NOT_FITTED = "Classifier has not been fitted";
     Classifier& Classifier::build(const std::vector<std::string>& features, const std::string& className, std::map<std::string, std::vector<int>>& states, const torch::Tensor& weights)
     {
         this->features = features;
@@ -87,14 +88,14 @@ namespace bayesnet {
     torch::Tensor Classifier::predict(torch::Tensor& X)
     {
         if (!fitted) {
-            throw std::logic_error("Classifier has not been fitted");
+            throw std::logic_error(CLASSIFIER_NOT_FITTED);
         }
         return model.predict(X);
     }
     std::vector<int> Classifier::predict(std::vector<std::vector<int>>& X)
     {
         if (!fitted) {
-            throw std::logic_error("Classifier has not been fitted");
+            throw std::logic_error(CLASSIFIER_NOT_FITTED);
         }
         auto m_ = X[0].size();
         auto n_ = X.size();
@@ -105,10 +106,31 @@ namespace bayesnet {
         auto yp = model.predict(Xd);
         return yp;
     }
+    torch::Tensor Classifier::predict_proba(torch::Tensor& X)
+    {
+        if (!fitted) {
+            throw std::logic_error(CLASSIFIER_NOT_FITTED);
+        }
+        return model.predict_proba(X);
+    }
+    std::vector<std::vector<double>> Classifier::predict_proba(std::vector<std::vector<int>>& X)
+    {
+        if (!fitted) {
+            throw std::logic_error(CLASSIFIER_NOT_FITTED);
+        }
+        auto m_ = X[0].size();
+        auto n_ = X.size();
+        std::vector<std::vector<int>> Xd(n_, std::vector<int>(m_, 0));
+        for (auto i = 0; i < n_; i++) {
+            Xd[i] = std::vector<int>(X[i].begin(), X[i].end());
+        }
+        auto yp = model.predict_proba(Xd);
+        return yp;
+    }
     float Classifier::score(torch::Tensor& X, torch::Tensor& y)
     {
         if (!fitted) {
-            throw std::logic_error("Classifier has not been fitted");
+            throw std::logic_error(CLASSIFIER_NOT_FITTED);
         }
         torch::Tensor y_pred = predict(X);
         return (y_pred == y).sum().item<float>() / y.size(0);
@@ -116,7 +138,7 @@ namespace bayesnet {
     float Classifier::score(std::vector<std::vector<int>>& X, std::vector<int>& y)
     {
         if (!fitted) {
-            throw std::logic_error("Classifier has not been fitted");
+            throw std::logic_error(CLASSIFIER_NOT_FITTED);
         }
         return model.score(X, y);
     }
@@ -145,6 +167,10 @@ namespace bayesnet {
     {
         return fitted ? model.getStates() : 0;
     }
+    int Classifier::getClassNumStates() const
+    {
+        return fitted ? model.getClassNumStates() : 0;
+    }
     std::vector<std::string> Classifier::topological_order()
     {
         return model.topological_sort();

src/BayesNet/Classifier.h

@@ -7,8 +7,31 @@
 
 namespace bayesnet {
     class Classifier : public BaseClassifier {
-    private:
+    public:
-        Classifier& build(const std::vector<std::string>& features, const std::string& className, std::map<std::string, std::vector<int>>& states, const torch::Tensor& weights);
+        Classifier(Network model);
+        virtual ~Classifier() = default;
+        Classifier& fit(std::vector<std::vector<int>>& X, std::vector<int>& y, const std::vector<std::string>& features, const std::string& className, std::map<std::string, std::vector<int>>& states) override;
+        Classifier& fit(torch::Tensor& X, torch::Tensor& y, const std::vector<std::string>& features, const std::string& className, std::map<std::string, std::vector<int>>& states) override;
+        Classifier& fit(torch::Tensor& dataset, const std::vector<std::string>& features, const std::string& className, std::map<std::string, std::vector<int>>& states) override;
+        Classifier& fit(torch::Tensor& dataset, const std::vector<std::string>& features, const std::string& className, std::map<std::string, std::vector<int>>& states, const torch::Tensor& weights) override;
+        void addNodes();
+        int getNumberOfNodes() const override;
+        int getNumberOfEdges() const override;
+        int getNumberOfStates() const override;
+        int getClassNumStates() const override;
+        torch::Tensor predict(torch::Tensor& X) override;
+        std::vector<int> predict(std::vector<std::vector<int>>& X) override;
+        torch::Tensor predict_proba(torch::Tensor& X) override;
+        std::vector<std::vector<double>> predict_proba(std::vector<std::vector<int>>& X) override;
+        status_t getStatus() const override { return status; }
+        std::string getVersion() override { return { project_version.begin(), project_version.end() }; };
+        float score(torch::Tensor& X, torch::Tensor& y) override;
+        float score(std::vector<std::vector<int>>& X, std::vector<int>& y) override;
+        std::vector<std::string> show() const override;
+        std::vector<std::string> topological_order()  override;
+        std::vector<std::string> getNotes() const override { return notes; }
+        void dump_cpt() const override;
+        void setHyperparameters(const nlohmann::json& hyperparameters) override; //For classifiers that don't have hyperparameters
     protected:
         bool fitted;
         int m, n; // m: number of samples, n: number of features
@@ -24,28 +47,8 @@ namespace bayesnet {
         virtual void buildModel(const torch::Tensor& weights) = 0;
         void trainModel(const torch::Tensor& weights) override;
         void buildDataset(torch::Tensor& y);
-    public:
+    private:
-        Classifier(Network model);
+        Classifier& build(const std::vector<std::string>& features, const std::string& className, std::map<std::string, std::vector<int>>& states, const torch::Tensor& weights);
-        virtual ~Classifier() = default;
-        Classifier& fit(std::vector<std::vector<int>>& X, std::vector<int>& y, const std::vector<std::string>& features, const std::string& className, std::map<std::string, std::vector<int>>& states) override;
-        Classifier& fit(torch::Tensor& X, torch::Tensor& y, const std::vector<std::string>& features, const std::string& className, std::map<std::string, std::vector<int>>& states) override;
-        Classifier& fit(torch::Tensor& dataset, const std::vector<std::string>& features, const std::string& className, std::map<std::string, std::vector<int>>& states) override;
-        Classifier& fit(torch::Tensor& dataset, const std::vector<std::string>& features, const std::string& className, std::map<std::string, std::vector<int>>& states, const torch::Tensor& weights) override;
-        void addNodes();
-        int getNumberOfNodes() const override;
-        int getNumberOfEdges() const override;
-        int getNumberOfStates() const override;
-        torch::Tensor predict(torch::Tensor& X) override;
-        status_t getStatus() const override { return status; }
-        std::string getVersion() override { return { project_version.begin(), project_version.end() }; };
-        std::vector<int> predict(std::vector<std::vector<int>>& X) override;
-        float score(torch::Tensor& X, torch::Tensor& y) override;
-        float score(std::vector<std::vector<int>>& X, std::vector<int>& y) override;
-        std::vector<std::string> show() const override;
-        std::vector<std::string> topological_order()  override;
-        std::vector<std::string> getNotes() const override { return notes; }
-        void dump_cpt() const override;
-        void setHyperparameters(const nlohmann::json& hyperparameters) override; //For classifiers that don't have hyperparameters
     };
 }
 #endif

src/BayesNet/Ensemble.cc

@@ -2,7 +2,10 @@
 
 namespace bayesnet {
 
-    Ensemble::Ensemble(bool predict_voting) : Classifier(Network()), n_models(0), predict_voting(predict_voting) {};
+    Ensemble::Ensemble(bool predict_voting) : Classifier(Network()), n_models(0), predict_voting(predict_voting)
+    {
+
+    };
     const std::string ENSEMBLE_NOT_FITTED = "Ensemble has not been fitted";
     void Ensemble::trainModel(const torch::Tensor& weights)
     {
@@ -37,7 +40,7 @@ namespace bayesnet {
         if (!fitted) {
             throw std::logic_error(ENSEMBLE_NOT_FITTED);
         }
-        return predict_voting ? do_predict_voting(X) : do_predict_prob(X);
+        return do_predict_voting(X);
 
     }
     torch::Tensor Ensemble::predict(torch::Tensor& X)
@@ -45,27 +48,32 @@ namespace bayesnet {
         if (!fitted) {
             throw std::logic_error(ENSEMBLE_NOT_FITTED);
         }
-        return predict_voting ? do_predict_voting(X) : do_predict_prob(X);
+        return do_predict_voting(X);
     }
-    torch::Tensor Ensemble::do_predict_prob(torch::Tensor& X)
+    torch::Tensor Ensemble::predict_proba(torch::Tensor& X)
     {
-        torch::Tensor y_pred = torch::zeros({ X.size(1), n_models }, torch::kFloat32);
+        auto n_states = getClassNumStates();
-        // auto threads{ std::vector<std::thread>() };
+        torch::Tensor y_pred = torch::zeros({ X.size(1), n_states }, torch::kFloat32);
-        // std::mutex mtx;
+        auto threads{ std::vector<std::thread>() };
-        // for (auto i = 0; i < n_models; ++i) {
+        std::mutex mtx;
-        //     threads.push_back(std::thread([&, i]() {
+        for (auto i = 0; i < n_models; ++i) {
-        //         auto ypredict = models[i]->predict(X);
+            threads.push_back(std::thread([&, i]() {
-        //         std::lock_guard<std::mutex> lock(mtx);
+                auto ypredict = models[i]->predict_proba(X);
-        //         y_pred.index_put_({ "...", i }, ypredict);
+                ypredict *= significanceModels[i];
-        //         }));
+                std::lock_guard<std::mutex> lock(mtx);
-        // }
+                y_pred.index_put_({ "...", i }, ypredict);
-        // for (auto& thread : threads) {
+                }));
-        //     thread.join();
+        }
-        // }
+        for (auto& thread : threads) {
+            thread.join();
+        }
+        auto sum = std::reduce(significanceModels.begin(), significanceModels.end());
+        y_pred /= sum;
         return y_pred;
     }
-    std::vector<int> Ensemble::do_predict_prob(std::vector<std::vector<int>>& X)
+    std::vector<std::vector<double>> Ensemble::predict_proba(std::vector<std::vector<int>>& X)
     {
+
         // long m_ = X[0].size();
         // long n_ = X.size();
         // vector<vector<int>> Xd(n_, vector<int>(m_, 0));
@@ -77,7 +85,7 @@ namespace bayesnet {
         //     y_pred.index_put_({ "...", i }, torch::tensor(models[i]->predict(Xd), torch::kInt32));
         // }
         // return voting(y_pred);
-        return std::vector<int>();
+        return std::vector<std::vector<double>>();
     }
     torch::Tensor Ensemble::do_predict_voting(torch::Tensor& X)
     {
@@ -105,14 +113,23 @@ namespace bayesnet {
             Xd[i] = std::vector<int>(X[i].begin(), X[i].end());
         }
         torch::Tensor y_pred = torch::zeros({ m_, n_models }, torch::kInt32);
+        auto threads{ std::vector<std::thread>() };
+        std::mutex mtx;
         for (auto i = 0; i < n_models; ++i) {
-            y_pred.index_put_({ "...", i }, torch::tensor(models[i]->predict(Xd), torch::kInt32));
+            threads.push_back(std::thread([&, i]() {
+                auto ypredict = models[i]->predict(Xd);
+                std::lock_guard<std::mutex> lock(mtx);
+                y_pred.index_put_({ "...", i }, torch::tensor(ypredict, torch::kInt32));
+                }));
+        }
+        for (auto& thread : threads) {
+            thread.join();
         }
         return voting(y_pred);
     }
     float Ensemble::score(torch::Tensor& X, torch::Tensor& y)
     {
-        auto y_pred = predict_voting ? do_predict_voting(X) : do_predict_prob(X);
+        auto y_pred = do_predict_voting(X);
         int correct = 0;
         for (int i = 0; i < y_pred.size(0); ++i) {
             if (y_pred[i].item<int>() == y[i].item<int>()) {
@@ -123,7 +140,7 @@ namespace bayesnet {
     }
     float Ensemble::score(std::vector<std::vector<int>>& X, std::vector<int>& y)
     {
-        auto y_pred = predict_voting ? do_predict_voting(X) : do_predict_prob(X);
+        auto y_pred = do_predict_voting(X);
         int correct = 0;
         for (int i = 0; i < y_pred.size(); ++i) {
             if (y_pred[i] == y[i]) {

src/BayesNet/Ensemble.h

@@ -12,10 +12,10 @@ namespace bayesnet {
         virtual ~Ensemble() = default;
         torch::Tensor predict(torch::Tensor& X) override;
         std::vector<int> predict(std::vector<std::vector<int>>& X) override;
+        torch::Tensor predict_proba(torch::Tensor& X) override;
+        std::vector<std::vector<double>> predict_proba(std::vector<std::vector<int>>& X) override;
         torch::Tensor do_predict_voting(torch::Tensor& X);
         std::vector<int> do_predict_voting(std::vector<std::vector<int>>& X);
-        torch::Tensor do_predict_prob(torch::Tensor& X);
-        std::vector<int> do_predict_prob(std::vector<std::vector<int>>& X);
         float score(torch::Tensor& X, torch::Tensor& y) override;
         float score(std::vector<std::vector<int>>& X, std::vector<int>& y) override;
         int getNumberOfNodes() const override;

src/BayesNet/Network.h

@@ -7,23 +7,6 @@
 
 namespace bayesnet {
     class Network {
-    private:
-        std::map<std::string, std::unique_ptr<Node>> nodes;
-        bool fitted;
-        float maxThreads = 0.95;
-        int classNumStates;
-        std::vector<std::string> features; // Including classname
-        std::string className;
-        double laplaceSmoothing;
-        torch::Tensor samples; // nxm tensor used to fit the model
-        bool isCyclic(const std::string&, std::unordered_set<std::string>&, std::unordered_set<std::string>&);
-        std::vector<double> predict_sample(const std::vector<int>&);
-        std::vector<double> predict_sample(const torch::Tensor&);
-        std::vector<double> exactInference(std::map<std::string, int>&);
-        double computeFactor(std::map<std::string, int>&);
-        void completeFit(const std::map<std::string, std::vector<int>>& states, const torch::Tensor& weights);
-        void checkFitData(int n_features, int n_samples, int n_samples_y, const std::vector<std::string>& featureNames, const std::string& className, const std::map<std::string, std::vector<int>>& states, const torch::Tensor& weights);
-        void setStates(const std::map<std::string, std::vector<int>>&);
     public:
         Network();
         explicit Network(float);
@@ -58,6 +41,23 @@ namespace bayesnet {
         void initialize();
         void dump_cpt() const;
         inline std::string version() { return  { project_version.begin(), project_version.end() }; }
+    private:
+        std::map<std::string, std::unique_ptr<Node>> nodes;
+        bool fitted;
+        float maxThreads = 0.95;
+        int classNumStates;
+        std::vector<std::string> features; // Including classname
+        std::string className;
+        double laplaceSmoothing;
+        torch::Tensor samples; // nxm tensor used to fit the model
+        bool isCyclic(const std::string&, std::unordered_set<std::string>&, std::unordered_set<std::string>&);
+        std::vector<double> predict_sample(const std::vector<int>&);
+        std::vector<double> predict_sample(const torch::Tensor&);
+        std::vector<double> exactInference(std::map<std::string, int>&);
+        double computeFactor(std::map<std::string, int>&);
+        void completeFit(const std::map<std::string, std::vector<int>>& states, const torch::Tensor& weights);
+        void checkFitData(int n_features, int n_samples, int n_samples_y, const std::vector<std::string>& featureNames, const std::string& className, const std::map<std::string, std::vector<int>>& states, const torch::Tensor& weights);
+        void setStates(const std::map<std::string, std::vector<int>>&);
     };
 }
 #endif

tests/TestBayesModels.cc

@@ -165,7 +165,6 @@ TEST_CASE("BoostAODE test used features in train note", "[BayesNet]")
         {"convergence", true},
         {"repeatSparent",true},
         {"select_features","CFS"},
-        {"tolerance", 3}
         });
     clf.fit(raw.Xv, raw.yv, raw.featuresv, raw.classNamev, raw.statesv);
     REQUIRE(clf.getNumberOfNodes() == 72);
@@ -175,3 +174,42 @@ TEST_CASE("BoostAODE test used features in train note", "[BayesNet]")
     REQUIRE(clf.getNotes()[1] == "Used features in train: 7 of 8");
     REQUIRE(clf.getNotes()[2] == "Number of models: 8");
 }
+TEST_CASE("TAN predict_proba", "[BayesNet]")
+{
+    auto raw = RawDatasets("iris", true);
+    auto clf = bayesnet::TAN();
+    clf.fit(raw.Xv, raw.yv, raw.featuresv, raw.classNamev, raw.statesv);
+    auto y_pred_proba = clf.predict_proba(raw.Xv);
+    auto y_pred = clf.predict(raw.Xv);
+    auto yt_pred_proba = clf.predict_proba(raw.Xt);
+    REQUIRE(y_pred.size() == y_pred_proba.size());
+    REQUIRE(y_pred.size() == yt_pred_proba.size(0));
+    REQUIRE(y_pred.size() == raw.yv.size());
+    REQUIRE(y_pred_proba[0].size() == 3);
+    REQUIRE(yt_pred_proba.size(1) == y_pred_proba[0].size());
+    for (int i = 0; i < y_pred_proba.size(); ++i) {
+        auto maxElem = max_element(y_pred_proba[i].begin(), y_pred_proba[i].end());
+        int predictedClass = distance(y_pred_proba[i].begin(), maxElem);
+        REQUIRE(predictedClass == y_pred[i]);
+        REQUIRE(yt_pred_proba[i].argmax().item<int>() == y_pred[i]);
+    }
+}
+
+// TEST_CASE("BoostAODE predict_proba", "[BayesNet]")
+// {
+//     auto raw = RawDatasets("iris", true);
+//     auto clf = bayesnet::BoostAODE();
+//     clf.fit(raw.Xv, raw.yv, raw.featuresv, raw.classNamev, raw.statesv);
+//     auto y_pred = clf.predict_proba(raw.Xv);
+//     REQUIRE(y_pred.size(0) == raw.yv.size(0));
+//     REQUIRE(y_pred.size(1) == 3);
+//     auto y_pred2 = clf.predict_proba(raw.Xv);
+//     REQUIRE(y_pred2.size(0) == raw.yv.size(0));
+//     REQUIRE(y_pred2.size(1) == 3);
+//     REQUIRE(y_pred.equal(y_pred2));
+//     for (int i = 0; i < y_pred.size(0); ++i) {
+//         for (int j = 0; j < y_pred.size(1); ++j) {
+//             REQUIRE(y_pred[i][j].item<float>() == y_pred2[i][j].item<float>());
+//         }
+//     }
+// }