diff --git a/sample/CMakeLists.txt b/sample/CMakeLists.txt index 000a88b..4a41a72 100644 --- a/sample/CMakeLists.txt +++ b/sample/CMakeLists.txt @@ -3,5 +3,6 @@ include_directories(${BayesNet_SOURCE_DIR}/src/BayesNet) include_directories(${BayesNet_SOURCE_DIR}/lib/Files) include_directories(${BayesNet_SOURCE_DIR}/lib/mdlp) include_directories(${BayesNet_SOURCE_DIR}/lib/argparse/include) +include_directories(${BayesNet_SOURCE_DIR}/lib/json/include) add_executable(BayesNetSample sample.cc ${BayesNet_SOURCE_DIR}/src/Platform/Folding.cc ${BayesNet_SOURCE_DIR}/src/Platform/Models.cc) target_link_libraries(BayesNetSample BayesNet ArffFiles mdlp "${TORCH_LIBRARIES}") \ No newline at end of file diff --git a/sample/sample.cc b/sample/sample.cc index 1045c2f..00395fd 100644 --- a/sample/sample.cc +++ b/sample/sample.cc @@ -3,6 +3,7 @@ #include #include #include +#include #include "ArffFiles.h" #include "BayesMetrics.h" #include "CPPFImdlp.h" @@ -141,111 +142,97 @@ int main(int argc, char** argv) /* * Begin Processing */ - auto ypred = torch::tensor({ 1,2,3,2,2,3,4,5,2,1 }); - auto y = torch::tensor({ 0,0,0,0,2,3,4,0,0,0 }); - auto weights = torch::ones({ 10 }, kDouble); - auto mask = ypred == y; - cout << "ypred:" << ypred << endl; - cout << "y:" << y << endl; - cout << "weights:" << weights << endl; - cout << "mask:" << mask << endl; - double value_to_add = 0.5; - weights += mask.to(torch::kDouble) * value_to_add; - cout << "New weights:" << weights << endl; - auto masked_weights = weights * mask.to(weights.dtype()); - double sum_of_weights = masked_weights.sum().item(); - cout << "Sum of weights: " << sum_of_weights << endl; - //weights.index_put_({ mask }, weights + 10); - // auto handler = ArffFiles(); - // handler.load(complete_file_name, class_last); - // // Get Dataset X, y - // vector& X = handler.getX(); - // mdlp::labels_t& y = handler.getY(); - // // Get className & Features - // auto className = handler.getClassName(); - // vector features; - // auto attributes = handler.getAttributes(); - // transform(attributes.begin(), attributes.end(), back_inserter(features), - // [](const pair& item) { return item.first; }); - // // Discretize Dataset - // auto [Xd, maxes] = discretize(X, y, features); - // maxes[className] = *max_element(y.begin(), y.end()) + 1; - // map> states; - // for (auto feature : features) { - // states[feature] = vector(maxes[feature]); - // } - // states[className] = vector(maxes[className]); - // auto clf = platform::Models::instance()->create(model_name); - // clf->fit(Xd, y, features, className, states); - // if (dump_cpt) { - // cout << "--- CPT Tables ---" << endl; - // clf->dump_cpt(); - // } - // auto lines = clf->show(); - // for (auto line : lines) { - // cout << line << endl; - // } - // cout << "--- Topological Order ---" << endl; - // auto order = clf->topological_order(); - // for (auto name : order) { - // cout << name << ", "; - // } - // cout << "end." << endl; - // auto score = clf->score(Xd, y); - // cout << "Score: " << score << endl; - // auto graph = clf->graph(); - // auto dot_file = model_name + "_" + file_name; - // ofstream file(dot_file + ".dot"); - // file << graph; - // file.close(); - // cout << "Graph saved in " << model_name << "_" << file_name << ".dot" << endl; - // cout << "dot -Tpng -o " + dot_file + ".png " + dot_file + ".dot " << endl; - // string stratified_string = stratified ? " Stratified" : ""; - // cout << nFolds << " Folds" << stratified_string << " Cross validation" << endl; - // cout << "==========================================" << endl; - // torch::Tensor Xt = torch::zeros({ static_cast(Xd.size()), static_cast(Xd[0].size()) }, torch::kInt32); - // torch::Tensor yt = torch::tensor(y, torch::kInt32); - // for (int i = 0; i < features.size(); ++i) { - // Xt.index_put_({ i, "..." }, torch::tensor(Xd[i], torch::kInt32)); - // } - // float total_score = 0, total_score_train = 0, score_train, score_test; - // Fold* fold; - // if (stratified) - // fold = new StratifiedKFold(nFolds, y, seed); - // else - // fold = new KFold(nFolds, y.size(), seed); - // for (auto i = 0; i < nFolds; ++i) { - // auto [train, test] = fold->getFold(i); - // cout << "Fold: " << i + 1 << endl; - // if (tensors) { - // auto ttrain = torch::tensor(train, torch::kInt64); - // auto ttest = torch::tensor(test, torch::kInt64); - // torch::Tensor Xtraint = torch::index_select(Xt, 1, ttrain); - // torch::Tensor ytraint = yt.index({ ttrain }); - // torch::Tensor Xtestt = torch::index_select(Xt, 1, ttest); - // torch::Tensor ytestt = yt.index({ ttest }); - // clf->fit(Xtraint, ytraint, features, className, states); - // auto temp = clf->predict(Xtraint); - // score_train = clf->score(Xtraint, ytraint); - // score_test = clf->score(Xtestt, ytestt); - // } else { - // auto [Xtrain, ytrain] = extract_indices(train, Xd, y); - // auto [Xtest, ytest] = extract_indices(test, Xd, y); - // clf->fit(Xtrain, ytrain, features, className, states); - // score_train = clf->score(Xtrain, ytrain); - // score_test = clf->score(Xtest, ytest); - // } - // if (dump_cpt) { - // cout << "--- CPT Tables ---" << endl; - // clf->dump_cpt(); - // } - // total_score_train += score_train; - // total_score += score_test; - // cout << "Score Train: " << score_train << endl; - // cout << "Score Test : " << score_test << endl; - // cout << "-------------------------------------------------------------------------------" << endl; - // } - // cout << "**********************************************************************************" << endl; - // cout << "Average Score Train: " << total_score_train / nFolds << endl; - // cout << "Average Score Test : " << total_score / nFolds << endl;return 0; + weights.index_put_({ mask }, weights + 10); + auto handler = ArffFiles(); + handler.load(complete_file_name, class_last); + // Get Dataset X, y + vector& X = handler.getX(); + mdlp::labels_t& y = handler.getY(); + // Get className & Features + auto className = handler.getClassName(); + vector features; + auto attributes = handler.getAttributes(); + transform(attributes.begin(), attributes.end(), back_inserter(features), + [](const pair& item) { return item.first; }); + // Discretize Dataset + auto [Xd, maxes] = discretize(X, y, features); + maxes[className] = *max_element(y.begin(), y.end()) + 1; + map> states; + for (auto feature : features) { + states[feature] = vector(maxes[feature]); + } + states[className] = vector(maxes[className]); + auto clf = platform::Models::instance()->create(model_name); + clf->fit(Xd, y, features, className, states); + if (dump_cpt) { + cout << "--- CPT Tables ---" << endl; + clf->dump_cpt(); + } + auto lines = clf->show(); + for (auto line : lines) { + cout << line << endl; + } + cout << "--- Topological Order ---" << endl; + auto order = clf->topological_order(); + for (auto name : order) { + cout << name << ", "; + } + cout << "end." << endl; + auto score = clf->score(Xd, y); + cout << "Score: " << score << endl; + auto graph = clf->graph(); + auto dot_file = model_name + "_" + file_name; + ofstream file(dot_file + ".dot"); + file << graph; + file.close(); + cout << "Graph saved in " << model_name << "_" << file_name << ".dot" << endl; + cout << "dot -Tpng -o " + dot_file + ".png " + dot_file + ".dot " << endl; + string stratified_string = stratified ? " Stratified" : ""; + cout << nFolds << " Folds" << stratified_string << " Cross validation" << endl; + cout << "==========================================" << endl; + torch::Tensor Xt = torch::zeros({ static_cast(Xd.size()), static_cast(Xd[0].size()) }, torch::kInt32); + torch::Tensor yt = torch::tensor(y, torch::kInt32); + for (int i = 0; i < features.size(); ++i) { + Xt.index_put_({ i, "..." }, torch::tensor(Xd[i], torch::kInt32)); + } + float total_score = 0, total_score_train = 0, score_train, score_test; + Fold* fold; + if (stratified) + fold = new StratifiedKFold(nFolds, y, seed); + else + fold = new KFold(nFolds, y.size(), seed); + for (auto i = 0; i < nFolds; ++i) { + auto [train, test] = fold->getFold(i); + cout << "Fold: " << i + 1 << endl; + if (tensors) { + auto ttrain = torch::tensor(train, torch::kInt64); + auto ttest = torch::tensor(test, torch::kInt64); + torch::Tensor Xtraint = torch::index_select(Xt, 1, ttrain); + torch::Tensor ytraint = yt.index({ ttrain }); + torch::Tensor Xtestt = torch::index_select(Xt, 1, ttest); + torch::Tensor ytestt = yt.index({ ttest }); + clf->fit(Xtraint, ytraint, features, className, states); + auto temp = clf->predict(Xtraint); + score_train = clf->score(Xtraint, ytraint); + score_test = clf->score(Xtestt, ytestt); + } else { + auto [Xtrain, ytrain] = extract_indices(train, Xd, y); + auto [Xtest, ytest] = extract_indices(test, Xd, y); + clf->fit(Xtrain, ytrain, features, className, states); + score_train = clf->score(Xtrain, ytrain); + score_test = clf->score(Xtest, ytest); + } + if (dump_cpt) { + cout << "--- CPT Tables ---" << endl; + clf->dump_cpt(); + } + total_score_train += score_train; + total_score += score_test; + cout << "Score Train: " << score_train << endl; + cout << "Score Test : " << score_test << endl; + cout << "-------------------------------------------------------------------------------" << endl; + } + cout << "**********************************************************************************" << endl; + cout << "Average Score Train: " << total_score_train / nFolds << endl; + cout << "Average Score Test : " << total_score / nFolds << endl;return 0; } \ No newline at end of file diff --git a/src/BayesNet/AODE.h b/src/BayesNet/AODE.h index 00965f6..78c9876 100644 --- a/src/BayesNet/AODE.h +++ b/src/BayesNet/AODE.h @@ -10,6 +10,7 @@ namespace bayesnet { AODE(); virtual ~AODE() {}; vector graph(const string& title = "AODE") const override; + void setHyperparameters(nlohmann::json& hyperparameters) override {}; }; } #endif \ No newline at end of file diff --git a/src/BayesNet/AODELd.h b/src/BayesNet/AODELd.h index aa67247..e85c2f6 100644 --- a/src/BayesNet/AODELd.h +++ b/src/BayesNet/AODELd.h @@ -16,6 +16,7 @@ namespace bayesnet { virtual ~AODELd() = default; vector graph(const string& name = "AODE") const override; static inline string version() { return "0.0.1"; }; + void setHyperparameters(nlohmann::json& hyperparameters) override {}; }; } #endif // !AODELD_H \ No newline at end of file diff --git a/src/BayesNet/BaseClassifier.h b/src/BayesNet/BaseClassifier.h index 5f1cbaa..130276a 100644 --- a/src/BayesNet/BaseClassifier.h +++ b/src/BayesNet/BaseClassifier.h @@ -1,6 +1,7 @@ #ifndef BASE_H #define BASE_H #include +#include #include namespace bayesnet { using namespace std; @@ -27,6 +28,7 @@ namespace bayesnet { const string inline getVersion() const { return "0.1.0"; }; vector virtual topological_order() = 0; void virtual dump_cpt()const = 0; + virtual void setHyperparameters(nlohmann::json& hyperparameters) = 0; }; } #endif \ No newline at end of file diff --git a/src/BayesNet/BoostAODE.cc b/src/BayesNet/BoostAODE.cc index eb8da07..1b8681f 100644 --- a/src/BayesNet/BoostAODE.cc +++ b/src/BayesNet/BoostAODE.cc @@ -2,11 +2,17 @@ #include "BayesMetrics.h" namespace bayesnet { - BoostAODE::BoostAODE() : Ensemble() {} + BoostAODE::BoostAODE() : Ensemble(), repeatSparent(false) {} void BoostAODE::buildModel(const torch::Tensor& weights) { // Models shall be built in trainModel } + void BoostAODE::setHyperparameters(nlohmann::json& hyperparameters) + { + if (hyperparameters.contains("repeatSparent")) { + repeatSparent = hyperparameters["repeatSparent"]; + } + } void BoostAODE::trainModel(const torch::Tensor& weights) { models.clear(); @@ -16,7 +22,6 @@ namespace bayesnet { auto X_ = dataset.index({ torch::indexing::Slice(0, dataset.size(0) - 1), "..." }); auto y_ = dataset.index({ -1, "..." }); bool exitCondition = false; - bool repeatSparent = false; vector featuresUsed; // Step 0: Set the finish condition // if not repeatSparent a finish condition is run out of features diff --git a/src/BayesNet/BoostAODE.h b/src/BayesNet/BoostAODE.h index b14c7c6..290b1a2 100644 --- a/src/BayesNet/BoostAODE.h +++ b/src/BayesNet/BoostAODE.h @@ -4,13 +4,16 @@ #include "SPODE.h" namespace bayesnet { class BoostAODE : public Ensemble { - protected: - void buildModel(const torch::Tensor& weights) override; - void trainModel(const torch::Tensor& weights) override; public: BoostAODE(); virtual ~BoostAODE() {}; vector graph(const string& title = "BoostAODE") const override; + void setHyperparameters(nlohmann::json& hyperparameters) override; + protected: + void buildModel(const torch::Tensor& weights) override; + void trainModel(const torch::Tensor& weights) override; + private: + bool repeatSparent; }; } #endif \ No newline at end of file diff --git a/src/BayesNet/CMakeLists.txt b/src/BayesNet/CMakeLists.txt index 435511c..2a120f3 100644 --- a/src/BayesNet/CMakeLists.txt +++ b/src/BayesNet/CMakeLists.txt @@ -1,5 +1,6 @@ include_directories(${BayesNet_SOURCE_DIR}/lib/mdlp) include_directories(${BayesNet_SOURCE_DIR}/lib/Files) +include_directories(${BayesNet_SOURCE_DIR}/lib/json/include) include_directories(${BayesNet_SOURCE_DIR}/src/BayesNet) include_directories(${BayesNet_SOURCE_DIR}/src/Platform) add_library(BayesNet bayesnetUtils.cc Network.cc Node.cc BayesMetrics.cc Classifier.cc diff --git a/src/BayesNet/KDB.h b/src/BayesNet/KDB.h index b997cdd..713b415 100644 --- a/src/BayesNet/KDB.h +++ b/src/BayesNet/KDB.h @@ -16,6 +16,7 @@ namespace bayesnet { public: explicit KDB(int k, float theta = 0.03); virtual ~KDB() {}; + void setHyperparameters(nlohmann::json& hyperparameters) override {}; vector graph(const string& name = "KDB") const override; }; } diff --git a/src/BayesNet/KDBLd.h b/src/BayesNet/KDBLd.h index 50a1b95..f463c80 100644 --- a/src/BayesNet/KDBLd.h +++ b/src/BayesNet/KDBLd.h @@ -13,6 +13,7 @@ namespace bayesnet { KDBLd& fit(torch::Tensor& X, torch::Tensor& y, vector& features, string className, map>& states) override; vector graph(const string& name = "KDB") const override; Tensor predict(Tensor& X) override; + void setHyperparameters(nlohmann::json& hyperparameters) override {}; static inline string version() { return "0.0.1"; }; }; } diff --git a/src/BayesNet/SPODE.h b/src/BayesNet/SPODE.h index 0a78830..c5a95e4 100644 --- a/src/BayesNet/SPODE.h +++ b/src/BayesNet/SPODE.h @@ -12,6 +12,7 @@ namespace bayesnet { explicit SPODE(int root); virtual ~SPODE() {}; vector graph(const string& name = "SPODE") const override; + void setHyperparameters(nlohmann::json& hyperparameters) override {}; }; } #endif \ No newline at end of file diff --git a/src/BayesNet/SPODELd.h b/src/BayesNet/SPODELd.h index b94fc6c..94957e6 100644 --- a/src/BayesNet/SPODELd.h +++ b/src/BayesNet/SPODELd.h @@ -13,6 +13,7 @@ namespace bayesnet { SPODELd& fit(torch::Tensor& dataset, vector& features, string className, map>& states) override; vector graph(const string& name = "SPODE") const override; Tensor predict(Tensor& X) override; + void setHyperparameters(nlohmann::json& hyperparameters) override {}; static inline string version() { return "0.0.1"; }; }; } diff --git a/src/BayesNet/TAN.h b/src/BayesNet/TAN.h index 91b5109..c8eea5f 100644 --- a/src/BayesNet/TAN.h +++ b/src/BayesNet/TAN.h @@ -3,7 +3,6 @@ #include "Classifier.h" namespace bayesnet { using namespace std; - using namespace torch; class TAN : public Classifier { private: protected: @@ -12,6 +11,7 @@ namespace bayesnet { TAN(); virtual ~TAN() {}; vector graph(const string& name = "TAN") const override; + void setHyperparameters(nlohmann::json& hyperparameters) override {}; }; } #endif \ No newline at end of file diff --git a/src/BayesNet/TANLd.h b/src/BayesNet/TANLd.h index c35e843..7ee5eb6 100644 --- a/src/BayesNet/TANLd.h +++ b/src/BayesNet/TANLd.h @@ -14,6 +14,7 @@ namespace bayesnet { vector graph(const string& name = "TAN") const override; Tensor predict(Tensor& X) override; static inline string version() { return "0.0.1"; }; + void setHyperparameters(nlohmann::json& hyperparameters) override {}; }; } #endif // !TANLD_H \ No newline at end of file diff --git a/src/Platform/Experiment.cc b/src/Platform/Experiment.cc index cdd6da6..88e3125 100644 --- a/src/Platform/Experiment.cc +++ b/src/Platform/Experiment.cc @@ -25,6 +25,7 @@ namespace platform { oss << std::put_time(timeinfo, "%H:%M:%S"); return oss.str(); } + Experiment::Experiment() : hyperparameters(json::parse("{}")) {} string Experiment::get_file_name() { string result = "results_" + score_name + "_" + model + "_" + platform + "_" + get_date() + "_" + get_time() + "_" + (stratified ? "1" : "0") + ".json"; @@ -124,6 +125,8 @@ namespace platform { auto result = Result(); auto [values, counts] = at::_unique(y); result.setSamples(X.size(1)).setFeatures(X.size(0)).setClasses(values.size(0)); + result.setHyperparameters(hyperparameters); + // Initialize results vectors int nResults = nfolds * static_cast(randomSeeds.size()); auto accuracy_test = torch::zeros({ nResults }, torch::kFloat64); auto accuracy_train = torch::zeros({ nResults }, torch::kFloat64); @@ -144,6 +147,10 @@ namespace platform { for (int nfold = 0; nfold < nfolds; nfold++) { auto clf = Models::instance()->create(model); setModelVersion(clf->getVersion()); + if (hyperparameters.size() != 0) { + clf->setHyperparameters(hyperparameters); + } + // Split train - test dataset train_timer.start(); auto [train, test] = fold->getFold(nfold); auto train_t = torch::tensor(train); @@ -153,12 +160,14 @@ namespace platform { auto X_test = X.index({ "...", test_t }); auto y_test = y.index({ test_t }); cout << nfold + 1 << ", " << flush; + // Train model clf->fit(X_train, y_train, features, className, states); nodes[item] = clf->getNumberOfNodes(); edges[item] = clf->getNumberOfEdges(); num_states[item] = clf->getNumberOfStates(); train_time[item] = train_timer.getDuration(); auto accuracy_train_value = clf->score(X_train, y_train); + // Test model test_timer.start(); auto accuracy_test_value = clf->score(X_test, y_test); test_time[item] = test_timer.getDuration(); diff --git a/src/Platform/Experiment.h b/src/Platform/Experiment.h index e2f1936..fe4415e 100644 --- a/src/Platform/Experiment.h +++ b/src/Platform/Experiment.h @@ -29,7 +29,8 @@ namespace platform { }; class Result { private: - string dataset, hyperparameters, model_version; + string dataset, model_version; + json hyperparameters; int samples{ 0 }, features{ 0 }, classes{ 0 }; double score_train{ 0 }, score_test{ 0 }, score_train_std{ 0 }, score_test_std{ 0 }, train_time{ 0 }, train_time_std{ 0 }, test_time{ 0 }, test_time_std{ 0 }; float nodes{ 0 }, leaves{ 0 }, depth{ 0 }; @@ -37,7 +38,7 @@ namespace platform { public: Result() = default; Result& setDataset(const string& dataset) { this->dataset = dataset; return *this; } - Result& setHyperparameters(const string& hyperparameters) { this->hyperparameters = hyperparameters; return *this; } + Result& setHyperparameters(const json& hyperparameters) { this->hyperparameters = hyperparameters; return *this; } Result& setSamples(int samples) { this->samples = samples; return *this; } Result& setFeatures(int features) { this->features = features; return *this; } Result& setClasses(int classes) { this->classes = classes; return *this; } @@ -59,7 +60,7 @@ namespace platform { const float get_score_train() const { return score_train; } float get_score_test() { return score_test; } const string& getDataset() const { return dataset; } - const string& getHyperparameters() const { return hyperparameters; } + const json& getHyperparameters() const { return hyperparameters; } const int getSamples() const { return samples; } const int getFeatures() const { return features; } const int getClasses() const { return classes; } @@ -85,11 +86,12 @@ namespace platform { bool discretized{ false }, stratified{ false }; vector results; vector randomSeeds; + json hyperparameters = "{}"; int nfolds{ 0 }; float duration{ 0 }; json build_json(); public: - Experiment() = default; + Experiment(); Experiment& setTitle(const string& title) { this->title = title; return *this; } Experiment& setModel(const string& model) { this->model = model; return *this; } Experiment& setPlatform(const string& platform) { this->platform = platform; return *this; } @@ -103,6 +105,7 @@ namespace platform { Experiment& addResult(Result result) { results.push_back(result); return *this; } Experiment& addRandomSeed(int randomSeed) { randomSeeds.push_back(randomSeed); return *this; } Experiment& setDuration(float duration) { this->duration = duration; return *this; } + Experiment& setHyperparameters(const json& hyperparameters) { this->hyperparameters = hyperparameters; return *this; } string get_file_name(); void save(const string& path); void cross_validation(const string& path, const string& fileName); diff --git a/src/Platform/main.cc b/src/Platform/main.cc index 6f9ce1c..a2a7cce 100644 --- a/src/Platform/main.cc +++ b/src/Platform/main.cc @@ -1,5 +1,6 @@ #include #include +#include #include "platformUtils.h" #include "Experiment.h" #include "Datasets.h" @@ -10,12 +11,14 @@ using namespace std; +using json = nlohmann::json; argparse::ArgumentParser manageArguments(int argc, char** argv) { auto env = platform::DotEnv(); argparse::ArgumentParser program("main"); program.add_argument("-d", "--dataset").default_value("").help("Dataset file name"); + program.add_argument("--hyperparameters").default_value("{}").help("Hyperparamters passed to the model in Experiment"); program.add_argument("-p", "--path") .help("folder where the data files are located, default") .default_value(string{ platform::Paths::datasets() }); @@ -59,6 +62,7 @@ argparse::ArgumentParser manageArguments(int argc, char** argv) auto seeds = program.get>("seeds"); auto complete_file_name = path + file_name + ".arff"; auto title = program.get("title"); + auto hyperparameters = program.get("hyperparameters"); if (title == "" && file_name == "") { throw runtime_error("title is mandatory if dataset is not provided"); } @@ -82,6 +86,7 @@ int main(int argc, char** argv) auto stratified = program.get("stratified"); auto n_folds = program.get("folds"); auto seeds = program.get>("seeds"); + auto hyperparameters =program.get("hyperparameters"); vector filesToTest; auto datasets = platform::Datasets(path, true, platform::ARFF); auto title = program.get("title"); @@ -106,6 +111,7 @@ int main(int argc, char** argv) experiment.setTitle(title).setLanguage("cpp").setLanguageVersion("14.0.3"); experiment.setDiscretized(discretize_dataset).setModel(model_name).setPlatform(env.get("platform")); experiment.setStratified(stratified).setNFolds(n_folds).setScoreName("accuracy"); + experiment.setHyperparameters(json::parse(hyperparameters)); for (auto seed : seeds) { experiment.addRandomSeed(seed); }