From 7b6d05b5acdd904cfe90cd49d5cdfe70fd0cf6d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Monta=C3=B1ana=20G=C3=B3mez?= Date: Sat, 4 Nov 2023 01:21:43 +0100 Subject: [PATCH] Complete Example --- src/PyClassifier.cc | 18 ++--- src/PyHelper.hpp | 171 +++++++++++++++++++++-------------------- src/PyWrap.cc | 41 ++++------ src/PyWrap.h | 10 ++- src/example.cpp | 181 ++++++++++++++++++++++++++------------------ src/main.cc | 8 +- 6 files changed, 227 insertions(+), 202 deletions(-) diff --git a/src/PyClassifier.cc b/src/PyClassifier.cc index 831c780..872560b 100644 --- a/src/PyClassifier.cc +++ b/src/PyClassifier.cc @@ -16,7 +16,9 @@ namespace pywrap { PyClassifier::~PyClassifier() { + std::cout << "Cleaning Classifier" << std::endl; pyWrap->clean(module, className); + std::cout << "Classifier cleaned" << std::endl; } PyObject* PyClassifier::toPyObject(torch::Tensor& data_tensor) { @@ -60,23 +62,19 @@ namespace pywrap { int n = XX.size(1); auto data_numpy = np::from_data(XX.data_ptr(), np::dtype::get_builtin(), p::make_tuple(m, n), p::make_tuple(sizeof(XX.dtype()) * 2 * n, sizeof(XX.dtype()) * 2), p::object()); print_array(data_numpy); - PyObject* Xp = data_numpy.ptr(); + CPyObject Xp = data_numpy.ptr(); std::cout << "Converting y to PyObject" << std::endl; auto y_numpy = np::from_data(y.data_ptr(), np::dtype::get_builtin(), p::make_tuple(m), p::make_tuple(sizeof(y.dtype()) * 2), p::object()); - PyObject* yp = y_numpy.ptr(); + CPyObject yp = y_numpy.ptr(); std::cout << "Calling fit" << std::endl; pyWrap->fit(module, this->className, Xp, yp); - Py_DECREF(Xp); - Py_DECREF(yp); return *this; } torch::Tensor PyClassifier::predict(torch::Tensor& X) { - PyObject* Xp = toPyObject(X); + CPyObject Xp = toPyObject(X); auto PyResult = pyWrap->predict(module, className, Xp); auto result = THPVariable_Unpack(PyResult); - Py_DECREF(Xp); - Py_DECREF(PyResult); return result; } double PyClassifier::score(torch::Tensor& X, torch::Tensor& y) @@ -90,13 +88,11 @@ namespace pywrap { int n = XX.size(1); auto data_numpy = np::from_data(XX.data_ptr(), np::dtype::get_builtin(), p::make_tuple(m, n), p::make_tuple(sizeof(XX.dtype()) * 2 * n, sizeof(XX.dtype()) * 2), p::object()); print_array(data_numpy); - PyObject* Xp = data_numpy.ptr(); + CPyObject Xp = data_numpy.ptr(); std::cout << "Converting y to PyObject" << std::endl; auto y_numpy = np::from_data(y.data_ptr(), np::dtype::get_builtin(), p::make_tuple(m), p::make_tuple(sizeof(y.dtype()) * 2), p::object()); - PyObject* yp = y_numpy.ptr(); + CPyObject yp = y_numpy.ptr(); auto result = pyWrap->score(module, className, Xp, yp); - Py_DECREF(Xp); - Py_DECREF(yp); return result; } diff --git a/src/PyHelper.hpp b/src/PyHelper.hpp index 64fc150..8ef80f5 100644 --- a/src/PyHelper.hpp +++ b/src/PyHelper.hpp @@ -3,92 +3,97 @@ #pragma once #include +#include -class CPyInstance { -public: - CPyInstance() - { - Py_Initialize(); - } - - ~CPyInstance() - { - Py_Finalize(); - } -}; - - -class CPyObject { -private: - PyObject* p; -public: - CPyObject() : p(NULL) - { - } - - CPyObject(PyObject* _p) : p(_p) - { - } - - - ~CPyObject() - { - Release(); - } - - PyObject* getObject() - { - return p; - } - - PyObject* setObject(PyObject* _p) - { - return (p = _p); - } - - PyObject* AddRef() - { - if (p) { - Py_INCREF(p); - } - return p; - } - - void Release() - { - if (p) { - Py_DECREF(p); +namespace pywrap { + namespace p = boost::python; + namespace np = boost::python::numpy; + class CPyInstance { + public: + CPyInstance() + { + Py_Initialize(); + np::initialize(); } - p = NULL; - } - - PyObject* operator ->() - { - return p; - } - - bool is() - { - return p ? true : false; - } - - operator PyObject* () - { - return p; - } - - PyObject* operator = (PyObject* pp) - { - p = pp; - return p; - } - - operator bool() - { - return p ? true : false; - } -}; + ~CPyInstance() + { + Py_Finalize(); + } + }; + class CPyObject { + private: + PyObject* p; + public: + CPyObject() : p(NULL) + { + } + + CPyObject(PyObject* _p) : p(_p) + { + } + + + ~CPyObject() + { + Release(); + } + + PyObject* getObject() + { + return p; + } + + PyObject* setObject(PyObject* _p) + { + return (p = _p); + } + + PyObject* AddRef() + { + if (p) { + Py_INCREF(p); + } + return p; + } + + void Release() + { + if (p) { + Py_XDECREF(p); + } + + p = NULL; + } + + PyObject* operator ->() + { + return p; + } + + bool is() + { + return p ? true : false; + } + + operator PyObject* () + { + return p; + } + + PyObject* operator = (PyObject* pp) + { + p = pp; + return p; + } + + operator bool() + { + return p ? true : false; + } + }; + +} /* namespace pywrap */ #endif \ No newline at end of file diff --git a/src/PyWrap.cc b/src/PyWrap.cc index 43082a3..099f915 100644 --- a/src/PyWrap.cc +++ b/src/PyWrap.cc @@ -5,7 +5,6 @@ #include #include #include -#include "PyHelper.hpp" namespace pywrap { namespace np = boost::python::numpy; @@ -34,12 +33,9 @@ namespace pywrap { PyWrap::~PyWrap() { - for (const auto& item : moduleClassMap) { - Py_DECREF(std::get<0>(item.second)); - Py_DECREF(std::get<1>(item.second)); - Py_DECREF(std::get<2>(item.second)); - } + std::cout << "Destruyendo PyWrap" << std::endl; Py_Finalize(); + std::cout << "PyWrap destruido" << std::endl; } void PyWrap::importClass(const std::string& moduleName, const std::string& className) @@ -50,15 +46,15 @@ namespace pywrap { return; } std::cout << "No estaba en el mapa" << std::endl; - PyObject* module = PyImport_ImportModule(moduleName.c_str()); + CPyObject module = PyImport_ImportModule(moduleName.c_str()); if (PyErr_Occurred()) { errorAbort("Could't import module " + moduleName); } - PyObject* classObject = PyObject_GetAttrString(module, className.c_str()); + CPyObject classObject = PyObject_GetAttrString(module, className.c_str()); if (PyErr_Occurred()) { errorAbort("Couldn't find class " + className); } - PyObject* instance = PyObject_CallObject(classObject, NULL); + CPyObject instance = PyObject_CallObject(classObject, NULL); if (PyErr_Occurred()) { errorAbort("Couldn't create instance of class " + className); } @@ -74,9 +70,6 @@ namespace pywrap { return; } std::cout << "--> Limpiando" << std::endl; - Py_DECREF(std::get<0>(result->second)); - Py_DECREF(std::get<1>(result->second)); - Py_DECREF(std::get<2>(result->second)); moduleClassMap.erase(result); std::cout << "Limpieza terminada" << std::endl; } @@ -100,14 +93,13 @@ namespace pywrap { std::string PyWrap::callMethodString(const std::string& moduleName, const std::string& className, const std::string& method) { std::cout << "Llamando método " << method << std::endl; - PyObject* instance = getClass(moduleName, className); - PyObject* result; + CPyObject instance = getClass(moduleName, className); + CPyObject result; if (!(result = PyObject_CallMethod(instance, method.c_str(), NULL))) errorAbort("Couldn't call method " + method); std::string value = PyUnicode_AsUTF8(result); std::cout << "Result: " << value << std::endl; - Py_DECREF(result); return value; } std::string PyWrap::version(const std::string& moduleName, const std::string& className) @@ -115,31 +107,30 @@ namespace pywrap { return callMethodString(moduleName, className, "version"); } - void PyWrap::fit(const std::string& moduleName, const std::string& className, PyObject* X, PyObject* y) + void PyWrap::fit(const std::string& moduleName, const std::string& className, CPyObject& X, CPyObject& y) { std::cout << "Llamando método fit" << std::endl; - PyObject* instance = getClass(moduleName, className); - PyObject* result; + CPyObject instance = getClass(moduleName, className); + CPyObject result; std::string method = "fit"; if (!(result = PyObject_CallMethodObjArgs(instance, PyUnicode_FromString(method.c_str()), X, y, NULL))) errorAbort("Couldn't call method fit"); - Py_DECREF(result); } - PyObject* PyWrap::predict(const std::string& moduleName, const std::string& className, PyObject* X) + CPyObject PyWrap::predict(const std::string& moduleName, const std::string& className, CPyObject& X) { std::cout << "Llamando método predict" << std::endl; - PyObject* instance = getClass(moduleName, className); - PyObject* result; + CPyObject instance = getClass(moduleName, className); + CPyObject result; std::string method = "predict"; if (!(result = PyObject_CallMethodObjArgs(instance, PyUnicode_FromString(method.c_str()), X, NULL))) errorAbort("Couldn't call method predict"); return result; // The caller has to decref the result } - double PyWrap::score(const std::string& moduleName, const std::string& className, PyObject* X, PyObject* y) + double PyWrap::score(const std::string& moduleName, const std::string& className, CPyObject& X, CPyObject& y) { std::cout << "Llamando método score" << std::endl; - PyObject* instance = getClass(moduleName, className); - PyObject* result; + CPyObject instance = getClass(moduleName, className); + CPyObject result; std::string method = "score"; if (!(result = PyObject_CallMethodObjArgs(instance, PyUnicode_FromString(method.c_str()), X, y, NULL))) errorAbort("Couldn't call method score"); diff --git a/src/PyWrap.h b/src/PyWrap.h index 2d3f835..99fb8f6 100644 --- a/src/PyWrap.h +++ b/src/PyWrap.h @@ -5,6 +5,8 @@ #include #include #include +#include "PyHelper.hpp" + namespace pywrap { /* @@ -27,11 +29,11 @@ namespace pywrap { // return THPVariable_Unpack(result); // }; // PyObject* callMethodArgs(const std::string& moduleName, const std::string& className, const std::string& method, PyObject* args); - void fit(const std::string& moduleName, const std::string& className, PyObject* X, PyObject* y); - PyObject* predict(const std::string& moduleName, const std::string& className, PyObject* X); + void fit(const std::string& moduleName, const std::string& className, CPyObject& X, CPyObject& y); + CPyObject predict(const std::string& moduleName, const std::string& className, CPyObject& X); std::string callMethodString(const std::string& moduleName, const std::string& className, const std::string& method); std::string version(const std::string& moduleName, const std::string& className); - double score(const std::string& moduleName, const std::string& className, PyObject* X, PyObject* y); + double score(const std::string& moduleName, const std::string& className, CPyObject& X, CPyObject& y); void clean(const std::string& moduleName, const std::string& className); void importClass(const std::string& moduleName, const std::string& className); // void doCommand2(); @@ -42,7 +44,7 @@ namespace pywrap { PyStatus initPython(); static PyWrap* wrapper; static std::mutex mutex; - std::map, std::tuple> moduleClassMap; + std::map, std::tuple> moduleClassMap; }; } /* namespace python */ #endif /* PYWRAP_H */ \ No newline at end of file diff --git a/src/example.cpp b/src/example.cpp index c39d233..2928540 100644 --- a/src/example.cpp +++ b/src/example.cpp @@ -1,123 +1,154 @@ -#include #include #include +#include "ArffFiles.h" #include +#include "PyHelper.hpp" -namespace p = boost::python; -namespace np = boost::python::numpy; -using namespace std; +namespace pywrap { + using namespace std; -void errorAbort(const std::string& message) -{ - std::cerr << message << std::endl; - PyErr_Print(); - exit(1); -} + void errorAbort(const std::string& message) + { + std::cerr << message << std::endl; + PyErr_Print(); + exit(1); + } -void print_array(np::ndarray& array) -{ - std::cout << "Array: " << std::endl; - std::cout << p::extract(p::str(array)) << std::endl; -} + void print_array(np::ndarray& array) + { + std::cout << "Array: " << std::endl; + std::cout << p::extract(p::str(array)) << std::endl; + } -np::ndarray to_numpy_matrix(torch::Tensor& input_data, np::dtype numpy_dtype) -{ - p::tuple shape = p::make_tuple(input_data.size(0), input_data.size(1)); - auto tensor_dtype = input_data.dtype(); - p::tuple stride = p::make_tuple(sizeof(tensor_dtype) * input_data.size(1), sizeof(tensor_dtype)); - auto dito = input_data.transpose(1, 0); - np::ndarray result = np::from_data(dito.data_ptr(), numpy_dtype, shape, stride, p::object()); - return result; -} -np::ndarray to_numpy_vector(torch::Tensor& input_data, np::dtype numpy_dtype) -{ - p::tuple shape = p::make_tuple(input_data.size(0)); - auto tensor_dtype = input_data.dtype(); - p::tuple stride = p::make_tuple(sizeof(tensor_dtype), sizeof(tensor_dtype)); - np::ndarray result = np::from_data(input_data.data_ptr(), numpy_dtype, shape, stride, p::object()); - return result; -} -void flat() -{ - double data[][4] = { {0.1, 0.2, 0.3, 0.4} , { 0.5, 0.6, 0.7, 0.8 }, { 0.9, 0.11, 0.12, 0.13 }, { 0.14, 0.15, 0.16, 0.17 }, { 0.18, 0.19, 0.21, 0.22 }, { 0.23, 0.24, 0.25, 0.26 }, { 0.27, 0.28, 0.29, 0.31 } }; - int labels[] = { 0, 1, 0, 1 , 0, 0, 1 }; - // cout << "Array data: (" << m << ", " << n << ") " << endl; - // for (int i = 0; i < m; ++i) { - // cout << "[ "; - // for (int j = 0; j < n; ++j) { - // cout << setw(4) << std::setprecision(2) << fixed << data[i][j] << " "; - // } - // cout << "]" << endl; - // } - // cout << "Array labels: " << endl; - // for (int i = 0; i < m; ++i) { - // cout << labels[i] << " "; - // } - // cout << endl; - // auto data_numpy = np::from_data(data, np::dtype::get_builtin(), p::make_tuple(m, n), p::make_tuple(sizeof(double) * n, sizeof(double)), p::object()); - // auto y_numpy = np::from_data(labels, np::dtype::get_builtin(), p::make_tuple(m), p::make_tuple(sizeof(int)), p::object()); -} + np::ndarray to_numpy_matrix(torch::Tensor& input_data, np::dtype numpy_dtype) + { + p::tuple shape = p::make_tuple(input_data.size(0), input_data.size(1)); + auto tensor_dtype = input_data.dtype(); + p::tuple stride = p::make_tuple(sizeof(tensor_dtype) * input_data.size(1), sizeof(tensor_dtype)); + auto dito = input_data.transpose(1, 0); + np::ndarray result = np::from_data(dito.data_ptr(), numpy_dtype, shape, stride, p::object()); + return result; + } + np::ndarray to_numpy_vector(torch::Tensor& input_data, np::dtype numpy_dtype) + { + p::tuple shape = p::make_tuple(input_data.size(0)); + auto tensor_dtype = input_data.dtype(); + p::tuple stride = p::make_tuple(sizeof(tensor_dtype), sizeof(tensor_dtype)); + np::ndarray result = np::from_data(input_data.data_ptr(), numpy_dtype, shape, stride, p::object()); + return result; + } + void flat() + { + double data[][4] = { {0.1, 0.2, 0.3, 0.4} , { 0.5, 0.6, 0.7, 0.8 }, { 0.9, 0.11, 0.12, 0.13 }, { 0.14, 0.15, 0.16, 0.17 }, { 0.18, 0.19, 0.21, 0.22 }, { 0.23, 0.24, 0.25, 0.26 }, { 0.27, 0.28, 0.29, 0.31 } }; + int labels[] = { 0, 1, 0, 1 , 0, 0, 1 }; + // cout << "Array data: (" << m << ", " << n << ") " << endl; + // for (int i = 0; i < m; ++i) { + // cout << "[ "; + // for (int j = 0; j < n; ++j) { + // cout << setw(4) << std::setprecision(2) << fixed << data[i][j] << " "; + // } + // cout << "]" << endl; + // } + // cout << "Array labels: " << endl; + // for (int i = 0; i < m; ++i) { + // cout << labels[i] << " "; + // } + // cout << endl; + // auto data_numpy = np::from_data(data, np::dtype::get_builtin(), p::make_tuple(m, n), p::make_tuple(sizeof(double) * n, sizeof(double)), p::object()); + // auto y_numpy = np::from_data(labels, np::dtype::get_builtin(), p::make_tuple(m), p::make_tuple(sizeof(int)), p::object()); + } + class Paths { + public: + static string datasets() + { + return "/home/rmontanana/Code/discretizbench/datasets/"; + } + }; + tuple, string, map>> loadDataset(const string& name, bool class_last) + { + auto handler = ArffFiles(); + handler.load(Paths::datasets() + static_cast(name) + ".arff", class_last); + // Get Dataset X, y + vector> X = handler.getX(); + vector y = handler.getY(); + // Get className & Features + auto className = handler.getClassName(); + vector features; + auto attributes = handler.getAttributes(); + transform(attributes.begin(), attributes.end(), back_inserter(features), [](const auto& pair) { return pair.first; }); + torch::Tensor Xd; + auto states = map>(); + Xd = torch::zeros({ static_cast(X.size()), static_cast(X[0].size()) }, torch::kFloat32); + for (int i = 0; i < features.size(); ++i) { + Xd.index_put_({ i, "..." }, torch::tensor(X[i], torch::kFloat32)); + } + return { Xd, torch::tensor(y, torch::kInt32), features, className, states }; + } + +} /* namespace pywrap */ +using namespace pywrap; int main(int argc, char** argv) { - Py_Initialize(); - np::initialize(); - int m = 7; - int n = 4; + auto [data_tensor, y_label, featuresx, classNamex, statesx] = loadDataset("iris", true); + //data_tensor = data_tensor.transpose(0, 1); + CPyInstance pInstance; + int m = data_tensor.size(0); + int n = data_tensor.size(1); + // int m = 7; + // int n = 4; // torch::Tensor data_tensor = torch::rand({ m, n }, torch::kFloat64); - torch::Tensor data_tensor = torch::tensor({ {0.1, 0.2, 0.3, 0.4} , { 0.5, 0.6, 0.7, 0.8 }, { 0.9, 0.11, 0.12, 0.13 }, { 0.14, 0.15, 0.16, 0.17 }, { 0.18, 0.19, 0.21, 0.22 }, { 0.23, 0.24, 0.25, 0.26 }, { 0.27, 0.28, 0.29, 0.31 } }, torch::kFloat32); + //torch::Tensor data_tensor = torch::tensor({ {0.1, 0.2, 0.3, 0.4} , { 0.5, 0.6, 0.7, 0.8 }, { 0.9, 0.11, 0.12, 0.13 }, { 0.14, 0.15, 0.16, 0.17 }, { 0.18, 0.19, 0.21, 0.22 }, { 0.23, 0.24, 0.25, 0.26 }, { 0.27, 0.28, 0.29, 0.31 } }, torch::kFloat32); // torch::Tensor y_label = torch::randint(0, 2, { m }, torch::kInt16); - torch::Tensor y_label = torch::tensor({ 17, 18, 19, 20 , 21, 22, 23 }, torch::kInt32); + //torch::Tensor y_label = torch::tensor({ 17, 18, 19, 20 , 21, 22, 23 }, torch::kInt32); cout << "Tensor data: (" << data_tensor.size(0) << ", " << data_tensor.size(1) << ") " << endl << data_tensor << endl; cout << "Tensor data sizes: " << data_tensor.sizes() << endl; - cout << "Tensor labels: " << y_label << endl; + // cout << "Tensor labels: " << y_label << endl; cout << "Tensor labels sizes: " << y_label.sizes() << endl; auto data_numpy = np::from_data(data_tensor.data_ptr(), np::dtype::get_builtin(), p::make_tuple(m, n), p::make_tuple(sizeof(data_tensor.dtype()) * 2 * n, sizeof(data_tensor.dtype()) * 2), p::object()); - auto y_numpy = np::from_data(y_label.data_ptr(), np::dtype::get_builtin(), p::make_tuple(m), p::make_tuple(sizeof(y_label.dtype()) * 2), p::object()); + data_numpy = data_numpy.transpose(); + auto y_numpy = np::from_data(y_label.data_ptr(), np::dtype::get_builtin(), p::make_tuple(n), p::make_tuple(sizeof(y_label.dtype()) * 2), p::object()); //auto y_numpy = np::from_data(y_label.data_ptr(), np::dtype::get_builtin(), p::make_tuple(m), p::make_tuple(sizeof(y_label.dtype()) * 4), p::object()); cout << "Numpy array data: " << endl; print_array(data_numpy); cout << "Numpy array labels: " << endl; print_array(y_numpy); cout << "primero" << endl; - PyObject* p = data_numpy.ptr(); - PyObject* yp = y_numpy.ptr(); + CPyObject p = data_numpy.ptr(); + CPyObject yp = y_numpy.ptr(); cout << "segundo" << endl; string moduleName = "stree"; string className = "Stree"; string method = "version"; - PyObject* module = PyImport_ImportModule(moduleName.c_str()); + CPyObject module = PyImport_ImportModule(moduleName.c_str()); if (PyErr_Occurred()) { errorAbort("Could't import module " + moduleName); } - PyObject* classObject = PyObject_GetAttrString(module, className.c_str()); + CPyObject classObject = PyObject_GetAttrString(module, className.c_str()); if (PyErr_Occurred()) { errorAbort("Couldn't find class " + className); } - PyObject* instance = PyObject_CallObject(classObject, NULL); + CPyObject instance = PyObject_CallObject(classObject, NULL); if (PyErr_Occurred()) { errorAbort("Couldn't create instance of class " + className); } - PyObject* result; + CPyObject result; if (!(result = PyObject_CallMethod(instance, method.c_str(), NULL))) errorAbort("Couldn't call method " + method); std::string value = PyUnicode_AsUTF8(result); cout << "Version: " << value << endl; + cout << "Calling fit" << endl; + p.AddRef(); + yp.AddRef(); method = "fit"; - if (!(result = PyObject_CallMethodObjArgs(instance, PyUnicode_FromString(method.c_str()), p, yp, NULL))) + if (!(result = PyObject_CallMethodObjArgs(instance, PyUnicode_FromString(method.c_str()), p.getObject(), yp.getObject(), NULL))) errorAbort("Couldn't call method fit"); + cout << "Calling score" << endl; method = "score"; - if (!(result = PyObject_CallMethodObjArgs(instance, PyUnicode_FromString(method.c_str()), p, yp, NULL))) + if (!(result = PyObject_CallMethodObjArgs(instance, PyUnicode_FromString(method.c_str()), p.getObject(), yp.getObject(), NULL))) errorAbort("Couldn't call method score"); float score = PyFloat_AsDouble(result); cout << "Score: " << score << endl; - Py_DECREF(result); - Py_DECREF(instance); - Py_DECREF(module); - Py_DECREF(p); - Py_DECREF(yp); - cout << "tercero" << endl; - return 0; } \ No newline at end of file diff --git a/src/main.cc b/src/main.cc index 6595a0b..1920650 100644 --- a/src/main.cc +++ b/src/main.cc @@ -51,9 +51,9 @@ int main(int argc, char* argv[]) cout << string(80, '-') << endl; cout << "X: " << X.sizes() << endl; cout << "y: " << y.sizes() << endl; - auto result = stree.fit(X, y, features, className, states); - cout << "Now calling score" << endl; - auto result2 = stree.score(X, y); - cout << "SVC score " << result2 << endl; + // auto result = stree.fit(X, y, features, className, states); + // cout << "Now calling score" << endl; + // auto result2 = stree.score(X, y); + // cout << "SVC score " << result2 << endl; return 0; } \ No newline at end of file