Passing numpy working partially

CMakeLists.txt

@@ -6,6 +6,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 find_package(Python3 3.11...3.11.9 COMPONENTS Interpreter Development REQUIRED)
 find_package(Torch REQUIRED)
+find_package(Boost REQUIRED COMPONENTS python3 numpy3)
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")
 

Makefile

@@ -4,7 +4,7 @@ SHELL := /bin/bash
 
 f_release = build_release
 f_debug = build_debug
-app_targets = main
+app_targets = main example
 test_targets = unit_tests_bayesnet unit_tests_platform
 n_procs = -j 16
 

src/CMakeLists.txt

@@ -1,8 +1,9 @@
 include_directories(${PyWrap_SOURCE_DIR}/lib/Files)
 include_directories(${Python3_INCLUDE_DIRS})
+include_directories(${TORCH_INCLUDE_DIRS})
 
 add_executable(main main.cc STree.cc SVC.cc PyClassifier.cc PyWrap.cc)
 add_executable(example example.cpp)
 
-target_link_libraries(main ${Python3_LIBRARIES} "${TORCH_LIBRARIES}" ${LIBTORCH_PYTHON} ArffFiles)
+target_link_libraries(main ${Python3_LIBRARIES} "${TORCH_LIBRARIES}" ${LIBTORCH_PYTHON} Boost::boost Boost::python Boost::numpy ArffFiles)
-target_link_libraries(example "${TORCH_LIBRARIES}" ArffFiles)
+target_link_libraries(example ${Python3_LIBRARIES} "${TORCH_LIBRARIES}" Boost::boost Boost::python Boost::numpy ArffFiles)

src/PyClassifier.cc

@@ -1,10 +1,13 @@
 #include "PyClassifier.h"
+#include <boost/python/numpy.hpp>
 #include <torch/csrc/autograd/python_variable.h>
 #include <torch/csrc/utils/tensor_numpy.h>
+//#include "tensorflow/python/lib/core/py_func.h"
 #include <iostream>
 
 namespace pywrap {
-
+    namespace p = boost::python;
+    namespace np = boost::python::numpy;
     PyClassifier::PyClassifier(const std::string& module, const std::string& className) : module(module), className(className)
     {
         pyWrap = PyWrap::GetInstance();
@@ -15,11 +18,23 @@ namespace pywrap {
     {
         pyWrap->clean(module, className);
     }
-    PyObject* PyClassifier::toPyObject(torch::Tensor& tensor)
+    PyObject* PyClassifier::toPyObject(torch::Tensor& data_tensor)
     {
-        return torch::utils::tensor_to_numpy(tensor);
+
-        //return THPVariable_Wrap(tensor);
+        // return torch::utils::tensor_to_numpy(data_tensor);
+        return THPVariable_Wrap(data_tensor);
+        //auto data_numpy = np::from_data(data_tensor.data_ptr(), np::dtype::get_builtin<float>(), p::make_tuple(m, n), p::make_tuple(sizeof(data_tensor.dtype()) * 2 * n, sizeof(data_tensor.dtype()) * 2), p::object());
+        // PyObject* numpyObject = data_numpy.ptr();
+
+        // return numpyObject;
     }
+    // PyObject* PyClassifier::toPyObjecty(torch::Tensor& data_tensor)
+    // {
+    //     //return THPVariable_Wrap(tensor);
+    //     auto y_numpy = np::from_data(data_tensor.data_ptr(), np::dtype::get_builtin<int32_t>(), p::make_tuple(m), p::make_tuple(sizeof(data_tensor.dtype()) * 2), p::object());
+    //     PyObject* numpyObject = y_numpy.ptr();
+
+    // }
     std::string PyClassifier::version()
     {
         return pyWrap->version(module, className);
@@ -29,16 +44,28 @@ namespace pywrap {
     {
         return pyWrap->callMethodString(module, className, method);
     }
+    void print_array(np::ndarray& array)
+    {
+        std::cout << "Array: " << std::endl;
+        std::cout << p::extract<char const*>(p::str(array)) << std::endl;
+    }
     PyClassifier& PyClassifier::fit(torch::Tensor& X, torch::Tensor& y, const std::vector<std::string>& features, const std::string& className, std::map<std::string, std::vector<int>>& states)
     {
         std::cout << "Converting X to PyObject" << std::endl;
         std::cout << "X.defined() = " << X.defined() << std::endl;
         //std::cout << "X.pyobj() = " << X.pyobj() << std::endl;
-        PyObject* Xp = toPyObject(X);
+        //PyObject* Xp = torch::utils::tensor_to_numpy(X);
+        auto XX = X.transpose(0, 1);
+        int m = XX.size(0);
+        int n = XX.size(1);
+        auto data_numpy = np::from_data(XX.data_ptr(), np::dtype::get_builtin<float>(), p::make_tuple(m, n), p::make_tuple(sizeof(XX.dtype()) * 2 * n, sizeof(XX.dtype()) * 2), p::object());
+        print_array(data_numpy);
+        PyObject* Xp = data_numpy.ptr();
         std::cout << "Converting y to PyObject" << std::endl;
-        PyObject* yp = toPyObject(y);
+        auto y_numpy = np::from_data(y.data_ptr(), np::dtype::get_builtin<int32_t>(), p::make_tuple(m), p::make_tuple(sizeof(y.dtype()) * 2), p::object());
+        PyObject* yp = y_numpy.ptr();
         std::cout << "Calling fit" << std::endl;
-        pyWrap->fit(module, className, Xp, yp);
+        pyWrap->fit(module, this->className, Xp, yp);
         Py_DECREF(Xp);
         Py_DECREF(yp);
         return *this;
@@ -54,8 +81,19 @@ namespace pywrap {
     }
     double PyClassifier::score(torch::Tensor& X, torch::Tensor& y)
     {
-        PyObject* Xp = toPyObject(X);
+        std::cout << "Converting X to PyObject" << std::endl;
-        PyObject* yp = toPyObject(y);
+        std::cout << "X.defined() = " << X.defined() << std::endl;
+        //std::cout << "X.pyobj() = " << X.pyobj() << std::endl;
+        //PyObject* Xp = torch::utils::tensor_to_numpy(X);
+        auto XX = X.transpose(0, 1);
+        int m = XX.size(0);
+        int n = XX.size(1);
+        auto data_numpy = np::from_data(XX.data_ptr(), np::dtype::get_builtin<float>(), p::make_tuple(m, n), p::make_tuple(sizeof(XX.dtype()) * 2 * n, sizeof(XX.dtype()) * 2), p::object());
+        print_array(data_numpy);
+        PyObject* Xp = data_numpy.ptr();
+        std::cout << "Converting y to PyObject" << std::endl;
+        auto y_numpy = np::from_data(y.data_ptr(), np::dtype::get_builtin<int32_t>(), p::make_tuple(m), p::make_tuple(sizeof(y.dtype()) * 2), p::object());
+        PyObject* yp = y_numpy.ptr();
         auto result = pyWrap->score(module, className, Xp, yp);
         Py_DECREF(Xp);
         Py_DECREF(yp);

src/PyHelper.hpp

@@ -0,0 +1,94 @@
+#ifndef PYHELPER_HPP
+#define PYHELPER_HPP
+#pragma once
+
+#include <Python.h>
+
+class CPyInstance {
+public:
+    CPyInstance()
+    {
+        Py_Initialize();
+    }
+
+    ~CPyInstance()
+    {
+        Py_Finalize();
+    }
+};
+
+
+class CPyObject {
+private:
+    PyObject* p;
+public:
+    CPyObject() : p(NULL)
+    {
+    }
+
+    CPyObject(PyObject* _p) : p(_p)
+    {
+    }
+
+
+    ~CPyObject()
+    {
+        Release();
+    }
+
+    PyObject* getObject()
+    {
+        return p;
+    }
+
+    PyObject* setObject(PyObject* _p)
+    {
+        return (p = _p);
+    }
+
+    PyObject* AddRef()
+    {
+        if (p) {
+            Py_INCREF(p);
+        }
+        return p;
+    }
+
+    void Release()
+    {
+        if (p) {
+            Py_DECREF(p);
+        }
+
+        p = NULL;
+    }
+
+    PyObject* operator ->()
+    {
+        return p;
+    }
+
+    bool is()
+    {
+        return p ? true : false;
+    }
+
+    operator PyObject* ()
+    {
+        return p;
+    }
+
+    PyObject* operator = (PyObject* pp)
+    {
+        p = pp;
+        return p;
+    }
+
+    operator bool()
+    {
+        return p ? true : false;
+    }
+};
+
+
+#endif

src/PyWrap.cc

@@ -4,8 +4,11 @@
 #include <iostream>
 #include <string>
 #include <map>
+#include <boost/python/numpy.hpp>
+#include "PyHelper.hpp"
 
 namespace pywrap {
+    namespace np = boost::python::numpy;
     PyWrap* PyWrap::wrapper = nullptr;
     std::mutex PyWrap::mutex;
 
@@ -26,7 +29,17 @@ namespace pywrap {
         if (PyStatus_Exception(status)) {
             throw std::runtime_error("Error initializing Python");
         }
+        np::initialize();
+    }
 
+    PyWrap::~PyWrap()
+    {
+        for (const auto& item : moduleClassMap) {
+            Py_DECREF(std::get<0>(item.second));
+            Py_DECREF(std::get<1>(item.second));
+            Py_DECREF(std::get<2>(item.second));
+        }
+        Py_Finalize();
     }
 
     void PyWrap::importClass(const std::string& moduleName, const std::string& className)
@@ -68,18 +81,10 @@ namespace pywrap {
         std::cout << "Limpieza terminada" << std::endl;
     }
 
-    PyWrap::~PyWrap()
+
-    {
-        for (const auto& item : moduleClassMap) {
-            Py_DECREF(std::get<0>(item.second));
-            Py_DECREF(std::get<1>(item.second));
-            Py_DECREF(std::get<2>(item.second));
-        }
-        Py_Finalize();
-    }
     void PyWrap::errorAbort(const std::string& message)
     {
-        std::cerr << message << std::endl;
+        std::cout << message << std::endl;
         PyErr_Print();
         exit(1);
     }
@@ -115,9 +120,8 @@ namespace pywrap {
         std::cout << "Llamando método fit" << std::endl;
         PyObject* instance = getClass(moduleName, className);
         PyObject* result;
-        const char method[] = "fit";
+        std::string method = "fit";
-
+        if (!(result = PyObject_CallMethodObjArgs(instance, PyUnicode_FromString(method.c_str()), X, y, NULL)))
-        if (!(result = PyObject_CallMethodObjArgs(instance, PyBytes_FromString(method), X, y, NULL)))
             errorAbort("Couldn't call method fit");
         Py_DECREF(result);
     }
@@ -126,8 +130,8 @@ namespace pywrap {
         std::cout << "Llamando método predict" << std::endl;
         PyObject* instance = getClass(moduleName, className);
         PyObject* result;
-        const char method[] = "predict";
+        std::string method = "predict";
-        if (!(result = PyObject_CallMethodObjArgs(instance, PyBytes_FromString(method), X, NULL)))
+        if (!(result = PyObject_CallMethodObjArgs(instance, PyUnicode_FromString(method.c_str()), X, NULL)))
             errorAbort("Couldn't call method predict");
         return result; // The caller has to decref the result
     }
@@ -136,8 +140,8 @@ namespace pywrap {
         std::cout << "Llamando método score" << std::endl;
         PyObject* instance = getClass(moduleName, className);
         PyObject* result;
-        const char method[] = "score";
+        std::string method = "score";
-        if (!(result = PyObject_CallMethodObjArgs(instance, PyBytes_FromString(method), X, y, NULL)))
+        if (!(result = PyObject_CallMethodObjArgs(instance, PyUnicode_FromString(method.c_str()), X, y, NULL)))
             errorAbort("Couldn't call method score");
         return PyFloat_AsDouble(result);
     }

src/example.cpp

@@ -1,60 +1,123 @@
-#include <torch/torch.h>
+#include <boost/python/numpy.hpp>
-#include "ArffFiles.h"
 #include <string>
 #include <iostream>
+#include <torch/torch.h>
 
+namespace p = boost::python;
+namespace np = boost::python::numpy;
 using namespace std;
-using namespace torch;
-class Test {
-public:
-    Test(const string& c) : c(c) {};
-    ~Test() { std::cout << "Destructor" << std::endl; };
 
-    template<typename T>
+void errorAbort(const std::string& message)
-    T callMethod(const T& parameter)
 {
-        std::cout << "Llamando a metodo" << std::endl;
+    std::cerr << message << std::endl;
-        return parameter;
+    PyErr_Print();
-    }
+    exit(1);
-private:
-    string c;
-};
-
-tuple<Tensor, Tensor, vector<string>, string, map<string, vector<int>>> loadDataset(const string& name, bool class_last)
-{
-    auto handler = ArffFiles();
-    handler.load(static_cast<string>(name) + ".arff", class_last);
-    // Get Dataset X, y
-    vector<vector<float>> X = handler.getX();
-    vector<int> y = handler.getY();
-    // // Get className & Features
-    auto className = handler.getClassName();
-    vector<string> features;
-    auto attributes = handler.getAttributes();
-    transform(attributes.begin(), attributes.end(), back_inserter(features), [](const auto& pair) { return pair.first; });
-    torch::Tensor Xd;
-    auto states = map<string, vector<int>>();
-    auto yt = torch::tensor(y, torch::kInt32);
-    Xd = torch::zeros({ static_cast<int>(X.size()), static_cast<int>(X[0].size()) }, torch::kFloat32);
-    for (int i = 0; i < features.size(); ++i) {
-        Xd.index_put_({ i, "..." }, torch::tensor(X[i], torch::kFloat32));
-    }
-    return make_tuple(Xd, yt, features, className, states);
 }
 
-int main()
+void print_array(np::ndarray& array)
 {
-    Test t("hola");
+    std::cout << "Array: " << std::endl;
-    cout << t.callMethod<string>("hola") << endl;
+    std::cout << p::extract<char const*>(p::str(array)) << std::endl;
-    cout << t.callMethod<int>(1) << endl;
+}
-    cout << t.callMethod<double>(7.3) << endl;
+
-    vector<vector<float>> X;
+np::ndarray to_numpy_matrix(torch::Tensor& input_data, np::dtype numpy_dtype)
-    vector<int> y = { 1, 2, 3 };
+{
-    X.push_back({ 1.1, 2.2, 3.3 });
+    p::tuple shape = p::make_tuple(input_data.size(0), input_data.size(1));
-    vector<float> v = { 1.1, 2.2, 3.3 };
+    auto tensor_dtype = input_data.dtype();
-    torch::Tensor matrix = torch::tensor(X[0], torch::kFloat32);
+    p::tuple stride = p::make_tuple(sizeof(tensor_dtype) * input_data.size(1), sizeof(tensor_dtype));
-    cout << "X:" << matrix << endl;
+    auto dito = input_data.transpose(1, 0);
-    cout << "y:" << torch::tensor(y, torch::kInt32) << endl;
+    np::ndarray result = np::from_data(dito.data_ptr(), numpy_dtype, shape, stride, p::object());
+    return result;
+}
+np::ndarray to_numpy_vector(torch::Tensor& input_data, np::dtype numpy_dtype)
+{
+    p::tuple shape = p::make_tuple(input_data.size(0));
+    auto tensor_dtype = input_data.dtype();
+    p::tuple stride = p::make_tuple(sizeof(tensor_dtype), sizeof(tensor_dtype));
+    np::ndarray result = np::from_data(input_data.data_ptr(), numpy_dtype, shape, stride, p::object());
+    return result;
+}
+void flat()
+{
+    double data[][4] = { {0.1, 0.2, 0.3, 0.4} , { 0.5, 0.6, 0.7, 0.8 }, { 0.9, 0.11, 0.12, 0.13 }, { 0.14, 0.15, 0.16, 0.17 }, { 0.18, 0.19, 0.21, 0.22 }, { 0.23, 0.24, 0.25, 0.26 }, { 0.27, 0.28, 0.29, 0.31 } };
+    int labels[] = { 0, 1, 0, 1 , 0, 0, 1 };
+    // cout << "Array data: (" << m << ", " << n << ") " << endl;
+    // for (int i = 0; i < m; ++i) {
+    //     cout << "[ ";
+    //     for (int j = 0; j < n; ++j) {
+    //         cout << setw(4) << std::setprecision(2) << fixed << data[i][j] << " ";
+    //     }
+    //     cout << "]" << endl;
+    // }
+    // cout << "Array labels: " << endl;
+    // for (int i = 0; i < m; ++i) {
+    //     cout << labels[i] << " ";
+    // }
+    // cout << endl;
+    // auto data_numpy = np::from_data(data, np::dtype::get_builtin<double>(), p::make_tuple(m, n), p::make_tuple(sizeof(double) * n, sizeof(double)), p::object());
+    // auto y_numpy = np::from_data(labels, np::dtype::get_builtin<int>(), p::make_tuple(m), p::make_tuple(sizeof(int)), p::object());
+}
+
+int main(int argc, char** argv)
+{
+    Py_Initialize();
+    np::initialize();
+    int m = 7;
+    int n = 4;
+    // torch::Tensor data_tensor = torch::rand({ m, n }, torch::kFloat64);
+    torch::Tensor data_tensor = torch::tensor({ {0.1, 0.2, 0.3, 0.4} , { 0.5, 0.6, 0.7, 0.8 }, { 0.9, 0.11, 0.12, 0.13 }, { 0.14, 0.15, 0.16, 0.17 }, { 0.18, 0.19, 0.21, 0.22 }, { 0.23, 0.24, 0.25, 0.26 }, { 0.27, 0.28, 0.29, 0.31 } }, torch::kFloat32);
+    // torch::Tensor y_label = torch::randint(0, 2, { m }, torch::kInt16);
+    torch::Tensor y_label = torch::tensor({ 17, 18, 19, 20 , 21, 22, 23 }, torch::kInt32);
+    cout << "Tensor data: (" << data_tensor.size(0) << ", " << data_tensor.size(1) << ") " << endl << data_tensor << endl;
+    cout << "Tensor data sizes: " << data_tensor.sizes() << endl;
+    cout << "Tensor labels: " << y_label << endl;
+    cout << "Tensor labels sizes: " << y_label.sizes() << endl;
+    auto data_numpy = np::from_data(data_tensor.data_ptr(), np::dtype::get_builtin<float>(), p::make_tuple(m, n), p::make_tuple(sizeof(data_tensor.dtype()) * 2 * n, sizeof(data_tensor.dtype()) * 2), p::object());
+    auto y_numpy = np::from_data(y_label.data_ptr(), np::dtype::get_builtin<int32_t>(), p::make_tuple(m), p::make_tuple(sizeof(y_label.dtype()) * 2), p::object());
+    //auto y_numpy = np::from_data(y_label.data_ptr(), np::dtype::get_builtin<int64_t>(), p::make_tuple(m), p::make_tuple(sizeof(y_label.dtype()) * 4), p::object());
+    cout << "Numpy array data: " << endl;
+    print_array(data_numpy);
+    cout << "Numpy array labels: " << endl;
+    print_array(y_numpy);
+    cout << "primero" << endl;
+    PyObject* p = data_numpy.ptr();
+    PyObject* yp = y_numpy.ptr();
+    cout << "segundo" << endl;
+    string moduleName = "stree";
+    string className = "Stree";
+    string method = "version";
+    PyObject* module = PyImport_ImportModule(moduleName.c_str());
+    if (PyErr_Occurred()) {
+        errorAbort("Could't import module " + moduleName);
+    }
+    PyObject* classObject = PyObject_GetAttrString(module, className.c_str());
+    if (PyErr_Occurred()) {
+        errorAbort("Couldn't find class " + className);
+    }
+    PyObject* instance = PyObject_CallObject(classObject, NULL);
+    if (PyErr_Occurred()) {
+        errorAbort("Couldn't create instance of class " + className);
+    }
+    PyObject* result;
+    if (!(result = PyObject_CallMethod(instance, method.c_str(), NULL)))
+        errorAbort("Couldn't call method " + method);
+
+    std::string value = PyUnicode_AsUTF8(result);
+    cout << "Version: " << value << endl;
+    method = "fit";
+    if (!(result = PyObject_CallMethodObjArgs(instance, PyUnicode_FromString(method.c_str()), p, yp, NULL)))
+        errorAbort("Couldn't call method fit");
+    method = "score";
+    if (!(result = PyObject_CallMethodObjArgs(instance, PyUnicode_FromString(method.c_str()), p, yp, NULL)))
+        errorAbort("Couldn't call method score");
+    float score = PyFloat_AsDouble(result);
+    cout << "Score: " << score << endl;
+    Py_DECREF(result);
+    Py_DECREF(instance);
+    Py_DECREF(module);
+    Py_DECREF(p);
+    Py_DECREF(yp);
+    cout << "tercero" << endl;
 
     return 0;
 }

src/main.cc

@@ -51,7 +51,9 @@ int main(int argc, char* argv[])
     cout << string(80, '-') << endl;
     cout << "X: " << X.sizes() << endl;
     cout << "y: " << y.sizes() << endl;
-    auto result = svc.fit(X, y, features, className, states).score(X, y);
+    auto result = stree.fit(X, y, features, className, states);
-    cout << "SVC score " << result << endl;
+    cout << "Now calling score" << endl;
+    auto result2 = stree.score(X, y);
+    cout << "SVC score " << result2 << endl;
     return 0;
 }