pybind11 with arrow gets incompatible function arguments

Question

#include <pybind11/pybind11.h>
#include <arrow/stl.h>
#include <arrow/table.h>
#include <string>
#include "Trade.h"
#include "TradeAggregator.h"
#include "arrow/python/pyarrow.h"
#include <iostream>
#include <Python.h>
namespace py = pybind11;
using std::to_string;
void print_table(PyObject *py_table)
{
    // convert pyobject to table
    auto status = arrow::py::unwrap_table(py_table);
    if (!status.ok())
    {
        std::cout << "Error converting pyarrow table to arrow table" << std::endl;
        return;
    }
    std::shared_ptr<arrow::Table> table = status.ValueOrDie();
    std::cout << "Table has " << table->num_rows() << " rows" << std::endl;
}
// Wrapper for return to pyarrow.
void aggregate_trades(PyObject *py_table)
{
    auto status = arrow::py::unwrap_table(py_table);
    // .....
}

PYBIND11_MODULE(mkt_data, m)
{
    arrow::py::import_pyarrow();
    m.doc() = "Market Data processing plugin ";
    py::class_<MktData::Trade>(m, "Trade")
        .def(py::init<MktData::event_id_t, MktData::timestamp_t, double, double, bool, MktData::event_id_t, MktData::event_id_t>())
        .def_readonly("trade_id", &MktData::Trade::trade_id)
        .def_readonly("timestamp", &MktData::Trade::timestamp)
        .def_readonly("price", &MktData::Trade::price)
        .def_readonly("quantity", &MktData::Trade::quantity)
        .def_readonly("consideration", &MktData::Trade::consideration)
        .def_readonly("is_buy", &MktData::Trade::is_buy)
        .def_readonly("first_id", &MktData::Trade::first_id)
        .def_readonly("last_id", &MktData::Trade::last_id)
        .def("__repr__", [](const MktData::Trade &t)
             { return "<Trade: trade_id=" + to_string(t.trade_id) + ", timestamp=" + to_string(t.timestamp) + ", price=" + to_string(t.price) + ", quantity=" + to_string(t.quantity) +
                      ", consideration=" + to_string(t.consideration) + ", is_buy=" + ((t.is_buy) ? "True" : "False") + ", first_id=" + to_string(t.first_id) + ", last_id=" + to_string(t.last_id) + ">"; });
    ;
    m.def("print_table", &print_table);
    m.def("aggregate_trades", &aggregate_trades, "Aggregate the trades in the arrow tables.");
}

Note that the print_table has been copied from ( pyarrow Table to PyObject* via pybind11 ), in which the author said she/he had successfully resolve the issue.

In my case, the code can be compiled, linked and imported into python, but both functions complain:

Input In [1], in <cell line: 7>()
      5 df = pd.read_csv('ADAUSDT-aggTrades-2022-09-01.zip')
      6 tbl = pa.Table.from_pandas(df)
----> 7 mkt_data.print_table(tbl)

TypeError: print_table(): incompatible function arguments. The following argument types are supported:
    1. (arg0: _object) -> None

Here is my CMakeLists.txt

cmake_minimum_required(VERSION 3.14)

if(${CMAKE_VERSION} VERSION_LESS 3.24)
    cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
else()
    cmake_policy(VERSION 3.24)
endif()

project(MarketDataProcessing VERSION 1.0
                              DESCRIPTION "Preprocessing Market Data"
                              LANGUAGES CXX)

# GoogleTest requires at least C++14
set(CMAKE_CXX_STANDARD 17)

# option(BUILD_PYTHON_MODULE "Build a mkt_data python module" ON)

include(FindPkgConfig)
find_package(Arrow REQUIRED)
add_library(mdprocessing SHARED include/Trade.h include/TradeAggregator.h include/utils.h 
                                src/Trade.cpp src/TradeAggregator.cpp )

target_include_directories(mdprocessing PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include )

target_link_libraries(mdprocessing arrow_shared)

# add_subdirectory(tests)

## Build Python module
find_package(pybind11 REQUIRED)

#add_library(hello_world hello_world.cpp)
set(PYBIND11_PYTHON_VERSION "3.9")
pybind11_add_module(mkt_data src/mkt_data_wrapper.cpp)
target_include_directories(mkt_data PUBLIC /home/ruihong/.python_venvs/learning/lib/python3.9/site-packages/pyarrow/include)
target_link_directories(mkt_data PUBLIC /home/ruihong/.python_venvs/learning/lib/python3.9/site-packages/pyarrow)
target_link_libraries(mkt_data PRIVATE mdprocessing  arrow_python)

Thank you so much!

Answer 1

I think you need to use pybind11::object instead of PyObject

void print_table(pybind11::object py_table)
{
    // convert pyobject to table
    auto status = arrow::py::unwrap_table(py_table.ptr());