帶有回調到 Python 的 Python 擴展模塊

Question

我的 Python 擴展模塊遇到了不可預測的行為，該模塊包含一個 C++ 庫，該庫啟動一個新的 pthread，並在完成一些工作后，將回調生成回調用方。 我已將其大大簡化為一個仍然可以演示此問題的簡單示例。 以下有時會產生一個Fatal Python error: PyEval_SaveThread: NULL tstate ，通常很快。 有時它會在tupledealoc上tupledealoc SIGSEGV。 偶爾會出現這種僵局。 我不知道為什么。 有沒有人有任何想法？

這是我的python測試代碼

import mymod
from time import sleep
from random import randrange

def my_cb1(s):
    print("Python cb %s" % (s));

for x in range(1,1000):
    num_cb = randrange(5) + 1
    print("Starting %d" % mymod.doit(my_cb1, "myid" + str(x), num_cb))

while True:
    sleep(1)

擴展模塊是：

#include <pthread.h>

#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include <stddef.h>

#include <iostream>
#include <map>
#include <deque>
#include <mutex>
#include <functional>
#include <thread>

static std::map<std::string, PyObject *> cb_map;
static std::mutex map_mtx;

struct fake_cb_info
{
  fake_cb_info() = delete;
  fake_cb_info(const unsigned long &num_cb, const std::string &id) :
    num_cb(num_cb), id(id)
  {
  }
  const unsigned long num_cb;
  const std::string id;
};
static std::deque<struct fake_cb_info> deq;
static std::mutex deq_mtx;

static bool is_worker_thread_running = false;
static std::thread worker_thread;

typedef std::function<void(const std::string &id, const std::string &s)> doit_cb_t;
static void internal_cb(const std::string &id, const std::string &s)
{
  std::scoped_lock<std::mutex> lk(map_mtx);

  if (0 != cb_map.count(id))
  {
      PyGILState_STATE gstate;
      gstate = PyGILState_Ensure();

      PyObject *arglist = Py_BuildValue("(s)", s.c_str());
      PyObject *result = PyObject_CallObject(cb_map.at(id), arglist);
      Py_DECREF(arglist);

      if (NULL == result)
      {
          if (NULL == PyErr_Occurred())
          {
            std::cerr << "Unknown error occurred in C callback" << std::endl;
          }
          else
          {
              PyErr_Print();
          }
      }
      else
      {
        Py_DECREF(result);
      }

      PyGILState_Release(gstate);
  }
  else
  {
    std::cerr << "Unknown callback id " << id << std::endl;
  }
}

void static worker()
{
  size_t x = 0;
  while(true)
  {
    std::scoped_lock<std::mutex> lk(deq_mtx);
    if (deq.size() == 0)
    {
      usleep(1000);
      continue;
    }

    auto info = deq.front();
    deq.pop_front();
    for (unsigned long i=0; i<info.num_cb; i++)
    {
      internal_cb(info.id, std::to_string(x++));
    }
  }
}

PyObject * _wrap_doit(void *self, PyObject *args, PyObject *kwargs)
{
    PyObject *py_retval;
    PyThreadState *py_thread_state = NULL;
    PyObject *cb;
    const char *id = NULL;
    Py_ssize_t id_len;
    std::string id_std;
    unsigned long num_callbacks;
    const char *keywords[] = {"cb_func", "id", "num_cb", NULL};

    if (!PyArg_ParseTupleAndKeywords(args, kwargs, (char *) "Os#k", (char **) keywords, &cb, &id, &id_len, &num_callbacks))
    {
        abort();
    }
    if (!PyCallable_Check(cb))
    {
        abort();
    }

    id_std = std::string(id, id_len);

    {
      std::scoped_lock<std::mutex> lk(map_mtx);
      if (0 == cb_map.count(id_std))
      {
        Py_INCREF(cb);
        cb_map.insert(std::make_pair(id_std, cb));

        // N.B. The corresponding Py_DECREF for the callback function PyObject
        // is intentionally not here. It is in another extension module method
        // that is not listed here (just trying to keep this example as small
        // and lean as possible)
      }
      else
      {
        std::cerr << "Only one callback for ID!" << std::endl;
        abort();
      }
    }

    if (PyEval_ThreadsInitialized ())
    {
      std::cout << "Saving thread" << std::endl;
      py_thread_state = PyEval_SaveThread();
    }

    {
      // Stash away the info so that we will know how many callbacks to
      // generate and sleep a bit. This is to simulate a real external library
      // doing work which will, in turn, generate callbacks
      struct fake_cb_info info(num_callbacks, id_std);
      std::scoped_lock<std::mutex> lk(deq_mtx);
      deq.push_back(info);

      if (!is_worker_thread_running)
      {
        std::cout << "@@@@ Creating a new thread\n";
        worker_thread = std::thread(&worker);
        pthread_setname_np(worker_thread.native_handle(), "worker_thread");
        worker_thread.detach();
        is_worker_thread_running = true;
      }

      usleep(10000);
    }

    if (py_thread_state)
    {
      std::cout << "Restoring thread" << std::endl;
      PyEval_RestoreThread(py_thread_state);
    }

    py_retval = Py_BuildValue((char *) "k", num_callbacks);
    return py_retval;
}

static PyMethodDef mymod_functions[] = {
    {
      (char *) "doit",
      (PyCFunction) _wrap_doit,
      METH_KEYWORDS | METH_VARARGS,
      "Generate requested number of multi-threaded callbacks.\n doit(callback_fn, id, num_callbacks)"
    },
    {NULL, NULL, 0, NULL}
};

static struct PyModuleDef moduledef = {
    PyModuleDef_HEAD_INIT,
    "mymod",
    "pthread test module",
    -1,
    mymod_functions,
};

#define MOD_ERROR NULL
#define MOD_INIT(name) PyObject* PyInit_##name(void)
#define MOD_RETURN(val) val

#if defined(__cplusplus)
extern "C"
#endif

#if defined(__GNUC__) && __GNUC__ >= 4
__attribute__ ((visibility("default")))
#endif


MOD_INIT(mymod)
{
    PyObject *m = PyModule_Create(&moduledef);
    if (m == NULL) {
        return MOD_ERROR;
    }
    return MOD_RETURN(m);
}

如果你想跳過編譯擴展模塊的setup.py步驟，這里有一個我用來構建它的 shell 腳本

#!/bin/bash -eux

obj='test_python_cmodule.o'
lib='mymod.cpython-36m-x86_64-linux-gnu.so'
if [ -f $obj ]; then
  rm $obj
fi
if [ -f $lib ]; then
  rm $lib
fi

g++ -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I.  -I/usr/include/python3.6m -c test_python_cmodule.cpp -o $obj -std=c++17 -Wno-unused-variable -g -O0

x86_64-linux-gnu-gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 $obj -lpthread -o $lib -lstdc++

Answer 1

觀察到該問題僅發生在 Python 3.6 而不是較新版本導致解決方案（發布在 comp.lang.python 上），即在這些較舊版本的 Python 中， PyEval_InitThreads()必須在主程序中至少顯式調用一次線。 在 Python3.7 中，它由 Python 自動調用，實際功能在 Python 3.9 中已棄用，計划從 Python 3.11 中的公共 API 中完全刪除