如何從 cython 訪問 python 共享 memory？

Question

我有兩個進程，一個創建共享 memory，第二個訪問它。 我正在使用 Python 3.8。

第二個進程可以選擇僅使用 python 函數或通過 cython 訪問共享的 memory。 cython 選項失敗 - 我得到一個 SIGSEGV。

是否有一些特殊的方法可以在 cython 中獲取共享的 memory？ 文檔似乎沒有顯示如何實際獲得 python 共享 memory。

過程 1 (setup_shm.py):

from multiprocessing import shared_memory
import numpy as np
import argparse
from package.get_shm import main as get_shm
from multiprocessing import Process


def main(cython=None):
    arr_share = np.array([[1.3424, 23.24324], [1.4234, .08682]], dtype='f')
    shape, dtype = arr_share.shape, arr_share.dtype
    # Create a shared memory of size arr_share.nbytes
    shm = shared_memory.SharedMemory(create=True, size=arr_share.nbytes)
    # Create array using the buffer of shm
    shm_np_array = np.ndarray(shape=shape, dtype=dtype, buffer=shm.buf)
    # Copy the data into the shared memory
    np.copyto(shm_np_array, arr_share)
    print('shared name {}'.format(shm.name))
    print('shared size {}'.format(arr_share.nbytes))
    print(arr_share)
    p = Process(target=get_shm, args=(shm.name,), kwargs={'cython': cython})
    p.start()
    p.join()
    shm.close()
    shm.unlink()


def setup():
    parser = argparse.ArgumentParser(description='Shared mem getter')
    parser.add_argument('--cython', action='store_true', help='Use cython to get shared mem')
    args = parser.parse_args()
    main(cython=args.cython)


if __name__ == "__main__":
    setup()

過程 2 (get_shm.py)：

from multiprocessing.shared_memory import SharedMemory
import numpy as np
import argparse
import sys
from c_get_shm import Vectors

def main(name, cython=None):
    val = None
    if cython:
        print('getting vector using cython - {}'.format(name))
        vectors = Vectors(name)
        try:
            val = vectors.data.base[0]
        except Exception as e:
            raise (e)
    else:
        shm = SharedMemory(name=name)
        np_array = np.ndarray(shape=(2, 2), dtype='f', buffer=shm.buf)
        print('getting vector using python - {}'.format(name))
        val = np_array[0]
        shm.close()
    print('val is {}'.format(val))
    sys.stdout.flush()
    return


def setup():
    parser = argparse.ArgumentParser(description='Shared mem getter')
    parser.add_argument('name', type=str, help='Shared memory name')
    parser.add_argument('--cython', action='store_true', help='Use cython to get shared mem')
    args = parser.parse_args()
    main(name=args.name, cython=args.cython)


if __name__ == "__main__":
    setup()

Cython class (c_get_shm.pyx)

#!python
#cython: language_level=3, boundscheck=False
from multiprocessing import shared_memory
cimport numpy as np
np.import_array()

cdef class Vectors:

    cdef public object shm_name
    cdef public float[:, :] data

    def __init__(self, shm_name):
        print('Vectors Class for getting shared memory - {}'.format(shm_name))
        self.__get_shared_array(shm_name)
        print('data is {}, {}'.format(self.data, self.data.shape))

    def __get_shared_array(self, shm_name: str):
        print('get_shared_array {}'.format(shm_name))
        shm = shared_memory.SharedMemory(name=shm_name)
        print('shm buf is {} size {}'.format(shm.buf, shm.buf.nbytes))
        self.data = np.ndarray(shape=(2, 2), dtype='f', buffer=shm.buf)

安裝程序.py

from setuptools import setup, Extension
from Cython.Build import cythonize

extensions = [
    Extension("c_get_shm", ["./c_get_shm.pyx"],
              libraries=['rt'])
]
setup(
    ext_modules=cythonize(extensions, gdb_debug=True)
)

我正在運行腳本

• python./setup_shm.py --cython
• python./setup_shm.py

我設法看到 SIGSEGV 的唯一方法是使用 strace：

Vectors Class for getting shared memory - psm_23cd04df
get_shared_array psm_23cd04df
shm buf is <memory at 0x7f8b4d66df40> size 16
data is <MemoryView of 'ndarray' object>, [2, 2, 0, 0, 0, 0, 0, 0]
[{WIFSIGNALED(s) && WTERMSIG(s) == SIGSEGV && WCOREDUMP(s)}], 0, NULL) = 868
--- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_DUMPED, si_pid=868, si_uid=1000, si_status=SIGSEGV, si_utime=0, si_stime=0} ---
close(5)                                = 0
munmap(0x7f8b65698000, 16)              = 0
close(4)                                = 0
unlink("/dev/shm/psm_23cd04df")         = 0

我試過 gdb python 並從那里運行腳本，但沒有堆棧跟蹤。

Answer 1

對於遇到類似問題的任何人，請確保在 Cython 中聲明共享的 memory。 我假設 SIGSEGV 是因為它不允許訪問 memory 除非你這樣做。

cdef class Vectors:

    cdef public object shm_name
    cdef public float[:, :] data
    # declare the shared memory in Cython!!
    cdef public object shm

    def __init__(self, shm_name):
        self.__get_shared_array(shm_name)

    def __get_shared_array(self, shm_name: str):
        self.shm = shared_memory.SharedMemory(name=shm_name)
        self.data = np.ndarray(shape=(2, 2), dtype='f', buffer=self.shm.buf)