Why I'm getting segmentation fault with python3 C-API and c FILE *

Question

I get segmentation fault when i try to pass opened python 3 file object to c routine via FILE * pointer. I try porting a python 2 code to python 3 and as so far i succeed this operation via the C-API PyFile_AsFile which is no longer exists in python 3.

a minimal c code would be demo.c

    #include <stdio.h>
    int writeArray2Integer(FILE * f) {
       fprintf(f, "test write\n");
       return 0;
    }

I compile it with

 gcc -DNDEBUG -g -O3 -Wall -fPIC -c demo.c -o build/demo.o
 gcc -shared build/demo.o -o build/demo.so

The Python 3 code calling my demo c library would be

   #!/usr/bin/env python
  
   import sys
   import ctypes
  
   _lib =  ctypes.CDLL('build/demo.so')
   _write_array_2integer_c = _lib.writeArray2Integer
   _write_array_2integer_c.argtypes = [ctypes.c_void_p]
   _write_array_2integer_c.restype = ctypes.c_int
   
   def writeArray2Integer(f):
       ctypes.pythonapi.PyObject_AsFileDescriptor.argtypes = [ctypes.py_object]
       ctypes.pythonapi.PyObject_AsFileDescriptor.restype =  ctypes.c_int
       fd = ctypes.pythonapi.PyObject_AsFileDescriptor(f) # Segmentation fault here !!
       pf = ctypes.pythonapi.fdopen(fd, "w")
       out = _write_array_2integer_c(pf)

   if __name__ == "__main__":
       f = open("/tmp/toto.txt", "w")
       writeArray2Integer(f)
       f.close()

there is the gdb trace

Program received signal SIGSEGV, Segmentation fault.
0x00007ffff7e254a5 in __GI__IO_fwrite (buf=buf@entry=0x7ffff75c5000, size=size@entry=1, count=count@entry=11, fp=0x558bf310)
    at iofwrite.c:37
37      iofwrite.c: No such file or directory.

and the gdb backtrace full

(gdb) bt full
#0  0x00007ffff7e254a5 in __GI__IO_fwrite (buf=buf@entry=0x7ffff75c5000, size=size@entry=1, count=count@entry=11, fp=0x558bf310)
    at iofwrite.c:37
        _IO_acquire_lock_file = <optimized out>
        request = 11
        written = 0
#1  0x00007ffff75c4141 in fprintf (__fmt=0x7ffff75c5000 "test write\n", __stream=<optimized out>)
    at /usr/include/x86_64-linux-gnu/bits/stdio2.h:100
No locals.
#2  writeArray2Integer (f=<optimized out>) at demo.c:4
No locals.
#3  0x00007ffff75dc630 in ffi_call_unix64 ()
   from /home/lahcen/Documents/thirdparty/deps/python3.7.5-linux/lib/python3.7/lib-dynload/../../libffi.so.6
No symbol table info available.
#4  0x00007ffff75dbfed in ffi_call ()
   from /home/lahcen/Documents/thirdparty/deps/python3.7.5-linux/lib/python3.7/lib-dynload/../../libffi.so.6
No symbol table info available.
#5  0x00007ffff75f201e in _call_function_pointer (argcount=1, resmem=0x7fffffffd3d0, restype=<optimized out>,
    atypes=0x7fffffffd390, avalues=0x7fffffffd3b0, pProc=0x7ffff75c4120 <writeArray2Integer>, flags=4353)
    at /usr/local/src/conda/python-3.7.5/Modules/_ctypes/callproc.c:829
        error_object = 0x0
        cc = 2
        _save = <optimized out>
        space = 0x7ffff761aab0
        cif = {abi = FFI_UNIX64, nargs = 1, arg_types = 0x7fffffffd390, rtype = 0x7ffff7636258, bytes = 0, flags = 10}
        _save = <optimized out>
        error_object = <optimized out>
        space = <optimized out>
        cif = {abi = <optimized out>, nargs = <optimized out>, arg_types = <optimized out>, rtype = <optimized out>,
          bytes = <optimized out>, flags = <optimized out>}
        cc = <optimized out>
        _py_xdecref_tmp = <optimized out>
        _py_decref_tmp = <optimized out>
        temp = <optimized out>
        temp = <optimized out>
#6  _ctypes_callproc (pProc=0x7ffff75c4120 <writeArray2Integer>, argtuple=<optimized out>, flags=4353, argtypes=<optimized out>,
    restype=0x55555594d140, checker=0x0) at /usr/local/src/conda/python-3.7.5/Modules/_ctypes/callproc.c:1186
        i = <optimized out>
        n = 1
        argcount = 1
        argtype_count = <optimized out>
        resbuf = 0x7fffffffd3d0
        args = <optimized out>
        pa = <optimized out>
        atypes = 0x7fffffffd390

Answer 1

1. I'd try to use file descriptor directly from python file object, using fileno() method.
2. After writing I think you should flush the stream, or you won't get the output written (until flush the output in only stored in memory buffer).

So to sum up writeArray2Integer function should look like this:

def writeArray2Integer(f):
   fd = f.fileno()
   pf = ctypes.pythonapi.fdopen(fd, "w")
   out = _write_array_2integer_c(pf)
   ctypes.pythonapi.fflush(pf)

Works for me with python 3;)

Update: Code below works for me both with python 2 and python 3. You can omit initializing artypes and restype in python 3, but without them python 2 gives segmentation fault (I think it's because it assumes arguments are 32-bit integers, which is not true with pointers on 64-bit system).

import ctypes
_lib =  ctypes.CDLL('build/demo.so')
_write_array_2integer_c = _lib.writeArray2Integer
_write_array_2integer_c.argtypes = [ ctypes.c_void_p ]

def writeArray2Integer(f):
    fd = f.fileno()

    fdopen = ctypes.pythonapi.fdopen
    fdopen.argtypes = [ ctypes.c_int, ctypes.c_void_p ]
    fdopen.restype = ctypes.c_void_p

    fflush = ctypes.pythonapi.fflush
    fflush.argtypes = [ ctypes.c_void_p ]
    fflush.restype = ctypes.c_int

    fp = fdopen(fd, "w")
    out = _write_array_2integer_c(fp)
    fflush(fp)

if __name__ == "__main__":
    f = open("toto.txt", "w")
    writeArray2Integer(f)
    f.close()