Pybind11 linking error: relocation R_X86_64_32 against hidden symbol `_ZTI9nan_error' can not be used when making a shared object

Question

Dear StackOverflow community,

I want to create a python module using pybind11 from a c++ code. In my case, the leading directory is called process_hclust. Inside, there is a src folder that possesses the needed c++ files.

* process_hclust (main folder)
  | setup.py
  | CMakeLists.txt
  * src
   | main.cpp
   ** hclust-cpp
     | CMakeLists.txt
     | fastcluster.h
     | fastcluster.cpp

• The main CMakeLists.txt adds the pybind11 module in "src/main.cpp" and links the library needed in the second CMakeLists.txt (as I understand).

cmake_minimum_required(VERSION 3.22.1)
project(process_hclust)

include(FetchContent)
FetchContent_Declare(
    pybind11
    GIT_REPOSITORY https://github.com/pybind/pybind11
    GIT_TAG        v2.2.3
)

FetchContent_GetProperties(pybind11)
if(NOT pybind11_POPULATED)
    FetchContent_Populate(pybind11)
    add_subdirectory(${pybind11_SOURCE_DIR} ${pybind11_BINARY_DIR})
endif()

pybind11_add_module(${PROJECT_NAME} src/main.cpp)

add_subdirectory(src/hclust-cpp)
target_link_libraries(${PROJECT_NAME} PRIVATE fastcluster)

• The second. CMakeLists.txt file inside hclust-cpp declares which files need to be linked.

add_library(fastcluster fastcluster.cpp fastcluster.h)

• Inside the main.cpp file, I am calling the header files using

#include "hclust-cpp/fastcluster.h"

The setup.py just calls cmake.

This code works on my local computer where I have python3.9 installed. It works by simple using:

pip install ./process_hclust

However , installing the module using my university machines with python3.7, I get the following error:

Processing ./cpp/process_hclust
  Preparing metadata (setup.py) ... done
Using legacy 'setup.py install' for process-hclust, since package 'wheel' is not installed.
Installing collected packages: process-hclust
  Attempting uninstall: process-hclust
    Found existing installation: process-hclust 0.0.0
    Uninstalling process-hclust-0.0.0:
      Successfully uninstalled process-hclust-0.0.0
  Running setup.py install for process-hclust ... error
  error: subprocess-exited-with-error
  
  × Running setup.py install for process-hclust did not run successfully.
  │ exit code: 1
  ╰─> [57 lines of output]
      running install
      running build
      running build_ext
      -- pybind11 v2.2.3
      -- Configuring done
      -- Generating done
      -- Build files have been written to: /afs/crc.nd.edu/user/j/jmarti53/Public/brain_project/LINKPROJECT/python/cpp/process_hclust/build/temp.linux-x86_64-3.7/process_hclust
      Consolidate compiler generated dependencies of target fastcluster
      [ 50%] Built target fastcluster
      Consolidate compiler generated dependencies of target process_hclust
      [ 75%] Linking CXX shared module ../../lib.linux-x86_64-3.7/process_hclust.cpython-37m-x86_64-linux-gnu.so
      /usr/bin/ld: src/hclust-cpp/libfastcluster.a(fastcluster.cpp.o): relocation R_X86_64_32 against hidden symbol `_ZTI9nan_error' can not be used when making a shared object
      /usr/bin/ld: final link failed: Nonrepresentable section on output
      collect2: error: ld returned 1 exit status
      gmake[2]: *** [CMakeFiles/process_hclust.dir/build.make:98: ../../lib.linux-x86_64-3.7/process_hclust.cpython-37m-x86_64-linux-gnu.so] Error 1
      gmake[1]: *** [CMakeFiles/Makefile2:118: CMakeFiles/process_hclust.dir/all] Error 2
      gmake: *** [Makefile:91: all] Error 2
      Traceback (most recent call last):
        File "<string>", line 36, in <module>
        File "<pip-setuptools-caller>", line 34, in <module>
        File "/afs/crc.nd.edu/user/j/jmarti53/Public/brain_project/LINKPROJECT/python/cpp/process_hclust/setup.py", line 137, in <module>
          python_requires=">=3.6",
        File "/afs/crc.nd.edu/user/j/jmarti53/Public/brain_project/LINKPROJECT/python/env/lib/python3.7/site-packages/setuptools/__init__.py", line 145, in setup
          return distutils.core.setup(**attrs)
        File "/opt/crc/p/python/3.7.3/gcc/lib/python3.7/distutils/core.py", line 148, in setup
          dist.run_commands()
        File "/opt/crc/p/python/3.7.3/gcc/lib/python3.7/distutils/dist.py", line 966, in run_commands
          self.run_command(cmd)
        File "/opt/crc/p/python/3.7.3/gcc/lib/python3.7/distutils/dist.py", line 985, in run_command
          cmd_obj.run()
        File "/afs/crc.nd.edu/user/j/jmarti53/Public/brain_project/LINKPROJECT/python/env/lib/python3.7/site-packages/setuptools/command/install.py", line 61, in run
          return orig.install.run(self)
        File "/opt/crc/p/python/3.7.3/gcc/lib/python3.7/distutils/command/install.py", line 545, in run
          self.run_command('build')
        File "/opt/crc/p/python/3.7.3/gcc/lib/python3.7/distutils/cmd.py", line 313, in run_command
          self.distribution.run_command(command)
        File "/opt/crc/p/python/3.7.3/gcc/lib/python3.7/distutils/dist.py", line 985, in run_command
          cmd_obj.run()
        File "/opt/crc/p/python/3.7.3/gcc/lib/python3.7/distutils/command/build.py", line 135, in run
          self.run_command(cmd_name)
        File "/opt/crc/p/python/3.7.3/gcc/lib/python3.7/distutils/cmd.py", line 313, in run_command
          self.distribution.run_command(command)
        File "/opt/crc/p/python/3.7.3/gcc/lib/python3.7/distutils/dist.py", line 985, in run_command
          cmd_obj.run()
        File "/afs/crc.nd.edu/user/j/jmarti53/Public/brain_project/LINKPROJECT/python/env/lib/python3.7/site-packages/setuptools/command/build_ext.py", line 78, in run
          _build_ext.run(self)
        File "/opt/crc/p/python/3.7.3/gcc/lib/python3.7/distutils/command/build_ext.py", line 340, in run
          self.build_extensions()
        File "/opt/crc/p/python/3.7.3/gcc/lib/python3.7/distutils/command/build_ext.py", line 449, in build_extensions
          self._build_extensions_serial()
        File "/opt/crc/p/python/3.7.3/gcc/lib/python3.7/distutils/command/build_ext.py", line 474, in _build_extensions_serial
          self.build_extension(ext)
        File "/afs/crc.nd.edu/user/j/jmarti53/Public/brain_project/LINKPROJECT/python/cpp/process_hclust/setup.py", line 122, in build_extension
          subprocess.check_call(["cmake", "--build", "."] + build_args, cwd=build_temp)
        File "/opt/crc/p/python/3.7.3/gcc/lib/python3.7/subprocess.py", line 347, in check_call
          raise CalledProcessError(retcode, cmd)
      subprocess.CalledProcessError: Command '['cmake', '--build', '.']' returned non-zero exit status 2.
      [end of output]
  
  note: This error originates from a subprocess, and is likely not a problem with pip.
  Rolling back uninstall of process-hclust
  Moving to /afs/crc.nd.edu/user/j/jmarti53/Public/brain_project/LINKPROJECT/python/env/lib/python3.7/site-packages/process_hclust-0.0.0-py3.7.egg-info
   from /afs/crc.nd.edu/user/j/jmarti53/Public/brain_project/LINKPROJECT/python/env/lib/python3.7/site-packages/~rocess_hclust-0.0.0-py3.7.egg-info
  Moving to /afs/crc.nd.edu/user/j/jmarti53/Public/brain_project/LINKPROJECT/python/env/lib/python3.7/site-packages/process_hclust.cpython-37m-x86_64-linux-gnu.so
   from /tmp/pip-uninstall-2ijatdvx/process_hclust.cpython-37m-x86_64-linux-gnu.so
error: legacy-install-failure

× Encountered error while trying to install package.
╰─> process-hclust

note: This is an issue with the package mentioned above, not pip.
hint: See above for output from the failure.

I believe the main problem is in the line

[ 75%] Linking CXX shared module ../../lib.linux-x86_64-3.7/process_hclust.cpython-37m-x86_64-linux-gnu.so
      /usr/bin/ld: src/hclust-cpp/libfastcluster.a(fastcluster.cpp.o): relocation R_X86_64_32 against hidden symbol `_ZTI9nan_error' can not be used when making a shared object
      /usr/bin/ld: final link failed: Nonrepresentable section on output

My small knowledge of pybind11, cmake, and building up python libraries makes me think that the problem appears when the program links the hclust-cpp directory, looking for the fastcluster.h header. Still, I don't understand completely what is happening.

I have been trying to solve this problem for some days, so any suggestion for what to do would be highly appreciated; and in case the problem is confusing and/or need more information, just let me know.

Best regards in advance!

Answer 1

My dear StackOverflow community,

I solved the problem. It got solved by adding SHARED into the second CMakeLists.txt

add_library(fastcluster SHARED fastcluster.cpp fastcluster.h)

Now, it works!!! I am so happy!!