Call main executable's functions from plugin compiled using Clang

Question

I'm writing a program (macOS, clang++ compiler, only AppleSilicon at the moment) that I can extend later by providing custom plugins (dynamic library, loaded at runtime) which use main program's public interface.

test.hpp - public interface:

#if defined(MAGIC_PLUGIN)
#  define MAGIC_EXPORT /* nothing */
#else
#  define MAGIC_EXPORT __attribute__((visibility("default")))
#endif

MAGIC_EXPORT
void testCall();

test.cpp - main programm:

#include <stdio.h>
#include <dlfcn.h>
#include "test.hpp"

// Declare a function to call from a loaded plugin
typedef void (* plugin_call_func)(void);

int main(int argc, char** argv) {
    // Load library
    const char* libraryName = "plugin.dylib";
    void* library = dlopen(libraryName, RTLD_NOW);
    if (library == nullptr) {
        printf("Cannot open library\n");
        return 1;
    }

    // Get function from loaded library
    plugin_call_func pluginCall = reinterpret_cast<plugin_call_func>(
                                    dlsym(library, "pluginCall"));
    if (pluginCall == nullptr) {
        printf("Cannot find the pluginCall function\n");
        return 2;
    }

    // Execute loaded function
    pluginCall();
    
    // Forget function and close library
    pluginCall = nullptr;
    auto libraryCloseResult = dlclose(library);
    if (libraryCloseResult != 0) {
        printf("Cannot close library\n");
        return 3;
    }

    return 0;
}

// Public function, should be called from a plugin
void testCall() {
    printf("Test call\n");
}

plugin.cpp - plugin's source:

#define MAGIC_PLUGIN

#include <stdio.h>
#include "test.hpp"

__attribute__((visibility("default")))
extern "C" void pluginCall(void) {
    printf("Plugin call\n");
    testCall();
}

First, I compile main app:

clang++ -std=c++20 -fvisibility=hidden -target arm64-apple-macos12 test.cpp -o test

The nm --defined-only test shows these symbols:

0000000100003ee4 T __Z8testCallv
0000000100000000 T __mh_execute_header
0000000100003dcc t _main

Mangled __Z8testCallv is what I need. Everything looks good so far. But then I try to compile the plugin as dynamic library...

clang++ -std=c++20 -fvisibility=hidden -dynamiclib -g -current_version 0.1 -target arm64-apple-macos12 plugin.cpp -o plugin.dylib

and get this error:

Undefined symbols for architecture arm64:
  "testCall()", referenced from:
      _pluginCall in plugin-38422c.o
ld: symbol(s) not found for architecture arm64
clang: error: linker command failed with exit code 1 (use -v to see invocation)

Well, it's kind of fair, I can understand this, because the dynamic library does not know that testCall is somewhere implemented. So I want to say it that it does not have to worry about testCall 's existence.

I tried to research how to do this, looked up man pages, read tons of stack overflow answers, and what I only found that works is adding these flags to linker:

-Wl,-undefined,dynamic_lookup

It works, the library compiles and the app works as expected. But I don't really want to use dynamic_lookup because it will mark every undefined symbol in the library as resolved, which may lead to some bad consequences. I want to tell the linker only about existence of the main program's public symbols.

What am I missing? Is there any better solution than dynamic_lookup ?

Answer 1

Your best bet is to manually do the work that's done by the dynamic library loader. That is: populating function pointers. After all, the plugin->main binding is already done manually, so doing the same thing the other way around makes sense.

You can make this process essentially transparent by carefully crafting the header shared by the plugin and main application. The only tricky part is handling ODR for plugins that are composed of multiple source files.

Since this is a C++ question, here's what it could look like with a RAII wrapper. The ODR conundrum is handled via the PLUGIN_MAIN macro that should only be defined in one of a plugin's sources.

test_plugin.hpp

using pluginCall_fn = void(*)();
using testCall_fn = void(*)();

#if defined(COMPILING_PLUGIN) || defined(PLUGIN_MAIN)
extern "C" {
  // Declare symbols provided by the plugin
  __attribute__((visibility("default"))) void pluginCall();
  
  // Declare pointers that will be populated by the main application
  __attribute__((visibility("default"))) extern testCall_fn testCall;
}

#ifdef PLUGIN_MAIN
// Only define the pointers in the TU with PLUGIN_MAIN defined.
testCall_fn testCall;
#endif

#else // In the main app.

#include <stdexcept>

// Declare "symbols" provided by the main application
void testCall();

struct loaded_library final {
  loaded_library(const char* libName) 
    : handle_(dlopen(libName, RTLD_NOW)) {
    if(!handle_) {
      throw std::runtime_error("failed to load plugin");
    }
  }

  ~loaded_library() {
    dlclose(handle_);
  }

  template<typename T>
  T get_symbol(const char* symbol) {
    T result = reinterpret_cast<T>(dlsym(handle_, symbol));
    if(!result) {
      throw std::runtime_error("missing symbol");
    }
    return result;
  }

private:
    void* handle_;
};

struct loaded_plugin final {
  loaded_plugin(const char* libName) 
    : lib_(libName) {

    // Load functions from plugin
    pluginCall = lib_.get_symbol<pluginCall_fn>("pluginCall");
    // ...

    // Assign callbacks to plugin
    *lib_.get_symbol<testCall_fn*>("testCall") = &testCall;

    // Call the plugin's init function here if applicable.
  }

  pluginCall_fn pluginCall;

private:
  loaded_library lib_;
};

#endif

plugin.cpp

#define PLUGIN_MAIN
#include "test_plugin.hpp"

#include <stdio.h>

void pluginCall() {
    printf("Plugin call\n");
    testCall();
}

test.cpp

#include "test_plugin.hpp"

int main(int argc, char** argv) {
    const char* libraryName = "plugin.dylib";

    loaded_plugin plugin(libraryName);

    plugin.pluginCall();
}

// Public function, should be called from a plugin
void testCall() {
    printf("Test call\n");
}