如何正确地将 c++ dll 中的多个 cv::Mat 传递给 opencvsharp Mat c#？

Question

I'm working on a project which required a dll file for another program written in c# to use (I'm not very familiar with the usage of C++/C#). For the very last step to complete my work, I have a problem with passing "multiple" cv::Mat from dll to C#.

I've found some examples on the Internet about C# using OpenCvSharp to receive a cv::Mat from dll, and it worked well like this in my code (it's simplified):

//original.hpp
extern "C" LIB_API cv::Mat* inference(unsigned char* img_pointer, long data_len);


//original.cpp
LIB_API cv::Mat* inference(unsigned char* img_pointer, long data_len)
{
    cv::Mat A;
    ..... // process that update A
    return new cv::Mat(A);
}

//original.cs
[DllImport(@"D:\Coco\Code\C_plus_plus\cpp_dll\x64\Release\cpp_dll.dll")]
private static extern IntPtr inference(byte[] img, long data_len);

static void Main()
{
    Intptr res = inference(X, Y);
    Mat A1 = new Mat(res);
    Cv2.ImShow("test1", A1);
    Cv2.WaitKey(2000);
}

Since it worked successfully, I planned to use the same syntax and pass the result through parameters with function, so that I can return multiple cv::Mat as I need, but this code doesn't work...

//rv1.hpp
extern "C" LIB_API void inference(unsigned char* img_pointer, long data_len, cv::Mat* res);

//rv1.cpp
LIB_API void inference(unsigned char* img_pointer, long data_len, cv::Mat *res)
{
    cv::Mat A;
    ..... // process that update A
    res = new cv::Mat(A);
}

//rv1.cs
[DllImport(@"D:\Coco\Code\C_plus_plus\cpp_dll\x64\Release\cpp_dll.dll")]
private static extern void inference(byte[] img, long data_len, out IntPtr res);

static void Main()
{
    Intptr res;
    inference(X, Y, out res);
    Mat A1 = new Mat(res);
    Cv2.ImShow("test1", A1);
    Cv2.WaitKey(2000);
}

I thought it was because I did the wrong assignment to the cv::Mat*, so it didn't get the correct address, then I revise the part of rv1.cpp, but this code doesn't work either...

// rv1_1.cpp
LIB_API void inference(unsigned char* img_pointer, long data_len, cv::Mat *res)
{
    cv::Mat A;
    ..... // process that update A
    res = &A;
}

(The error is System.AccessViolationException: Attempted to read or write protected memory )

---

Then I came up with another way, passing a vector of cv::Mat* with the function's parameters , and the code is like:

//rv2.hpp
extern "C" LIB_API void inference(unsigned char* img_pointer, long data_len, cv::Mat ***data_1, long* len);

// rv2.cpp
LIB_API void inference(unsigned char* img_pointer, long data_len, cv::Mat ***data_1, long* len)
{
    std::vector<cv::Mat*> vec_mat;
    cv::Mat A;
    cv::Mat B;
    ..... // process that update A, B
    vec_mat.push_back(new cv::Mat(A));
    vec_mat.push_back(new cv::Mat(B));

    *len = vec_mat.size();
    auto size = (*len) * sizeof(cv::Mat*);
    *data_1 = static_cast<cv::Mat**>(CoTaskMemAlloc(size));
    memcpy(*data_1, vec_mat.data(), size);
}

//rv2.cs
[DllImport(@"D:\Coco\Code\C_plus_plus\cpp_dll\x64\Release\cpp_dll.dll")]
private static extern void inference(byte[] img, long data_len, out IntPtr[] data, out int len);

static void Main()
{
    IntPtr[] sss1;
    int itemsCount;
    inference(image_byte_array, ms.Length, out sss1, out itemsCount);
    for (int i = 0; i < itemsCount; i++) // index out of range (the length of ss1 is "1")
    {
       Mat A3 = new Mat(sss1[i]);
       Cv2.ImShow("test3", A3);
       Cv2.WaitKey(2000);
    }
}

The thing is, I expected the returned vector should have 2 items in it, but it turned out to have only one item.

(When I loop through the IntPtr[ ], it only get 1 item, and then stop with the error like "index out of range")

I know there must be some syntax error in my code, but I have no idea where are they and how to correct them...

(And it seems to be some very basic syntax problems with the usage of pointer... )

---

Since the method above can still get the "first" item of the vector, I currently can pass "multiple" cv::Mat* in this way:

(Which is really stupid and improper to code like this...)

//rv3.hpp
extern "C" LIB_API void inference(unsigned char* img_pointer, long data_len, cv::Mat*** data_1, cv::Mat ***data_2);

// rv3.cpp
LIB_API void inference(unsigned char* img_pointer, long data_len, cv::Mat ***data_1, cv::Mat ***data_2)
{
    std::vector<cv::Mat*> vec_mat1;
    std::vector<cv::Mat*> vec_mat2;

    cv::Mat A;
    cv::Mat B;
    ..... // process that update A, B
    vec_mat1.push_back(new cv::Mat(A));
    vec_mat2.push_back(new cv::Mat(B));

    auto size = (*len) * sizeof(cv::Mat*);
    *data_1 = static_cast<cv::Mat**>(CoTaskMemAlloc(size));
    *data_2 = static_cast<cv::Mat**>(CoTaskMemAlloc(size));
    memcpy(*data_1, vec_mat1.data(), size);
    memcpy(*data_2, vec_mat2.data(), size);

}

//rv3.cs
[DllImport(@"D:\Coco\Code\C_plus_plus\cpp_dll\x64\Release\cpp_dll.dll")]
private static extern void inference(byte[] img, long data_len, out IntPtr[] data_1, out IntPtr[] data_2);

static void Main()
{
    IntPtr[] sss1, sss2;
    int itemsCount;
    inference(image_byte_array, ms.Length, out sss1, out sss2);

    Mat A3 = new Mat(sss1[0]);
    Cv2.ImShow("test3", A3);
    Cv2.WaitKey(2000);

    Mat A4 = new Mat(sss2[0]);
    Cv2.ImShow("test4", A4);
    Cv2.WaitKey(2000);
}

As I said above, I don't think this is a proper way to pass multiple cv::Mat* from dll to C#.

In my opinion, it should be like:

1. pass multiple cv::Mat* with the parameters of the function

2. pass a vector of multiple cv::Mat* in it with the parameters of the function

   ( Not multiple vectors of only one cv::Mat* in each )

But I really have no idea how to revise the code correctly, so any advise or help is really appreciated...

(Thanks in advance for finish reading my messy question description!)

Answer 1

Since you already return a pointer to a cv::Mat you can also make it a dynamic array.

LIB_API void inference(unsigned char* img_pointer, long data_len, cv::Mat*& res, int& img_count, int& mat_type_size)
{
    img_count = 10;
    mat_type_size = sizeof(cv::Mat);
    res = new cv::Mat[img_count]

    for (int i = 0; i < img_count; i++)
    {
        // process each mat 
        cv::Mat& A = res[i];
    }
}

Note that

• cv::Mat*& res is now of a reference to a pointer. Just passing a pointer doesn't work since you just reassign the pointer to a new address.
• int& img_count is also a reference so you can return the actual number of images that you have allocated back to C#.
• int& mat_type_size simply says how many bytes a cv::Mat object is. This is necessary to correctly increment the C# IntPtr to point to the next image in the array.

In your C# code you should be able to import it like this (my knowledge about marshalling is limited):

[DllImport(@"D:\Coco\Code\C_plus_plus\cpp_dll\x64\Release\cpp_dll.dll")]
private static extern void inference(byte[] img, long data_len, out IntPtr images, ref int img_count, out int mat_type_size);

and use it like this:

static void Main()
{   
    int imgCount = 5;
    inference(new byte[10], 10, out var imgPtrs, ref imgCount, out var matTypeSize);

    List<Mat> images = new List<Mat>();
    for (int i = 0; i < imgCount; i++)
            images.Add(new Mat(IntPtr.Add(imgPtrs, i * matTypeSize)));
    // ...
}

I've tested the code and it works. This is how I am using it:

Example

C++

// hpp
extern "C" __declspec(dllexport) void inference(unsigned char* img_pointer, long data_len, cv::Mat * &res, int& img_count, int& mat_type_size);

// cpp
void inference(unsigned char* img_pointer, long data_len, cv::Mat*& res, int& img_count, int& mat_type_size)
{
    mat_type_size = sizeof(cv::Mat);
    res = new cv::Mat[img_count];

    for (int i = 0; i < img_count; i++)
    {
        // process each mat 
        cv::Mat& A = res[i];
        A.create(100, 100, CV_8UC1);
        cv::circle(A, {50, 50}, 10 * i, 255, -1);
    }
}

C#

static class Program
{
    [DllImport(@"Cpp.dll")]
    private static extern void inference(byte[] img, long data_len, out IntPtr images, ref int img_count, out int mat_type_size);


    static void Main(string[] args)
    {            
        int imgCount = 5;
        inference(new byte[10], 10, out var imgPtrs, ref imgCount, out var matTypeSize);

        List<Mat> images = new List<Mat>();
        for (int i = 0; i < imgCount; i++)
                images.Add(new Mat(IntPtr.Add(imgPtrs, i * matTypeSize)));

        foreach (var img in images)
        {
            Cv2.ImShow("Test", img);
            Cv2.WaitKey();
        }            
    }
}

Answer 2

You can use just one function for passing Mat object between two languages.

C++ Function

//for conversion from c++ to cs
//this variable must be delete after using
std::vector<uchar>* vec = new std::vector<uchar>;
void convertMat2CS(cv::Mat income_mat, uchar** ptr, int* length)
{
    cv::imencode(".png", income_mat, *vec);
    *ptr = &vec[0][0];
    *length = static_cast<int>(vec->size());
}

C# Side

[DllImport(dll)]
public static extern void convertMat2CS(out IntPtr ptr, out int len);

void Main(){
    convertMat2CS(out IntPtr ptr, out int length);
    byte[] pngImageBytes = new byte[length];    
    Marshal.Copy(ptr, pngImageBytes, 0, length);
    Mat mat = new Mat();
    CvInvoke.Imdecode(pngImageBytes, LoadImageType.AnyColor, mat);
    CvInvoke.Imshow("Test_" + showCount, mat);
    CvInvoke.WaitKey(1);

}

Answer 3

how can I pass an image cv::Mat OpenCV of this function in the form of a mat through the C++ DLL to C# and receive it in the form of a PictureBox in C# by opencvsharp? I want to Convert Mat object from C++ opencv to C# opencvsharp.but I don't how to do.

C++/opencv-------Mat------------>DLL------->C#/opencvsharp---------->Mat.

At first,I Wrote one DLL for C++ as following:

//C++ DLL -----unmanaged

c++:

 extern "C" __declspec(dllexport) cv::Mat draw_objects(const cv::Mat& bgr, const std::vector<Object>& objects)

{ static const char* class_names[] = { "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush" };

static const unsigned char colors[81][3] = {
        {56,  0,   255},
        {226, 255, 0},
        {0,   94,  255},
        {0,   37,  255},
        {0,   255, 94},
        {255, 226, 0},
        {0,   18,  255},
        {255, 151, 0},
        {170, 0,   255},
        {0,   255, 56},
        {255, 0,   75},
        {0,   75,  255},
        {0,   255, 169},
        {255, 0,   207},
        {75,  255, 0},
        {207, 0,   255},
        {37,  0,   255},
        {0,   207, 255},
        {94,  0,   255},
        {0,   255, 113},
        {255, 18,  0},
        {255, 0,   56},
        {18,  0,   255},
        {0,   255, 226},
        {170, 255, 0},
        {255, 0,   245},
        {151, 255, 0},
        {132, 255, 0},
        {75,  0,   255},
        {151, 0,   255},
        {0,   151, 255},
        {132, 0,   255},
        {0,   255, 245},
        {255, 132, 0},
        {226, 0,   255},
        {255, 37,  0},
        {207, 255, 0},
        {0,   255, 207},
        {94,  255, 0},
        {0,   226, 255},
        {56,  255, 0},
        {255, 94,  0},
        {255, 113, 0},
        {0,   132, 255},
        {255, 0,   132},
        {255, 170, 0},
        {255, 0,   188},
        {113, 255, 0},
        {245, 0,   255},
        {113, 0,   255},
        {255, 188, 0},
        {0,   113, 255},
        {255, 0,   0},
        {0,   56,  255},
        {255, 0,   113},
        {0,   255, 188},
        {255, 0,   94},
        {255, 0,   18},
        {18,  255, 0},
        {0,   255, 132},
        {0,   188, 255},
        {0,   245, 255},
        {0,   169, 255},
        {37,  255, 0},
        {255, 0,   151},
        {188, 0,   255},
        {0,   255, 37},
        {0,   255, 0},
        {255, 0,   170},
        {255, 0,   37},
        {255, 75,  0},
        {0,   0,   255},
        {255, 207, 0},
        {255, 0,   226},
        {255, 245, 0},
        {188, 255, 0},
        {0,   255, 18},
        {0,   255, 75},
        {0,   255, 151},
        {255, 56,  0},
        {245, 255, 0}
};

int color_index = 0;

cv::Mat image = bgr.clone();

for (int i = 0; i < objects.size(); i++) {
    const Object& obj = objects[i];

    const unsigned char* color = colors[color_index % 80];
    color_index++;

    cv::Scalar cc(color[0], color[1], color[2]);

    fprintf(stderr, "%d = %.5f at %.2f %.2f %.2f x %.2f\n", obj.label, obj.prob,
        obj.rect.x, obj.rect.y, obj.rect.width, obj.rect.height);
    
    for(int y = 0; y < image.rows; y++){
        uchar* image_ptr = image.ptr(y);
        const float* mask_ptr = obj.cv_mask.ptr<float>(y);  
        for(int x = 0; x < image.cols; x++){
            if(mask_ptr[x] >= 0.5)
            {
                image_ptr[0] = cv::saturate_cast<uchar>(image_ptr[0] * 0.5 + color[2] * 0.5);
                image_ptr[1] = cv::saturate_cast<uchar>(image_ptr[1] * 0.5 + color[1] * 0.5);
                image_ptr[2] = cv::saturate_cast<uchar>(image_ptr[2] * 0.5 + color[0] * 0.5);
            }
            image_ptr += 3;
        }
    }

    cv::rectangle(image, obj.rect, cc, 2);

    char text[256];
    sprintf(text, "%s %.1f%%", class_names[obj.label], obj.prob * 100);

    int baseLine = 0;
    cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);

    int x = obj.rect.x;
    int y = obj.rect.y - label_size.height - baseLine;
    if (y < 0)
        y = 0;
    if (x + label_size.width > image.cols)
        x = image.cols - label_size.width;

    cv::rectangle(image, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)),
        cc, -1);

    cv::putText(image, text, cv::Point(x, y + label_size.height),
        cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(255, 255, 255));
}

return image;

}

c#:

[DllImport(path_to_my_dll,  CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]
static extern opencvsharp.Mat draw_objects(opencvsharp.Mat bgr, array[] objects)