OpenCV real-time stereocam image processing problems (post-calibration)

Question

I have built my own stereocamera from two USB cameras (with an annoying autofocus). I have calibrated them using /opencv/samples/cpp/stereo_calib.cpp', which produced an extrinsics/intrinsics file with an RMS Error of 0.4818 and an average reprojection error of 0.5426

I am now trying to run realtime depth mapping using Ptr<StereoBM> bm in openCV. I am getting useless images and I'm wondering if anyone can point me in the right direction of where to look. I'm also wondering if the autofocusing of the cameras every so often can change the focal length and therefore change the calibration, altering the results.

Code and pictures below.

Bonus if anyone can recommend any more robust methods than StereoBM for matching in openCV :)

---

Image of checkerboard calibration:

Left camera image:

Post StereoBM Processing:

Snippet of Code:

//Set up stereoBM
    Ptr<StereoBM> bm = StereoBM::create(16,9);

    //Read intrinsice parameters
    string intrinsic_filepath = "/home/will/Desktop/repos3.0/stereo-vision/Stereo-Vision/intrinsics.yml";
    FileStorage fs(intrinsic_filepath, FileStorage::READ);
    if(!fs.isOpened())
    {
        printf("Failed to open intrinsics.yml");
        return -1;
    }
    Mat M1, D1, M2, D2;
    fs["M1"] >> M1;
    fs["D1"] >> D1;
    fs["M2"] >> M2;
    fs["D2"] >> D2;

    //Read Extrinsic Parameters
    string extrinsic_filepath = "/home/will/Desktop/repos3.0/stereo-vision/Stereo-Vision/extrinsics.yml";
    fs.open(extrinsic_filepath, FileStorage::READ);
    if(!fs.isOpened())
    {
        printf("Failed to open extrinsics");
        return -1;
    }

    Mat R, T, R1, P1, R2, P2;
    fs["R"] >> R;
    fs["T"] >> T;

    Mat frame1,frame2, gray1, gray2;
    int counter = 0;

    capture >> frame1;
    capture >> frame2;

    Size img_size = frame1.size();
    Rect roi1, roi2;
    Mat Q;

    stereoRectify( M1, D1, M2, D2, img_size, R, T, R1, R2, P1, P2, Q, CALIB_ZERO_DISPARITY, -1, img_size, &roi1, &roi2 );

    Mat map11, map12, map21, map22;
    initUndistortRectifyMap(M1, D1, R1, P1, img_size, CV_16SC2, map11, map12);
    initUndistortRectifyMap(M2, D2, R2, P2, img_size, CV_16SC2, map21, map22);

    int numberOfDisparities = 16;
    int SADWindowSize = 9;

    bm->setROI1(roi1);
    bm->setROI2(roi2);
    bm->setPreFilterCap(31);
    bm->setBlockSize(SADWindowSize);
    bm->setMinDisparity(0);
    bm->setNumDisparities(numberOfDisparities);
    bm->setTextureThreshold(10);
    bm->setUniquenessRatio(15);
    bm->setSpeckleWindowSize(100);
    bm->setSpeckleRange(32);
    bm->setDisp12MaxDiff(1);

    while(1){
        capture >> frame1;
        capture2 >> frame2;
        imshow("Cam1", frame1);
        imshow("Cam2", frame2);

        /************************* STEREO ***********************/

        cvtColor(frame1, gray1, CV_RGB2GRAY);
        cvtColor(frame2, gray2, CV_RGB2GRAY);

        int64 t = getTickCount();

        Mat img1r, img2r;
        remap(gray1, img1r, map11, map12, INTER_LINEAR);
        remap(gray2, img2r, map21, map22, INTER_LINEAR);

        Mat disp, disp8;
        bm->compute(img1r, img2r, disp);
        t = getTickCount() - t;
        printf("Time elapsed: %fms\n", t*1000/getTickFrequency());

        disp.convertTo(disp8, CV_8U, 255/(numberOfDisparities*16.));
        imshow("disparity", disp8);
    }

Answer 1

So, half a pixel of RMS error means your calibration is basically garbage.

In your calibration image I noticed that the target is not even flat. If I can see that, the camera can too, but the calibration model will still assume it's flat, and that will make the optimizer very sad.

Suggest you take a look at this answer about calibration.