如何使用 opencv.js 对齐图像

Question

I am following this example from Satya Mallick

I hosted a test here https://icollect.money/opencv_align#

Problem: the findHomography() succeeds but the warpPerspective() fails with an 'unhandled exception'

I suspect that the homography is wrong as it looks like its an empty array:

h: Mat {$$: {…}}
cols: 0
data: Uint8Array(0)
data8S: Int8Array(0)
data16S: Int16Array(0)
data16U: Uint16Array(0)
data32F: Float32Array(0)
data64F: Float64Array(0)
matSize: Array(0)
   rows: 0

I included the cpp code from the referenced article (above) inline with the javascript code:

    function Align_img() {
    
                //im2 is the original reference image we are trying to align to
                let im2 = cv.imread(image_A_element);
                //im1 is the image we are trying to line up correctly
                let im1 = cv.imread(image_B_element);
    
                //17            Convert images to grayscale
                //18            Mat im1Gray, im2Gray;
                //19            cvtColor(im1, im1Gray, CV_BGR2GRAY);
                //20            cvtColor(im2, im2Gray, CV_BGR2GRAY);
                let im1Gray = new cv.Mat();
                let im2Gray = new cv.Mat();
                cv.cvtColor(im1, im1Gray, cv.COLOR_BGRA2GRAY);
                cv.cvtColor(im2, im2Gray, cv.COLOR_BGRA2GRAY);
    
                //22            Variables to store keypoints and descriptors
                //23            std::vector<KeyPoint> keypoints1, keypoints2;
                //24            Mat descriptors1, descriptors2;
                let keypoints1 = new cv.KeyPointVector();
                let keypoints2 = new cv.KeyPointVector();
                let descriptors1 = new cv.Mat();
                let descriptors2 = new cv.Mat();
    
                //26            Detect ORB features and compute descriptors.
                //27            Ptr<Feature2D> orb = ORB::create(MAX_FEATURES);
                //28            orb->detectAndCompute(im1Gray, Mat(), keypoints1, descriptors1);
                //29            orb->detectAndCompute(im2Gray, Mat(), keypoints2, descriptors2);
                var orb = new cv.ORB(5000); 
                orb.detectAndCompute(im1Gray, new cv.Mat(), keypoints1, descriptors1);
                orb.detectAndCompute(im2Gray, new cv.Mat(), keypoints2, descriptors2);
    
                //31            Match features.
                //32            std::vector<DMatch> matches;
                //33            Ptr<DescriptorMatcher> matcher = DescriptorMatcher::create("BruteForce-Hamming");
                //34            matcher->match(descriptors1, descriptors2, matches, Mat());
                let bf = new cv.BFMatcher(cv.NORM_HAMMING, true);
                let matches = new cv.DMatchVector();
                bf.match(descriptors1, descriptors2, matches);
    
                //36            Sort matches by score
                //37            std::sort(matches.begin(), matches.end());
                //39            Remove not so good matches
                //40            const int numGoodMatches = matches.size() * GOOD_MATCH_PERCENT;
                //41            matches.erase(matches.begin()+numGoodMatches, matches.end());
                let good_matches = new cv.DMatchVector();
                for (let i = 0; i < matches.size(); i++) {
                    if (matches.get(i).distance < 30) {
                        good_matches.push_back(matches.get(i));
                    }
                }
    
                //44            Draw top matches
                //45            Mat imMatches;
                //46            drawMatches(im1, keypoints1, im2, keypoints2, matches, imMatches);
                //47            imwrite("matches.jpg", imMatches);
                let imMatches = new cv.Mat();
                let color = new cv.Scalar(0,255,0, 255);
                cv.drawMatches(im1, keypoints1, im2, keypoints2, good_matches, imMatches, color);
                cv.imshow('imageCompareMatches', imMatches);
    
                //50            Extract location of good matches
                //51            std::vector<Point2f> points1, points2;
                //53            for( size_t i = 0; i < matches.size(); i++ )
                //54            {
                //55                points1.push_back( keypoints1[ matches[i].queryIdx ].pt );
                //56                points2.push_back( keypoints2[ matches[i].trainIdx ].pt );
                //57            }
    
                let points1 = [];
                let points2 = [];

                for (let i = 0; i < good_matches.size(); i++) {
                    points1.push(keypoints1.get(good_matches.get(i).queryIdx ).pt );
                    points2.push(keypoints2.get(good_matches.get(i).trainIdx ).pt );
                }

                //59            Find homography
                //60            h = findHomography( points1, points2, RANSAC );
                //The first 2 arguments to findHomography need to be matArray so you must convert your point1 and point2 to matArray
                //reference: https://docs.opencv.org/3.4/d9/d0c/group__calib3d.html#ga4abc2ece9fab9398f2e560d53c8c9780
//*********** the issue seems to be here in how mat1 and mat2 are created *****
                let mat1 = cv.matFromArray(points1.length, 2, cv.CV_32F, points1); 
                let mat2 = cv.matFromArray(points2.length, 2, cv.CV_32F, points2); 
                let h = cv.findHomography(mat1, mat2, cv.RANSAC);
    
                //62          Use homography to warp image
                //63          warpPerspective(im1, im1Reg, h, im2.size());
                let image_B_final_result = new cv.Mat();
                cv.warpPerspective(im1, image_B_final_result, h, im2.size());
    
                cv.imshow('imageAligned', image_B_final_result);
    
                matches.delete();
                bf.delete();
                orb.delete();
                descriptors1.delete();
                descriptors2.delete();
                keypoints1.delete();
                keypoints2.delete();
                im1Gray.delete();
                im2Gray.delete();
                h.delete();
                image_B_final_result.delete();
                mat1.delete();
                mat2.delete();
            }

Answer 1

for (let i = 0; i < good_matches.size(); i++) {
    points1.push(keypoints1.get(good_matches.get(i).queryIdx).pt);
    points2.push(keypoints2.get(good_matches.get(i).trainIdx).pt);
}

shoud be below code

for (let i = 0; i < good_matches.size(); i++) {
    points1.push(keypoints1.get(good_matches.get(i).queryIdx).pt.x);
    points1.push(keypoints1.get(good_matches.get(i).queryIdx).pt.y);
    points2.push(keypoints2.get(good_matches.get(i).trainIdx).pt.x);
    points2.push(keypoints2.get(good_matches.get(i).trainIdx).pt.y);
}

I push staggered data into points[] , then it's work!