消除光流算法产生的噪声？

Question

I'm using the function cvCalcOpticalFlowPyrLK() to detect and tracking moving objects in the video. I use Canny() function to get features to track. But the result is not good, it had very much noise from cvCalcOpticalFlowPyrLK() .

Please see my code below:

#include <windows.h>
#include <stdio.h>
#include <vector>
#include <opencv/cv.h>
#include <opencv/highgui.h>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>

using namespace std;
using namespace cv;

double Distance(Point& p1, Point& p2)
{
    return abs(p2.x - p1.x) + abs(p2.y - p1.y);
}

VOID EliminateBgrLines(vector<vector<Point>>* srcCountours, vector<vector<Point>>* dstCountours,
    CvPoint2D32f* frame1_features, CvPoint2D32f* frame2_features, char* optical_flow_found_feature)
{
    Point p, q;
    int featureIndex = 0;
    int numContours = srcCountours->size();
    int* removeContoursIndex = new int[numContours];
    int countRemoveContours = 0;

    vector<vector<Point>>::iterator Iter;
    int iterIndex = 0;
    for(Iter = srcCountours->begin() ; Iter != srcCountours->end() ; Iter++)
    {
        vector<Point> contour = *Iter;
        int numPoints = contour.size();
        int countBgrPoints = 0;
        int countTotalPointsFound = 0;
        double totalDistance = 0;

        for(int k = 0 ; k < numPoints; k++)
        {               
            if(optical_flow_found_feature[featureIndex] == 0)  
            {
                featureIndex ++;
                countBgrPoints ++; //The points that can not find in the next frame will be treated as background point
                continue;       
            }
            countTotalPointsFound ++;

            p. x = (int)frame1_features[featureIndex].x;
            p. y = (int)frame1_features[featureIndex].y;
            q. x = (int)frame2_features[featureIndex].x;
            q. y = (int)frame2_features[featureIndex].y;            
            featureIndex ++;

            double d = Distance(p, q);
            totalDistance += d;
            double opticalFlowThreshold = 1;
            if(d < opticalFlowThreshold)
                countBgrPoints ++;
        }

        int eliminateBgrLineThreshold = 40; //40 %
        if ( (double)countBgrPoints/numPoints > (eliminateBgrLineThreshold/100) ) //eliminateBgrLineThreshold(%) is bgr point
        {
            removeContoursIndex[countRemoveContours] = iterIndex;
            countRemoveContours ++;
        }
        iterIndex++;
    }       
    for(int i = 0 ; i < numContours; i++)
    {           
        if(trackingCoreGlobal->ISContainInArray(i, removeContoursIndex, countRemoveContours) == false)
            dstCountours->insert(dstCountours->end(), srcCountours->at(i));
    }
    delete []removeContoursIndex;       
}

int main()
{
    long current_frame = 0 ;
    CvSize frame_size;
    long number_of_frames = 0;
    CvCapture *input_video = NULL;

    input_video = cvCaptureFromFile(videoPath);
    if  (input_video == NULL)
        return false;
    cvQueryFrame(input_video);

    frame_size.height = (int)cvGetCaptureProperty(input_video, CV_CAP_PROP_FRAME_HEIGHT);
    frame_size.width = (int)cvGetCaptureProperty(input_video, CV_CAP_PROP_FRAME_WIDTH);

    /*  Go to the end of the AVI (The fraction is "1") */
    cvSetCaptureProperty( input_video, CV_CAP_PROP_POS_AVI_RATIO, 1.0) ;
    number_of_frames = (int)cvGetCaptureProperty( input_video, CV_CAP_PROP_POS_FRAMES) ;
    /*  Return to the beginning */
    cvSetCaptureProperty( input_video, CV_CAP_PROP_POS_FRAMES , 0.0) ;

    int number_of_features = 0;
    CvPoint2D32f* frame1_features = new CvPoint2D32f[NUMBER_OF_FEATURES];
    CvPoint2D32f* frame2_features = new CvPoint2D32f[NUMBER_OF_FEATURES];
    char* optical_flow_found_feature = new char[NUMBER_OF_FEATURES];
    float* optical_flow_feature_error = new float[NUMBER_OF_FEATURES];

    while(isStopped == false)
    {
        static  IplImage * frame  = NULL, * frame1_1C = NULL, * frame2_1C = 
            NULL, * eig_image  = NULL, * temp_image  = NULL, * pyramid1 = NULL, * pyramid2 = NULL;
        /*  Go to the frame we want.   Important if multiple frames are queried in
        * the loop which they of course are for optical flow.   Note that the very
        * first call to this is actually not needed . ( Because the correct position
        * is set outsite the for ()  loop.)
        */
        cvSetCaptureProperty( input_video, CV_CAP_PROP_POS_FRAMES, current_frame) ;
        /*  Get the next frame of the video.
        * IMPORTANT !   cvQueryFrame()  always returns a pointer to the _ same_
        * memory location.   So successive calls :
        * frame1 = cvQueryFrame();
        * frame2 = cvQueryFrame();
        * frame3 = cvQueryFrame();
        * will result in ( frame1 ==  frame2 &&  frame2 ==  frame3 ) being true.
        * The solution is to make a copy of the cvQueryFrame() output .
        */

        frame = cvQueryFrame(input_video);
        if(frame == NULL)
            return false;
        frame1_1C = cvCreateImage(frame_size, IPL_DEPTH_8U, 1);
        if(frame1_1C ==  NULL)
            return false;
        cvConvertImage(frame, frame1_1C);

        /* Get the second frame of video. Sample principles as the first */
        frame = cvQueryFrame(input_video);
        if(frame == NULL)
            return false;
        frame2_1C = cvCreateImage(frame_size, IPL_DEPTH_8U, 1);
        if(frame2_1C ==  NULL)
            return false;
        cvConvertImage(frame, frame2_1C);

        /*Preparation : Allocate the necessary storage*/
        eig_image = cvCreateImage(frame_size, IPL_DEPTH_32F, 1);
        if(eig_image ==  NULL)
            return false;
        temp_image = cvCreateImage(frame_size, IPL_DEPTH_32F, 1);
        if(temp_image ==  NULL)
            return false;

        number_of_features = 0;

        /*  Actually run the Shi and Tomasi algorithm !!
        * "frame1 _ 1 C " is the input image.
        * "eig _ image" and " temp_ image" are just workspace for the algorithm .
        * The first ".01" specifies the minimum quality of the features  ( based on the 
        eigenvalues ).
        * The second  ".01" specifies the minimum Euclidean distance between features .
        * "NULL" means use the entire input image.   You could point to a part of the 
        image.
        * WHEN THE ALGORITHM RETURNS:
        * "frame1 _ features" will contain the feature points.
        * "number _ of_ features" will be set to a value < = NUMBER_OF_FEATURES indicating the number of 
        feature points found.
        */

        //Get Features by using Canny Edges - Begin
        int ratio = 3;
        int kernel_size = 3;
        int cannyThreshold = 40;
        Mat detected_edges;
        Mat frame1_1C_Mat = cvarrToMat(frame1_1C);
        // Reduce noise with a kernel 3x3
        blur(frame1_1C_Mat, detected_edges, Size(3,3)) ;
        // Canny detector
        Canny(detected_edges, detected_edges, cannyThreshold, cannyThreshold*ratio, kernel_size);

        RNG rng(12345);
        vector<vector<Point>> contours;
        vector<Vec4i> hierarchy;
        findContours(detected_edges, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, Point(0, 0));

        DrawContours(contours, "Before Removing Bgr Contours"); //This function will draw the result contours

        //Copy feature for tracking by optical flow
        int numContours = contours.size();
        for(int i = 0 ; i < numContours; i++)
        {
            vector<Point> contour = contours.at(i);
            int numPoints = contour.size();
            for(int k = 0 ; k < numPoints; k++)
            {
                Point point = contour.at(k);
                frame1_features[number_of_features].x = point.x;
                frame1_features[number_of_features].y = point.y;
                number_of_features++;
            }
        }
        /*
        This termination criteria tells the algorithm to stop when it has either done 
        20 iterations or when epsilon is better than 0.3, you can play with these parameters 
        for speed vs accuracy but these values work pretty well in many situations.
        */
        CvTermCriteria optical_flow_termination_criteria = cvTermCriteria(CV_TERMCRIT_ITER|CV_TERMCRIT_EPS, 20, 0.3) ;

        /*  This is some workspace for the algorithm.
        The algorithm actually carves the image into pyramids of different resolutions
        */
        pyramid1 = cvCreateImage(frame_size, IPL_DEPTH_8U, 1);
        if(pyramid1 ==  NULL)
            return false;
        pyramid2 = cvCreateImage(frame_size, IPL_DEPTH_8U, 1);
        if(pyramid2 ==  NULL)
            return false;

        /*Actually run Pyramidal Lucas Kanade Optical Flow!!
        * "frame1_1C" is the first frame with the known features.
        * "frame2_1C" is the second frame where we want to find the first frame's features.
        * "pyramid1" and "pyramid2" are workspace for the algorithm.
        * "frame1_features" are the features from the first frame.
        * "frame2_features" is the (outputted) locations of those features in the second frame.
        * "number_ of_features" is the number of features in the frame 1 _ features array .
        * "optical_flow_window" is the size of the window to use to avoid the aperture problem.
        * "5" is the maximum number of pyramids to use. 0 would be just one level.
        * "optical_flow_found_feature" is as described above (non-zero iff feature found by the flow).
        * "optical_flow_feature_error" is as described above (error in the flow for this feature).
        * "optical_flow_termination_criteria" is as described above (how long the algorithm should look).
        * "0" means disable enhancements. (For example , the second array isn't pre-initialized with guesses)
        */      

        /*This is the window size to use to avoid the aperture problem (see slide "Optical Flow: Overview")*/
        int opticalFlowWindowSize = 21;
        int pyramidLevel = 5;
        CvSize optical_flow_window = cvSize(opticalFlowWindowSize, opticalFlowWindowSize);      
        cvCalcOpticalFlowPyrLK(frame1_1C, frame2_1C, pyramid1, pyramid2, frame1_features, 
            frame2_features, number_of_features, optical_flow_window, pyramidLevel, 
            optical_flow_found_feature, optical_flow_feature_error, 
            optical_flow_termination_criteria, 0) ;     

        vector<vector<Point>> afterRemoveBgrItemContours;
        EliminateBgrLines(&contours, &afterRemoveBgrItemContours, frame1_features, frame2_features, optical_flow_found_feature);

        DrawContours(afterRemoveBgrItemContours, "After Removing Bgr Contours"); //This function will draw the result contours

        current_frame++;
        if (current_frame >= number_of_frames - 1)  
            current_frame = 0;

        detected_edges.release();
        frame1_1C_Mat.release();
        detected_edges.release();
        cvReleaseImage(&frame1_1C);
        cvReleaseImage(&frame2_1C);
        cvReleaseImage(&eig_image);
        cvReleaseImage(&temp_image);
        cvReleaseImage(&pyramid1);
        cvReleaseImage(&pyramid2);      
    } //End while(true)

    delete []frame1_features;
    delete []frame2_features;
    delete []optical_flow_found_feature;
    delete []optical_flow_feature_error;
}

I had try with many thresholds but it can not eliminate noise well.

Here is some results that I had try:

I had try with many other thresholds but it still have much noise, Someone can show me the way to improve my algorithm for the better result?

Many thanks,

T&T

Answer 1

Canny edges do not make not very good features for Optical Flow. You should try the very well-named goodFeaturesToTrack() instead.

Answer 2

You already use a blur kernel to remove the noise, but there are quite a few options out there (gausian blur, median blur, billateral filtering, etc). You might want to experiment with those to see if these improve your results. Note that using a blur will remove certain edges (and thus features). Changing the kernel size wil have a pretty big impact on your noise removal as well. You could also try to change the kernel size for the canny detector.

I don't really think that canny edges are well suited for tracking. There are far better options available.