OpenCV splitting camera feed into grid and determining colour

Question

I've written a piece of code to take my camera feed, split it into a grid (like a chess board) and evaluate each square for colour.

The code i currently have looks like this:

using namespace std;
using namespace cv;



//Standard Dilate and erode functions to improve white/black areas in Binary Image
// Pointer &thresh used so it affects threshImg so it can be used in tracking.
void morphOps(Mat &thresh){

    //Increases size of black to remove unwanted white specks outside of object
    Mat erodeElement = getStructuringElement( MORPH_RECT,Size(3,3));

    //Increases white-area size to remove holes in object
    Mat dilateElement = getStructuringElement( MORPH_RECT,Size(8,8));

    erode(thresh,thresh,erodeElement);
    erode(thresh,thresh,erodeElement);


    dilate(thresh,thresh,dilateElement);
    dilate(thresh,thresh,dilateElement);

}


//Tracking for the Filtered Object
void trackFilteredObject(int noteNum, string colourtype, Mat &thresh ,Mat HSVImage, Mat &cam){

    vector<Brick> Bricks;

    Mat temp;
    thresh.copyTo(temp);

    threshold(temp, thresh, 120, 255, 3);           //3 = Threshold to Zero
    int whitePixs = countNonZero(thresh);
    int cols = thresh.cols;
    int rows = thresh.rows;
    int imgSize = (rows*cols)/0.75;

    if(whitePixs > imgSize){


        Brick Brick;

        Brick.setColour(colourtype);
        Brick.setnoteNum(noteNum);

        Bricks.push_back(Brick);



    }

int main(int argc, char* argv[])
    {

    /// Create a window
    namedWindow("window", CV_WINDOW_AUTOSIZE );

        while(1){
//initialtes camera, sets capture resolution
VideoCapture capture;
capture.open(1);
capture.set(CV_CAP_PROP_FPS, 30);
capture.set(CV_CAP_PROP_FRAME_WIDTH,640);
capture.set(CV_CAP_PROP_FRAME_HEIGHT,480);

Mat cam;


// Saves camera image to Matrix "cam"


capture.read(cam);


//Sets Widths and Heights based on camera resolution (cam.cols/cam.rows retrieves this)

    int Width = cam.cols;
    int gridWidth = Width/16;
    int Height = cam.rows;
    int gridHeight = Height/16;


//Splits image into 256 squares going left to right through rows and descending vertically. (16 squares per row for 4/4 pattern)

    Mat BigImage;
    Mat HSVImage;

    // Converts cam to HSV pallete
    cvtColor(cam, HSVImage, COLOR_BGR2HSV);

    Size smallSize(gridWidth,gridHeight);
   std::vector<Mat> smallImages;

    for (int y = 0; y < HSVImage.rows; y += smallSize.height)
    {
        for (int x = 0; x < HSVImage.cols; x += smallSize.width)
        {
            cv::Rect rect =  cv::Rect(x,y, smallSize.width, smallSize.height);
            //Saves the matrix to vector
            smallImages.push_back(cv::Mat(HSVImage, rect));

        }
    }

    for (int i = 0; i < smallImages.size(); i++){

        Mat HSV;
        smallImages.at(i).copyTo(HSV);
        int noteNum = i;
        Mat threshImg;

        inRange(HSV,Scalar(0,0,0),Scalar(255,255,255),threshImg);
        morphOps(threshImg); //erodes image
        string colour = "Red";
        trackFilteredObject(noteNum,colour,threshImg,HSV,cam);

        inRange(HSV,Scalar(0,0,0),Scalar(255,255,255),threshImg);
        morphOps(threshImg); // threshold = mat after erosion/dilation
        colour = "yellow";
        trackFilteredObject(noteNum,colour,threshImg,HSV,cam);

        inRange(HSV,Scalar(0,0,0),Scalar(255,255,255),threshImg);
        morphOps(threshImg);
        colour = "Black";
        trackFilteredObject(noteNum,colour,threshImg,HSV,cam);

        inRange(HSV,Scalar(0,0,0),Scalar(255,255,255),threshImg);
        morphOps(threshImg); // threshold = mat after erosion/dilation
        colour = "White";
        trackFilteredObject(noteNum,colour,threshImg,HSV,cam);

        inRange(HSV,Scalar(0,0,0),Scalar(255,255,255),threshImg);
        morphOps(threshImg); // threshold = mat after erosion/dilation
        colour = "Green";
        trackFilteredObject(noteNum,colour,threshImg,HSV,cam);


    }
            imshow("window", cam);

        }

    return 0;
}

At the moment the code takes quite a long time to execute a full loop (about 1.5 seconds) but i ideally need it to run as close to real time as possible for a music application.

Could anyone suggest why it takes so long to execute? Is there a better way to evaluate the colour of each square?

My class is as follows:

//Brick.h
#include <string>

using namespace std;


class Brick{

public:
    Brick(void);
    ~Brick(void);


    string getColour();
    void setColour(string whatColour);

    int getnoteNum();
    void setnoteNum(int whatnoteNum);


private:

    int noteNum;
    string colour;



};

///

Brick.cpp
#include <stdio.h>
#include <Brick.h>


Brick::Brick(void){


}


Brick::~Brick(void){

}



// get/set Colour
////////////////////////////////

string Brick::getColour(){

    return Brick::colour;


}



void Brick::setColour(string whatColour){


    Brick::colour = whatColour;

}





// get/set Note Number
////////////////////////////////

int Brick::getnoteNum(){

    return Brick::noteNum;


}



void Brick::setnoteNum(int whatnoteNum){


    Brick::noteNum = whatnoteNum;

}

I will be so grateful to anyone who replies!

Thank you.

Answer 1

Try hard to not use erode and dilate. These operations are extremely time intensive. I'm quite confident that they are the bottleneck in your program.

There are some measures you can take:

1. Downscaling(or downsampling) the image. Ideally, you want the downscaled image's pixel to be of the same order of magnitude of a grid square's size.
2. Remove dilate and erode.
3. Off-topic: Bugfix. Fix the inRange() parameters used. Consult the HSV color space diagram and normalize to your space. eg extracting "green pixels" would correspond to inRange(HSV,Scalar(80f*255/360,0.3*255,0.3*255),Scalar(160f*255/360,255,255),threshImg);