Comic Balloon Detection: How can I count white pixels inside a vector<RotatedRect> Ellipse in OpenCV?

Question

I've been looking everywhere I can for the answer but I can't find one.

I'm making a comic balloon detection program and I need to find an ellipse that have a specific percentage of white inside the contour (percentage is to be decided later), thus why I need to count the white pixels inside the contour and I don't know how.

I have tried countNonZero() but since the parameter of that is an array it doesn't accept my minEllipse[i] or contours[i] that are declared as vector<RotatedRect> .

Below is the code:

// Modified version of thresold_callback function 
// from http://docs.opencv.org/doc/tutorials/imgproc/shapedescriptors/bounding_rotated_ellipses/bounding_rotated_ellipses.html
            Mat fittingEllipse(int, void*, Mat inputImage)
            {
                Mat threshold_output;
                vector<vector<Point> > contours;
                vector<Vec4i> hierarchy;
                int numberOfCaptions = 0;

                // Detect edges using Threshold
                threshold(inputImage, threshold_output, 224, 250, THRESH_BINARY);

                findContours(inputImage, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, Point(0, 0));

                vector<RotatedRect> minEllipse(contours.size());
                Mat drawing = Mat::zeros(inputImage.size(), CV_8UC3);

                for (int i = 0; i < contours.size(); i++)
                {
                    if (contours[i].size() > 5)
                        minEllipse[i] = fitEllipse(Mat(contours[i]));
                }

                int totalContourSize = 0, whitepixels, blackpixels;

                //Draw ellipse/caption
                for (int i = 0; i < contours.size(); i++)
                {
                    Scalar color = Scalar(255, 0, 0);

                    if (minEllipse[i].size.height >= inputImage.rows / 8 && //IJIP-290-libre.pdf
                        minEllipse[i].size.width >= inputImage.cols / 10 && //IJIP-290-libre.pdf
                        minEllipse[i].size.height < inputImage.rows / 3  &&
                        minEllipse[i].size.width < inputImage.cols / 3 &&
                        (
                        (minEllipse[i].angle >= 0 && minEllipse[i].angle <= 10) ||
                        (minEllipse[i].angle >= 80 && minEllipse[i].angle <= 100) ||
                        (minEllipse[i].angle >= 170 && minEllipse[i].angle <= 190) ||
                        (minEllipse[i].angle >= 260 && minEllipse[i].angle <= 280) ||
                        (minEllipse[i].angle >= 350 && minEllipse[i].angle <= 360)
                        )) {

                        ellipse(drawing, minEllipse[i], color, -1, 8);
                    }
                }

                drawing = binarizeImage(drawing);
                return drawing;
            } // end of fittingEllipse


            Mat CaptionDetection(Mat inputImage){
                Mat outputImage, binaryImage, captionDetectImage;

                binaryImage = captionDetectImage = binarizeImage(inputImage);
                threshold(captionDetectImage, captionDetectImage, 224, 250, 0); //IJIP-290-libre.pdf

                GaussianBlur(captionDetectImage, captionDetectImage, Size(9, 9), 0, 0);
                captionDetectImage = fittingEllipse(0, 0, captionDetectImage);

                //binaryImage = invertImage(binaryImage);

                outputImage = inputImage;

                for (int i = 0; i < inputImage.rows; i++) {
                    for (int j = 0; j < inputImage.cols; j++) {
                        if (captionDetectImage.at<uchar>(i, j) == 0) {
                            outputImage.at<Vec3b>(i, j)[0] = outputImage.at<Vec3b>(i, j)[1] = outputImage.at<Vec3b>(i, j)[2] = 0;
                        }
                    }
                }

                return outputImage;
            } // end of CaptionDetection

The very bulky if statement yields me only 53% accuracy of getting the comic balloon detection (not to mention all the false detections), that's why I need to get the percentage of white pixels in the contours that are found to get a higher percentage.

EDIT:

My desired output would be the whole manga page would be black except the comic balloons and then count the number of white and black pixels there

ONLY on the CaptionDetection function should I count the number of pixels for each captions

FINAL ANSWER

I edited the code that user Kornel gave

            Mat fittingEllipse(int, void*, Mat inputImage)
            {
                Mat outputImage;
                vector<Vec4i> hierarchy;
                int numberOfCaptions = 0;

                // Detect edges using Threshold
                threshold(inputImage, inputImage, 224, 250, THRESH_BINARY);

                findContours(inputImage, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, Point(0, 0));

                vector<RotatedRect> minEllipse(contours.size());

                for (int i = 0; i < contours.size(); i++)
                {
                    if (contours[i].size() > 5)
                        minEllipse[i] = fitEllipse(Mat(contours[i]));
                }

                //Draw ellipse/caption
                outputImage = Mat::zeros(inputImage.size(), CV_8UC3);
                for (int i = 0; i < contours.size(); i++)
                {
                    Scalar color = Scalar(255, 255, 255);
                    Mat drawing = Mat::zeros(inputImage.size(), CV_8UC3);

                    ellipse(drawing, minEllipse[i], color, -1, 8);

                    drawing = binarizeImage(drawing);
                    int area = countNonZero(drawing);

                    if ((area >= 10000 && area <= 40000) &&
                        (
                        (minEllipse[i].angle >= 0 && minEllipse[i].angle <= 10) ||
                        (minEllipse[i].angle >= 80 && minEllipse[i].angle <= 100) ||
                        (minEllipse[i].angle >= 170 && minEllipse[i].angle <= 190) ||
                        (minEllipse[i].angle >= 260 && minEllipse[i].angle <= 280) ||
                        (minEllipse[i].angle >= 350 && minEllipse[i].angle <= 360)
                        )){
                        ellipse(outputImage, minEllipse[i], color, -1, 8);
                        captionMask[captionCount] = drawing;
                        captionCount++;
                    }
                }

                imwrite((string)SAVE_FILE_DEST + "out.jpg", outputImage);

                return outputImage;
            } // end of fittingEllipse
            Mat replaceROIWithOrigImage(Mat inputImg, Mat mask, int k){
                Mat outputImage = inputImg;
                Mat maskImg = mask;
                imwrite((string)SAVE_FILE_DEST + "inputbefore[" + to_string(k) + "].jpg", inputImg);
                for (int i = 0; i < inputImg.rows; i++) {
                    for (int j = 0; j < inputImg.cols; j++) {

                        if (maskImg.at<uchar>(i, j) == 0) {
                            inputImg.at<Vec3b>(i, j)[0] = inputImg.at<Vec3b>(i, j)[1] = inputImg.at<Vec3b>(i, j)[2] = 0;
                        }

                    }
                }
                imwrite((string)SAVE_FILE_DEST + "maskafter[" + to_string(k) + "].jpg", inputImg);
                return inputImg;
            }

            Mat CaptionDetection(Mat inputImage){
                Mat outputImage, binaryImage, captionDetectImage;

                binaryImage = captionDetectImage = binarizeImage(inputImage);
                threshold(captionDetectImage, captionDetectImage, 224, 250, 0); //IJIP-290-libre.pdf

                GaussianBlur(captionDetectImage, captionDetectImage, Size(9, 9), 0, 0);
                captionDetectImage = fittingEllipse(0, 0, captionDetectImage);

                for (int i = 0; i < captionCount; i++){

                    Mat replacedImg = replaceROIWithOrigImage(inputImage.clone(), captionMask[i], i);

                    int area = countNonZero(binarizeImage(replacedImg));

                    cout << area << endl;
                }

                return outputImage;
            } // end of CaptionDetection

The if condition in fittingEllipse() is to be edited for better accuracy later.

Thank you for your help and time user a-Jays and Kornel!

Answer 1

Say you have a rotated rectangle rRect which defines an ellipse just like minEllipse[i] in your code.

First of all, the area of it can be estimated by the closed formula area = a * b * PI , where a and b are the semi-major and semi-minor axes (1⁄2 of the ellipse's major and minor axes), respectively, so:

cv::RotatedRect rRect(cv::Point2f(100.0f, 100.0f), cv::Size2f(100.0f, 50.0f), 30.0f);
float area = (rRect.size.width / 2.0f) * (rRect.size.height / 2.0f) * M_PI;

Or a bit shorter:

float area = (rRect.size.area() / 4.0f) * M_PI;

---

Alternatively, you can simply draw it over a mask by cv::ellipse() , ie:

cv::Mat mask = cv::Mat::zeros(200, 200, CV_8UC1);
cv::ellipse(mask, rRect, cv::Scalar::all(255), -1);

And you count the non-zero elements as the usual:

int area = cv::countNonZero(mask);