Python opencv, color/shape based detection

Question

I'm trying to detect some objects in a picture based on their shape and color.

This is the original image, and I want to locate the two pink mugs (highlighted in green)

I'm using a color mask to isolate the two mugs, and the result is pretty good, as you can see here:

The problem is that there could be other objects with similar colors that get detected as well, as the red chair in the bottom right part.

I can tweak with the parameters for the color mask better...for example, I can isolate the color more specifically, use dilation/erosion to reduce noise. But relying only on colors is not ideal, and it's prone to errors. For example, if I simply turn the chair slightly, the lighting on it changes and I get noise again.

To make everything a little more robust, I've been trying to further select the mugs by using their shape with cv2.approxPolyDP, but I'm often unable to separate the mug from noisy regions. The mug shape identified by the color mask is often not very precise, so the approximating polygon can be formed by up to 10 segments, which makes it useless to separate it from noise.

This is the code I'm using:

import cv2
import numpy as np 


def main():
    cap = cv2.VideoCapture(0)
    cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)

    cv2.namedWindow("Color selection")
    cv2.createTrackbar("Low_H", "Color selection", 114, 255, nothing)
    cv2.createTrackbar("Low_S", "Color selection", 76, 255, nothing)
    cv2.createTrackbar("Low_V", "Color selection", 145, 255, nothing)
    cv2.createTrackbar("Up_H", "Color selection", 170, 255, nothing)
    cv2.createTrackbar("Up_S", "Color selection", 255, 255, nothing)
    cv2.createTrackbar("Up_V", "Color selection", 255, 255, nothing)
    cv2.createTrackbar("N_erosion", "Color selection", 4, 50, nothing)
    cv2.createTrackbar("epsilon", "Color selection", 2, 20, nothing)
    cv2.createTrackbar("Area_min", "Color selection", 52, 500, nothing)
    cv2.createTrackbar("Area_max", "Color selection", 1800, 4000, nothing)


    while True:
        ret, frame = cap.read()
        frame_hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
        frame_hsv_blur = cv2.GaussianBlur(frame_hsv, (7, 7), 0)

        ## parameters selection
        l_h = cv2.getTrackbarPos("Low_H", "Color selection")
        l_s = cv2.getTrackbarPos("Low_S", "Color selection")
        l_v = cv2.getTrackbarPos("Low_V", "Color selection")
        u_h = cv2.getTrackbarPos("Up_H", "Color selection")
        u_s = cv2.getTrackbarPos("Up_S", "Color selection")
        u_v = cv2.getTrackbarPos("Up_V", "Color selection")
        N_erode = cv2.getTrackbarPos("N_erosion", "Color selection")
        eps = cv2.getTrackbarPos("epsilon", "Color selection")/100
        area_min = cv2.getTrackbarPos("Area_min", "Color selection")
        area_max = cv2.getTrackbarPos("Area_max", "Color selection")
        N_erode = N_erode if N_erode>0 else 1

        lower_values = np.array([l_h, l_s, l_v])
        upper_values = np.array([u_h, u_s, u_v])
        mask = cv2.inRange(frame_hsv_blur, lower_values, upper_values)
        kernel = np.ones((N_erode,N_erode), np.uint8)
        mask = cv2.erode(mask, kernel)

        ## find contours in image based on color mask
        contours, hierarchy = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
        for contour in contours:
            area = cv2.contourArea(contour)
            perimeter = cv2.arcLength(contour, True)
            approx = cv2.approxPolyDP(contour, eps*perimeter, True)
            x,y,w,h = cv2.boundingRect(contour)
            if (area_min < area < area_max) and (2<len(approx)):
                x_0 = int(x+w/2)
                y_0 = int(y+h/2)
                frame = cv2.putText(frame, str(len(approx)), (x,y), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255,0,0), thickness=3)
                frame = cv2.circle(frame, (x_0, y_0), 10, (255,255,50), -1)

        cv2.imshow("tracking", frame)
        cv2.imshow("mask", mask)


        key = cv2.waitKey(1)
        if key == ord('q'):
            break
        elif key == ord('s'):
            cv2.imwrite("saved_image.jpg", frame)

    cv2.destroyAllWindows()
    cap.release()


def nothing(x):
    pass


if __name__ == '__main__':
    main()

So, again, my main question regards the shape detection . I was wondering if I could try a different approach to better exploit the fact that I'm looking for a very specific shape, maybe using something else than cv2.approxPolyDP. Any suggestions?

Answer 1

I checked your picture. This is what I could do best in 5 min. practice a little more and the better result will show up.

import cv2
import numpy as np 
from google.colab.patches import cv2_imshow


img = cv2.imread("/content/JSjbB.png")
cv2_imshow(img)
img_output = img
cv2_imshow(img_output)
frame_hsv = cv2.cvtColor(img_output, cv2.COLOR_BGR2HSV)
frame_hsv_blur = cv2.GaussianBlur(frame_hsv, (7, 7), 0)
#print(frame_hsv_blur)
lower_values = np.array([130, 100,0])
upper_values = np.array([170,255, 255])
mask = cv2.inRange(frame_hsv_blur, lower_values, upper_values)
cv2_imshow(mask)
kernel = np.ones((5,5),np.uint8)
erosion = cv2.erode(mask,kernel,iterations = 3)
dilation = cv2.dilate(erosion,kernel,iterations = 3)
cv2_imshow(dilation)

The result is something like this: