Handwritten in order digit recognition and extraction with Python-OpenCV

Question

I want to extract the digits inside the boxes in order.

Original image

I used the watershed algorithm to separate digits connected to boxes but it wont't contour digits properly, instead it selects only some part of digits.

#To get in big box that contain smaller boxes from the image
img = cv2.imread('1_6.png',0)
img = cv2.GaussianBlur(img,(3,3),1)
_,img  = cv2.threshold(img,240,255,cv2.THRESH_BINARY)
img = cv2.GaussianBlur(img,(11,11),1)
edges = cv2.Canny(img,100,200)
_,c,h = cv2.findContours(edges.copy(),cv2.RETR_CCOMP,cv2.CHAIN_APPROX_NONE)
img = cv2.imread('1_6.png')
temp_c = sorted(c,key=cv2.contourArea,reverse=True)

#Select the big box
epsilon = 0.0001*cv2.arcLength(temp_c[0],True)
approx = cv2.approxPolyDP(temp_c[0],epsilon,True)

#Crop big box
pts = approx.copy()
rect = cv2.boundingRect(pts)
x,y,w,h = rect
croped = img[y:y+h, x:x+w].copy()

## (2) make mask
pts = pts - pts.min(axis=0)

mask = np.ones(croped.shape[:2], np.uint8)
cv2.drawContours(mask, [pts], -1, (255, 255, 255), -1, cv2.LINE_AA)

## (3) do bit-op
dst = cv2.bitwise_and(croped, croped, mask=mask)


gray = cv2.cvtColor(dst,cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(gray,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)
kernel = np.ones((1,1),np.uint8)
opening = cv2.morphologyEx(thresh,cv2.MORPH_CLOSE,kernel, iterations = 2)

sure_bg = cv2.dilate(opening,kernel,iterations=1)

dist_transform = cv2.distanceTransform(opening,cv2.DIST_L2,5)

ret, sure_fg = cv2.threshold(dist_transform,0.3*dist_transform.max(),255,0)

sure_fg = np.uint8(sure_fg)

unknown = cv2.subtract(sure_bg,sure_fg)

ret, markers = cv2.connectedComponents(sure_fg)

# Add one to all labels so that sure background is not 0, but 1
markers = markers+1

# Now, mark the region of unknown with zero
markers[unknown==255] = 0

plt.imshow(markers,cmap="gray")

img = dst.copy()
markers = cv2.watershed(dst,markers)
img[markers == -1] = [0,0,255]

Current result

Answer 1

Here's my approach. I will try to be as detailed as possible:

• Convert image to grayscale
• Perform canny edge detection
• Remove horizontal and vertical lines to isolate characters
• Perform morphological operations to enhance letters
• Find contours
• Filter contours using contour area and aspect ratio
• Sort contours from left to right to extract the digits in order
• Iterate through sorted contours and extract ROI

First we perform Canny edge detection using cv2.Canny()

Next the goal is to remove the vertical and horizontal lines we can isolate the digits. We begin by creating various kernels, each targeted at a horizontal, vertical, or general orientation

vertical_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1,2))
horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (2,1))
kernel = cv2.getStructuringElement(cv2.MORPH_CROSS, (3,3))

We begin by removing the horizontal lines with cv2.erode()

Now we dilate the vertical lines with cv2.dilate()

Next we remove the vertical lines

Now note we have almost nothing left, so we have to restore the digits by dilating

From here we find contours using cv2.findContours() . We filter using cv2.contourArea() and by aspect ratio to obtain bounding boxes.

Now to extract the digits in order, we use imutils.contours.sort_contours()

Finally, we extract the ROI for each digit and save the image. Here's a screenshot of the saved ROI's in order

import cv2
import numpy as np
from imutils import contours

image = cv2.imread('1.png')
original = image.copy()
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
canny = cv2.Canny(gray, 130, 255, 1)

vertical_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1,2))
horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (2,1))
kernel = cv2.getStructuringElement(cv2.MORPH_CROSS, (3,3))
erode = cv2.erode(canny, vertical_kernel)
cv2.imshow('remove horizontal', erode)
dilate = cv2.dilate(erode, vertical_kernel, iterations=5)
cv2.imshow('dilate vertical', dilate)
erode = cv2.erode(dilate, horizontal_kernel, iterations=1)
cv2.imshow('remove vertical', erode)
dilate = cv2.dilate(erode, kernel, iterations=4)
cv2.imshow('dilate horizontal', dilate)

cnts = cv2.findContours(dilate, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]

digit_contours = []
for c in cnts:
    area = cv2.contourArea(c)
    peri = cv2.arcLength(c, True)
    approx = cv2.approxPolyDP(c, 0.01 * peri, True)
    x,y,w,h = cv2.boundingRect(approx)
    aspect_ratio = w / float(h)

    if (aspect_ratio >= 0.4 and aspect_ratio <= 1.3):
        if area > 150:
            ROI = original[y:y+h, x:x+w]
            cv2.rectangle(image,(x,y),(x+w,y+h),(0,255,0),2)
            digit_contours.append(c)

sorted_digit_contours = contours.sort_contours(digit_contours, method='left-to-right')[0]
contour_number = 0
for c in sorted_digit_contours:
    x,y,w,h = cv2.boundingRect(c)
    ROI = original[y:y+h, x:x+w]
    cv2.imwrite('ROI_{}.png'.format(contour_number), ROI)
    contour_number += 1

cv2.imshow('canny', canny)
cv2.imshow('image', image)
cv2.waitKey(0)