OpenCV copy irregular contour region to another image after contour rotation

Question

I am working with an image with distorted/rotated texts. I need to rotate these text blobs back to the horizontal level (0 degrees) before I can run OCR on them. I managed to fix the rotation issue but now I need to find a way to copy the contents of the original contour to the rotated matrix.

Here are a few things I've done to extract and fix the rotation issue:

1. Find contour
2. Heavy dilation and remove non-text lines
3. Find the contour angle and do angle correction in the polar space.

I have tried using affine transformation to rotate the rectangle text blobs but it ended up cropping out some of the texts because some of the text blobs are irregular. Result here

Blue dots in the contours are centroids, the numbers are contour angles. How can I copy the content of unrotated contour, rotate them and copy to a new image?

Code

def getContourCenter(contour):
    M = cv2.moments(contour)
    if M["m00"] != 0:
        cx = int(M['m10']/M['m00'])
        cy = int(M['m01']/M['m00'])
    else:
        return 0, 0
    return int(cx), int(cy)

def rotateContour(contour, center: tuple, angle: float):

    def cart2pol(x, y):
        theta = np.arctan2(y, x)
        rho = np.hypot(x, y)
        return theta, rho

    def pol2cart(theta, rho):
        x = rho * np.cos(theta)
        y = rho * np.sin(theta)
        return x, y

    # Translating the contour by subtracting the center with all the points
    norm = contour - [center[0], center[1]]

    # Convert the points to polar co-ordinates, add the rotation, and convert it back to Cartesian co-ordinates.
    coordinates = norm[:, 0, :]
    xs, ys = coordinates[:, 0], coordinates[:, 1]
    thetas, rhos = cart2pol(xs, ys)

    thetas = np.rad2deg(thetas)
    thetas = (thetas + angle) % 360
    thetas = np.deg2rad(thetas)

    # Convert the new polar coordinates to cartesian co-ordinates
    xs, ys = pol2cart(thetas, rhos)
    norm[:, 0, 0] = xs
    norm[:, 0, 1] = ys

    rotated = norm + [center[0], center[1]]
    rotated = rotated.astype(np.int32)

    return rotated


def straightenText(image, vis):

    # create a new mat
    mask = 0*np.ones([image.shape[0], image.shape[1], 3], dtype=np.uint8)

    # invert pixel index arrangement and dilate aggressively
    dilate = cv2.dilate(~image, ImageUtils.box(33, 1))

    # find contours
    _, contours, hierarchy = cv2.findContours(dilate, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
    for contour in contours:
        [x, y, w, h] = cv2.boundingRect(contour)
        if w > h:

            # find contour angle and centers
            (x, y), (w, h), angle = cv2.minAreaRect(contour)
            cx, cy = getContourCenter(contour)

            # fix angle returned
            if w < h:
                angle = 90 + angle

            # fix contour angle
            rotatedContour = rotateContour(contour, (cx, cy), 0-angle)

            cv2.drawContours(vis, contour, -1, (0, 255, 0), 2)
            cv2.drawContours(mask, rotatedContour, -1, (255, 0, 0), 2)
            cv2.circle(vis, (cx, cy), 2, (0, 0, 255), 2, 8) # centroid
            cv2.putText(vis, str(round(angle, 2)), (cx, cy), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255,0,0), 2)

Answer 1

Here is one way, which is the simplest way I can think to do it in Python/OpenCV, though may not be optimal in speed.

• Create a white empty image for the desired output. (So we have black text on white background in case you need to do OCR)

• Get the rotated bounding rectangle of your contour in your input.

• Get the normal bounding rectangle of your contour in the output.

• Get the 4 bounding box corners for each.

• Compute an affine transform matrix between the two sets of 4 corner points.

• Warp the (whole) input image to the same size (non-optimal).

• Use the output bounding box dimensions and upper left corner with numpy slicing to transfer the region in the warped image to the same region in the white output image.

• Repeat for each text contour using the resulting image in place of the original white image as the new destintination image.

So here is a simulation to show you how.

Source Text Image:

Source Text Image with Red Rotated Rectangle:

Desired Bounding Rectangle in White Destination Image:

Text Transferred To White Image into Desired Rectangle Region:

Code:

import cv2
import numpy as np

# Read source text image.
src = cv2.imread('text_on_white.png')
hs, ws, cs = src.shape

# Read same text image with red rotated bounding box drawn.
src2 = cv2.imread('text2_on_white.png')

# Read white image showing desired output bounding box.
src2 = cv2.imread('text2_on_white.png')

# create white destination image
dst = np.full((hs,ws,cs), (255,255,255), dtype=np.uint8)

# define coordinates of bounding box in src
src_pts = np.float32([[51,123], [298,102], [300,135], [54,157]])

# size and placement of text in dst is (i.e. bounding box):
xd = 50
yd = 200
wd = 249
hd = 123
dst_pts = np.float32([[50,200], [298,200], [298,234], [50,234]])

# get rigid affine transform (no skew)
# use estimateRigidTransform rather than getAffineTransform so can use all 4 points
matrix = cv2.estimateRigidTransform(src_pts, dst_pts, 0)

# warp the source image
src_warped = cv2.warpAffine(src, matrix, (ws,hs), cv2.INTER_AREA, borderValue=(255,255,255))

# do numpy slicing on warped source and place in white destination
dst[yd:yd+hd, xd:xd+wd] = src_warped[yd:yd+hd, xd:xd+wd]

# show results
cv2.imshow('SRC', src)
cv2.imshow('SRC2', src2)
cv2.imshow('SRC_WARPED', src_warped)
cv2.imshow('DST', dst)
cv2.waitKey(0)
cv2.destroyAllWindows()

# save results
cv2.imwrite('text_on_white_transferred.png', dst)

Answer 2

To extract ONLY the content of a single contour, and not its larger bounding box, you can create a mask by drawing a filled contour and then applying that to the original image. In your case you would need something like this:

# prepare the target image
resX,resY = image.shape[1],image.shape[0]
target = np.zeros((resY, resX , 3), dtype=np.uint8)  
target.fill(255)  # make it entirely white

# find the contours
allContours,hierarchy = cv2.findContours(image, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)

# then perform rotation, etc, per contour
for contour in allContours:
  # create empty mask
  mask = np.zeros((resY, resX , 1), dtype=np.uint8)  

  # draw the contour filled into the mask
  cv2.drawContours(mask, [contour], -1, (255),  thickness=cv2.FILLED) 
  
  # copy the relevant part into a new image 
  # (you might want to use bounding box here for more efficiency)
  single = cv2.bitwise_and(image, image, mask=mask)   

  # then apply your rotation operations both on the mask and the result
  single = doContourSpecificOperation(single)
  mask = doContourSpecificOperation(mask)

  # then, put the result into your target image (which was originally white)
  target = cv2.bitwise_and(target, single, mask=mask)