Python - Detect a QR code from an image and crop using OpenCV

Question

I'm working on a project using Python(3.7) and OpenCV in which I have an Image(captured using the camera) of a document with a QR code placed on it.

This QR code has 6 variables respectively as:

1. Size of QR code image

2. Top

3. Right

4. Bottom

5. Left

6. Unit

---

Latest Update:

Here are the steps I need to perform in the same order:

1. Detect the qr code & decode it to read size values
2. So, if the size of QR-code(image) is not equal to the size which is mentioned inside it then scale the image to equal both size values.
3. Then crop the image towards all sides from QR code image according to the values mentioned inside qr code.

I have tried this code:

def decodeAndCrop(inputImage):
    print(str(inputImage))
    image = cv2.imread(str(inputImage))
    qrCodeDetector = cv2.QRCodeDetector()
    decodedText, points, _ = qrCodeDetector.detectAndDecode(image)
    qr_data = decodedText.split(",")
    print("qr data from fucntion: {}".format(qr_data))
    if points is not None:
        pts = len(points)
    # print(pts)
    for i in range(pts):
        nextPointIndex = (i + 1) % pts
        if str(inputImage) == "scaled_img.jpg":
            cv2.line(
                image,
                tuple(points[i][0]),
                tuple(points[nextPointIndex][0]),
                (255, 0, 0),
                5,
            )
        print(points[i][0])
        width = int(
            math.sqrt(
                (points[0][0][0] - points[1][0][0]) ** 2
                + (points[0][0][1] - points[1][0][1]) ** 2
            )
        )
        height = int(
            math.sqrt(
                (points[1][0][0] - points[2][0][0]) ** 2
                + (points[1][0][1] - points[2][0][1]) ** 2
            )
        )
        print("height and width after scaling: {} {}".format(height, width))
        if not str(inputImage) == "scaled_img.jpg":
            scaled_img = None
            if width == qr_data[0] and height == qr_data[0]:
                print("Sizes are equal")
                # Add the extension values to points and crop
                y = int(points[0][0][1]) - int(qr_data[1])
                x = int(points[0][0][0]) - int(qr_data[4])
                roi = image[
                    y : y + height + int(qr_data[3]), x : x + width + int(qr_data[2])
                ]
                scaled_img = cv2.imwrite("scaled_img.jpg", roi)
                return scaled_img
            else:
                print(
                    "Width and height  "
                    + str(width)
                    + "x"
                    + str(height)
                    + "  not equal to "
                    + str(qr_data[0])
                    + "x"
                    + str(qr_data[0])
                )
                if height > int(qr_data[0]):
                    scale_width = int(width) - int(qr_data[0])
                    scale_height = int(height) - int(qr_data[0])
                    print(f"scaled width: {scale_width} scaled height: {scale_height}")
                    dimension = (scale_width, scale_height)
                    scaled_img = cv2.resize(
                        image, dimension, interpolation=cv2.INTER_AREA
                    )
                    print("new img dims: {}".format(scaled_img.shape))
                    cv2.imshow("scaled image:", scaled_img)
                    cv2.imwrite("scaled_img.jpg", scaled_img)
                elif height < int(qr_data[0]):
                    scale_width = int(qr_data[0]) - width
                    scale_height = int(qr_data[0] - height)
                    print(f"scaled width: {scale_width} scaled height: {scale_height}")
                    dimension = (scale_width, scale_height)
                    scaled_img = cv2.resize(
                        image, dimension, interpolation=cv2.INTER_AREA
                    )
                    print("new img dims: {}".format(scaled_img.shape))
                    cv2.imshow("scaled image:", scaled_img)
                    cv2.imwrite("scaled_img.jpg", scaled_img)
                    cv2.imshow("final output:", roi)
                return scaled_img

        else:
            y = int(points[0][0][1]) - int(qr_data[1])
            x = int(points[0][0][0]) - int(qr_data[4])
            print(" x and y")
            print(x)
            print(y)
            roi = image[
                y : y + height + int(qr_data[3]), x : x + width + int(qr_data[2])
            ]
            final_img = cv2.imwrite("finalized_image.jpg", roi)
            cv2.imshow("finalized image:", final_img)
            return final_img


if __name__ == "__main__":
    image_to_crop = decodeAndCrop("example_input_1.jpg")
    final_image = decodeAndCrop("scaled_img.jpg")
    cv2.imshow("Cropped:", image_to_crop)
    # cv2.imshow("Final: ", final_image)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

---

The code above gives an error as: final_img = cv2.imwrite("finalized_image.jpg", roi) cv2.error: OpenCV(4.2.0) /Users/travis/build/skvark/opencv-python/opencv/modules/imgcodecs/src/loadsave.cpp:715: error: (-215:Assertion failed) !_img.empty() in function 'imwrite'

---

End of Latest Update:

---

An example decoded information of a QR code is as: 100, 20, 40, 60, 20, px

Now, I need to detect the QR code from this document image and in the first step I need to compare the size of QR code in captured image of document with the size which is mentioned in the decoded information for example if in the captured image the size of the QR image is 90X90px and the size from decoded info is 100X100px we need to compare that.

Then, in the second step I have to crop the complete image by using the Top, Right, Bottom & Left variables accordingly. According to the above example we need to crop the image from the position of detected QR code to 20px Top, 40px Right, 60px Bottom and 20px Right. I have added an example Image below.

I have done to decode the QR code information but how can I take the detected QR code area as a seprate image and compare it's size with the mentioned size and then crop the Image accordingly?

Here's what I have tried so far:

import cv2

image = cv2.imread('/Users/abdul/PycharmProjects/QScanner/images/second.jpg')

qrCodeDetector = cv2.QRCodeDetector()
decodedText, points, _ = qrCodeDetector.detectAndDecode(image)
qr_data = decodedText.split(',')
qr_size = qr_data[0]
top = qr_data[1]
right = qr_data[2]
bottom = qr_data[3]
left = qr_data[4]

print(f'Size: {qr_size}' + str(qr_data[5]))
print(f'Top: {top}')
print(f'Right: {right}')
print(f'Bottom: {bottom}')
print(f'Left: {left}')
if points is not None:
    pts = len(points)
    print(pts)
    for i in range(pts):
        nextPointIndex = (i+1) % pts
        cv2.line(image, tuple(points[i][0]), tuple(points[nextPointIndex][0]), (255,0,0), 5)
        print(points[i][0])
    print(decodedText)    
    cv2.imshow("Image", image)
    cv2.waitKey(0)
    cv2.destroyAllWindows()
else:
    print("QR code not detected")

Here's an example Image:

and here's a sample of input image:

Answer 1

Here's a simple approach using thresholding, morphological operations, and contour filtering.

1. Obtain binary image. Load image, grayscale, Gaussian blur , Otsu's threshold

2. Connect individual QR contours. Create a rectangular structuring kernel with cv2.getStructuringElement then perform morphological operations with cv2.MORPH_CLOSE .

3. Filter for QR code. Find contours and filter using contour approximation , contour area , and aspect ratio .

---

Detected QR code

Extracted QR code

From here you can compare the QR code with your reference information

Code

import cv2
import numpy as np

# Load imgae, grayscale, Gaussian blur, Otsu's threshold
image = cv2.imread('1.jpg')
original = image.copy()
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
blur = cv2.GaussianBlur(gray, (9,9), 0)
thresh = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]

# Morph close
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5,5))
close = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel, iterations=2)

# Find contours and filter for QR code
cnts = cv2.findContours(close, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]
for c in cnts:
    peri = cv2.arcLength(c, True)
    approx = cv2.approxPolyDP(c, 0.04 * peri, True)
    x,y,w,h = cv2.boundingRect(approx)
    area = cv2.contourArea(c)
    ar = w / float(h)
    if len(approx) == 4 and area > 1000 and (ar > .85 and ar < 1.3):
        cv2.rectangle(image, (x, y), (x + w, y + h), (36,255,12), 3)
        ROI = original[y:y+h, x:x+w]
        cv2.imwrite('ROI.png', ROI)

cv2.imshow('thresh', thresh)
cv2.imshow('close', close)
cv2.imshow('image', image)
cv2.imshow('ROI', ROI)
cv2.waitKey()     

Answer 2

I got the width and height data using points and compare it with the qr_data size. Then cropped the QR according to needed.

import cv2
import math  

image = cv2.imread('/ur/image/directory/qr.jpg')

qrCodeDetector = cv2.QRCodeDetector()
decodedText, points, _ = qrCodeDetector.detectAndDecode(image)
qr_data = decodedText.split(',')
qr_size = qr_data[0]
top = qr_data[1]
right = qr_data[2]
bottom = qr_data[3]
left = qr_data[4]

if points is not None:
    pts = len(points)
    print(pts)
    for i in range(pts):
        nextPointIndex = (i+1) % pts
        cv2.line(image, tuple(points[i][0]), tuple(points[nextPointIndex][0]), (255,0,0), 5)
        print(points[i][0])

    width = int(math.sqrt((points[0][0][0]-points[1][0][0])**2 + (points[0][0][1]-points[1][0][1])**2))
    height = int(math.sqrt((points[1][0][0]-points[2][0][0])**2 + (points[1][0][1]-points[2][0][1])**2))

    # Compare the size
    if(width==qr_data[0] and height==qr_data[0]):
        print("Sizes are equal")
    else:
        print("Width and height  " + str(width) + "x" +  str(height) + "  not equal to " 
        + str(qr_data[0]) + "x" + str(qr_data[0]))

    # Add the extension values to points and crop
    y = int(points[0][0][1]) - int(qr_data[1])
    x = int(points[0][0][0]) - int(qr_data[4])
    roi = image[y:y+height + int(qr_data[3]), x:x+width + int(qr_data[2])]
    print(decodedText)    
    cv2.imshow("Image", image)
    cv2.imshow("Crop", roi)
    cv2.waitKey(0)
    cv2.destroyAllWindows()
else:
    print("QR code not detected")

Result:

Answer 3

So, you mainly have 3 problems here.

1. If the image is rotated with an angle \\theta,
2. If the sheet is one a plane. (ie, in the images, the upper line doesn't seem to be linear. But it should not be a big deal.)
3. The black borders. Will you always have those or may it be a different background? This is important because without cropping out those, you won't be able to get a reasonable result.

I improved your code a little bit and removed the border pixels:

import cv2
import matplotlib.pyplot as plt    
import math
import numpy as np

image = cv2.imread('/Users/samettaspinar/Public/im.jpg')

qrCodeDetector = cv2.QRCodeDetector()
decodedText, points, _ = qrCodeDetector.detectAndDecode(image)
qr_data = decodedText.split(',')
qr_size = int(qr_data[0])
top = int(qr_data[1])
right = int(qr_data[2])
bottom = int(qr_data[3])
left = int(qr_data[4])

print(f'Size: {qr_size}' + str(qr_data[5]))
print(f'Top: {top}')
print(f'Right: {right}')
print(f'Bottom: {bottom}')
print(f'Left: {left}')

plt.imshow(image)
plt.show()

dists = [] #This is for estimating distances between corner points.
           #I will average them to find ratio of pixels in image vs qr_size  
           #in the optimal case, all dists should be equal

if points is not None:
    pts = len(points)
    for i in range(pts):
        p1 = points[i][0]
        p2 = points[(i+1) % pts][0]

        dists.append(math.sqrt((p1[0]-p2[0])**2 + (p1[1]-p2[1])**2))

        print('line', tuple(p1), tuple(p2))
        image = cv2.line(image, tuple(p1), tuple(p2), (255,0,0), 5)
else:
    print("QR code not detected")

print('distances: ', dists)


# Remove the black border pixels. I had a simple idea for this
# Get the average intensity of the gray image
# If count the row average of the first half that are less than intensity/2. 
# It approx gives number of black borders on the left. etc.  
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
inten = np.mean(gray)

x = np.mean(gray, axis=0) # finds the vertical average
y = np.mean(gray, axis=1) # finds horizontal average

bl_left = np.sum([x[:int(col/2)] < inten/2])
bl_right = np.sum([x[int(col/2)+1:] < inten/2])

bl_top = np.sum([y[:int(row/2)] < inten/2])
bl_bottom = np.sum([y[int(row/2)+1:] < inten/2])

print('black margins: ', bl_left, bl_right, bl_top, bl_bottom)

# Estimate how many pixel you will crop out
ratio = np.mean(dists)/ int(qr_size)
print('actual px / qr_size in px: ', ratio)

row,col,dim = image.shape

top, left, right, bottom = int(top*ratio), int(left*ratio), int(right*ratio), int(bottom*ratio)
top += bl_top
left += bl_left
right += bl_right
bottom += bl_bottom

print('num pixels to be cropped: ', top, left, right, bottom)

gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
image2 = image[top:row-bottom, left:col-right, :]

plt.imshow(image2)
plt.show()

Notice that I ignored the rotation issue. If there is rotation, you can find the angle by calculating the tangents/arctan where I calculated the distances.

Answer 4

For QR detection and parsing

import cv2
import sys

filename = sys.argv[1]

# read the QRCODE image
#in case if QR code is not black/white it is better to convert it into grayscale
img = cv2.imread(filename, 0)# Zero means grayscale
img_origin = cv2.imread(filename)

# initialize the cv2 QRCode detector
detector = cv2.QRCodeDetector()

# detect and decode
data, bbox, straight_qrcode = detector.detectAndDecode(img)

# if there is a QR code
if bbox is not None:
    print(f"QRCode data:\n{data}")
    # display the image with lines
    # length of bounding box
    n_lines = len(bbox[0])#Cause bbox = [[[float, float]]], we need to convert fload into int and loop over the first element of array
    bbox1 = bbox.astype(int) #Float to Int conversion
    for i in range(n_lines):
        # draw all lines
        point1 = tuple(bbox1[0, [i][0]])
        point2 = tuple(bbox1[0, [(i+1) % n_lines][0]])
        cv2.line(img_origin, point1, point2, color=(255, 0, 0), thickness=2)


    # display the result
    cv2.imshow("img", img)
    cv2.waitKey(0)
    cv2.destroyAllWindows()
    
else:
    print("QR code not detected")