How to maintain a higher FPS while enlarging an image using cv2.resize()

Question

I am running a simple code to measure the FPS of my live streaming (using webcam) . The FPS decreases when i resize the image to a larger frame. Is there any way to maintain the FPS while enlarging the frame (through resize function) at the same time. Or is it an inevitable trade off ?

This is the code for face recognition using the face_recognition library. The FPS(frames per second) slow down when i resize to a larger size. Is there any way to maintain a higher FPS while also enlarging the image using cv2.resize() ?

import face_recognition
import cv2

video_capture = cv2.VideoCapture(0)
#video_capture.set(cv2.CAP_PROP_FPS, 30)
# Load a sample picture and learn how to recognize it.
obama_image = face_recognition.load_image_file("osama LinkedIN.jpg")

obama_face_encoding = face_recognition.face_encodings(obama_image)[0]


# Load a second sample picture and learn how to recognize it.
imran_shafqat_image = face_recognition.load_image_file("haris intern3.jpg")
imran_shafqat_face_encoding = face_recognition.face_encodings(imran_shafqat_image)[0]


# Create arrays of known face encodings and their names
known_face_encodings = [
    obama_face_encoding,
    imran_shafqat_face_encoding,
   # obama_face_encoding2
   # biden_face_encoding
]
known_face_names = [
    "Osama Naeem",
    "Imran Shafqat"
 #   "random guy2"
]

# Initialize some variables
face_locations = []
face_encodings = []
face_names = []
process_this_frame = True
fxx = 1.5
fyy = 1.5
while True:
    # Grab a single frame of video
    ret, frame = video_capture.read()

    # Resize frame of video to 1/4 size for faster face recognition processing
    small_frame = cv2.resize(frame, (0, 0), fx=fxx, fy=fyy)

    # Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses)
    rgb_small_frame = small_frame[:, :, ::-1]
    #rgb_small_frame = frame[:, :, ::-1]
    # Only process every other frame of video to save time
    if process_this_frame:
        # Find all the faces and face encodings in the current frame of video
        face_locations = face_recognition.face_locations(rgb_small_frame)
        face_encodings = face_recognition.face_encodings(rgb_small_frame, face_locations)

        face_names = []
        for face_encoding in face_encodings:
            # See if the face is a match for the known face(s)
            matches = face_recognition.compare_faces(known_face_encodings, face_encoding)
            print ("match = ", matches)
            name = "Unknown"

            # If a match was found in known_face_encodings, just use the first one.
            if True in matches:
                first_match_index = matches.index(True)
                name = known_face_names[first_match_index]

            face_names.append(name)

    process_this_frame = not process_this_frame


    # Display the results
    for (top, right, bottom, left), name in zip(face_locations, face_names):
        # Scale back up face locations since the frame we detected in was scaled to 1/4 size
        top *= (1/fxx)
        right *= (1/fxx)
        bottom *= (1/fyy)
        left *= (1/fyy)

        # Draw a box around the face
        cv2.rectangle(frame, (round(left), round(top)), (round(right), round(bottom)), (0, 0, 255), 2)

        # Draw a label with a name below the face

        #cv2.rectangle(frame, (round(left) - 35, round(bottom) - 40), (round(right), round(bottom)), (0, 0, 255), cv2.FILLED)
        font = cv2.FONT_HERSHEY_DUPLEX
        cv2.putText(frame, name, (round(left) + 6, round(bottom) - 6), font, 0.5, (255, 255, 255), 1)

    # Display the resulting image
    cv2.imshow('Video', frame)

    # Hit 'q' on the keyboard to quit!
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

# Release handle to the webcam
video_capture.release()
cv2.destroyAllWindows()

The code works fine, but i want to maintain the FPS at the same rate when i enlarge it to a larger size.

Answer 1

When you enlarge a image using cv2.resize() , you create a larger image which increases the processing time per frame. Essentially, your program must do additional work to process more pixels. However, a possible solution which allows you to increase your FPS could be to use multithreading . This method will allow you to increase FPS by I/O latency reduction as opposed to reducing the amount of time needed to process each resized frame. The idea is to separate reading frames into its own independent thread while you do processing in the main thread. Here's a widget showing how to separate reading frames and processing into individual threads.

from threading import Thread
import cv2, time

class VideoStreamWidget(object):
    def __init__(self, src=0):
        self.capture = cv2.VideoCapture(src)
        # Start the thread to read frames from the video stream
        self.thread = Thread(target=self.update, args=())
        self.thread.daemon = True
        self.thread.start()

    def update(self):
        # Read the next frame from the stream in a different thread
        while True:
            if self.capture.isOpened():
                (self.status, self.frame) = self.capture.read()
            time.sleep(.01)

    def show_frame(self):
        # Display frames in main program
        cv2.imshow('frame', self.frame)
        key = cv2.waitKey(1)
        if key == ord('q'):
            self.capture.release()
            cv2.destroyAllWindows()
            exit(1)

if __name__ == '__main__':
    video_stream_widget = VideoStreamWidget()
    while True:
        try:
            video_stream_widget.show_frame()
        except AttributeError:
            pass