Selecting ROI for Object Tracking from uninterrupted video stream with Python OpenCV

Question

I was following this tutorial https://www.pyimagesearch.com/2018/07/30/opencv-object-tracking/ that allows you to create an ROI from a static screenshot of the video stream to do basic object tracking (not detection).

I was wondering if it was possible to grab a bounding box from the webcamera stream as it was playing (without pause or opening a new window). I understand that the standard cv2.selectROI() triggers a seperate window to open and pauses the video stream, which is the opposite of what I am trying to go for. A helpful post from 2018 pointed me towards creating a mouse event clicker and its gotten me to the point where I can create rectangles on the live video feed, but the opencv tracker seems to spaz out when I give it the coordinates of my BBox. Python OpenCV select ROI on video/stream while its playing

I've tried working with some examples posted already, but they are pretty outdated as of 2022 and I got to the point where I can trigger the tracker but it doesnt seem to lay the initial BBox around where I selected with my mouse clicks. Any help would be greatly appreciated!

(I set it up to test with 2 left mouse clicks create the BBox and a single right mouse click removes it/stops the tracker)

import cv2, sys

(major_ver, minor_ver, subminor_ver) = (cv2.__version__).split('.')
print(cv2.__version__)
video = cv2.VideoCapture(r'C:\Users\longc\Downloads\race_test.mp4')
cv2.namedWindow('Frame', cv2.WINDOW_NORMAL)
import selectinwindow

# Our ROI, defined by two points
p1, p2 = None, None
state = 0
bbox = (0,50,100,150)
# Set up tracker.
# Instead of MIL, you can also use
cv2.namedWindow("Frame", cv2.WINDOW_NORMAL)
tracker_types = ['BOOSTING', 'MIL','KCF', 'TLD', 'MEDIANFLOW', 'CSRT', 'MOSSE']
tracker_type = tracker_types[6]

if int(major_ver) < 4 and int(minor_ver) < 3:
    tracker = cv2.cv2.Tracker_create(tracker_type)
else:
    if tracker_type == 'BOOSTING':
        tracker = cv2.TrackerBoosting_create()
    if tracker_type == 'MIL':
        tracker = cv2.TrackerMIL_create()
    if tracker_type == 'KCF':
        tracker = cv2.TrackerKCF_create()
    if tracker_type == 'TLD':
        tracker = cv2.TrackerTLD_create()
    if tracker_type == 'MEDIANFLOW':
        tracker = cv2.TrackerMedianFlow_create()
    if tracker_type == 'CSRT':
        tracker = cv2.TrackerCSRT_create()
    if tracker_type == 'MOSSE':
        tracker = cv2.legacy.TrackerMOSSE_create()


# Called every time a mouse event happen
def on_mouse(event, x, y, flags, userdata):
    global state, p1, p2, bbox, val, tracker, frame

    # Left click
    if event == cv2.EVENT_LBUTTONUP:
        # Select first point
        if state == 0:
            p1 = (x, y)
            state += 1
        # Select second point
        elif state == 1:
            p2 = (x, y)
            state += 1
            bbox = p1 + p2
            print(p1,p2)
            print(bbox)
            # Initialize tracker with first frame and bounding box
            if int(major_ver) < 4 and int(minor_ver) < 3:
                tracker = cv2.cv2.Tracker_create(tracker_type)
            else:
                if tracker_type == 'BOOSTING':
                    tracker = cv2.TrackerBoosting_create()
                if tracker_type == 'MIL':
                    tracker = cv2.TrackerMIL_create()
                if tracker_type == 'KCF':
                    tracker = cv2.TrackerKCF_create()
                if tracker_type == 'TLD':
                    tracker = cv2.TrackerTLD_create()
                if tracker_type == 'MEDIANFLOW':
                    tracker = cv2.TrackerMedianFlow_create()
                if tracker_type == 'CSRT':
                    tracker = cv2.TrackerCSRT_create()
                if tracker_type == 'MOSSE':
                    tracker = cv2.legacy.TrackerMOSSE_create()
            val = tracker.init(frame, bbox)
    # Right click (erase current ROI)
    if event == cv2.EVENT_RBUTTONUP:
        p1, p2 = None, None
        state = 0
        val = tracker.init(frame, bbox)

# Register the mouse callback
cv2.setMouseCallback('Frame', on_mouse)


test_counter = 0
while video.isOpened():
    val, frame = video.read()
    if not val:
        break
    # If a ROI is selected, draw it
    if state > 1:
        #cv2.rectangle(frame, p1, p2, (255, 0, 0), 1)
        # Read a new frame
        val, frame = video.read()
        if not val:
            break

        # Start timer
        timer = cv2.getTickCount()

        # Update tracker
        val, bbox = tracker.update(frame)

        # Calculate Frames per second (FPS)
        fps = cv2.getTickFrequency() / (cv2.getTickCount() - timer);

        # Draw bounding box
        if val:
            # Tracking success
            p1 = (int(bbox[0]), int(bbox[1]))
            p2 = (int(bbox[0] + bbox[2]), int(bbox[1] + bbox[3]))
            cv2.rectangle(frame, p1, p2, (255, 0, 0), 2, 1)
            cv2.putText(frame, "Tracking Successful", (100, 80), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2)
        else:
            # Tracking failure
            cv2.putText(frame, "Tracking failure detected", (100, 80), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 0, 255), 2)

        # Display tracker type on frame
        cv2.putText(frame, tracker_type + " Tracker", (100, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2);

        # Display FPS on frame
        cv2.putText(frame, "FPS : " + str(int(fps)), (100, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255, 127, 0), 2);

    # Show image
    cv2.imshow('Frame', frame)

    # Let OpenCV manage window events
    key = cv2.waitKey(50)
    # If ESCAPE key pressed, stop
    if key == 27:
        video.release()

    # Delayed test console
    test_counter += 1
    if test_counter % 30 == 0:
        print("Current BBox: {}".format(str(bbox)))
        print(val)
        #print(frame)
        print(tracker)

Answer 1

one of the possible solution can be like this, select ROI and save the corresponding image and then perform re-identification on the stream and when the re-identification is successful then you can re-initialize the tracker on current location.