Double imshow in python script with opencv

Question

I have following code:

import numpy as np
import cv2
import pickle
import rtsp
import PIL as Image

face_cascade = cv2.CascadeClassifier('cascades\data\haarcascade_frontalface_alt2.xml')
recognizer = cv2.face.LBPHFaceRecognizer_create()
recognizer.read("trainner.yml")

labels = {"person_name": 1}
with open("labels.pickle", 'rb') as f:
    og_labels = pickle.load(f)
    labels = {v:k for k,v in og_labels.items()}

url = 'rtsp://user:pass@xxx.xxx.x.xxx:YYYY/stream0/mobotix.mjpeg'
with rtsp.Client(url) as client:
  client.preview()
  while True:
    frame = client.read(raw=True)
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    faces = face_cascade.detectMultiScale(gray, scaleFactor=1.3, minNeighbors=5)
    for (x,y,w,h) in faces:
      print(x,y,w,h)
      roi_gray = gray[y:y+h, x:x+w]
      roi_color = frame[y:y+h, x:x+w]
        
      #recognize?
      id_, conf = recognizer.predict(roi_gray)
      if conf>=45: # and conf <=85:
        print(id_)
        print(labels[id_])
        font = cv2.FONT_HERSHEY_SIMPLEX
        name = labels[id_]
        color = (0,0,255)
        stroke = 2
        cv2.putText(frame, name, (x,y), font, 1, color, stroke, cv2.LINE_AA)

      #img_item = "my-image.png"
      #cv2.imwrite(img_item, roi_gray)

      color = (0, 0, 255)
      stroke = 2
      end_cord_x = x + w
      end_cord_y = y + h
      cv2.rectangle(frame, (x,y), (end_cord_x, end_cord_y), color, stroke)

    cv2.imshow('IP cam',frame)
   
    if cv2.waitKey(20) & 0xFF == ord('q'):
        break

cv2.destroyAllWindows()

Everything is working fine, I'm running my code and first it's opening the Cam view of "client.preview", face detection is not working at this time. When I close this one, the IPcam windows opens and everything is working. (I'm still getting a lot of missed frames by the RTSP stream but no direct issue).

If I leave the code "client.preview" out of it, I'm getting an error from opencv because of src_empty. If I try to change the code to "client.read()" idem an error occurs from opencv because of src_empty.

How should I fix this?

Answer 1

Before the rtsp client has received its first frame, the read call on the rtsp client will return None. When you call "client.preview" the time taken for the window to open gives the rtsp client enough time to receive the first frame from the stream before the first read is called. All calls to read beyond this will always return a frame, which is why everything works when it is included in the code. The solution to your problem would be check the result from read function of the rtsp client to ensure that you have received a frame before processing it.

import numpy as np
import cv2
import pickle
import rtsp
import PIL as Image
import time

face_cascade = cv2.CascadeClassifier('cascades\data\haarcascade_frontalface_alt2.xml')
recognizer = cv2.face.LBPHFaceRecognizer_create()
recognizer.read("trainner.yml")

labels = {"person_name": 1}
with open("labels.pickle", 'rb') as f:
    og_labels = pickle.load(f)
    labels = {v:k for k,v in og_labels.items()}

url = 'rtsp://user:pass@xxx.xxx.x.xxx:YYYY/stream0/mobotix.mjpeg'
with rtsp.Client(url) as client:
  client.preview()
  while True:
    frame = client.read(raw=True)
    if frame is not None:
        time.sleep(.10)
    else:
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        faces = face_cascade.detectMultiScale(gray, scaleFactor=1.3, minNeighbors=5)
        for (x,y,w,h) in faces:
        print(x,y,w,h)
        roi_gray = gray[y:y+h, x:x+w]
        roi_color = frame[y:y+h, x:x+w]
            
        #recognize?
        id_, conf = recognizer.predict(roi_gray)
        if conf>=45: # and conf <=85:
            print(id_)
            print(labels[id_])
            font = cv2.FONT_HERSHEY_SIMPLEX
            name = labels[id_]
            color = (0,0,255)
            stroke = 2
            cv2.putText(frame, name, (x,y), font, 1, color, stroke, cv2.LINE_AA)

        #img_item = "my-image.png"
        #cv2.imwrite(img_item, roi_gray)

        color = (0, 0, 255)
        stroke = 2
        end_cord_x = x + w
        end_cord_y = y + h
        cv2.rectangle(frame, (x,y), (end_cord_x, end_cord_y), color, stroke)

        cv2.imshow('IP cam',frame)
    
        if cv2.waitKey(20) & 0xFF == ord('q'):
            break

cv2.destroyAllWindows()

Answer 2

Ok I understand that the RTSP protocol is first sending empty frames and building up the buffer.

with rtsp.Client(url) as client:
  while True:
    frame = client.read(raw=True)
    if not frame:
        time.sleep(.30)
    else:
      gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
      faces = face_cascade.detectMultiScale(gray, scaleFactor=1.3, minNeighbors=5)
      for (x,y,w,h) in faces:
        
      cv2.imshow('IP cam',frame)
      
      if cv2.waitKey(20) & 0xFF == ord('q'):
        break

cv2.destroyAllWindows()

I deleted the line : client.preview() because of a double function. The code is starting and after period I receive following error:

if not frame:
ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()