QT Opencv 人脸检测在 Python 中不起作用？

Question

I am working on a qt project where I have created window which will display the live frames from a usb webcam using opencv .I also need to detect faces in the live feed and thus I am using haar-cascading method for this. I have created the UI part in qt-designer and then have converted it into the .py file. I am then importing this file in another app.py and using app.py for all the logic part. Below is the content of gui.py file:

from PyQt5 import QtCore, QtGui, QtWidgets


class Ui_MainWindow(object):
    def setupUi(self, MainWindow):
        MainWindow.setObjectName("MainWindow")
        MainWindow.resize(800, 400)
        self.centralwidget = QtWidgets.QWidget(MainWindow)
        self.centralwidget.setObjectName("centralwidget")
        self.groupBox = QtWidgets.QGroupBox(self.centralwidget)
        self.groupBox.setGeometry(QtCore.QRect(10, 10, 381, 370))
        self.groupBox.setTitle("")
        self.groupBox.setObjectName("groupBox")
        self.pushButton = QtWidgets.QPushButton(self.groupBox)
        self.pushButton.setGeometry(QtCore.QRect(150, 160, 75, 23))
        self.pushButton.setObjectName("pushButton")
        self.groupBox_2 = QtWidgets.QGroupBox(self.centralwidget)
        self.groupBox_2.setGeometry(QtCore.QRect(400, 10, 391, 370))
        self.groupBox_2.setTitle("")
        self.groupBox_2.setObjectName("groupBox_2")
        self.label = QtWidgets.QLabel(self.groupBox_2)
        self.label.setGeometry(QtCore.QRect(10, 10, 371, 360))
        self.label.setText("")
        self.label.setObjectName("label")
        MainWindow.setCentralWidget(self.centralwidget)
        self.menubar = QtWidgets.QMenuBar(MainWindow)
        self.menubar.setGeometry(QtCore.QRect(0, 0, 800, 21))
        self.menubar.setObjectName("menubar")
        MainWindow.setMenuBar(self.menubar)
        self.statusbar = QtWidgets.QStatusBar(MainWindow)
        self.statusbar.setObjectName("statusbar")
        MainWindow.setStatusBar(self.statusbar)

        self.retranslateUi(MainWindow)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)

    def retranslateUi(self, MainWindow):
        _translate = QtCore.QCoreApplication.translate
        MainWindow.setWindowTitle(_translate("MainWindow", "BIOT "))
        self.pushButton.setText(_translate("MainWindow", "PushButton"))


if __name__ == "__main__":
    import sys
    app = QtWidgets.QApplication(sys.argv)
    MainWindow = QtWidgets.QMainWindow()
    ui = Ui_MainWindow()
    ui.setupUi(MainWindow)
    MainWindow.show()
    sys.exit(app.exec_())

and below is the code for app.py which handles all the logic part:

import sys
import cv2
import os
import imutils
from PyQt5.QtWidgets import QApplication, QMainWindow
from PyQt5.QtGui import QImage
from PyQt5.QtGui import QPixmap
from PyQt5.QtCore import QTimer
from ui.gui import Ui_MainWindow


curr_path = os.path.dirname(os.path.abspath(__file__))


class ROCKET(QMainWindow, Ui_MainWindow):
    def __init__(self):
        QMainWindow.__init__(self)
        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)

        self.cap = cv2.VideoCapture(1)
        self.face_detect = cv2.CascadeClassifier(os.path.join(curr_path, 'models', 'haarcascade_frontalface_default.xml'))

        self.timer = QTimer()
        self.timer.timeout.connect(self.view_cam)
        self.timer.start(20)
        self.ui.pushButton.setText("Stop")

    def __del__(self):
        self.timer.stop()
        self.cap.release()
        self.ui.pushButton.setText("Start")

    def view_cam(self):

        ret, image = self.cap.read()
        image = imutils.resize(image, width=371, height=360)
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
        height, width, channel = image.shape
        faces = self.face_detect.detectMultiScale(gray, 1.3, 5)
        for (x, y, width, height) in faces:
            print("face detected")
            cv2.rectangle(image, (x, y), (x + width, y + height), (255, 0, 0), 2)
        step = channel * width
        qImg = QImage(image.data, width, height, step, QImage.Format_RGB888)
        self.ui.label.setPixmap(QPixmap.fromImage(qImg))


app = QApplication(sys.argv)
main_window = ROCKET()
main_window.show()
sys.exit(app.exec_())

as you can see in the above code, I have imported cascade classifier in __init__ and I have also started a timer which is connected to view_cam function. In view_cam function, I am reading the frames, detecting and displaying the result. The problem here is that as soon as it starts detecting the faces, it should draw the bounding box rectangle across the face but instead of that, it looks like below:

When there is no face, it normally shows the live frame and works fine but as soon as the face is detected, it starts showing above zig zag kind of lines. I am not very expert in Qt . Can anyone please guide me here as to what I am doing. Please help. Thanks (Please ignore stop button, its not doing anything)

Answer 1

The problem is that the memoryview (image.data) changes when the image is modified when using cv2.rectangle, that can be seen if the following comparison is made throwing an AssertionError :

last_mv = image.data
for (x, y, width, height) in faces:
    print("face detected")
    cv2.rectangle(image, (x, y), (x + width, y + height), (255, 0, 0), 2)
    current_mv = image.data
    assert last_mv == current_mv

And changing the memoryview also changes the shape that is synchronized with height, width, channel, and you can check that with:

height, width, channel = image.shape
print("before", height, width, channel)

faces = self.face_detect.detectMultiScale(gray, 1.3, 5)

for (x, y, width, height) in faces:
    print("face detected")
    cv2.rectangle(image, (x, y), (x + width, y + height), (255, 0, 0), 2)

step = channel * width
print("after", height, width, channel)

Output:

before 208 371 3
face detected
after 112 112 3
before 208 371 3
face detected
after 110 110 3
before 208 371 3
face detected
after 108 108 3

As you can see the shape before and after cv2.rectangle are different.

The solution is to calculate the geometry after making all the changes.

def view_cam(self):
    ret, image = self.cap.read()
    image = imutils.resize(image, width=371, height=360)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

    faces = self.face_detect.detectMultiScale(gray, 1.3, 5)

    for (x, y, width, height) in faces:
        cv2.rectangle(image, (x, y), (x + width, y + height), (255, 0, 0), 2)

    
    step = channel * width
    qImg = QImage(image.data, width, height, step, QImage.Format_RGB888)
    self.ui.label.setPixmap(QPixmap.fromImage(qImg))