在 HTML 中获取网络摄像头视频并通过 socketio 发送到 flask 服务器

Question

I'm trying to capture webcam video in the client side and sent frames to the server to process it. I'm newbie in JS and I'm having some problems. I tried to use OpenCV.js to get the data but I didn't understand how to get it, in Python we can make

cap = cv2.VideoCapture(0)
ret, frame = cap.read()

and the frame is an 2d-array with the image, but how can I get each frame (as 2d array) to send using OpenCV.js?

I have this code on the client side:

<script type="text/javascript">
function onOpenCvReady() {
    cv['onRuntimeInitialized'] = () => {
        var socket = io('http://localhost:5000');

        socket.on('connect', function () {
            console.log("Connected...!", socket.connected)
        });

        const video = document.querySelector("#videoElement");

        video.width = 500;
        video.height = 400;

        if (navigator.mediaDevices.getUserMedia) {
            navigator.mediaDevices.getUserMedia({ video: true })
                .then(function (stream) {
                    video.srcObject = stream;
                    video.play();

                })
                .catch(function (err0r) {
                    console.log(err0r)
                    console.log("Something went wrong!");
                });
        }

                    let src = new cv.Mat(video.height, video.width, cv.CV_8UC4);
                    let cap = new cv.VideoCapture(video);

                    const FPS = 15;
                    function processVideo() {

                        let begin = Date.now();
                        cap.read(src);

                        handle_socket(src['data']);

                        // schedule next one.
                        let delay = 1000 / FPS - (Date.now() - begin);
                        setTimeout(processVideo, delay);
                    }
                    // schedule first one.
                    setTimeout(processVideo, 0);

        function handle_socket(src) {

            socket.emit('event', { info: 'I\'m connected!', data: src });

        }

    }
}

</script>

Answer 1

My solution was:

// Select HTML video element where the webcam data is
const video = document.querySelector("#videoElement");

// returns a frame encoded in base64
const getFrame = () => {
    const canvas = document.createElement('canvas');
    canvas.width = video.videoWidth;
    canvas.height = video.videoHeight;
    canvas.getContext('2d').drawImage(video, 0, 0);
    const data = canvas.toDataURL('image/jpeg');
    return data;
}


// Send data over socket
function handle_socket(data, dst) {
    socket.emit('event', data, function (res) {
        if (res !== 0) {
            results = res;
        }
    });
}

frame64 = getFrame()
handle_socket(frame64);