如何在普通的 Javascript 网络应用程序中使用经过训练的 TensorFlow 模型

Question

I've been following this tutorial: https://docs.annotations.ai/workshops/object-detection/6.html

And got to step 6, once I get to the webapp example it's done in ReactJS and I can't figure out how to convert it to plain JS for our particular use case. I was able to get this far:

scripts.js

var videoRef = document.getElementById("video");

if(navigator.mediaDevices.getUserMedia) {
  try {
    let stream = await navigator.mediaDevices.getUserMedia({ audio: false, video: { facingMode: { exact: "environment" } } });
  } catch(error) {}

  navigator.mediaDevices.enumerateDevices()
  .then(function(devices) {

    // Remove non camera devices
    for(var i = devices.length - 1; i>=0; i--) {
      var device = devices[i];
      if(device.kind != 'videoinput') {
        devices.splice(i, 1);
      }
      if(!device.kind) {
        devices.splice(i, 1);
      }
    }

    // Force camera to back camera for mobile devices
    var activeDevice = devices[0];
    for(var i in devices) {
      var device = devices[i];
      if(device.label) {
        if(device.label.toLowerCase().indexOf('back') > -1) {
          activeDevice = device;
        }
      }
    }

    const constraints = {video: {deviceId: activeDevice.deviceId ? {exact: activeDevice.deviceId} : undefined}, audio: false};

    const webcamPromise = navigator.mediaDevices
    .getUserMedia(constraints)
    .then(stream => {
      window.stream = stream;
      videoRef.srcObject = stream;

      return new Promise(resolve => {
        videoRef.onloadedmetadata = () => {
          resolve();
        };
      });
    }, (error) => {
      console.error(error, 'camera error');
    });

    const loadlModelPromise = cocoSsd.load({modelUrl: 'https://nanonets.s3-us-west-2.amazonaws.com/uploadedfiles/87be4e38-b40d-4217-898b-fd619319c2e4/ssd/model.json'});

    Promise.all([loadlModelPromise, webcamPromise])
    .then(values => {
      detectFromVideoFrame(values[0], videoRef);
    })
    .catch(error => {
      console.error(error, 'error loading promises');
    })

  })
}


function detectFromVideoFrame(model, video) {
  model.detect(video).then(predictions => {

    console.log(predictions, 'predictions found');

    requestAnimationFrame(() => {
      detectFromVideoFrame(model, video);
    });
  }, (error) => {
    console.error(error, 'Tensorflow Error');
  });
};

In the HTML I include a coco-ssd.js file which I also believe I need to modify, but i'm not sure how to generate that file:

<script src="/lib/coco-ssd.js"></script>
<script src="https://unpkg.com/babel-standalone@6.26.0/babel.min.js"></script>

That code works with a pre-defined coco-ssd model but from following the tutorial I can't figure out how to use my own model, here is the files that were generated:

Now I need to find out how to use those files in my Javascript above.

I think I need to change these lines:

const loadlModelPromise = cocoSsd.load({modelUrl: 'https://nanonets.s3-us-west-2.amazonaws.com/uploadedfiles/87be4e38-b40d-4217-898b-fd619319c2e4/ssd/model.json'});

And include a different coco-ssd.js file:

<script src="/lib/coco-ssd.js"></script>

But it's not clear what files to include from the generated folder structure, that's what I'm getting stuck on.

Answer 1

You can use the script for TensorFlow js & coco SSD

<!-- Load TensorFlow.js. This is required to use coco-ssd model. -->
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs"> </script>
<!-- Load the coco-ssd model. -->
<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/coco-ssd"> </script>

Step 1: Create an index.html file
index.html
Note: You can use the modelUrl functionality to load your own model hosted somewhere

<!-- Load TensorFlow.js. This is required to use coco-ssd model. -->
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs"> </script>
<!-- Load the coco-ssd model. -->
<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/coco-ssd"> </script>

<!-- Replace this with your image. Make sure CORS settings allow reading the image! -->
<img id="img" src="https://images.pexels.com/photos/20787/pexels-photo.jpg?auto=compress&cs=tinysrgb&dpr=1&w=500" crossorigin="anonymous"/>
<!-- Place your code in the script tag below. You can also use an external .js file -->
<script>
  // Notice there is no 'import' statement. 'cocoSsd' and 'tf' is
  // available on the index-page because of the script tag above.

  const img = document.getElementById('img');

  // Load the model.
  //Note: cocoSsd.load() will also work on this without parameter. It will default to the coco ssd model
  cocoSsd.load({ modelUrl: 'PATH TO MODEL JSON FILE' }).then(model => {
    // detect objects in the image.
    model.detect(img).then(predictions => {
      console.log('Predictions: ', predictions);
    });
  });
</script>

Step 2: Let's say you have a nodejs server to run the index file and serve it locally by accessing localhost:3000.
Server.js

const express = require('express');
app = express();

app.get('/',function(req,res) {
    res.sendFile('./index.html', { root: __dirname });
});
const port = 3000
app.listen(port, function(){
    console.log(`Listening at port ${port}`);
})
'''