Completely Different prediction results from .h5 keras model and .json tensorflow.js model

Question

so, My model is giving me results with decent accuracy for test images

import cv2
from IPython.display import display, Javascript
from google.colab.output import eval_js
from base64 import b64decode

import matplotlib.pyplot as plt
face_haar_cascade = cv2.CascadeClassifier('/content/gdrive/My Drive/New FEC Facial Expression/haarcascade_frontalface_default.xml')
from IPython.display import Image
try:
 filename = '/content/gdrive/My Drive/photo-1533227268428-f9ed0900fb3b.jpg'
 img = cv2.imread(filename)

 gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

 faces = face_haar_cascade.detectMultiScale(gray, 1.3,6)
 print('faces', faces)
 for(x,y,w,h) in faces:
   cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)
   roi_gray = gray[y:y+h, x:x+w]
   roi_color = img[y:y+h, x:x+w]
   plt.grid(None)
   plt.xticks([])
   plt.yticks([])
   imgplot = plt.imshow(img)
 # Show the image which was just taken.
 # display(Image(filename))
except Exception as err:
 # Errors will be thrown if the user does not have a webcam or if they do not
 # grant the page permission to access it.
 print(str(err))


import cv2
import sys

imagePath ='/content/gdrive/My Drive/photo-1533227268428-f9ed0900fb3b.jpg'
image = cv2.imread(imagePath)
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

faceCascade = cv2.CascadeClassifier('/content/gdrive/My Drive/New FEC Facial Expression/haarcascade_frontalface_default.xml')
faces = faceCascade.detectMultiScale(
   gray,
   scaleFactor=1.3,
   minNeighbors=3,
   minSize=(30, 30)
)

print("[INFO] Found {0} Faces.".format(len(faces)))

for (x, y, w, h) in faces:
   cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
   roi_color = image[y:y + h, x:x + w]
   print("[INFO] Object found. Saving locally.")
   cv2.imwrite('/content/gdrive/My Drive/converted Images/faces.jpg', roi_color)

status = cv2.imwrite('faces_detected.jpg', image)
print("[INFO] Image faces_detected.jpg written to filesystem: ", status)
# from skimage import io
from keras.preprocessing import image
img = image.load_img('/content/gdrive/My Drive/converted Images/faces.jpg', color_mode = "grayscale", target_size=(48, 48))
x = image.img_to_array(img)
x = np.expand_dims(x, axis = 0)
x /= 255
show_img=image.load_img('/content/gdrive/My Drive/converted Images/faces.jpg', grayscale=False, target_size=(200, 200))
plt.gray()
plt.imshow(show_img)
plt.show()
if len(faces): 
 custom = model.predict(x)
 index = np.argmax(custom[0])
 emotion1 = custom[0][index]*100
 print(custom)
 print(emotion_label_to_text[index],' => ',  emotion1)
else:
 print('No Face Detected')

This is giving good results and the output for the same result is correct, the image I inserted was happy image and opencv is used to detect the face and crop it then use that cropped image to put into the model and is giving me good results,

But the tf.js part I converted keras model to .json using tfjs converter and I wrote following code

 const classifier = new cv.CascadeClassifier(cv.HAAR_FRONTALFACE_ALT2);
    try {
        const canvImg = await canvas.loadImage(
            path.join(__dirname, `images/${req.file.filename}`)
        );
        const image = await cv.imread(path.join(__dirname, `/images/${req.file.filename}`));
        const classifier = new cv.CascadeClassifier(cv.HAAR_FRONTALFACE_ALT2);
        const { objects, numDetections } = classifier.detectMultiScale(image.bgrToGray());
        if (!objects.length) {
            return next({
                msg: 'No face detected'
            })
        } else {
            const model = await tf.loadLayersModel(
                "http://localhost:8000/models/model.json"
            );
            const obj = objects[0]
            const cnvs = canvas.createCanvas(48, 48);
            const ctx = cnvs.getContext("2d");
            ctx.drawImage(canvImg, obj.x, obj.y, obj.width, obj.height, 0, 0, cnvs.width, cnvs.height);
            var tensor = tf.browser
                .fromPixels(cnvs)
                .mean(2)
                .toFloat()
                .expandDims(-1)
                .expandDims(0, 'None')



            const prediction = await model.predict(tensor).data();
            console.log(prediction);
            var emotions = [
                "angry",
                "disgust",
                "fear",
                "happy",
                "sad",
                "surprise",
                        ];
            var index = Object.values(prediction).findIndex(
                (p) => p === Math.max(...Object.values(prediction))
            );
            res.status(200).json(emotions[index])
            fs.unlink(
                path.join(process.cwd(), "./faceDetection/images/" + req.file.filename),
                function(err, removed) {
                    if (err) console.log("file removing err");
                    else console.log("file removed");
                }
            );
        }

    } catch (e) {
        return next(e)
    }

I used opencv4nodejs for the detection of image, canvas to crop the image (canvas giving me good result for cropping the face part) and tf.js for predicting, But the output is giving me same results everytime where among all those keys in object one of them will get 1 (fear in this case) and keep giving me same result for the same image that I tested in keras.

Am I doing something wrong in manipulation of tensor?

Answer 1

One possible reason. In python you "normalize" your image input to [0,1] using x /= 255 . You are not doing it in Javascript.

Answer 2

The preprocessing in js is different than the one in python.

In python the image is normalized by dividing it by 255

In Js the image is transformed to grayscale by computing the mean over the third axis (mean(2)). Here is what the tensor should be:

 const tensor = tf.browser.fromPixels(cnvs)
  .div(255)
  .toFloat()
  .expandDims(0)