MLKit Object Detection is not detecting objects

Question

MLKit by Google (without Firebase) is new, so I'm having trouble. I'm trying to follow this example here: https://developers.google.com/ml-kit/vision/object-detection/custom-models/android

The app opens fine, & the camera works (As in, I can see things). But the actual detection doesn't seem to work.

Am I missing part of the code to actually detect the object? Or is it an issue with the implementation of CameraX or ImageInput?

package com.example.mlkitobjecttest;

import androidx.annotation.NonNull;
import androidx.appcompat.app.AppCompatActivity;
import androidx.camera.core.Camera;
import androidx.camera.core.CameraSelector;
import androidx.camera.core.CameraX;
import androidx.camera.core.ImageAnalysis;
import androidx.camera.core.ImageProxy;
import androidx.camera.core.Preview;
import androidx.camera.core.impl.PreviewConfig;
import androidx.camera.lifecycle.ProcessCameraProvider;
import androidx.camera.view.PreviewView;
import androidx.core.app.ActivityCompat;
import androidx.core.content.ContextCompat;
import androidx.lifecycle.LifecycleOwner;

import android.content.pm.PackageManager;
import android.graphics.Rect;
import android.media.Image;
import android.os.Bundle;
import android.text.Layout;
import android.util.Rational;
import android.util.Size;
import android.view.View;
import android.widget.TextView;
import android.widget.Toast;

import com.google.android.gms.tasks.OnFailureListener;
import com.google.android.gms.tasks.OnSuccessListener;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.mlkit.common.model.LocalModel;
import com.google.mlkit.vision.common.InputImage;
import com.google.mlkit.vision.objects.DetectedObject;
import com.google.mlkit.vision.objects.ObjectDetection;
import com.google.mlkit.vision.objects.ObjectDetector;
import com.google.mlkit.vision.objects.custom.CustomObjectDetectorOptions;

import org.w3c.dom.Text;

import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class MainActivity extends AppCompatActivity {

    private class YourAnalyzer implements ImageAnalysis.Analyzer {

        @Override
        @androidx.camera.core.ExperimentalGetImage
        public void analyze(ImageProxy imageProxy) {

            Image mediaImage = imageProxy.getImage();
            if (mediaImage != null) {
                InputImage image =
                        InputImage.fromMediaImage(mediaImage, imageProxy.getImageInfo().getRotationDegrees());
                // Pass image to an ML Kit Vision API
                // ...
                LocalModel localModel =
                        new LocalModel.Builder()
                                .setAssetFilePath("mobilenet_v1_1.0_128_quantized_1_default_1.tflite")
                                // or .setAbsoluteFilePath(absolute file path to tflite model)
                                .build();

                CustomObjectDetectorOptions customObjectDetectorOptions =
                        new CustomObjectDetectorOptions.Builder(localModel)
                                .setDetectorMode(CustomObjectDetectorOptions.SINGLE_IMAGE_MODE)
                                .enableMultipleObjects()
                                .enableClassification()
                                .setClassificationConfidenceThreshold(0.5f)
                                .setMaxPerObjectLabelCount(3)
                                .build();

                ObjectDetector objectDetector =
                        ObjectDetection.getClient(customObjectDetectorOptions);

                objectDetector
                        .process(image)
                        .addOnFailureListener(new OnFailureListener() {
                            @Override
                            public void onFailure(@NonNull Exception e) {
                                //Toast.makeText(getApplicationContext(), "Fail. Sad!", Toast.LENGTH_SHORT).show();
                                //textView.setText("Fail. Sad!");
                                imageProxy.close();
                            }
                        })
                        .addOnSuccessListener(new OnSuccessListener<List<DetectedObject>>() {
                            @Override
                            public void onSuccess(List<DetectedObject> results) {

                                for (DetectedObject detectedObject : results) {
                                    Rect box = detectedObject.getBoundingBox();


                                    for (DetectedObject.Label label : detectedObject.getLabels()) {
                                        String text = label.getText();
                                        int index = label.getIndex();
                                        float confidence = label.getConfidence();
                                        textView.setText(text);
                                        


                                }}
                                imageProxy.close();
                            }
                        });

            }
            //ImageAnalysis.Builder.fromConfig(new ImageAnalysisConfig).setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST);

        }

    }


    PreviewView prevView;
    private ListenableFuture<ProcessCameraProvider> cameraProviderFuture;
    private ExecutorService executor = Executors.newSingleThreadExecutor();
    TextView textView;

    private int REQUEST_CODE_PERMISSIONS = 101;
    private String[] REQUIRED_PERMISSIONS = new String[]{"android.permission.CAMERA"};
   /* @NonNull
    @Override
    public CameraXConfig getCameraXConfig() {
        return CameraXConfig.Builder.fromConfig(Camera2Config.defaultConfig())
                .setCameraExecutor(ContextCompat.getMainExecutor(this))
                .build();
    }
*/
    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);

        prevView = findViewById(R.id.viewFinder);
        textView = findViewById(R.id.scan_button);

        if(allPermissionsGranted()){
            startCamera();
        }else{
            ActivityCompat.requestPermissions(this, REQUIRED_PERMISSIONS, REQUEST_CODE_PERMISSIONS);
        }

    }

    private void startCamera() {
        cameraProviderFuture = ProcessCameraProvider.getInstance(this);
        cameraProviderFuture.addListener(new Runnable() {
            @Override
            public void run() {
                try {
                    ProcessCameraProvider cameraProvider = cameraProviderFuture.get();
                    bindPreview(cameraProvider);
                } catch (ExecutionException | InterruptedException e) {
                    // No errors need to be handled for this Future.
                    // This should never be reached.
                }
            }
        }, ContextCompat.getMainExecutor(this));


    }

    void bindPreview(@NonNull ProcessCameraProvider cameraProvider) {

        Preview preview = new Preview.Builder()
                .build();

        CameraSelector cameraSelector = new CameraSelector.Builder()
                .requireLensFacing(CameraSelector.LENS_FACING_BACK)
                .build();

        preview.setSurfaceProvider(prevView.createSurfaceProvider());

        ImageAnalysis imageAnalysis =
                new ImageAnalysis.Builder()
                        .setTargetResolution(new Size(1280, 720))
                        .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
                        .build();
        imageAnalysis.setAnalyzer(ContextCompat.getMainExecutor(this), new YourAnalyzer());

        Camera camera = cameraProvider.bindToLifecycle((LifecycleOwner)this, cameraSelector, preview, imageAnalysis);


    }



    private boolean allPermissionsGranted() {
        for(String permission: REQUIRED_PERMISSIONS){
            if(ContextCompat.checkSelfPermission(this, permission) != PackageManager.PERMISSION_GRANTED){
                return false;
            }
        }
        return true;
    }

    @Override
    public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {

        if(requestCode == REQUEST_CODE_PERMISSIONS){
            if(allPermissionsGranted()){
                startCamera();
            } else{
                Toast.makeText(this, "Permissions not granted by the user.", Toast.LENGTH_SHORT).show();
                this.finish();
            }
        }
    }

}

Answer 1

Nothing is detected because you defined the wrong path to tflite model file. You emulator or physical device cannot resolve given path as it doesn't exists on mobile device: C:\\Users\\dude\\Documents\\mlkitobjecttest\\app\\src\\main\\assets\\mobilenet_v1_1.0_128_quantized_1_default_1.tflite

Copy your model mobilenet_v1_1.0_128_quantized_1_default_1.tflite into assets directory under you app's project src/main directory.

If you do not have that directory just create a new one named assets .

At the end it should look like this:

After that fix LocalModel initialization code:

LocalModel localModel =
    new LocalModel.Builder()
    .setAssetFilePath("mobilenet_v1_1.0_128_quantized_1_default_1.tflite")
    // or .setAbsoluteFilePath(absolute file path to tflite model)
    .build();

Update: one more issue found

ImageAnalysis instance was not bound to CameraProvider :

...
ImageAnalysis imageAnalysis = ...
    
Camera camera = cameraProvider.bindToLifecycle((LifecycleOwner)this, cameraSelector, preview); // imageAnalysis is not used

To fix it just pass as the last argument imageAnalysis variable into bindToLifecycle method:

Camera camera = cameraProvider.bindToLifecycle((LifecycleOwner)this, cameraSelector, preview, imageAnalysis);

Second update: another one issue found

MLKit cannot process an image because it was closed while it was processing or right before processing started. I'm talking about imageProxy.close() line of code declared inside of public void analyze(ImageProxy imageProxy) .

Java documentation of close() method:

/**
 * Free up this frame for reuse.
 * <p>
 * After calling this method, calling any methods on this {@code Image} will
 * result in an {@link IllegalStateException}, and attempting to read from
 * or write to {@link ByteBuffer ByteBuffers} returned by an earlier
 * {@link Plane#getBuffer} call will have undefined behavior. If the image
 * was obtained from {@link ImageWriter} via
 * {@link ImageWriter#dequeueInputImage()}, after calling this method, any
 * image data filled by the application will be lost and the image will be
 * returned to {@link ImageWriter} for reuse. Images given to
 * {@link ImageWriter#queueInputImage queueInputImage()} are automatically
 * closed.
 * </p>
 */

To fix that move imageProxy.close() into failure and success listeners:

objectDetector
    .process(image)
    .addOnFailureListener(new OnFailureListener() {
        @Override
        public void onFailure(@NonNull Exception e) {
            Toast.makeText(getApplicationContext(), "Fail. Sad!", Toast.LENGTH_LONG).show();
            ...
            imageProxy.close();
        }
    })
    .addOnSuccessListener(new OnSuccessListener<List<DetectedObject>>() {
        @Override
        public void onSuccess(List<DetectedObject> results) {
            Toast.makeText(getBaseContext(), "Success...", Toast.LENGTH_LONG).show();
            ...
            imageProxy.close();
        }
    });

The fixed solution was tested with image classification model from Tensorflow and test was successful.