如何在 Xamarin 應用程序（android）中將數據提供給 TesnorFlow lite model？

Question

我正在嘗試使用Xamarin開發人類活動識別應用程序。 I have developed a TensorFlow model and converted that to Tensorflow Lite and able to load the model to the app but not sure how to load the input data.

輸入數據：來自手機的加速度計和陀螺儀傳感器數據。 形狀： 128*9 。

我有 SensorModel class 如下

public class SensorModel
    {
        public double Acc_X { get; set; }
        public double Acc_Y { get; set; }
        public double Acc_Z { get; set; }
        public double Gyro_X { get; set; }
        public double Gyro_Y { get; set; }
        public double Gyro_Z { get; set; }
        public double Acc_Total_X { get; set; }
        public double Acc_Total_Y { get; set; }
        public double Acc_Total_Z { get; set; }
    }

我的分類器方法如下：

public List<ActivityModel> Classify(List<SensorModel> sensorData)
        {
            var assetDescriptor = Application.Context.Assets.OpenFd("converted_model.tflite");
            var inputStream = new FileInputStream(assetDescriptor.FileDescriptor);

            var mappedByteBuffer = inputStream.Channel.Map(FileChannel.MapMode.ReadOnly, assetDescriptor.StartOffset, assetDescriptor.DeclaredLength);
            var interpreter = new Xamarin.TensorFlow.Lite.Interpreter(mappedByteBuffer);

            //sensor data to byte array
            BinaryFormatter bf = new BinaryFormatter();
            MemoryStream ms = new MemoryStream();
            bf.Serialize(ms, sensorData);
            var input = ms.ToArray();

            //var sr = new StreamReader(Application.Context.Assets.Open("activity_labels.txt"));
            //var labels = sr.ReadToEnd().Split('\n').Select(s => s.Trim()).Where(s => !string.IsNullOrEmpty(s)).ToList();
            var labels = new List<string> { "1", "2", "3", "4", "5", "6" };

            var outputLocations = new float[labels.Count];
            var output = Java.Lang.Object.FromArray(outputLocations);

            try
            {
                interpreter.Run(input, output);
            }
            catch (Exception ex)
            {
                throw new Exception(ex.ToString());
            }
            var classificationResult = output.ToArray<float[]>();

            ////Map the classificationResult to the labels and sort the result to find which label has the highest probability
            var classificationModelList = new List<ActivityModel>();

            for (var i = 0; i < labels.Count; i++)
            {
                var label = labels[i]; classificationModelList.Add(new ActivityModel(label, classificationResult[0][i]));
            }

            return classificationModelList;
        }

我的輸入數據的大小應該是 128*9。 這是一個傳感器數據列表。 （類型浮點或雙）。

但是得到這個異常。

{Java.Lang.IllegalArgumentException: Cannot convert between a TensorFlowLite tensor with type FLOAT32 and a Java object of type [B (which is compatible with the TensorFlowLite type UINT8).
  at Java.Interop.JniEnvironment+InstanceMethods.CallVoidMethod (Java.Interop.JniObjectReference instance, Java.Interop.JniMethodInfo method, Java.Interop.JniArgumentValue* args) [0x0006e] in <dac4c5a4b77f4e61a5e6d9d3050dfb9f>:0 
  at Java.Interop.JniPeerMembers+JniInstanceMethods.InvokeAbstractVoidMethod (System.String encodedMember, Java.Interop.IJavaPeerable self, Java.Interop.JniArgumentValue* parameters) [0x00014] in <dac4c5a4b77f4e61a5e6d9d3050dfb9f>:0 
  at Xamarin.TensorFlow.Lite.Interpreter.Run (Java.Lang.Object input, Java.Lang.Object output) [0x00053] in <4ff04480b90243d7b71dc70e02d1dbfb>:0 
  at HAR_Test_2.ActivityClassifier.Classify (System.Collections.Generic.List`1[T] sensorData) [0x000cf] in C:\Users\Nandan\Documents\Visual Studio 2019\Projects\HAR_Test_2\HAR_Test_2\ActivityClassifier.cs:47 
  --- End of managed Java.Lang.IllegalArgumentException stack trace ---
java.lang.IllegalArgumentException: Cannot convert between a TensorFlowLite tensor with type FLOAT32 and a Java object of type [B (which is compatible with the TensorFlowLite type UINT8).
    at org.tensorflow.lite.Tensor.throwIfTypeIsIncompatible(Tensor.java:316)
    at org.tensorflow.lite.Tensor.getInputShapeIfDifferent(Tensor.java:218)
    at org.tensorflow.lite.NativeInterpreterWrapper.run(NativeInterpreterWrapper.java:137)
    at org.tensorflow.lite.Interpreter.runForMultipleInputsOutputs(Interpreter.java:311)
    at org.tensorflow.lite.Interpreter.run(Interpreter.java:272)
    at crc6455763083a08ecf13.MainActivity.n_onSensorChanged(Native Method)
    at crc6455763083a08ecf13.MainActivity.onSensorChanged(MainActivity.java:75)
    at android.hardware.SystemSensorManager$SensorEventQueue.dispatchSensorEvent(SystemSensorManager.java:833)
    at android.os.MessageQueue.nativePollOnce(Native Method)
    at android.os.MessageQueue.next(MessageQueue.java:326)
    at android.os.Looper.loop(Looper.java:160)
    at android.app.ActivityThread.main(ActivityThread.java:6669)
    at java.lang.reflect.Method.invoke(Native Method)
    at com.android.internal.os.RuntimeInit$MethodAndArgsCaller.run(RuntimeInit.java:493)
    at com.android.internal.os.ZygoteInit.main(ZygoteInit.java:858)
}
    base: {Java.Lang.RuntimeException}
    JniPeerMembers: {Android.Runtime.XAPeerMembers}

誰能建議我應該如何將數據提供給解釋器？

Answer 1

解決了這個問題。 我必須用 Array of float[] 為解釋器提供數據，但我傳遞的是 SensorModel 列表。 現在我將SensorModel 列表轉換為float[] ，它就像一個魅力。 我在這里添加代碼

public List<ActivityModel> Classify(List<SensorModel> sensorData)
        {
            var assetDescriptor = Application.Context.Assets.OpenFd("converted_model.tflite");
            var inputStream = new FileInputStream(assetDescriptor.FileDescriptor);

            var mappedByteBuffer = inputStream.Channel.Map(FileChannel.MapMode.ReadOnly, assetDescriptor.StartOffset, assetDescriptor.DeclaredLength);
            var interpreter = new Xamarin.TensorFlow.Lite.Interpreter(mappedByteBuffer);

            var InputArray = PrepareInputArray(sensorData).ToArray();
            var Input_tensor = InputArray.SelectMany(x => x).ToArray();

            var sr = new StreamReader(Application.Context.Assets.Open("activity_labels.txt"));
            var labels = sr.ReadToEnd().Split('\n').Select(s => s.Trim()).Where(s => !string.IsNullOrEmpty(s)).ToList();

            var outputLocations = new float[1][] { new float[labels.Count] };
            var outputs = Java.Lang.Object.FromArray(outputLocations);
            try
            {
                interpreter.Run(Input_tensor, outputs);
            }
            catch (Exception ex)
            {
                throw new Exception(ex.ToString());
            }
            var classificationResult = outputs.ToArray<float[]>();

            ////Map the classificationResult to the labels and sort the result to find which label has the highest probability
            var classificationModelList = new List<ActivityModel>();

            for (var i = 0; i < labels.Count; i++)
            {
                var label = labels[i]; classificationModelList.Add(new ActivityModel(label, classificationResult[0][i]));
            }

            return classificationModelList;
        }

        private List<List<float>> PrepareInputArray(List<SensorModel> sensorData)
        {
            List<List<float>> record = new List<List<float>>();

            sensorData.ForEach(el =>
            {
                record.Add(new List<float> { el.Acc_X, el.Acc_Y, el.Acc_Z, el.Gyro_X, el.Gyro_Y, el.Gyro_Z, el.Acc_Total_X, el.Acc_Total_Y, el.Acc_Total_Z });
            });
            return record;
        }

謝謝各位朋友。

Answer 2

我不熟悉 Xamarin 應用程序，因為代碼是 c# 並調用 Java。 從代碼來看，您似乎有多個輸入，因此需要runForMultipleInputsOutputs而不是run ：

https://www.tensorflow.org/lite/api_docs/java/org/tensorflow/lite/Interpreter#runForMultipleInputsOutputs(java.lang.Object[],%20java.util.Map%3Cjava.lang.Integer,%20java. lang.Object%3E)