SEGMENTATION FAULT when running inference on tflite_runtime converted model

Question

I am trying to convert and run a small keras model with tflite_runtime. Converting to tflite works and running inference with tf.lite also works well, however when using the interpreter from tflite_runtime.interpreter I get "segmentation fault: 11" and no other error messages. Any ideas on how to solve? I need this to run without tensorflow only with tflite-runtime

I am on macOS with: python 3.8.5 tensorflow 2.7.0 tflite_runtime 2.5.0

The model is for detecting hand poses from a set of landmarks:

Model: "sequential"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 dropout (Dropout)           (None, 42)                0         
                                                                 
 dense (Dense)               (None, 50)                2150      
                                                                 
 dropout_1 (Dropout)         (None, 50)                0         
                                                                 
 dense_1 (Dense)             (None, 50)                2550      
                                                                 
 dense_2 (Dense)             (None, 5)                 255       
                                                                 
=================================================================
Total params: 4,955
Trainable params: 4,955
Non-trainable params: 0
_________________________________________________________________

Code for converting:

saved_model_dir = './save_at_500.h5'
model = tf.keras.models.load_model(saved_model_dir)

df = pd.read_csv('./test_hand_data_2.csv')
gt = np.array([])
lmk = np.array([])
gt = np.append(gt, df['pose'].to_numpy()-1)
lmk = np.append(lmk, df.loc[:,'lx0':'ly20'].to_numpy())
lmk = np.reshape(lmk,(gt.shape[0],42))

def representative_dataset():
  for data in tf.data.Dataset.from_tensor_slices((lmk)).batch(1).take(100):
    yield [tf.dtypes.cast(data, tf.float32)]

converter = tf.lite.TFLiteConverter.from_keras_model(model)
converter.optimizations = [tf.lite.Optimize.DEFAULT]
converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]
converter.allow_custom_ops=True
converter.representative_dataset = representative_dataset
tflite_quant_model = converter.convert()
open( 'model.tflite' , 'wb' ).write(tflite_quant_model)

Code for running inference: (Note the commented out code that works)

import cv2
import numpy as np
import tflite_runtime.interpreter as tflite
#import tensorflow as tf
import pandas as pd
import os
from time import time



def main():
    model_path = os.path.join(path,'models/6-10/model.tflite')

    #interpreter = tf.lite.Interpreter(model_path) # this works!
    interpreter = tflite.Interpreter(model_path=model_path) # segmentation fault here

    interpreter.allocate_tensors()
    input_details = interpreter.get_input_details()
    output_details = interpreter.get_output_details()

    print('INPUT\n', input_details)
    print('\n OUTPUT\n',output_details)

    lmk,gt = get_data()
    input_data = [lmk[0]]
    print(input_data)

    interpreter.set_tensor(input_details[0]['index'], input_data)

     # Execute the inference
    t1=time()
    interpreter.invoke()
    t2=time()

    output_data = interpreter.get_tensor(output_details[0]['index'])
    print(output_data)
    print('Inference time:',t2-t1,'s')

if __name__ == "__main__":
    main()

Thanks!

Answer 1

Simply read from the website TFLite-Interpreter you need to create a correct target method.

Sample: I create a test and simple convert it to the TFLite by model.save and read its input/output for proving the save and recalling works.

import os
from os.path import exists

import tensorflow as tf
import tflite

import h5py

import matplotlib.pyplot as plt

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]
None
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
physical_devices = tf.config.experimental.list_physical_devices('GPU')
assert len(physical_devices) > 0, "Not enough GPU hardware devices available"
config = tf.config.experimental.set_memory_growth(physical_devices[0], True)
print(physical_devices)
print(config)

os.environ['TF_GPU_THREAD_MODE']='gpu_private'
os.environ['TF_GPU_THREAD_COUNT']='1'

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Variables
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
SEQUENCE_LENGTH = 1
IMAGE_HEIGHT = 32
IMAGE_WIDTH = 32
CHANNELS = 3

num_classes = 10

list_label_actual = [ 'Candidt Kibt', 'Pikaploy' ]

checkpoint_path = "F:\\models\\checkpoint\\" + os.path.basename(__file__).split('.')[0] + "\\TF_DataSets_01.h5"
checkpoint_dir = os.path.dirname(checkpoint_path)

database_buffer = "F:\\models\\buffer\\" + os.path.basename(__file__).split('.')[0] + "\\TF_DataSets_01.h5"
database_buffer_dir = os.path.dirname(database_buffer)

saved_model_path = "F:\\models\\checkpoint\\" + os.path.basename(__file__).split('.')[0] + "\\DekDee naja"
saved_model_dir = os.path.dirname(saved_model_path)

if not exists(checkpoint_dir) : 
    os.mkdir(checkpoint_dir)
    print("Create directory: " + checkpoint_dir)
    
if not exists(database_buffer_dir) : 
    os.mkdir(database_buffer_dir)
    print("Create directory: " + database_buffer_dir)
    
if not exists(saved_model_dir) : 
    os.mkdir(saved_model_dir)
    print("Create directory: " + saved_model_dir)

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Class
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
class MyLSTMLayer( tf.keras.layers.LSTM ):
    def __init__(self, units, return_sequences, return_state):
        super(MyLSTMLayer, self).__init__( units, return_sequences=return_sequences, return_state=return_state )
        self.num_units = units
        self.return_sequences = return_sequences
        self.return_state = return_state

    def build(self, input_shape):
        self.w = self.add_weight(
            shape=(input_shape[-1], self.units),
            initializer="random_normal",
            trainable=True,
        )
        self.b = self.add_weight(
            shape=(self.units,), initializer="random_normal", trainable=True
        )

    def call(self, inputs):
    
        return tf.matmul(inputs, self.w) + self.b

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Dataset
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""       
(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.cifar10.load_data()
# Create hdf5 file
hdf5_file = h5py.File(database_buffer, mode='w')

# Train images
hdf5_file['x_train'] = train_images
hdf5_file['y_train'] = train_labels

# Test images
hdf5_file['x_test'] = test_images
hdf5_file['y_test'] = test_labels

hdf5_file.close()

# Visualize dataset train sample
hdf5_file = h5py.File(database_buffer,  mode='r')

x_train = hdf5_file['x_train'][0: 10000]
x_test = hdf5_file['x_test'][0: 100]
y_train = hdf5_file['y_train'][0: 10000]
y_test = hdf5_file['y_test'][0: 100]
        
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Model Initialize
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
model = tf.keras.models.Sequential([
    tf.keras.layers.InputLayer(input_shape=( 32, 32, 3 ), name="input_01"),
    tf.keras.layers.Normalization(mean=3., variance=2., name="normal_01"),
    tf.keras.layers.Normalization(mean=4., variance=6., name="normal_02"),
    tf.keras.layers.Reshape((32 * 32, 3), name="reshape_01"),
    MyLSTMLayer(96, False, False),
    tf.keras.layers.Flatten(name="flattern_01"),
    tf.keras.layers.Dense(192, activation='relu', name="dense_02"),
    tf.keras.layers.Dense(10, name="dense_03"),
])

model.summary()

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Optimizer
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
optimizer = tf.keras.optimizers.Nadam(
    learning_rate=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-07,
    name='Nadam'
)
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Loss Fn
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""                               
lossfn = tf.keras.losses.SparseCategoricalCrossentropy(
    from_logits=False,
    reduction=tf.keras.losses.Reduction.AUTO,
    name='sparse_categorical_crossentropy'
)

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Model Summary
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
model.compile(optimizer=optimizer, loss=lossfn, metrics=['accuracy'])

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Model Training
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
history = model.fit( x_train, y_train, epochs=50, steps_per_epoch=1 )

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Model Save
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
model.save(
    saved_model_dir,
    overwrite=True,
    include_optimizer=True,
    save_format=None,
    signatures=None,
    options=None,
    save_traces=True
)

tf_lite_model_converter = tf.lite.TFLiteConverter.from_keras_model(
    model
)
tflite_model = tf_lite_model_converter.convert()

# Save the model.
with open( saved_model_dir + '\\model.tflite', 'wb' ) as f:
    f.write(tflite_model)
    
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Question
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
model_path = saved_model_dir + '\\model.tflite'


interpreter = tf.lite.Interpreter(model_path) # segmentation fault here

interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()

print('INPUT\n', input_details)
print('\n OUTPUT\n',output_details)

Output:

INFO: Created TensorFlow Lite XNNPACK delegate for CPU.
INPUT
 [{'name': 'serving_default_input_01:0', 'index': 0, 'shape': array([ 1, 32, 32,  3]), 'shape_signature': array([-1, 32, 32,  3]), 'dtype': <class 'numpy.float32'>, 'quantization': (0.0, 0), 'quantization_parameters': {'scales': array([], dtype=float32), 'zero_points': array([], dtype=int32), 'quantized_dimension': 0}, 'sparsity_parameters': {}}]

 OUTPUT
 [{'name': 'StatefulPartitionedCall:0', 'index': 16, 'shape': array([ 1, 10]), 'shape_signature': array([-1, 10]), 'dtype': <class 'numpy.float32'>, 'quantization': (0.0, 0), 'quantization_parameters': {'scales': array([], dtype=float32), 'zero_points': array([], dtype=int32), 'quantized_dimension': 0}, 'sparsity_parameters': {}}]

Answer 2

Had a similar issue on my Raspberry Pi 4. Pip installed tflite-runtime v2.5.0, and I have been making my models of TF v2.9.1. The latest version of tflite_runtime is 2.10.0.

So I specified its version when installing tflite_runtime.

sudo python3 -m  pip install tflite-runtime==2.10.0