CNN prediction using tensorflow

Question

I'm quite new to python and tensorflow, but already managed to build, train and validate a CNN with my own database of images saved as tf.records. Now I want the model to read in a single picture and predict in real-time. Therefore, I wanted to modify my validation script by getting rid of the parser (which decoded my images saved as tf.records) and didn't batch the input images, since I only want to predict one. Somehow I always get the following Error:

TypeError: Value passed to parameter 'input' has DataType uint8 not in list of allowed values: float16, bfloat16, float32, float64

I took a closer look at the script I used to create the tf.records and compared them to the parser I used in the scripts for training and validation, but wasn't able to find the mistake.

I would be thankful, if you could help me to find the mistake or show me an easier way to predict the classes with an already trained CNN.

import tensorflow as tf
import cv2

num_classes = 2
crop_top = 5
crop_bottom = 10
crop_sides = 5
img_size_height = 80
img_size_width = 100
model_dir = "./2cv_128fc"


def load_image():
    img = cv2.imread('./dir_pred_img/img_2.jpg')
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    img = cv2.resize(img, (img_size_width + (2 * crop_sides), img_size_height + crop_top + crop_bottom),
                     interpolation=cv2.INTER_CUBIC)
    img = img[crop_top:(img_size_height + crop_top), crop_sides:(img_size_width + crop_sides)]
    features = {'image': img}
    return features


def conv_nn(input_layer):
    conv_1 = tf.layers.conv2d(inputs=input_layer, name='conv_layer_1', filters=32, kernel_size=3, padding='same',
                              activation=tf.nn.relu)

    pool_1 = tf.layers.max_pooling2d(inputs=conv_1, pool_size=2, strides=2)
    conv_2 = tf.layers.conv2d(inputs=pool_1, name='conv_layer_2', filters=32, kernel_size=3, padding='same',
                              activation=tf.nn.relu)

    pool_2 = tf.layers.max_pooling2d(inputs=conv_2, pool_size=2, strides=2)
    flatten = tf.contrib.layers.flatten(pool_2)
    fc_layer = tf.layers.dense(inputs=flatten, name='fully_connected_layer', units=128, activation=tf.nn.relu)
    fc_layer = tf.layers.dropout(fc_layer, rate=0.5, noise_shape=None, seed=None)
    output_layer = tf.layers.dense(inputs=fc_layer, name='output_layer', units=num_classes)
    return output_layer


def model_fn(features):
    input_layer = features["image"]
    input_layer = tf.identity(input_layer, name="input_tensor")
    input_layer = tf.reshape(input_layer, [-1, img_size_height, img_size_width, 1])  # 1.tensor 2.shape
    input_layer = tf.identity(input_layer, name="input_tensor_reshaped")

    logits = conv_nn(input_layer)

    pred = tf.nn.softmax(logits=logits)
    return pred


model = tf.estimator.Estimator(model_fn=model_fn, model_dir=model_dir)
prediction = list(model.predict(input_fn=load_image))
print(prediction[0])

full error message:

WARNING:tensorflow:Input graph does not use tf.data.Dataset or contain a QueueRunner. That means predict yields forever. This is probably a mistake.
Traceback (most recent call last):
  File "C:/Users/Dell/PycharmProjects/create_data/pred_img.py", line 54, in <module>
    prediction = list(model.predict(input_fn=load_image))
  File "C:\Users\Dell\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\estimator\estimator.py", line 577, in predict
    features, None, model_fn_lib.ModeKeys.PREDICT, self.config)
  File "C:\Users\Dell\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\estimator\estimator.py", line 1195, in _call_model_fn
    model_fn_results = self._model_fn(features=features, **kwargs)
  File "C:/Users/Dell/PycharmProjects/create_data/pred_img.py", line 47, in model_fn
    logits = conv_nn(input_layer)
  File "C:/Users/Dell/PycharmProjects/create_data/pred_img.py", line 27, in conv_nn
    activation=tf.nn.relu)
  File "C:\Users\Dell\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\layers\convolutional.py", line 417, in conv2d
    return layer.apply(inputs)
  File "C:\Users\Dell\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 817, in apply
    return self.__call__(inputs, *args, **kwargs)
  File "C:\Users\Dell\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\layers\base.py", line 374, in __call__
    outputs = super(Layer, self).__call__(inputs, *args, **kwargs)
  File "C:\Users\Dell\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 757, in __call__
    outputs = self.call(inputs, *args, **kwargs)
  File "C:\Users\Dell\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\layers\convolutional.py", line 194, in call
    outputs = self._convolution_op(inputs, self.kernel)
  File "C:\Users\Dell\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\ops\nn_ops.py", line 868, in __call__
    return self.conv_op(inp, filter)
  File "C:\Users\Dell\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\ops\nn_ops.py", line 520, in __call__
    return self.call(inp, filter)
  File "C:\Users\Dell\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\ops\nn_ops.py", line 204, in __call__
    name=self.name)
  File "C:\Users\Dell\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\ops\gen_nn_ops.py", line 1043, in conv2d
    data_format=data_format, dilations=dilations, name=name)
  File "C:\Users\Dell\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 609, in _apply_op_helper
    param_name=input_name)
  File "C:\Users\Dell\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 60, in _SatisfiesTypeConstraint
    ", ".join(dtypes.as_dtype(x).name for x in allowed_list)))
TypeError: Value passed to parameter 'input' has DataType uint8 not in list of allowed values: float16, bfloat16, float32, float64

Answer 1

The following code descirbes how you can implement this with keras.

import keras
from keras.layers import Input, Convolution2D, MaxPooling2D, Cropping2D, Dense, Flatten, Dropout
from keras.preprocessing import image
from keras.models import Model
from keras.optimizers import Adam
import cv2
import numpy as np

# parameters
num_classes = 2
crop_top = 5
crop_bottom = 10
crop_sides = 5
img_size_height = 80
img_size_width = 100
channels = 3
input_shape = (img_size_height, img_size_width, channels)
activation = 'relu'
learning_rate = 0.0001

if num_classes == 2:
    loss = 'binary_crossentropy'
else:
    loss = 'categorical_crossentropy'


test_image = image.load_img('./data/img.png')
test_image = image.img_to_array(test_image)
input_shape = test_image.shape

def model(input_shape=input_shape):

    inputs = Input(shape=input_shape)

   # cropping=((pixels_from_top, pixels_from_bottom), (pixels_from_left, pixels_from_right))
    cropping = Cropping2D(cropping=((crop_top, crop_bottom), (crop_sides, crop_sides)))(inputs)
    conv_1 = Convolution2D(32, kernel_size=(3, 3), padding='same', activation=activation)(cropping)
    pool_1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(conv_1)
    conv_2 = Convolution2D(32, kernel_size=(3, 3), padding='same', activation=activation)(pool_1)
    pool_2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(conv_2)

    flatten = Flatten()(pool_2)
    dense_1 = Dense(128, activation=activation, name='fully_connected_layer')(flatten)
    dropout = Dropout(0.5)(dense_1)

    outputs = Dense(num_classes, activation='softmax')(dropout)

    model = Model(inputs=inputs, outputs=outputs)

    adam = Adam(lr=learning_rate)
    model.compile(optimizer=adam, loss=loss, metrics=['acc', 'mse', 'mae'])
    model.summary()

    return model

test_image = np.expand_dims(test_image, axis=0)
model = model(input_shape)
# note that without training we will only get random results
prediction = model.predict_on_batch(test_image)
print(prediction)

Answer 2

I found a solution. I added following line to the end of def load_image() and returned dataset instead of feautures.

dataset = tf.data.Dataset.from_tensors(features)
return dataset