Categorize a not trained object as unknown object in machine learning using python

Question

I'm new in machine learning. Currently I'm working on a fruit/vegetable classification project using deep learning with keras. I was able to train the model. So far, it can detect already trained objects correctly but when I'm providing an other fruit/vegetable which was not trained, its predicts from the trained model whereas, It should recognize it as unknown fruit.

Suppose, I trained a model with Apple, Potato and Banana images. As long as I provide the images of Apple/Potato/Banana its predicts correctly. But the moment I provide a image of an Orange, it predicts as an Potato or lemon predicts as an Apple. Here is the code snippet that I'm using to train & predicts fruits:

import os.path
import numpy as np
np.random.seed(123)
from keras import applications
from keras.models import Sequential
from keras.layers import Dropout, Flatten, Dense
from keras.utils.np_utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator, img_to_array, load_img
from keras.layers.normalization import BatchNormalization
import matplotlib.pyplot as plt
import math
import cv2   

#dimensions of images
img_width, img_height = 224, 224  

#file paths & directories
top_model_weights_path = 'bottleneck_fc_model.h5'
bottleneck_train_path = 'bottleneck_features_train.npy'
bottleneck_validation_path = 'bottleneck_features_validation.npy'
train_data_dir = 'data/train'
validation_data_dir = 'data/validation/'

#hyperparameters
epochs = 10
batch_size = 16


def save_bottleneck_features():
    model = applications.VGG16(include_top=False, weights='imagenet')

    datagen = ImageDataGenerator(rescale=1./255,
                                 shear_range=0.2,
                                 zoom_range=0.2,
                                 horizontal_flip=True)

    generator = datagen.flow_from_directory(train_data_dir,
                                            target_size=(img_width, img_height),
                                            batch_size=batch_size,
                                            shuffle=False)

    no_train_samples = len(generator.filenames)

    predict_size_train = int(math.ceil(no_train_samples / batch_size))
    bottleneck_features_train = model.predict_generator(generator, predict_size_train)
    np.save(bottleneck_train_path, bottleneck_features_train)

    datagen = ImageDataGenerator(rescale=1./255)

    generator = datagen.flow_from_directory(validation_data_dir,
                                            target_size=(img_width, img_height),
                                            batch_size=batch_size,
                                            class_mode=None,
                                            shuffle=False)

    no_validation_samples = len(generator.filenames)

    predict_size_validation = int(math.ceil(no_validation_samples / batch_size))
    bottleneck_features_validation = model.predict_generator(generator, predict_size_validation)
    np.save(bottleneck_validation_path, bottleneck_features_validation)



def train_top_model():
    datagen_top = ImageDataGenerator(rescale=1./255)

    generator_top = datagen_top.flow_from_directory(train_data_dir, 
                                                    target_size=(img_width, img_height),
                                                    batch_size=batch_size,
                                                    class_mode='categorical',
                                                    shuffle=False)

    num_classes = len(generator_top.class_indices)

    # save the class indices to use later in predictions
    np.save('class_indices.npy', generator_top.class_indices)

    # get the class labels for the training data, in the original order
    train_labels = generator_top.classes
    # convert the training labels to categorical vectors
    train_labels = to_categorical(train_labels, num_classes=num_classes)

    generator_top = datagen_top.flow_from_directory(validation_data_dir,
                                                    target_size=(img_width, img_height),
                                                    batch_size=batch_size,
                                                    class_mode=None,
                                                    shuffle=False)

    validation_labels = generator_top.classes
    validation_labels = to_categorical(validation_labels, num_classes=num_classes)

    # load the bottleneck features saved earlier
    train_data = np.load('bottleneck_features_train.npy')
    validation_data = np.load('bottleneck_features_validation.npy')


    # build the model
    model = Sequential()
    model.add(Flatten(input_shape=train_data.shape[1:]))
    model.add(BatchNormalization())
    model.add(Dense(256, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(num_classes, activation='softmax'))

    model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])

    history = model.fit(train_data, train_labels, epochs=epochs, batch_size=batch_size)

    model.save_weights(top_model_weights_path)


def predict(image_path):
    class_dictionary = np.load('class_indices.npy').item()
    #print("Values: ",class_dictionary)

    num_classes = len(class_dictionary)

    orig = cv2.imread(image_path)

    print('[INFO] loading and preprocessing image...')
    image = load_img(image_path, target_size=(224, 224))
    image = img_to_array(image)
    image = image / 255
    image = np.expand_dims(image, axis=0)

    model = applications.VGG16(include_top=False, weights='imagenet')

    bottleneck_prediction = model.predict(image)

    # build top model
    model = Sequential()
    model.add(Flatten(input_shape=bottleneck_prediction.shape[1:]))
    model.add(BatchNormalization())
    model.add(Dense(256, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(num_classes, activation='sigmoid'))

    model.load_weights(top_model_weights_path)

    # use the bottleneck prediction on the top model to get the final classification
    class_predicted = model.predict_classes(bottleneck_prediction)

    probabilities = model.predict_proba(bottleneck_prediction)

    inID = class_predicted[0]

    inv_map = {v: k for k, v in class_dictionary.items()}

    label = inv_map[inID]

    print("Image ID: {}, Label: {}".format(inID, label))

    cv2.putText(orig, "Predicted: {}".format(label), (10, 30),
                cv2.FONT_HERSHEY_PLAIN, 1.5, (0, 0, 0), 2)

    cv2.imshow('Classification', orig)
    cv2.imwrite('predicted.jpg', orig)
    cv2.waitKey(0)


if __name__ == '__main__': 
    if not os.path.exists(bottleneck_train_path):
        save_bottleneck_features()
    if not os.path.exists(top_model_weights_path):
        train_top_model()

    image_path = 'fruits/orange.jpg'  
    predict(image_path)

How to overcome this situation? Any help will be appreciated.

Answer 1

When training with loss='categorical_crossentropy' , and activation='softmax' at the last layer, each class is represented by 1 node in the graph at the last layer. softmax ensures the value of all the nodes is summed to 1, by normalizing all the values.
categorial crossentropy assumes the node with the highest value is the predicted class, and compares it with the labels.

For example, after predicting a sample you might can have the following values for nodes: 0.33 banana, 0.33 apple, 0.34 potato , thus potato will be the class selected for the prediction.

If you wish to have an 'unknown class' prediction, you should add it as another possible class, and have some samples labeled as unknown class during training.

Edit:
Note that while what i suggested above can work for you, training it can be very difficulty, as it might require a larger number of samples, both known and unkown, as the 'unkown' features are harder to generalize, and might damage the other classes generalization.

A more practical way would be to define a threshold, as i mentioned earlier, after predicting you recieve some value for each one of the possible labels. When picking the label, this value is the 'confidence' score of the label.
Examining the example above, we will pick potato with 0.34 confidence.
You could define some threshold, say 0.6, and when assuming the label for the image, check the different confidence score, if the highest (the 'chosen') label is lower then the threshold, you can mark it as 'unknown fruit'.

Another Edit
Several good works on the subject as come up since that question, for future reference, this: https://github.com/hendrycks/outlier-exposure

would be a good place to start.