Keras model.predict 给出不一致的值

Question

I have trained a basic classifier when tested on model.evaluate it produces the same metrics every time.

When using model.predict to check the validation data, I get different values each time the model.predict line is run. I can't figure out for me why this is happening?

No training is done between each 'model.predict(validation_data)' run.

# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""
import tensorflow as tf
import math 
import tensorflow_hub as hub
import numpy as np
import matplotlib.pyplot as plt
from tensorflow import keras
from tensorflow.keras import layers
from sklearn.metrics import roc_curve
from tensorflow.keras.preprocessing.image import ImageDataGenerator



train_examples = 425
test_examples = 245
validation_examples = 245
# train_examples = 20
# test_examples = 20
# validation_examples = 20
img_height = img_width = 224
batch_size = 32
epochs = 100

#create matrices to store training accuracy data for multiple epochs
val_store = []
test_store = []
train_store = []

#loop over n epochs to determine number of epochs required to avoid overfitting
#for epoch_tot in range(1,25):

#NasNet
model = keras.Sequential([
    hub.KerasLayer("https://tfhub.dev/google/imagenet/nasnet_mobile/feature_vector/4",
    trainable = True),
    layers.Dense(1, activation = "sigmoid"),
    ])

# model = keras.models.load_model('isic_model2/')

train_datagen = ImageDataGenerator(
    rescale = 1.0/255,
    rotation_range = 15,
    zoom_range = (0.95, 0.95),
    horizontal_flip = True,
    vertical_flip = True,
    data_format = "channels_last",
    dtype = tf.float32,
    )

validation_datagen = ImageDataGenerator(rescale=1.0/255, dtype=tf.float32)
test_datagen = ImageDataGenerator(rescale=1.0/255, dtype=tf.float32)

train_gen = train_datagen.flow_from_directory(
    "ClassifierData/Training/",
    target_size = (img_height, img_width),
    batch_size=batch_size,
    color_mode = "rgb",
    class_mode = "binary",
    shuffle = True,
    seed = 123,
    )

validation_gen = validation_datagen.flow_from_directory(
    "ClassifierData/Validation/",
    target_size = (img_height, img_width),
    batch_size=batch_size,
    color_mode = "rgb",
    class_mode = "binary",
    shuffle = True,
    seed = 123,
    )
    
test_gen = test_datagen.flow_from_directory(
    "ClassifierData/Test/",
    target_size = (img_height, img_width),
    batch_size=batch_size,
    color_mode = "rgb",
    class_mode = "binary",
    shuffle = True,
    seed = 123,
    )

METRICS = [
    keras.metrics.BinaryAccuracy(name="accuracy"),
    keras.metrics.Precision(name="precision"),
    keras.metrics.Recall(name="recall"),
    keras.metrics.AUC(name='auc'),
    ]
    
model.compile(
    optimizer = keras.optimizers.Adam(lr=3e-4),
    loss = [keras.losses.BinaryCrossentropy(from_logits=False)],
    metrics = METRICS,
    )

# model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
#     filepath = ("checkpoints1"),
#     monitor = 'val_auc',
#     save_freq = 'epoch',
#     verbose = 1,
#     )


history = model.fit(
    train_gen,
    epochs=epochs,
    verbose = 1,
    steps_per_epoch = train_examples // batch_size,
    validation_data=validation_gen,
    validation_steps=validation_examples // batch_size,
    callbacks = [keras.callbacks.ModelCheckpoint("isic_model4")]
    #callbacks = [model_checkpoint_callback],
    )

    
def plot_roc(labels, data):
    predictions = model.predict(data)
    fp, tp, _ = roc_curve(labels, predictions)
    
    plt.plot(100*fp, 100*tp)
    plt.xlabel("False positives [%]")
    plt.ylabel("True positives [%]")
    plt.show()
    
test_labels = np.array([])
num_batches = 0

for _, y in test_gen:
    test_labels = np.append(test_labels, y)
    num_batches += 1
    if num_batches == math.ceil(test_examples / batch_size):
        break

plot_roc(test_labels, test_gen)
val_eval = model.evaluate(validation_gen, verbose = 1)
test_eval = model.evaluate(test_gen, verbose=1)        
train_eval = model.evaluate(train_gen, verbose=1)


#plot auc against number of epochs
train_loss = history.history['loss']
val_loss   = history.history['val_loss']
train_acc  = history.history['auc']
val_acc    = history.history['val_auc']
train_prec = history.history['precision']
val_prec = history.history['val_precision']

xc         = range(epochs)
plt.figure()
plt.plot(xc, train_acc)
plt.plot(xc, val_acc)
plt.figure()
plt.plot(xc, train_loss)
plt.plot(xc, val_loss)

#Save important data to csv
import pandas
df = pandas.DataFrame(data={"train_loss": train_loss,
                            "val_loss": val_loss,
                            'train_acc': train_acc,
                            'val_acc' : val_acc,
                            'train_prec': train_prec,
                            'val_prec': val_prec,
                            'xc' : xc
                            })
df.to_csv("./accuracy.csv", sep=',',index=False)

val_predict1 = model.evaluate(validation_datagen, verbose = 1)
val_predict2 = model.evaluate(validation_datagen, verbose = 1)
y_true_labels = history.classes

Answer 1

try setting shuffle = False in validation_gen and test_gen. Not sure why evaluating gives the same answer but predict does not. Maybe evaluate resets the generator and predict doesn't. What you normally want is to go through the test samples or validation samples exactly once. Let's say you have 500 samples. You can set the batch size to say 50 and in the model predict set steps to 10. below is the code to determine the batch size and steps given the number of samples=length such that batch_size X steps = length. Note if the number of samples is a prime number, the batch size will be 1 and steps will equal length. Term b_max in the code below is an integer denoting the maximum batch size you will allow based on your memory capacity.

batch_size=sorted([int(length/n) for n in range(1,length+1) if length % n ==0 and length/n<=b_max],reverse=True)[0]  
steps=int(length/batch_size)

Using these values ensures you go through the samples once and the generator ends up back at the beginning.