Keras model training hanging on first epoch

Question

I have been trying to train a keras model but it keeps getting stuck at the start of the first epoch. The worst thing is that it is not throwing any errors. I am training on a GTX 1050TI

Below is a sample of my code:

import tensorflow as tf
import os

from tensorflow import keras
from keras_preprocessing.image import ImageDataGenerator
from keras_applications.xception import Xception


import matplotlib.pyplot as plt


train_dir='C:\\Users\\AYERHAN MSUGHTER\\Documents\\chest_xray\\train'
single='C:\\Users\\AYERHAN MSUGHTER\\Documents\\chest_xray\\train\\person_1_bacteria_1'
validation_dir='C:\\Users\\AYERHAN MSUGHTER\\Documents\\chest_xray\\validation'
test_dir='C:\\Users\\AYERHAN MSUGHTER\\Documents\\chest_xray\\test'
shape_size=71
training_batch_size=10
validation_batch_size=10
target_size=([71,71])

#this is the image data generator that will generate additonal data for us
train_datagen=ImageDataGenerator(rescale=1/255,rotation_range=5,width_shift_range=0.1,
                                 height_shift_range=0.05,shear_range=0.1,
                                 zoom_range=0.15,horizontal_flip=True,
                                 vertical_flip= False,
                                 fill_mode='reflect')

validation_test_datagen= ImageDataGenerator(rescale=1/255)

train_generator=train_datagen.flow_from_directory(directory=train_dir,classes=('NORMAL','PNEUMONIA'), batch_size=training_batch_size,target_size=target_size,class_mode='categorical',shuffle=True,seed=5566)



validation_generator=validation_test_datagen.flow_from_directory(directory=validation_dir,
                                                classes=('NORMAL','PNEUMONIA'),
                                                 target_size=target_size,
                                                      batch_size=training_batch_size,
                                                      class_mode='categorical',
                                                      shuffle=True,
                                                      seed=5566)

test_generator=validation_test_datagen.flow_from_directory(directory=train_dir,
                                                classes=['NORMAL','PNEUMONIA'],
                                                 target_size=(shape_size,shape_size),
                                                      batch_size=1,
                                                      class_mode='categorical',
                                                      shuffle=True,
                                                      seed=5566)
conv_base=tf.keras.applications.Xception(weights='imagenet',include_top=False,input_shape=(shape_size,shape_size,3))
for layer in conv_base.layers[:-4]:
    layer.trainable = False
for layer in conv_base.layers:
    print(layer,layer.trainable)


import os
layer_input=tf.keras.layers.InputLayer(input_shape=[shape_size,shape_size,3],dtype=tf.float32,name='input_layer')
layer_global_average_pooling_2d=tf.keras.layers.AveragePooling2D()
layer_dense1=tf.keras.layers.Dense(units=1024,activation='relu', name='fc1')
layer_dropout1=tf.keras.layers.Dropout(rate=0.3,name='dropout1')
layer_dense2=tf.keras.layers.Dense(units=512,activation='relu', name='fc2')
layer_dropout2=tf.keras.layers.Dropout(rate=0.3,name='dropout2')
layer_dense3=tf.keras.layers.Dense(units=2,activation='relu', name='fc3')

#this would be our model

model= tf.keras.Sequential([
    layer_input,
    conv_base,
    layer_global_average_pooling_2d,
    layer_dense1,
    layer_dropout1,
    layer_dense2,
    layer_dropout2,
    layer_dense3

         ])

training_step_size= (len(list(train_dir     )) / training_batch_size  )
validation_step_size= (len(list(validation_dir  )) / training_batch_size  )
print(validation_step_size)

weight_adjustment=len(list(os.path.join(train_dir, '//NORMAL//' ))) / len(list(os.path.join(train_dir, '/PNEUMONIA/', )))
model.compile(

  loss      = "binary_crossentropy"        ,

  optimizer = tf.keras.optimizers.RMSprop(lr=1e-5) ,

  metrics   = ["accuracy"]

)

history=model.fit_generator(

  train_generator                                      ,

  steps_per_epoch  = training_step_size             ,

 # class_weight     = [1,weight_adjustment],

  epochs           = 30                                ,

  validation_data  = validation_generator              ,

  validation_steps = 2,
    verbose=1
)

here is the output

"C:\Users\AYERHAN MSUGHTER\Anaconda3\python.exe" "C:/Users/AYERHAN MSUGHTER/PycharmProjects/test/test.py"
Found 0 images belonging to 2 classes.
Found 0 images belonging to 2 classes.
Found 0 images belonging to 2 classes.
WARNING:tensorflow:From C:\Users\AYERHAN MSUGHTER\Anaconda3\lib\site-packages\tensorflow\python\ops\init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
2019-11-29 22:16:25.367761: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX AVX2
2019-11-29 22:16:25.370526: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library nvcuda.dll
2019-11-29 22:16:26.168159: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1640] Found device 0 with properties: 
name: GeForce GTX 1050 Ti major: 6 minor: 1 memoryClockRate(GHz): 1.62
pciBusID: 0000:01:00.0
2019-11-29 22:16:26.168581: I tensorflow/stream_executor/platform/default/dlopen_checker_stub.cc:25] GPU libraries are statically linked, skip dlopen check.
2019-11-29 22:16:26.169445: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1763] Adding visible gpu devices: 0
2019-11-29 22:16:26.657806: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1181] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-11-29 22:16:26.658125: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1187]      0 
2019-11-29 22:16:26.658300: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1200] 0:   N 
2019-11-29 22:16:26.659042: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1326] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 3000 MB memory) -> physical GPU (device: 0, name: GeForce GTX 1050 Ti, pci bus id: 0000:01:00.0, compute capability: 6.1)
<tensorflow.python.keras.engine.input_layer.InputLayer object at 0x000001BCDBD9E748> False
<tensorflow.python.keras.layers.convolutional.Conv2D object at 0x000001BCDD8F5198> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDD8F5F60> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCD6911C50> False
<tensorflow.python.keras.layers.convolutional.Conv2D object at 0x000001BCD6911710> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCD69D96D8> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCD69D9DA0> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCD6A663C8> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCD6ABE208> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCD6AE6160> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCD6AF82B0> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCD6B719B0> False
<tensorflow.python.keras.layers.convolutional.Conv2D object at 0x000001BCD69E5240> False
<tensorflow.python.keras.layers.pooling.MaxPooling2D object at 0x000001BCDDE11198> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCD6A5B780> False
<tensorflow.python.keras.layers.merge.Add object at 0x000001BCDDE2FA58> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDDEA5828> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDDEA5860> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDDF28550> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDDF32DD8> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDDF49F60> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDDFBC978> False
<tensorflow.python.keras.layers.convolutional.Conv2D object at 0x000001BCDDE4E438> False
<tensorflow.python.keras.layers.pooling.MaxPooling2D object at 0x000001BCDDFCB160> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDDEA5048> False
<tensorflow.python.keras.layers.merge.Add object at 0x000001BCDDFEA0B8> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE05DDD8> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDE05D978> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDE0E0470> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE0E0BA8> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDE0FCDA0> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDE172860> False
<tensorflow.python.keras.layers.convolutional.Conv2D object at 0x000001BCDE009B38> False
<tensorflow.python.keras.layers.pooling.MaxPooling2D object at 0x000001BCDE17F208> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDE05D128> False
<tensorflow.python.keras.layers.merge.Add object at 0x000001BCDE1A46A0> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE1C2C88> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDE1EE7B8> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDE218550> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE21FE10> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDE230FD0> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDE2A8A90> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE2B5278> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDE2C2908> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDE33DEB8> False
<tensorflow.python.keras.layers.merge.Add object at 0x000001BCDE34F550> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE1C2588> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDE367F28> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDE3E6748> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE3EF0F0> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDE3FFC88> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDE474BA8> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE485390> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDE48EA20> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDE520048> False
<tensorflow.python.keras.layers.merge.Add object at 0x000001BCDE5206A0> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE355D30> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDE5368D0> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDE5B2828> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE5BD390> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDE5CAEB8> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDE640CC0> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE651438> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDE66D860> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDE6E6128> False
<tensorflow.python.keras.layers.merge.Add object at 0x000001BCDE6E6780> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE536F60> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDE723160> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDE77A940> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE787128> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDE794748> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDE80BDA0> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE81E588> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDE825C18> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDE8B6240> False
<tensorflow.python.keras.layers.merge.Add object at 0x000001BCDE8B6898> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE703F98> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDE8F13C8> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDE945A58> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE957240> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDE960898> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDE9D8EB8> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE9EB550> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDE9F1D30> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDEA81358> False
<tensorflow.python.keras.layers.merge.Add object at 0x000001BCDEA81A20> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDE8D1BE0> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDEABCFD0> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDEB15B70> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDEB23358> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDEB2B978> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDEBA7FD0> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDEBBD7B8> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDEBC1E10> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDEC50438> False
<tensorflow.python.keras.layers.merge.Add object at 0x000001BCDEC50B70> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDEA9BD30> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDEC86B38> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDECE2C88> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDECEE470> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDECF5AC8> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDFD58128> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDFD58780> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDFD5CEF0> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDFDE8588> False
<tensorflow.python.keras.layers.merge.Add object at 0x000001BCDFDF5518> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDEC6AB38> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDFE28DA0> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDFE7FDA0> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDFE8D588> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDFE94BE0> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDFF24240> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCDFF24898> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCDFF2AC18> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCDFFB7668> False
<tensorflow.python.keras.layers.merge.Add object at 0x000001BCDFFC4D30> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCE005D278> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCE005D978> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCE00E0710> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCE00EF3C8> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCE00FDE80> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCE0173B70> False
<tensorflow.python.keras.layers.convolutional.Conv2D object at 0x000001BCDFFD6550> False
<tensorflow.python.keras.layers.pooling.MaxPooling2D object at 0x000001BCE0186358> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCE004D9B0> False
<tensorflow.python.keras.layers.merge.Add object at 0x000001BCE01A57F0> False
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCE01D6518> False
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCE021CE48> False
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCE0227780> True
<tensorflow.python.keras.layers.convolutional.SeparableConv2D object at 0x000001BCE0230E10> True
<tensorflow.python.keras.layers.normalization.BatchNormalization object at 0x000001BCE02BDEF0> True
<tensorflow.python.keras.layers.core.Activation object at 0x000001BCE02D2588> True
5.7
Epoch 1/30

Answer 1

It looks like the generator is worng since they display 0 images found. You should also set shuffle = False for validation and test generators, otherwise the order of the labels won't match the images.

The number of steps for training and validation should be integers, use int() to change it or round up/down.