Transfer learning using keras

Question

I am trying to classify the medical images taken from publicly available datasets. I used transfer learning for this task. Initially, when I had used the following code on the same dataset with VGG, Resnet, Densenet, and Inception, the accuracy was above 85% without fine-tuning (TensorFlow version 1.15.2.) Now, after upgrading the TensorFlow to 2.x, when I try the same code on the same dataset, the accuracy is never crossing 32%. Can anyone please help me rectify the issue? Is it something to do with the TensorFlow version or something else? I have tried varying the learning rate, fine-tuning the model, etc. Is this the issue with batch normalization error in Keras?

import os
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "2"
import sys

import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow.keras.applications.resnet50 import ResNet50, preprocess_input
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.applications.densenet import DenseNet121
from tensorflow.keras.applications.vgg16 import VGG16
from tensorflow.keras.applications.inception_v3 import InceptionV3
import cv2
import glob
from tensorflow.keras.layers import Input, Dense, Flatten, Conv2D, MaxPooling2D, Dropout
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
import tensorflow.keras as keras
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping
import numpy as np
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from keras import backend as k
from mlxtend.evaluate import confusion_matrix
from mlxtend.plotting import plot_confusion_matrix
import math
import pandas as pd
from openpyxl import load_workbook


save_dir = '../new_tran/'

def extract_outputs(cnf_matrix, mdl_name):
  cnf_matrix_T=np.transpose(cnf_matrix)
  recall = np.diag(cnf_matrix) / np.sum(cnf_matrix, axis = 1)
  precision = np.diag(cnf_matrix) / np.sum(cnf_matrix, axis = 0)
  n_class=3
  TP=np.zeros(n_class)
  FN=np.zeros(n_class)
  FP=np.zeros(n_class)
  TN=np.zeros(n_class)
  for i in range(n_class):
    TP[i]=cnf_matrix[i,i]
    FN[i]=np.sum(cnf_matrix[i])-cnf_matrix[i,i]
    FP[i]=np.sum(cnf_matrix_T[i])-cnf_matrix[i,i]
    TN[i]=np.sum(cnf_matrix)-TP[i]-FP[i]-FN[i]
  P=TP+FN
  N=FP+TN
  classwise_sensitivity=np.true_divide(TP,P)
  classwise_specificity=np.true_divide(TN,N)
  classwise_accuracy=np.true_divide((TP+TN), (P+N))
  OS=np.mean(classwise_sensitivity)
  OSp=np.mean(classwise_specificity)
  OA=np.sum(np.true_divide(TP,(P+N)))
  Px=np.sum(P)
  TPx=np.sum(TP)
  FPx=np.sum(FP)
  TNx=np.sum(TN)
  FNx=np.sum(FN)
  Nx=np.sum(N)
  pox=OA
  pex=((Px*(TPx+FPx))+(Nx*(FNx+TNx)))/(math.pow((TPx+TNx+FPx+FNx),2))
  kappa_overall=[np.true_divide(( pox-pex ), ( 1-pex )),np.true_divide(( pex-pox ), ( 1-pox ))]
  kappa=np.max(kappa_overall)
  Rcl=np.mean(recall)
  Prcn=np.mean(precision)
  #######--------------------- Print all scores
  print('classwise_sen',classwise_sensitivity*100)
  print('classwise_spec',classwise_specificity*100)
  print('classwise_acc',classwise_accuracy*100)
  print('overall_sen',OS*100)
  print('overall_spec',OSp*100)
  print('overall_acc',OA*100)
  print('overall recall', Rcl)
  print('overall precision',Prcn)
  f1score=(2 * Prcn * Rcl) / (Prcn + Rcl)
  print('overall F1-score',f1score )
  print('Kappa',kappa)
 

def preProcess(X):
    X = X.astype('float32')
    # scale from [0,255] to [-1,1]
    X = (X - 127.5) / 127.5
    return X

train_datagen = ImageDataGenerator(preprocessing_function=preProcess)
test_datagen = ImageDataGenerator(preprocessing_function=preProcess)

IMG_SIZE = 256
batch_size = 16
train_data_dir = '../data/train/'
test_dir = '../data/test/'
val_dir = '../data/val/'
train_generator = train_datagen.flow_from_directory(
    train_data_dir,
    target_size=(IMG_SIZE , IMG_SIZE),
    batch_size=16,
    class_mode='sparse')
valid_generator = train_datagen.flow_from_directory(
    val_dir,
    target_size=(IMG_SIZE , IMG_SIZE),
    batch_size=16,
    class_mode='sparse')
test_generator = train_datagen.flow_from_directory(
    test_dir,
    target_size=(IMG_SIZE , IMG_SIZE),
    batch_size=16,
    class_mode='sparse')


test_im=np.concatenate([test_generator.next()[0] for i in range(test_generator.__len__())])
test_lb=np.concatenate([test_generator.next()[1] for i in range(test_generator.__len__())])

t_x, t_y = next(train_generator)

checkpoint1 = ModelCheckpoint(save_dir+"best_res.hdf5", monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
checkpoint2  = EarlyStopping(monitor='val_loss', patience=10, verbose=0, mode='min')
callbacks_list1 = [checkpoint1, checkpoint2]



def new_model():
    img_in = Input(t_x.shape[1:])             
    model = DenseNet121(include_top= False , 
                layers=tf.keras.layers, 
                weights='imagenet',      
                input_tensor= img_in, 
                input_shape= t_x.shape[1:],
                pooling ='avg') 
    x = model.output  
    predictions = Dense(3, activation="softmax", name="predictions")(x)    
    model = Model(inputs=img_in, outputs=predictions)
    return model

model1 = new_model()
opt = Adam(lr=3E-4)
model1.compile(optimizer = opt, loss = 'sparse_categorical_crossentropy',metrics = ['accuracy'])

history1 = model1.fit(train_generator, 
                                  validation_data = valid_generator, 
                                  epochs = 200,
                                  callbacks=callbacks_list1)
model1.load_weights(save_dir+'best_res.hdf5')
model1.compile(optimizer = opt, loss = 'sparse_categorical_crossentropy',metrics = ['accuracy'])

y_pred1 = model1.predict(test_im)
pred_class1=np.argmax(y_pred1,axis=1)
print('accuracy = ',accuracy_score(pred_class1,test_lb))
cm = confusion_matrix(y_target=test_lb,y_predicted=pred_class1, binary=False)
print(cm)
fig, ax = plot_confusion_matrix(conf_mat=cm)
plt.gcf().savefig(save_dir+"resnet.png", dpi=144) 
plt.close()
extract_outputs(cm, 'Resnet')

Here are some of the screenshots of the output from tensorflow 2.x

Answer 1

Basically the flow_from_directory by default shuffle the data and you didn't change it.

Just add shuffle=False to your test_generator should be enough.

Like

test_generator = train_datagen.flow_from_directory(
    test_dir,
    target_size=(IMG_SIZE, IMG_SIZE),
    batch_size=16,
    class_mode='sparse',
    shuffle=False)

Or if you really want to have it shuffled then test_im and test_lb have to be in the same order. For example

test_im = []
test_lb = []
for im, lb in test_generator:
    test_im.append(im)
    test_lb.append(lb)
test_im = np.array(test_im)
test_lb = np.array(test_lb)