Keras, Python. High accuracy model classifies incorrectly all the time

Question

I have 2 classses. 0=dogs, 1=nondogs. 8800 images (150,150 pixels) in training and 4400 images for validation. 4400 dogs, 4400non dogs in training. 2200 dogs, 2200 nondogs in validation. Nondog images contain random images of boats, trees, pianos, etc. I've trained my network to a great accuracy of 87%+. Plots: AccvsValAcc - http://imgur.com/a/6y6DG LossVSValLoss - http://imgur.com/a/QGZQx

My network:

#model dog/nondog
model = Sequential()
model.add(Convolution2D(16, 3, 3, input_shape=(3, img_width, img_height)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(16, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(32, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(64, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
              optimizer='rmsprop',
              metrics=['accuracy'])

I have 1 node at the end of model as I understand Im dealing with binary classification problem if I need to classify dog from nondog only.

Problem I'm facing is when I model.predict an unseen dog picture to system it always classifies it as nondog. Did I approach this problem incorrectly? If my accuracy is so high, can anyone explain to me why it NEVER classifies a dog picture as a dog? Is there any changes you could recommend to my network or approach?

EDIT: Originaly Ive trained on 70x70 images. Just finished retraining on 150x150 images. Also instead of model.predict Im using now model.predict_classes. BUT it still the same problem. Result is always a non-dog result on every image i tried. :(

EDIT2: Full code:

    # -*- coding: utf-8 -*-
"""
Created on Thu Jan 26 16:21:36 2017

@author: PoLL
"""

from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Convolution2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
import numpy as np
import matplotlib.pyplot as plt
import matplotlib

from PIL import Image
import numpy as np

from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
import PIL
from PIL import Image
#draw rect
import matplotlib.patches as patches






#########################################################################################################
#VALUES
# dimensions of images.
img_width, img_height = 150,150
train_data_dir = 'data1/train'
validation_data_dir = 'data1/validation'
nb_train_samples = 8800 #1000 cats/dogs
nb_validation_samples = 4400 #400cats/dogs
nb_epoch = 20
#########################################################################################################


#model dog/nondog
model = Sequential()
model.add(Convolution2D(16, 3, 3, input_shape=(3, img_width, img_height)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(16, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(32, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(64, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
              optimizer='rmsprop',
              metrics=['accuracy'])








#augmentation configuration for training
train_datagen = ImageDataGenerator(
        rescale=1./255,
        shear_range=0.2,
        zoom_range=0.2,
        horizontal_flip=True)
############################################################################################
#PRINT MODEL
from keras.utils.visualize_util import plot
plot(model, to_file='C:\Users\PoLL\Documents\Python Scripts\catdog\model.png')  
##########################################################################################################
#TEST AUGMENTATION
img = load_img('data/train/cats/cat.0.jpg')  # this is a PIL image
x = img_to_array(img)  # this is a Numpy array with shape (3, 150, 150)
x = x.reshape((1,) + x.shape)  # this is a Numpy array with shape (1, 3, 150, 150)

# the .flow() command below generates batches of randomly transformed images
# and saves the results to the `preview/` directory
i = 0
for batch in train_datagen.flow(x, batch_size=1,
                          save_to_dir='data/TEST AUGMENTATION', save_prefix='cat', save_format='jpeg'):
    i += 1
    if i > 20:
        break  # otherwise the generator would loop indefinitely
##########################################################################################################

# only rescaling
test_datagen = ImageDataGenerator(rescale=1./255)

#PREPARE TRAINING DATA
train_generator = train_datagen.flow_from_directory(
        train_data_dir, #data/train
        target_size=(img_width, img_height),  #RESIZE to 150/150
        batch_size=32,
        class_mode='binary')  #since we are using binarycrosentropy need binary labels

#PREPARE VALIDATION DATA
validation_generator = test_datagen.flow_from_directory(
        validation_data_dir,  #data/validation
        target_size=(img_width, img_height), #RESIZE 150/150
        batch_size=32,
        class_mode='binary')


#START model.fit
history =model.fit_generator(
        train_generator, #train data
        samples_per_epoch=nb_train_samples,
        nb_epoch=nb_epoch,
        validation_data=validation_generator,  #validation data
        nb_val_samples=nb_validation_samples)

############################################################################################
#LOAD WEIGHTS
model.load_weights('savedweights2.h5')
############################################################################################
#check labels 0=cat 1=dog
#dog = 0, nondog =1
labels = (train_generator.class_indices)
print(labels)
############################################################################################
#TESTING
#load test DOG
img=load_img('data/prediction/catordog/dog.1234.jpg')
#reshape to 1,3,150,150
img = np.array(img).reshape((1,3,img_width, img_height))
plt.imshow(img.reshape((150, 150, 3)))
print(model.predict_classes(img))
#load test CAT
img2=load_img('data/prediction/catordog/cat.187.jpg')
#reshape to 1,3,150,150
img2 = np.array(img2).reshape((1,3,img_width, img_height))
plt.imshow(img2.reshape((150, 150, 3)))
print(model.predict_classes(img))
print(model.predict_classes(img2))


############################################################################################
#RESIZE IMAGES
baseheight = 70
basewidth = 70
img = Image.open('data/prediction/catordog/dog.1297.jpg')
wpercent = (basewidth / float(img.size[0]))
hsize = int((float(img.size[1]) * float(wpercent)))
img = img.resize((basewidth, hsize), PIL.Image.ANTIALIAS)
img.save('resized_dog.jpg')
############################################################################################

#load test DOG
img=load_img('resized_dog.jpg')
#reshape to 1,3,150,150
img = np.array(img).reshape((1,3,img_width, img_height))
plt.imshow(img.reshape((70, 70, 3)))

print(model.predict(img))

#plt.imshow(image)
print(img.shape)
############################################################################################
##### WINDOW BOX  TO GO THROUGH THIS IMAGE
image=load_img('finddog/findadog2.jpg')
image= np.array(image).reshape((600,1050,3))
plt.imshow(image)
print(image.shape)

############################################################################################
############################################################################################
#OBJECT IS HERE

#object x,y,w,h,
object0 =  (140, 140, 150,150)
object1 =  (340, 340, 150,150)
#object2 = (130,130,150,150)
objloc  = []
objloc.append(object0)
objloc.append(object1)
#objloc.append(object2)



#SLIDING WINDOW
def find_a_dog(image, step=20, window_sizes=[70]):
    boxCATDOG = 0  
    locations = []
    for win_size in window_sizes:
        #top =y, left =x
        for Y in range(0, image.shape[0] - win_size + 1, step):
            for X in range(0, image.shape[1] - win_size + 1, step):
                # compute the (top, left, bottom, right) of the bounding box
                box = (Y, X, Y + win_size, X + win_size)
                # crop
                cropped_img = image[box[0]:box[2], box[1]:box[3]]
                #reshape cropped image by window
                cropped_img = np.array(cropped_img).reshape((1,3,70,70))
                #classify it
                boxCATDOG = predict_function(cropped_img)              
                if boxCATDOG ==0:
                 #   print('box classified as dog') 
                    #save location of it                 
                    locations.append(box)
                    print("found dog")
    return locations                                 



############################################################################################                   
#FUNCTIONS   #
def predict_function(x):
    result = model.predict_classes(x)
    if result==1:
        return 1
    else:
        return 0   
#SHOW CROPPED IMAGE
def show_image(im):
   plt.imshow(im.reshape((150,150,3)))
#SHOW INPUT IMAGE
def show_ori_image(im):
   plt.imshow(im.reshape((600,1050,3)))

def draw_obj_loc(image,objectloc):
    fix,ax = plt.subplots(1)
    ax.imshow(image)
    for l in objloc:
        rectG = patches.Rectangle((l[0],l[1]),l[2],l[3],linewidth=1,edgecolor='G',facecolor='none')
        ax.add_patch(rectG)
    print len(objectloc)

    #draw box when classifies as dog
def draw_boxes(image, locations):
    fix,ax = plt.subplots(1)
    ax.imshow(image)       
    for l in locations:
        print l
        rectR = patches.Rectangle((l[1],l[0]),150,150,linewidth=1,edgecolor='R',facecolor='none')
        ax.add_patch(rectR)
    print len(locations)

def draw_both(image, locations,objectloc):
    fix,ax = plt.subplots(1)
    ax.imshow(image)
    for l in objloc:
        rectG = patches.Rectangle((l[0],l[1]),l[2],l[3],linewidth=1,edgecolor='G',facecolor='none')
        ax.add_patch(rectG)
    for l in locations:
        print l
        rectR = patches.Rectangle((l[1],l[0]),150,150,linewidth=1,edgecolor='R',facecolor='none')
        ax.add_patch(rectR)
#check if overlaps
def check_overlapping(image,locations,objloc):   

    for ol in objloc:
        objX = (ol[0])
        objY = (ol[1])
        objW = (ol[2])
        objH = (ol[3])

        for ok in locations:
            X=(ok[0])
            Y=(ok[1])
   # for l in locations:
        #  if (objX+objW<X or X+150<objX or objY+objH<Y or Y+150<objY):
            if (objX+objW<X or X+150<objX or objY+objH<Y or Y+150<objY):
                # Intersection = Empty
                       #no overlapping, false positive
                       print('THERES NO OVERLAPPING :',objloc.index(ol))
                        #                                              
            else:
                        #Intersection = Not Empty
                        print('THERE IS OVERLAPPING WITH OBJECT: ',objloc.index(ol), 'WITH BOX NUMBER: ',locations.index(ok))



############################################################################################       
#get locations from image
locations = find_a_dog(image)
#show where windowslide classifed as positive
draw_boxes(image,locations)
#show where objects actually are
draw_obj_loc(image,objloc)
#check for overlapping between slider classification and actual
check_overlapping(image,locations,objloc)
#drawboth boxes
draw_both(image, locations,objloc)







#GREEN RECT
#   X,Y    X+W,Y
      ######
      #    #
      #    #
      ######
#   X,Y+H   X+W,Y+H


#WINDOW
#  Y1,X1   Y1+W,X1
      ######
      #    #
      #    #
      ######
# Y1,X+H   Y1+W,X1+H

###REMOVED FUNCTIONS
##DRAW RECT RED
def draw_rect(im,Y,X):
    fig,ax = plt.subplots(1)
    ax.imshow(im)
    rect = patches.Rectangle((Y,X),150,150,linewidth=1,edgecolor='r',facecolor='none')
    ax.add_patch(rect) 
   # im =plt.savefig('rect.jpg')

######OBJECT LOCATION AND H W GREEN
def draw_box_object(im,X,Y,W,H):
    fig,ax = plt.subplots(1)
    ax.imshow(im)
    rect = patches.Rectangle((X,Y),W,H,linewidth=1,edgecolor='G',facecolor='none')
    ax.add_patch(rect)
  #  im = plt.savefig('boxfordog.jpg')


################################################################################################
#PLOT
#ACC VS VAL_ACC
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy ACC VS VAL_ACC')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
#LOSS VS VAL_LOSS
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss LOSS vs VAL_LOSS')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
################################################################################################
#SAVE WEIGHTS
model.save_weights('savedweights.h5')
#70x70
model.save_weights('savedweights2.h5')
#150x150
model.save_weights('savedweights3.h5')

I appologise for messy code, lots of changes occur often..

Answer 1

On which dataset are you measuring the accuracy ? Id suggest performing "machine learning diagnostics" using learning curves and other performance metrics lice precision and recall, this would help you determine if you are suffering overfitting and give you some guidelines.

Also perform "error analysis", take some examples your model gets wrong and see if theres any pattern in them.