tensorflow VGG16 network accuracy and loss doesn't change

Question

I try to handcraft a VGG16 model with using tensorflow for training with my own dataset, which only have 2 classes, so I find VGG16 network architecture and the way how to train network on github as reference

But somehow the code I found doesn't work, so I try to fixed the code.

After I try the code for a long time, the accuracy and loss doesn't change among all epochs, and I can't find bug in this code.

Can someone help me to figure out where i messed up.

The code which I build my VGG16 network is following:

def build_network(height, width, channel):
    x = tf.placeholder(tf.float32,shape = [None, height, width, channel], name = 'input')
    y = tf.placeholder(tf.int64, shape=[None, 2], name = 'labels_placeholders')

    def weight_variable(shape, name = "weights"):
        initial = tf.truncated_normal(shape, dtype = tf.float32, stddev = 0.1)
        return tf.Variable(initial, name = name)

    def bias_variable(shape, name = "biases"):
        initial = tf.constant(0.1, dtype = tf.float32, shape = shape)
        return tf.Variable(initial, name = name)

    def conv2d(input, w):
        return tf.nn.conv2d(input, w, [1,1,1,1], padding = "SAME")

    def pool_max(input):
        return tf.nn.max_pool(input, ksize = [1,2,2,1], strides = [1,2,2,1], padding = "SAME", name = "pool1")

    def fc(input, w, b):
        return tf.matmul(input,w)+b

    with tf.name_scope('conv1_1') as scope:
        kernel = weight_variable([3, 3, 3, 64])
        biases = bias_variable([64])
        output_conv1_1 = tf.nn.relu(conv2d(x,kernel) + biases, name = scope)

    with tf.name_scope("conv1_2") as scope:
        kernel = weight_variable([3,3,64,64])
        biases = bias_variable([64])
        output_conv1_2 = tf.nn.relu(conv2d(output_conv1_1, kernel) + biases, name = scope)

    pool1 = pool_max(output_conv1_2)

    with tf.name_scope("conv2_1") as scope:
        kernel = weight_variable([3,3,64,128])
        biases = bias_variable([128])
        output_conv2_1 = tf.nn.relu(conv2d(pool1, kernel) + biases, name = scope)

    with tf.name_scope("conv2_2") as scope:
        kernel = weight_variable([3,3,128,128])
        biases = bias_variable([128])
        output_conv2_2 = tf.nn.relu(conv2d(output_conv2_1, kernel) + biases, name = scope)


    pool2 = pool_max(output_conv2_2)

    with tf.name_scope("conv3_1") as scope:
        kernel = weight_variable([3,3,128,256])
        biases = bias_variable([256])
        output_conv3_1 = tf.nn.relu(conv2d(pool2,kernel)+biases, name = scope)

    with tf.name_scope("conv3_2") as scope:
        kernel = weight_variable([3,3,256,256])
        biases = bias_variable([256])
        output_conv3_2 = tf.nn.relu(conv2d(output_conv3_1, kernel) + biases, name = scope)

    with tf.name_scope("conv3_3") as scope:
        kernel = weight_variable ([3,3,256,256])
        biases = bias_variable([256])
        output_conv3_3 = tf.nn.relu(conv2d(output_conv3_2, kernel) + biases, name = scope)

    pool3 = pool_max(output_conv3_3)

    with tf.name_scope("conv4_1") as scope:
        kernel = weight_variable([3,3,256,512])
        biases = bias_variable([512])
        output_conv4_1 = tf.nn.relu(conv2d(pool3,kernel)+ biases, name = scope)

    with tf.name_scope('conv4_2') as scope:
        kernel = weight_variable([3, 3, 512, 512])
        biases = bias_variable([512])
        output_conv4_2 = tf.nn.relu(conv2d(output_conv4_1, kernel) + biases, name=scope)

    with tf.name_scope('conv4_3') as scope:
        kernel = weight_variable([3, 3, 512, 512])
        biases = bias_variable([512])
        output_conv4_3 = tf.nn.relu(conv2d(output_conv4_2, kernel) + biases, name=scope)

    pool4 = pool_max(output_conv4_3)


    with tf.name_scope('conv5_1') as scope:
        kernel = weight_variable([3, 3, 512, 512])
        biases = bias_variable([512])
        output_conv5_1 = tf.nn.relu(conv2d(pool4, kernel) + biases, name=scope)

    with tf.name_scope('conv5_2') as scope:
        kernel = weight_variable([3, 3, 512, 512])
        biases = bias_variable([512])
        output_conv5_2 = tf.nn.relu(conv2d(output_conv5_1, kernel) + biases, name=scope)

    with tf.name_scope('conv5_3') as scope:
        kernel = weight_variable([3, 3, 512, 512])
        biases = bias_variable([512])
        output_conv5_3 = tf.nn.relu(conv2d(output_conv5_2, kernel) + biases, name=scope)

    pool5 = pool_max(output_conv5_3)


    with tf.name_scope('fc6') as scope:
        shape = int(np.prod(pool5.get_shape()[1:]))
        kernel = weight_variable([shape, 4096])
        biases = bias_variable([4096])
        pool5_flat = tf.reshape(pool5, [-1, shape])
        output_fc6 = tf.nn.relu(fc(pool5_flat, kernel, biases), name=scope)

    with tf.name_scope('fc7') as scope:
        kernel = weight_variable([4096, 4096])
        biases = bias_variable([4096])
        output_fc7 = tf.nn.relu(fc(output_fc6, kernel, biases), name=scope)

    with tf.name_scope('fc8') as scope:
        kernel = weight_variable([4096, 2])
        biases = bias_variable([2])
        output_fc8 = tf.nn.relu(fc(output_fc7, kernel, biases), name=scope)

    finaloutput = tf.nn.softmax(output_fc8, name="softmax")
    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=output_fc8, labels=y))

    #define optimizer
    optimize = tf.train.AdamOptimizer(learning_rate= 1e-3).minimize(cost)
    #optimize = tf.train.GradientDescentOptimizer(learning_rate = 0.5).minimize(cost)
    prediction_labels = tf.argmax(finaloutput, axis = 1, name = "output")
    read_labels = tf.argmax(y, axis = 1)
    correct_prediction = tf.equal(prediction_labels, read_labels)

    # correct rate
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    correct_times_in_batch = tf.reduce_sum(tf.cast(correct_prediction, tf.int32))
    return dict(
        x=x,
        y=y,
        optimize = optimize,
        correct_prediction = correct_prediction,
        correct_times_in_batch = correct_times_in_batch,
        cost=cost,
        accuracy = accuracy
    )

def train_network(graph, batch_size, num_epochs, pb_file_path):
    init = tf.global_variables_initializer()

    with tf.Session() as sess:
        sess.run(init)
        epoch_delta = 2
        train_epoch_size = int(x_train.shape[0]/batch_size)
        test_epoch_size = int(x_val.shape[0]/batch_size)
        for epoch in range(num_epochs):

            for i in range(train_epoch_size):
                train_batch_x = x_train[i*batch_size:(i+1)*batch_size]
                train_batch_y = y_train[i*batch_size:(i+1)*batch_size]
                train_batch_y = np.asarray(train_batch_y).reshape(-1,1)
                train_batch_y_ohe  = keras.utils.to_categorical(train_batch_y, num_classes=2).astype(int)
                feed_dict = {
                    graph["x"]: np.reshape(train_batch_x,(batch_size, 224, 224, 3)),
                    graph["y"]: train_batch_y_ohe
                }
                sess.run(graph['optimize'], feed_dict = feed_dict)
                loss = sess.run(graph['cost'], feed_dict = feed_dict)

                accuracy_ = sess.run(graph['accuracy'],feed_dict = feed_dict)

            print("epoch{}, loss:{}, accuracy:{}".format(epoch, loss, accuracy_))

this bug bother me for a long time

Answer 1

The combination of some of your chosen parameters most probably is your issue here; first things to consider:

• Change the stddev in your weight initialization to 0.01 (0.1 is a huge value), ie:

   initial = tf.truncated_normal(shape, dtype = tf.float32, stddev = 0.01)

• Change your bias initialization to zero, ie:

   initial = tf.constant(0.0, dtype = tf.float32, shape = shape)

• As already suggested, use

  optimize = tf.train.GradientDescentOptimizer(learning_rate=LR).minimize(cost)

with a sensible learning rate LR (start with 0.01 or 0.001).