ValueError: Shape must be rank 2 but is rank 4 for 'in_top_k/InTopKV2' (op: 'InTopKV2') with input shapes: [?,28,28,10], [?], []

Question

I'm new to Tensorflow and I am trying to train on MNIST. However, the code fails on

correct = tf.nn.in_top_k(logits, tf.argmax(y, axis=1), 1)

with the error "ValueError: Shape must be rank 2 but is rank 4 for 'in_top_k/InTopKV2' (op: 'InTopKV2') with input shapes: [?,28,28,10], [?], []"

What is going on here, and what do I need to know to make this compatible with different architectures in the future? I've included the entire file below.

import tensorflow as tf
import numpy as np
from tensorflow.python.framework import graph_util
from tensorflow.python.framework import graph_io

tf.reset_default_graph()

x = tf.placeholder(tf.float32, shape=(None, 28, 28), name='x_input')
y = tf.placeholder(tf.float32, shape=(None, 10), name='y_label')
y = tf.stop_gradient(y, name="stop_gradient_y")

input_layer = tf.reshape(x, [-1, 28, 28, 1], name='x_reshaped')

fc_layer1 = tf.layers.dense(
        inputs=input_layer, units=1024, activation=tf.nn.relu, name='fc_layer_1')

fc_layer2 = tf.layers.dense(
        inputs=fc_layer1, units=512, activation=tf.nn.relu, name='fc_layer_2')

fc_layer3 = tf.layers.dense(
        inputs=fc_layer2, units=512, activation=tf.nn.relu, name='fc_layer_3')

fc_layer4 = tf.layers.dense(
        inputs=fc_layer3, units=512, activation=tf.nn.relu, name='fc_layer_4')

fc_layer5 = tf.layers.dense(
        inputs=fc_layer4, units=512, activation=tf.nn.relu, name='fc_layer_5')

logits = tf.layers.dense(inputs=fc_layer5, units=10, name='logits')
classes = tf.argmax(input=logits, axis=1, name='classes')
probabilities = tf.nn.softmax(logits, name="probabilities_out")
loss = tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=logits, name='loss_func')
grad = tf.gradients(loss, x)
grad_out = tf.identity(grad, name='gradient_out')

optimizer = tf.train.AdamOptimizer()
train_op = optimizer.minimize(loss)
correct = tf.nn.in_top_k(logits, tf.argmax(y, axis=1), 1)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))

(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()
x_train = x_train/np.float32(255)
y_train = y_train.astype(np.int32)
x_test = x_test/np.float32(255)
y_test = y_test.astype(np.int32)

y_train = tf.keras.utils.to_categorical(y_train, 10)
y_test = tf.keras.utils.to_categorical(y_test, 10)

num_epochs = 100
batch_size = 100

init = tf.global_variables_initializer()

with tf.Session() as sess:
    init.run()
    for epoch in range(num_epochs):
        print('Epoch: {}'.format(epoch))
        for i in range(x_train.shape[0] // batch_size):
            batch_indices = np.random.randint(x_train.shape[0], size=batch_size)
            x_batch = x_train[batch_indices]
            y_batch = y_train[batch_indices]
            sess.run(train_op, feed_dict={x: x_batch, y: y_batch})
        acc_test = accuracy.eval(feed_dict={x: x_test, y: y_test})
        print(epoch, "Test accuracy:", acc_test)

    constant_graph = graph_util.convert_variables_to_constants(
            sess, 
            sess.graph.as_graph_def(), 
            ['probabilities_out', 'gradient_out'])

    graph_io.write_graph(constant_graph, '.', 'mnist_gradient_fc_without.pb', as_text=False)

Answer 1

Thanks JacoSolari and Jarom Allen. For the benefit of community providing complete working code here

import tensorflow as tf
import numpy as np
from tensorflow.python.framework import graph_util
from tensorflow.python.framework import graph_io

tf.reset_default_graph()

x = tf.placeholder(tf.float32, shape=(None, 28, 28), name='x_input')
y = tf.placeholder(tf.float32, shape=(None, 10), name='y_label')
y = tf.stop_gradient(y, name="stop_gradient_y")

input_layer = tf.reshape(x, [-1, 784 ], name='x_reshaped')


fc_layer1 = tf.layers.dense(
        inputs=input_layer, units=1024, activation=tf.nn.relu, name='fc_layer_1')

fc_layer2 = tf.layers.dense(
        inputs=fc_layer1, units=512, activation=tf.nn.relu, name='fc_layer_2')

fc_layer3 = tf.layers.dense(
        inputs=fc_layer2, units=512, activation=tf.nn.relu, name='fc_layer_3')

fc_layer4 = tf.layers.dense(
        inputs=fc_layer3, units=512, activation=tf.nn.relu, name='fc_layer_4')

fc_layer5 = tf.layers.dense(
        inputs=fc_layer4, units=512, activation=tf.nn.relu, name='fc_layer_5')

logits = tf.layers.dense(inputs=fc_layer5, units=10, name='logits')
classes = tf.argmax(input=logits, axis=1, name='classes')
probabilities = tf.nn.softmax(logits, name="probabilities_out")
loss = tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=logits, name='loss_func')
grad = tf.gradients(loss, x)
grad_out = tf.identity(grad, name='gradient_out')

optimizer = tf.train.AdamOptimizer()
train_op = optimizer.minimize(loss)
correct = tf.nn.in_top_k(logits, tf.argmax(y, axis=1), 1)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))

(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()
x_train = x_train/np.float32(255)
y_train = y_train.astype(np.int32)
x_test = x_test/np.float32(255)
y_test = y_test.astype(np.int32)

y_train = tf.keras.utils.to_categorical(y_train, 10)
y_test = tf.keras.utils.to_categorical(y_test, 10)

num_epochs = 5
batch_size = 100

init = tf.global_variables_initializer()

with tf.Session() as sess:
    init.run()
    for epoch in range(num_epochs):
        print('Epoch: {}'.format(epoch))
        for i in range(x_train.shape[0] // batch_size):
            batch_indices = np.random.randint(x_train.shape[0], size=batch_size)
            x_batch = x_train[batch_indices]
            y_batch = y_train[batch_indices]
            sess.run(train_op, feed_dict={x: x_batch, y: y_batch})
        acc_test = accuracy.eval(feed_dict={x: x_test, y: y_test})
        print(epoch, "Test accuracy:", acc_test)

    constant_graph = graph_util.convert_variables_to_constants(
            sess, 
            sess.graph.as_graph_def(), 
            ['probabilities_out', 'gradient_out'])

    graph_io.write_graph(constant_graph, '.', 'mnist_gradient_fc_without.pb', as_text=False)

Output:

Epoch: 0
0 Test accuracy: 0.9527
Epoch: 1
1 Test accuracy: 0.9683
Epoch: 2
2 Test accuracy: 0.9731
Epoch: 3
3 Test accuracy: 0.9776
Epoch: 4
4 Test accuracy: 0.9821
INFO:tensorflow:Froze 12 variables.
INFO:tensorflow:Converted 12 variables to const ops.