Tensorflow MNIST using csv: low accuracy

Question

I am trying to implement MNIST image classification in Tensorflow using CSV input. The accuracy is very low at around 10 percent. I am using 3 fully connected layers with softmax entropy at the output. I converted the mnist datasets from here into csv to use as input using the below code:

import idx2numpy
import numpy as np
# Reading
filename = 'train-images'
flext1 = '.idx3-ubyte'
flext2 = '.csv'
ndarr = idx2numpy.convert_from_file(filename+flext1).reshape(60000, 784)
np.savetxt(filename,ndarr,delimiter=',')

Then I one-hot encoded the labels as below:

import tensorflow as tf
import numpy as np
import pandas as pd

trainimages=pd.read_csv("train-images.csv",header=None,delimiter=',').values
trainlabels=pd.read_csv("train-labels.csv",header=None,delimiter=',').values

testimages=pd.read_csv("t10k-images.csv",header=None,delimiter=',').values
testlabels=pd.read_csv("t10k-labels.csv",header=None,delimiter=',').values

label_ph=tf.placeholder(tf.int64,shape=(None,1))
reshapeval=tf.Variable(100)
onehotencoding=tf.one_hot(label_ph,depth=10)
datalabels=tf.reshape(onehotencoding,shape=(reshapeval,10))

with tf.Session() as sess:
  onehottrainlabels=sess.run(datalabels,feed_dict={label_ph:trainlabels,reshapeval:trainlabels.shape[0]})
  onehottestlabels=sess.run(datalabels,feed_dict={label_ph:testlabels,reshapeval:testlabels.shape[0]})
  print("datalabels:",onehottrainlabels)
  print("datalabels:",onehottestlabels)
  #sess.run(datalabels)

np.savetxt("mnist_train_labels_onehot.csv",onehottrainlabels,delimiter=',')
np.savetxt("mnist_test_labels_onehot.csv",onehottestlabels,delimiter=',')

I tried to use the csv files generated as shown above to try and train a image classification model as shown below:

import tensorflow as tf
import pandas as pd
import numpy as np

trainimages=pd.read_csv("train-images.csv",header=None,delimiter=',').values
trainlabels=pd.read_csv("mnist_train_labels_onehot.csv",header=None,delimiter=',').values
testimages=pd.read_csv("t10k-images.csv",header=None,delimiter=',').values
testlabels=pd.read_csv("mnist_test_labels_onehot.csv",header=None,delimiter=',').values
#%%
train_num=len(trainimages)
n_epochs=400
BATCH_SIZE = 50
N_FEATURES = 785
n_inputs =28*28
n_hidden1=300
n_hidden2=100
n_outputs = 10
learning_rate=0.01
X=tf.placeholder(tf.float32,shape=(None,n_inputs),name='X')
y=tf.placeholder(tf.int64,shape=(None,10),name='y')  
print("stage1")

with tf.name_scope("dnn"):
    hidden1=tf.contrib.layers.fully_connected(X,n_hidden1,scope="hidden1")
    hidden2=tf.contrib.layers.fully_connected(hidden1,n_hidden2,scope="hidden2")
    logits=tf.contrib.layers.fully_connected(hidden2,n_outputs,scope="outputs",activation_fn=None)
with tf.name_scope("loss"):
    xentropy =tf.nn.softmax_cross_entropy_with_logits(labels=y,logits=logits)
    loss =tf.reduce_mean(xentropy,name="loss")
    loss_summary=tf.summary.scalar("log_loss",loss)
with tf.name_scope("train"):
    optimizer=tf.train.GradientDescentOptimizer(learning_rate)
    training_op=optimizer.minimize(loss)
with tf.name_scope("eval"):
    correct=tf.equal(tf.argmax(y, 1), tf.argmax(logits, 1))#tf.nn.in_top_k(logits,y,1)
    accuracy=tf.reduce_mean(tf.cast(correct,tf.float32))

init =tf.global_variables_initializer()
saver =tf.train.Saver()
print("stage 2")

with tf.Session() as sess:
  sess.run(init)
  print("stage 3")
  for epoch in range(n_epochs):
    #print("stage 4")
    for iteration in range(train_num//BATCH_SIZE):
      #print("stage 5")
      ind = np.random.choice(trainimages.shape[0],BATCH_SIZE)
      x_train_batch = trainimages[ind]
      y_train_batch = trainlabels[ind]
      sess.run(training_op,feed_dict={X:x_train_batch,y:y_train_batch})
      #print("stage 6")
    if epoch%20==0:
      #print(x_train_batch, y_train_batch)
      acc_train=sess.run(accuracy,feed_dict={X:x_train_batch,y:y_train_batch})
      acc_test=sess.run(accuracy,feed_dict={X:testimages,y:testlabels})

      print(epoch, "Train accuracy:", acc_train,"Test accuracy:",acc_test)

The output accuracy never increases about 0.15. I would really appreciate it if someone could advice me on what I am doing wrong here.

For reference, the above graph construction is based on the example given in the Hands-on Machine Learning with Tensorflow book. Below is the original code which produces an accuracy of 98 percent. Of course I have modified the code a little bit to be able to accept csv input.

import tensorflow as tf
import numpy as np
import pandas as pd
import random
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/tmp/data/")


#%%
tf.reset_default_graph()
print("I reset graph")
n_inputs =28*28
n_hidden1=300
n_hidden2=100
n_outputs = 10
learning_rate=0.01
X=tf.placeholder(tf.float32,shape=(None,n_inputs),name='X')
y=tf.placeholder(tf.int64,shape=(None),name='y')                     



with tf.name_scope("dnn"):
    hidden1=tf.layers.dense(X,n_hidden1,name="hidden1",activation=tf.nn.relu)
    hidden2=tf.layers.dense(hidden1,n_hidden2,name="hidden2",activation=tf.nn.relu)
    logits=tf.layers.dense(hidden2,n_outputs,name="outputs")
with tf.name_scope("loss"):
    xentropy =tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,logits=logits)
    loss =tf.reduce_mean(xentropy,name="loss")
    #loss_summary=tf.summary.scalar("log_loss",loss)
with tf.name_scope("train"):
    optimizer=tf.train.GradientDescentOptimizer(learning_rate)
    training_op=optimizer.minimize(loss)
with tf.name_scope("eval"):
    correct=tf.nn.in_top_k(logits,y,1)
    accuracy=tf.reduce_mean(tf.cast(correct,tf.float32))

init =tf.global_variables_initializer()
saver =tf.train.Saver()

#==============================================================================
# from tensorflow.examples.tutorials.mnist import input_data
# mnist=input_data.read_data_sets("/tmp/data/")
#==============================================================================

n_epochs=400
batch_size=100
print("graph construction over i start train and test")
with tf.Session() as sess:
    sess.run(init)
    for epoch in range(n_epochs):
        for iteration in range(mnist.train.num_examples//batch_size):
            X_batch, y_batch = mnist.train.next_batch(batch_size)

            #print(X_batch, y_batch)
            sess.run(training_op,feed_dict={X:X_batch,y:y_batch})
        print("iteration over")
        if epoch%5==0:
          acc_train=sess.run(accuracy,feed_dict={X:X_batch,y:y_batch})
          acc_test=sess.run(accuracy,feed_dict={X: mnist.validation.images, y: mnist.validation.labels})
          print(epoch, "Train accuracy:", acc_train,"Test accuracy:",acc_test)
    save_path=saver.save(sess,"./my_model_final.ckpt")

Additional information: I printed out the neural network output logits for a few of the iterations after reducing the batch size. The outputs seem to saturate after 10 or so iterations. I tried changing the optimizer to AdamOptimizer but still the accuracy won't improve. Below shows a sample of the logits outputs.

iteration  7
Y_train: [[0 1 0 0 0 0 0 0 0 0]
 [0 1 0 0 0 0 0 0 0 0]
 [0 0 0 0 0 0 0 0 1 0]] 
 Logits: [[  2.26569266e+17   3.05248736e+22  -2.50115512e+23  -4.39891557e+14
    1.11553920e+23   3.07485097e+14   2.20944474e+17   4.47584162e+13
    1.08036284e+23   4.83180834e+14]
 [  3.86803140e+17   8.75195504e+22  -7.17081147e+23  -2.68349530e+15
    3.19824929e+23   8.67874780e+14   3.76970654e+17   4.52455091e+13
    3.09735965e+23   3.87887019e+14]
 [  2.38161056e+17   3.84007123e+22  -3.14640962e+23  -6.05196997e+14
    1.40332858e+23   3.16957379e+14   2.32241818e+17   4.01556080e+13
    1.35906927e+23   4.00997575e+14]]
Softmax_out: [  8.10290437e+22   2.32305379e+23   4.42593055e+21]
Accuracy: [False False False]


 iteration  8
Y_train: [[1 0 0 0 0 0 0 0 0 0]
 [0 1 0 0 0 0 0 0 0 0]
 [0 0 0 1 0 0 0 0 0 0]] 
 Logits: [[             inf              inf   1.66566635e+35              inf
              -inf   1.03125672e+30   1.62685693e+35   6.78127837e+30
   -4.95936289e+35   4.14895906e+30]
 [  1.38760797e+38   1.10565560e+38   3.23025301e+34   2.54545823e+38
              -inf   1.99993203e+29   3.15498898e+34   1.31510396e+30
   -9.61777107e+34   8.04613877e+29]
 [             inf              inf   2.54806070e+35              inf
              -inf   1.57756931e+30   2.48869187e+35   1.03736909e+31
   -7.58660917e+35   6.34688775e+30]]
Softmax_out: [ nan  nan  nan]
Accuracy: [ True False False]


 iteration  9
Y_train: [[0 0 0 0 0 1 0 0 0 0]
 [0 1 0 0 0 0 0 0 0 0]
 [0 0 1 0 0 0 0 0 0 0]] 
 Logits: [[ nan  nan  nan  nan  nan  nan  nan  nan  nan  nan]
 [ nan  nan  nan  nan  nan  nan  nan  nan  nan  nan]
 [ nan  nan  nan  nan  nan  nan  nan  nan  nan  nan]]
Softmax_out: [ nan  nan  nan]
Accuracy: [False False False]

Answer 1

10% accuracy in a ten-category problem is random output; my guess is that your call to reshape is scrambling your input. For example, consider the following code:

import tensorflow as tf

initial = tf.constant([[1, 2, 3, 4],[5, 6, 7, 8]])
reshaped = tf.reshape(initial, [4, 2])

with tf.Session() as sess:
    init = tf.global_variables_initializer()
    sess.run(init)

    print(sess.run(initial))
    print(sess.run(reshaped))

I'm turning a 2x4 tensor into a 4x2 tensor, which makes it sound like I'm transposing it, but in fact the output is

[[1 2 3 4]
 [5 6 7 8]]
[[1 2]
 [3 4]
 [5 6]
 [7 8]]

I'd recommend using matplotlib to display a few of your examples to see if they still look like numbers. It would also be a good idea to output the labels to make sure they still match.

Answer 2

I think you should build layer like follow.

Convo with 28 input + relu + dropout(0.2)

Convo with 32 input features + relu + dropout(0.2)

Convo with 32 input features + relu + dropout(0.2)

flatten()

Dense or Fully Connected layer with 512 dendries + dropout(0.5)

Dense or Fully Connected with 128 output dendries with softmax function

I had achieved 99.7% accuracy using my own but mnist type dataset.

Check this out : Text Classification using Keras

Note : I had 36 classes. 10 digits and 26 alphabets