Batching issue in tensorflow

Question

I am just switching from caffe to tensorflow. I have this very initial example in tensorflow, which does not have batching. I am gonna use mini batch, but I am stuck in. It seems that it requires batch, queue and coordinate. I do not know exactly how I can use them.

I appreciate if you could explain me in my code how I can use batching,

import tensorflow as tf
import numpy as np
import scipy.io as sio
import h5py

sess = tf.InteractiveSession()

train_mat = h5py.File('Basket_train_data_binary.mat')
test_mat = h5py.File('Basket_test_data_binary.mat')

train_mat = train_mat["binary_train"].value
test_mat = test_mat["binary_test"].value

Train = np.transpose(train_mat)
Test = np.transpose(test_mat)

# import the data

# placeholders, which are the training data
x = tf.placeholder(tf.float32, shape=[None, 42])
y_ = tf.placeholder(tf.float32, shape=[None])


nnodes = 10
# define the variables
W1 = tf.Variable(tf.zeros([43,nnodes]))
b1 = tf.Variable(tf.zeros([nnodes]))

W2 = tf.Variable(tf.zeros([nnodes,1]))
b2 = tf.Variable(tf.zeros([1]))

# initilize the variables
sess.run(tf.initialize_all_variables())

# placeholders, which are the training data                                                                                                      
x = tf.placeholder(tf.float32, shape=[None, 43])
y_ = tf.placeholder(tf.float32, shape=[None])


nnodes = 10
# define the variables                                                                                                                           
W1 = tf.Variable(tf.zeros([43,nnodes]))
b1 = tf.Variable(tf.zeros([nnodes]))

W2 = tf.Variable(tf.zeros([nnodes,1]))
b2 = tf.Variable(tf.zeros([1]))

# Passing global_step to minimize() will increment it at each step.
global_step = tf.Variable(0, trainable=False)

# initilize the variables                                                                                                                       
sess.run(tf.initialize_all_variables())

# prediction function (just one layer)                                                                                                           
layer1 = tf.nn.sigmoid(tf.matmul(x,W1) + b1)
y = tf.matmul(layer1,W2) + b2

# cost function 
cost_function = tf.reduce_sum(tf.square(y_ - y))

alpha = 2
l2regularization = tf.reduce_sum(tf.square(W1)) +             tf.reduce_sum(tf.square(b1)) +tf.reduce_sum(tf.square(W2)) + tf.reduce_sum(tf.square(b2))
loss = cost_function + alpha*l2regularization

# define the learning_rate and its decaying procedure.
decay_rate = 0.00005
starter_learning_rate = 0.0000009
learning_rate = tf.train.exponential_decay(starter_learning_rate,     global_step,10000, decay_rate, staircase=True)


# define the training paramters and model, gradient model and feeding the function
train_step =     tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)

# Train the Model for 1000 times. by defining the batch number we determine that it is sgd
bth_sz = 100
for i in range(2):
  train_step.run(feed_dict={x:Train[1:100,0:43] , y_:Train[1:100,43]})

print "y"
sess.run([tf.Print(y,[y])],feed_dict={x:Train[0:100,0:43] ,     y_:Train[1:100,43]})
print "y_"
sess.run([tf.Print(y_,[y_])],feed_dict={x:Train[0:100,0:43] , y_:Train[1:100,43]})
print "W1" 
sess.run([tf.Print(W1,[W1])],feed_dict={x:Train[0:100,0:43] ,     y_:Train[1:100,43]})
print "W2" 
sess.run([tf.Print(W2,[W2])],feed_dict={x:Train[0:100,0:43] ,     y_:Train[1:100,43]})
print "b1" 
sess.run([tf.Print(b1,[b1])],feed_dict={x:Train[0:100,0:43] , y_:Train[1:100,43]})

# evaluation
# it returns 1, if both y and y_ are equal. 
correct_prediction = tf.reduce_sum(tf.square(y_ - y))

# calculate the accuracy
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

# print tset loss 
print(accuracy.eval(feed_dict={x: Test[:,0:43], y_: Test[:,43]}))

# print training loss 
print sess.run(cost_function,feed_dict={x: Train[:,0:43], y_: Train[:,43]})

Right now, I am using x:Train[1:100,0:43] to select the first 100 records. I also do not want to write the mini-batch selection by myself.

Thanks in advance, Afshin

Answer 1

It is pretty easy to use queue, but yeah it is unfortunately not obvious from tutorial (as for me). So below I provide example of using queue for using batches for training:

# capacity - upper bound on the nu,ber fo elements
queue = tf.FIFOQueue(capacity, [tf.float64, tf.float64])

# here we create placeholder for data and op that enqueues them
enq_x, enq_y = tf.placeholder(tf.float64), tf.placeholder(tf.float64)
enqueue_op = queue.enqueue_many([enq_x, enq_y])

# here we dequeue data from queue for further usage
bth_sz = 100
x, y_ = queue.dequeue_many(bth_sz)

# here you initialize your variables and loss for training that use x and y_ ...

# Note that you can enqueue data any size. For example if 
# you have big data set you can divide it into several parts
# and enqueue each part in different threads
sess.run(enqueue_op, feed_dict={enq_x: Train[,0:43], enq_y: Train[,43]})

for _ in range(2):
    # Note that you can make btch_sz as placeholder and provide it through feed_dict here
    sess.run(train_step)

I hope this was helpful!

EDITED : enq_y - placeholder instead of constant

EDITED :

Train = np.random.rand(100, 44)

tf.reset_default_graph()
# capacity - upper bound on the nu,ber fo elements
queue = tf.FIFOQueue(500, [tf.float64, tf.float64], shapes=[[43], []])

# here we create placeholder for data and op that enqueues them
enq_x, enq_y = tf.placeholder(tf.float64, shape=[None, 43]), tf.placeholder(tf.float64, shape=[None])
enqueue_op = queue.enqueue_many([enq_x, enq_y])

bth_sz = tf.placeholder(tf.int32)
x, y_ = queue.dequeue_many(bth_sz)

# here you initialize your variables and loss for training that use x and y_
# ...

# Note that you can enqueue data any size. For example if you have big data set you can divide it
# and enqueue each part in different threads
with tf.Session() as sess:
    sess.run(enqueue_op, feed_dict={enq_x: Train[:,0:43], enq_y: Train[:,43]})

    sess.run([x, y_], feed_dict={bth_sz: 10})