Tensorflow next word predictor bug

Question

I have the following code

 flags = tf.flags
logging = tf.logging

flags.DEFINE_string('model', 'small',
                    'A type of model. Possible options are: small, medium, large.'
                    )
flags.DEFINE_string('data_path', None, 'data_path')
flags.DEFINE_string('checkpoint_dir', 'ckpt', 'checkpoint_dir')
flags.DEFINE_bool('use_fp16', False,
                'Train using 16-bit floats instead of 32bit floats')
flags.DEFINE_bool('train', False, 'should we train or test')

FLAGS = flags.FLAGS


def data_type():
    return tf.float16 if FLAGS.use_fp16 else tf.float32


class PTBModel(object):
    """The PTB model."""

    def __init__(self, is_training, config):
        self.batch_size = batch_size = config.batch_size
        self.num_steps = num_steps = config.num_steps
        size = config.hidden_size
        vocab_size = config.vocab_size

        self._input_data = tf.placeholder(tf.float32, [batch_size,
                                                    num_steps])
        self._targets = tf.placeholder(tf.int32, [batch_size,
                                                num_steps])

        # Slightly better results can be obtained with forget gate biases
        # initialized to 1 but the hyperparameters of the model would need to be
        # different than reported in the paper.

        lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(size, forget_bias=0.0,
                                                state_is_tuple=True)
        if is_training and config.keep_prob < 1:
            lstm_cell = tf.nn.rnn_cell.DropoutWrapper(lstm_cell,
                                                    output_keep_prob=config.keep_prob)
        cell = tf.nn.rnn_cell.MultiRNNCell([lstm_cell]
                                        * config.num_layers, state_is_tuple=True)

        self._initial_state = cell.zero_state(batch_size, data_type())

        with tf.device('/cpu:0'):
            embedding = tf.get_variable('embedding', [vocab_size,
                                                    size], dtype=data_type())
            inputs = tf.nn.embedding_lookup(embedding, self._input_data)

        if is_training and config.keep_prob < 1:
            inputs = tf.nn.dropout(inputs, config.keep_prob)

            # Simplified version of tensorflow.models.rnn.rnn.py's rnn().
            # This builds an unrolled LSTM for tutorial purposes only.
            # In general, use the rnn() or state_saving_rnn() from rnn.py.
            #
            # The alternative version of the code below is:
            #
            # from tensorflow.models.rnn import rnn

        inputs = [tf.squeeze(input_, [1]) for input_ in tf.split(inputs, num_steps, axis=1)]

        (outputs, state) = tf.nn.rnn(cell, inputs, initial_state=self._initial_state)

        # outputs = []
        # state = self._initial_state
        # with tf.variable_scope("RNN"):
        #  for time_step in range(num_steps):
        #    if time_step > 0: tf.get_variable_scope().reuse_variables()
        #    (cell_output, state) = cell(inputs[:, time_step, :], state)
        #    outputs.append(cell_output)

        output = tf.reshape(tf.concat(outputs, axis=1), [-1, size])
        softmax_w = tf.get_variable('softmax_w', [size, vocab_size],
                                    dtype=data_type())
        softmax_b = tf.get_variable('softmax_b', [vocab_size],
                                    dtype=data_type())
        logits = tf.matmul(output, softmax_w) + softmax_b

        loss = tf.nn.seq2seq.sequence_loss_by_example([logits],
                                                    [tf.reshape(self._targets, [-1])], [tf.ones([batch_size
                                                                                                * num_steps],
                                                                                                dtype=data_type())])
        self._cost = cost = tf.reduce_sum(loss) / batch_size
        self._final_state = state

        # RANI

        self.logits = logits

        if not is_training:
            return

        self._lr = tf.Variable(0.0, trainable=False)
        tvars = tf.trainable_variables()
        (grads, _) = tf.clip_by_global_norm(tf.gradients(cost, tvars),
                                            config.max_grad_norm)
        optimizer = tf.train.GradientDescentOptimizer(self._lr)
        self._train_op = optimizer.apply_gradients(zip(grads, tvars))

        self._new_lr = tf.placeholder(tf.float32, shape=[],
                                    name='new_learning_rate')
        self._lr_update = tf.assign(self._lr, self._new_lr)

    def assign_lr(self, session, lr_value):
        session.run(self._lr_update, feed_dict={self._new_lr: lr_value})


    ...

However, When I run it, I get the following errors

File "ptb_word_lm.py", line 349, in <module>
    tf.app.run()
File "C:\Users\Josh Goldman\AppData\Local\Programs\Python\Python35\lib\site-packages\tensorflow\python\platform\app.py", line 48, in run
    _sys.exit(main(_sys.argv[:1] + flags_passthrough))
File "ptb_word_lm.py", line 299, in main
    m = PTBModel(is_training=True, config=config)
File "ptb_word_lm.py", line 60, in __init__
    inputs = tf.nn.embedding_lookup(embedding, self._input_data)
File "C:\Users\Josh Goldman\AppData\Local\Programs\Python\Python35\lib\site-packages\tensorflow\python\ops\embedding_ops.py", line 122, in embedding_lookup
    return maybe_normalize(_do_gather(params[0], ids, name=name))
File "C:\Users\Josh Goldman\AppData\Local\Programs\Python\Python35\lib\site-packages\tensorflow\python\ops\embedding_ops.py", line 42, in _do_gather
    return array_ops.gather(params, ids, name=name)
File "C:\Users\Josh Goldman\AppData\Local\Programs\Python\Python35\lib\site-packages\tensorflow\python\ops\gen_array_ops.py", line 1179, in gather
    validate_indices=validate_indices, name=name)
File "C:\Users\Josh Goldman\AppData\Local\Programs\Python\Python35\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 589, in apply_op
    param_name=input_name)
File "C:\Users\Josh Goldman\AppData\Local\Programs\Python\Python35\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 60, in _SatisfiesTypeConstraint
    ", ".join(dtypes.as_dtype(x).name for x in allowed_list)))
TypeError: Value passed to parameter 'indices' has DataType float32 not in list of allowed values: int32, int64

Someone, please help me. I have all my packages upgraded to the newest version. I'm using the correct interpreter. I'm sorry if the error is very simple. I'm only 13 and am very new to programming. By the way, this code is not mine; I got it from Github.

Answer 1

The error is due to tensorflow version, syntax of tf.split is changed in the newer version. there is another same problem with tf.concat

# replace this line with the following one
inputs = [tf.squeeze(input_, [1]) for input_ in tf.split(1, num_steps, inputs)]
# this support `tensorflow >= 1.0.0`
inputs = [tf.squeeze(input_, [1]) for input_ in tf.split(inputs, num_steps, axis=1)]

# Also use dtype float32 for inputs
self._input_data = tf.placeholder(tf.float32, [batch_size,
                                                 num_steps])

# replace this line
output = tf.reshape(tf.concat(1, outputs), [-1, size])
# with this one
output = tf.reshape(tf.concat(outputs, axis=1), [-1, size])