char-rnn（多层递归神经网络）实现得到IndexError

Question

I am trying to implement this char-rnn.py with slight changes in my system.

This is my full code:

from keras.models import Sequential
from keras.layers import Dense, Activation,TimeDistributedDense, Dropout
from keras.layers import LSTM
from keras.optimizers import RMSprop
from keras.utils.data_utils import get_file
import numpy

# Obtain the corpus of character sequence to train from.
# Here it is just the sequence 123456789 repeated 100000 times.
x = "123456789"*1000

# Construct a dictionary, and the reverse dictionary for the participating chars.
# '*" is a 'start-sequence' character.
dct = ['*'] + list(set(x))
max_features = len(dct)
rev_dct = [(j, i) for i, j in enumerate(dct)]
rev_dct = dict(rev_dct)

# Convert the characters to their dct indexes.
x = [rev_dct[ch] for ch in x]

# Divide the corpuse to substrings of length 200.
n_timestamps = 200
x = x[:len(x)- len(x) % n_timestamps]
x = numpy.array(x, dtype='int32').reshape((-1, n_timestamps))

# Generate input and ouput per substring, as an indicator matrix.
y = numpy.zeros((x.shape[0], x.shape[1], max_features), dtype='int32')
for i in numpy.arange(x.shape[0]):
    for j in numpy.arange(x.shape[1]):
        y[i, j, x[i, j]] = 1

# Shift-1 the input sequences to the right, and make them start with '*'.
x = numpy.roll(y, 1, axis=1)
x[:, 0, :] = 0
x[:, 0, 0] = 1

# Build the model.

model = Sequential()
model.add(LSTM(256, return_sequences=True, batch_input_shape=x.shape))
model.add(Dense(max_features))
model.add(Activation('softmax'))

optimizer = RMSprop(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)


model.fit(x, y, batch_size=100, nb_epoch=1)

# Sample 128 sentences (200 characters each) from model.

def mnrnd(probs):
    rnd = numpy.random.random()
    for i in xrange(len(probs)):
        rnd -= probs[i]
        if rnd <= 0:
            return i
    return i

sentences = numpy.zeros((45, n_timestamps, max_features))
sentences[:, 0, 0] = 1

# Start sampling char-sequences. At each iteration i the probability over
# the i-th character of each sequences is computed.
for i in numpy.arange(n_timestamps):
    probs = model.predict_proba(sentences)[:,i,:]
    # Go over each sequence and sample the i-th character.
    for j in numpy.arange(len(sentences)):
        sentences[j, i+1, mnrnd(probs[j, :])] = 1
sentences = [sentence[1:].nonzero()[1] for sentence in sentences]

# Convert to readable text.
text = []
for sentence in sentences:
    text.append(''.join([dct[word] for word in sentence]))

But I am getting this error:

Traceback (most recent call last):
  File "char-rnn.py", line 70, in <module>
    sentences[j, i+1, mnrnd(probs[j, :])] = 1
IndexError: index 200 is out of bounds for axis 1 with size 200

Answer 1

It looks like it is trying to run over a sequence that is longer than the data.

Looking at your code, this is the area throwing the error:

for i in numpy.arange(n_timestamps):
probs = model.predict_proba(sentences)[:,i,:]
# Go over each sequence and sample the i-th character.
for j in numpy.arange(len(sentences)):
    sentences[j, i+1, mnrnd(probs[j, :])] = 1

The problem is probably that your data is n_timestamps long, but you are trying to predict the n_timestamps + 1 character (when you are predicting i +1).

try reducing the length of your loop by one as follows:

for i in numpy.arange(n_timestamps - 1):
probs = model.predict_proba(sentences)[:,i,:]
# Go over each sequence and sample the i-th character.
for j in numpy.arange(len(sentences)):
    sentences[j, i+1, mnrnd(probs[j, :])] = 1

Answer 2

There might be a problem with the way you manipulate X, I tried to run your code but encountered different errors than the one you mentioned.

I'm adding a different example on an LSTM which does what you wanted and is based on the example here .

Here is the code:

from __future__ import print_function
from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.layers import LSTM
from keras.optimizers import RMSprop
import numpy as np
import random
import sys

text = "123456789"*1000
print('corpus length:', len(text))

chars = sorted(list(set(text)))
print('total chars:', len(chars))
char_indices = dict((c, i) for i, c in enumerate(chars))
indices_char = dict((i, c) for i, c in enumerate(chars))

# cut the text in semi-redundant sequences of maxlen characters
maxlen = 200
sentences = []
next_chars = []
for i in range(0, len(text) - maxlen):
    sentences.append(text[i: i + maxlen])
    next_chars.append(text[i + maxlen])
print('nb sequences:', len(sentences))

print('Vectorization...')
X = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)
y = np.zeros((len(sentences), len(chars)), dtype=np.bool)
for i, sentence in enumerate(sentences):
    for t, char in enumerate(sentence):
        X[i, t, char_indices[char]] = 1
    y[i, char_indices[next_chars[i]]] = 1


# build the model: a single LSTM
print('Build model...')
model = Sequential()
model.add(LSTM(128, input_shape=(maxlen, len(chars))))
model.add(Dense(len(chars)))
model.add(Activation('softmax'))

optimizer = RMSprop(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)


def sample(preds, temperature=1.0):
    # helper function to sample an index from a probability array
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)

# train the model, output generated text after each iteration
for iteration in range(1, 60):
    print()
    print('-' * 50)
    print('Iteration', iteration)
    model.fit(X, y, batch_size=128, nb_epoch=1)

    start_index = random.randint(0, len(text) - maxlen - 1)

    for diversity in [0.2, 0.5, 1.0, 1.2]:
        print()
        print('----- diversity:', diversity)

        generated = ''
        sentence = text[start_index: start_index + maxlen]
        generated += sentence
        print('----- Generating with seed: "' + sentence + '"')
        sys.stdout.write(generated)

        for i in range(400):
            x = np.zeros((1, maxlen, len(chars)))
            for t, char in enumerate(sentence):
                x[0, t, char_indices[char]] = 1.

            preds = model.predict(x, verbose=0)[0]
            next_index = sample(preds, diversity)
            next_char = indices_char[next_index]

            generated += next_char
            sentence = sentence[1:] + next_char

            sys.stdout.write(next_char)
            sys.stdout.flush()
        print()