Keras model.predict gives different results to model.evalute

Question

I am training a Keras LSTM for Named Entity Recognition. There is a bi-LSTM at both the word and character level.

During training, the accuracy on both the train and test set are high. After training, I can run model.evaluate() on the test set and get a high score with an accuracy of 99%.

However, if I use model.predict() on X_test , the model just predicts that each sequence is an array of zeros (which I used for padding), and I have an accuracy of 60% and an f1 score of 0. Here is the code:

class BiLSTM:

    def __init__(self):
        self.annotations = load_annotations('data/annotations.p')
        self.cache = format_data(self.annotations)
        self.cv_sets = cross_val_sets(self.cache['padded_sents'],     self.cache['padded_labels'])
        self.embedding_matrix = get_embedding_matrix(self.cache['word_to_integer'])
        self.char_cache = format_char_data(self.annotations,
                                           self.cache['word_to_integer'].keys(),
                                           self.cache['max_sequence_length'])
        self.model = None

    def fit_model(self, X_train, y_train, X_char_train):

        #Extract parameters from the cache
        word_to_integer = self.cache['word_to_integer']
        n_words = self.cache['n_words']
        n_tags = self.cache['n_tags']
        max_sequence_length = self.cache['max_sequence_length']
        X_char = self.char_cache['X_char']
        max_len_char = self.char_cache['max_len_char']
        n_chars = self.char_cache['n_chars']

        # (among top max_features most common words)
        batch_size = 32


        #Word input
        word_in = Input(shape=(max_sequence_length,))

        # Word embedding matrix
        embedding_matrix = get_embedding_matrix(word_to_integer)

        # Word Embedding layer
        embedding_layer = Embedding(input_dim=n_words + 1,
                                output_dim=200,
                                weights=[embedding_matrix],
                                input_length=max_sequence_length,
                                trainable=False)(word_in)

        # input and embeddings for characters
        char_in = Input(shape=(max_sequence_length, max_len_char,))
        emb_char = TimeDistributed(Embedding(input_dim=n_chars + 2, output_dim=10,
                                         input_length=max_len_char, mask_zero=True))(char_in)

        # character LSTM to get word encodings by characters
        char_enc = TimeDistributed(LSTM(units=20, return_sequences=False,
                                    recurrent_dropout=0.5))(emb_char)

        # main LSTM
        x = concatenate([embedding_layer, char_enc])
        x = SpatialDropout1D(0.3)(x)
        main_lstm = Bidirectional(LSTM(units=50, return_sequences=True,
                                   recurrent_dropout=0.6))(x)
        out = TimeDistributed(Dense(n_tags + 1, activation="softmax"))(main_lstm)

        model = Model([word_in, char_in], out)
        optimizer = Adam(lr=0.01, beta_1=0.9, beta_2=0.999)
        model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=["acc"])


        model.fit([X_train, X_char_train], y_train,
                        batch_size=32, epochs=5, validation_split=0.2, verbose=1)

        self.model = model


    def run(self, cutoff = 0.8):
        sents = self.cache['padded_sents']
        labels = self.cache['padded_labels']
        # Train a model
        cutoff = int(sents.shape[0]*0.8)
        X_train = sents[:cutoff]
        X_test = sents[cutoff:]
        y_train = labels[:cutoff]
        y_test = labels[cutoff:]
        X_char_train = np.array(self.char_cache['X_char'])[:cutoff]
        X_char_test = np.array(self.char_cache['X_char'])[cutoff:]

        self.fit_model(X_train, y_train, X_char_train)

        # Accuracy metrics
        loss, accuracy = self.model.evaluate([X_test, X_char_test], y_test)

        print(accuracy)

        probs = self.model.predict([X_test, X_char_test])

        predicted = probs.argmax(axis=-1)
        actual = y_test.argmax(axis=-1)
        accuracy, f1 = get_metrics(actual, predicted, self.cache['integer_to_label'])
        print('acc: {}, f1: {}'.format(accuracy, f1))


if __name__ == "__main__":
    lstm = BiLSTM()
    lstm.run()

I have done a pretty extensive search and can't find the solution. Any help appreciated, thanks!

Answer 1

Have you tried printing out individual predictions for a single sequence like

predicted = probs[0].argmax(axis = -1)

and comparing it with

actual = y_test[0].argmax(axis = -1)

Comparing this result will help you in debugging. If this prediction is same then there must be some issue with get_metrics . Try this and post the feedback.