RNN 和 CNN-RNN 不会正确训练，总是预测一类

Question

I am currently developing a model to detect emotion from text using deep learning algorithms. I have a relatively small labelled dataset(~7500) with 7 different emotions as classes. I developed a CNN and achieved an accuracy of ~63% but when I tried to apply a RNN, using LSTM, and a CNN-RNN, also using LSTM, they just don't seem to train properly at all and always end up predicting the same class. I believe my models to be fundamentally sound but with some mistakes with the parameters. I have the dataset split up into 85% for training, with a further 20% of that for validation, and the remaining 15% for testing. My embedding matrix is developed using the word representations from the Google News word2vec and the word index is developed using keras Tokenizer.

Dataset breakdown:

Emotion

anger 1086

disgust 1074

fear 1086

guilt 1062

joy 1089

sadness 1080

shame 1058

CNN implementation

def make_model(kernel_sizes, num_filters, dropout, hidden_units):

    submodels = []
    for kernel_size in kernel_sizes:
        submodel = Sequential()

        submodel.add(Embedding(input_dim = input_dim,
                            output_dim   = output_dim,
                            weights      = [embedding_matrix],
                            input_length = max_len,
                            trainable    = True))

        submodel.add(Conv1D(filters=num_filters, kernel_size=kernel_size, padding='same',activation='relu',strides=1))
        submodel.add(GlobalMaxPooling1D())
        submodels.append(submodel)

    submodel_outputs = [model.output for model in submodels]    
    submodel_inputs = [model.input for model in submodels]

    merged = Concatenate(axis=1)(submodel_outputs)
    x = Dropout(dropout)(merged)

    if(hidden_units > 0):
        x = Dense(hidden_units, activation='relu')(x)
        x = Dropout(dropout)(x)

    x = Dense(7,activation='softmax', kernel_initializer="uniform")(x)
    out = Activation('sigmoid')(x)

    model = Model(submodel_inputs, out)
    model.compile(loss='categorical_crossentropy',optimizer='rmsprop',metrics=['acc'])

    return model

def fit_model(model, kernel_sizes, num_epochs, batch_size, x_train, y_train):

    x_train = [x_train]*len(kernel_sizes)

    history = model.fit(x_train, y_train, batch_size=batch_size, epochs=num_epochs, validation_split=0.2)

    return history

kernel_sizes  = [2,6]
num_filters   = 100
dropout       = 0.6
num_hidden    = 270
callbacks     = callbacks_list
num_epochs    = 15
batch_size = 64
model = make_model(kernel_sizes, num_filters, dropout, num_hidden)
print(model.summary())
history = fit_model(model, kernel_sizes, num_epochs, batch_size, x_train, y_train)

Model: "model_1"

---

Layer (type) Output Shape Param # Connected to

embedding_1_input (InputLayer) (None, 179) 0

---

embedding_2_input (InputLayer) (None, 179) 0

---

embedding_1 (Embedding) (None, 179, 300) 2729400 embedding_1_input[0][0]

---

embedding_2 (Embedding) (None, 179, 300) 2729400 embedding_2_input[0][0]

---

conv1d_1 (Conv1D) (None, 179, 100) 60100 embedding_1[0][0]

---

conv1d_2 (Conv1D) (None, 179, 100) 180100 embedding_2[0][0]

---

global_max_pooling1d_1 (GlobalM (None, 100) 0 conv1d_1[0][0]

---

global_max_pooling1d_2 (GlobalM (None, 100) 0 conv1d_2[0][0]

---

concatenate_1 (Concatenate) (None, 200) 0 global_max_pooling1d_1[0][0]
global_max_pooling1d_2[0][0]

---

dropout_1 (Dropout) (None, 200) 0 concatenate_1[0][0]

---

dense_1 (Dense) (None, 270) 54270 dropout_1[0][0]

---

dropout_2 (Dropout) (None, 270) 0 dense_1[0][0]

---

dense_2 (Dense) (None, 7) 1897 dropout_2[0][0]

---

activation_1 (Activation) (None, 7) 0 dense_2[0][0]

Total params: 5,755,167 Trainable params: 5,755,167 Non-trainable params: 0

---

Training and Validation results for CNN

CNN confusion matrix

---

RNN Implementation

def make_model(lstm_units, dropout, hidden_units):

    model = Sequential()   

    model.add(Embedding(input_dim = input_dim,
                        output_dim   = output_dim,
                        weights      = [embedding_matrix],
                        input_length = max_len,
                        trainable    = False))

    model.add(LSTM(lstm_units))

    model.add(Dropout(dropout))

    if(hidden_units > 0):
        model.add(Dense(hidden_units, activation='elu'))
        model.add(Dropout(dropout))

    model.add(Dense(7,activation='softmax', kernel_initializer="uniform"))
    model.add(Activation('sigmoid'))

    model.compile(loss='categorical_crossentropy',optimizer='rmsprop',metrics=['acc'])

    return model

lstm_units = 120
dropout = 0.5
hidden_units = 550
callbacks = [tensorboard, early]
num_epochs = 20
batch_size = 60

model = make_model(lstm_units, dropout, hidden_units)
print(model.summary())
history = fit_model(model, num_epochs, batch_size, x_train, y_train)

Model: "sequential_6"

---

Layer (type) Output Shape Param #

embedding_6 (Embedding) (None, 179, 300) 2729400

---

lstm_8 (LSTM) (None, 120) 202080

---

dropout_5 (Dropout) (None, 120) 0

---

dense_6 (Dense) (None, 550) 66550

---

dropout_6 (Dropout) (None, 550) 0

---

dense_7 (Dense) (None, 7) 3857

---

activation_3 (Activation) (None, 7) 0

Total params: 3,001,887 Trainable params: 272,487 Non-trainable params: 2,729,400

---

RNN training and validation scores

RNN confusion matrix

---

CNN-RNN implementation

def make_model(kernel_sizes, num_filters, dropout, hidden_units, lstm_units):

    submodels = []
    for kernel_size in kernel_sizes:
        submodel = Sequential()

        submodel.add(Embedding(input_dim = input_dim,
                            output_dim   = output_dim,
                            weights      = [embedding_matrix],
                            input_length = max_len,
                            trainable    = True))

        submodel.add(Conv1D(filters=num_filters, kernel_size=kernel_size, padding='same',activation='relu',strides=1))
        submodel.add(MaxPooling1D(pool_size=2, strides = 2))
        submodel.add(Dropout(dropout))
        submodel.add(LSTM(lstm_units)) 
        submodels.append(submodel)

    submodel_outputs = [model.output for model in submodels]    
    submodel_inputs = [model.input for model in submodels]

    merged = Concatenate(axis=1)(submodel_outputs)
    x = Dropout(dropout)(merged)

    if(hidden_units > 0):
        x = Dense(hidden_units, activation='relu')(x)
        x = Dropout(dropout)(x)

    x = Dense(7,activation='softmax', kernel_initializer="uniform")(x)
    out = Activation('sigmoid')(x)

    model = Model(submodel_inputs, out)
    model.compile(loss='categorical_crossentropy',optimizer='rmsprop',metrics=['acc'])

    return model

kernel_sizes  = [2,3,6]
num_filters   = 100
dropout       = 0.6
num_hidden    = 270
lstm_units = 80
callbacks     = [tensorboard, early]
num_epochs    = 20
batch_size = 64

model = make_model(kernel_sizes, num_filters, dropout, num_hidden, lstm_units)
print(model.summary())
history = fit_model(model, kernel_sizes, num_epochs, batch_size, x_train, y_train)

Model: "model_2"

---

Layer (type) Output Shape Param # Connected to

embedding_8_input (InputLayer) (None, 179) 0

---

embedding_9_input (InputLayer) (None, 179) 0

---

embedding_10_input (InputLayer) (None, 179) 0

---

embedding_8 (Embedding) (None, 179, 300) 2729400 embedding_8_input[0][0]

---

embedding_9 (Embedding) (None, 179, 300) 2729400 embedding_9_input[0][0]

---

embedding_10 (Embedding) (None, 179, 300) 2729400 embedding_10_input[0][0]

---

conv1d_8 (Conv1D) (None, 179, 100) 60100 embedding_8[0][0]

---

conv1d_9 (Conv1D) (None, 179, 100) 90100 embedding_9[0][0]

---

conv1d_10 (Conv1D) (None, 179, 100) 180100 embedding_10[0][0]

---

max_pooling1d_7 (MaxPooling1D) (None, 89, 100) 0 conv1d_8[0][0]

---

max_pooling1d_8 (MaxPooling1D) (None, 89, 100) 0 conv1d_9[0][0]

---

max_pooling1d_9 (MaxPooling1D) (None, 89, 100) 0 conv1d_10[0][0]

---

dropout_9 (Dropout) (None, 89, 100) 0 max_pooling1d_7[0][0]

---

dropout_10 (Dropout) (None, 89, 100) 0 max_pooling1d_8[0][0]

---

dropout_11 (Dropout) (None, 89, 100) 0 max_pooling1d_9[0][0]

---

lstm_2 (LSTM) (None, 80) 57920 dropout_9[0][0]

---

lstm_3 (LSTM) (None, 80) 57920 dropout_10[0][0]

---

lstm_4 (LSTM) (None, 80) 57920 dropout_11[0][0]

---

concatenate_3 (Concatenate) (None, 240) 0 lstm_2[0][0]
lstm_3[0][0]
lstm_4[0][0]

---

dropout_12 (Dropout) (None, 240) 0 concatenate_3[0][0]

---

dense_3 (Dense) (None, 270) 65070 dropout_12[0][0]

---

dropout_13 (Dropout) (None, 270) 0 dense_3[0][0]

---

dense_4 (Dense) (None, 7) 1897 dropout_13[0][0]

---

activation_2 (Activation) (None, 7) 0 dense_4[0][0]

Total params: 8,759,227 Trainable params: 8,759,227 Non-trainable params: 0

---

CNN-RNN training and validation scores CNN-RNN confusion matrix

I understand there is no magic formula to neural networks and no one size fits all approach, I am just looking for some guidance in the areas which I may have made mistakes in when implementing the CNN-RNN and RNN.

Apologies in advance for any formatting errors as this is my first question asked. If there is any other info required please let me know.

Thanks very much.

Answer 1

I can't say this will solve all your issues, but something that is definitely wrong is your repeated use of the sigmoid activation right after a softmax activation, while your classification problem has 7 classes. The sigmoid activation can only separate two classes.

For instance:

model.add(Dense(7,activation='softmax', kernel_initializer="uniform"))
model.add(Activation('sigmoid'))

You should just remove the sigmoid activation the three times you did this.

Answer 2

First of all, your CNN implementation is over-enthusiastic, have you come up with the architecture by experimenting with multiple designs or just chose it?

Usually, when multiple heads are chosen they are fed a slightly variation of the input, not the exact same copy so maybe your multi-head design is not the most optimal choice, it introduces too many unnecessary parameters and can lead to overfitting and from your loss curve which is evident.

You used categorical crossentropy but used sigmoid after softmax, which is also not how things are done. Just use the softmax activation and get rid of the sigmoid.

Is the confusion matrix for test set? Then, it seems your test split is too easy, as the model is so much overfitted it should perform poorly. So, try to find a better test split by making sure not too much similar data falls in both training and testing.

It's always better to fine-tune your simpler model before going to complicated ones. As your LSTM model didn't perform well, it doesn't make sense to try a even more complicated model (CNN-LSTM). Your LSTM model didn't converge, the reasons can be many (the obvious ones being the incorrect usage of activation layer).

def make_model(lstm_units, dropout, hidden_units):

    model = Sequential()   

    model.add(Embedding(input_dim = input_dim,
                        output_dim   = output_dim,
                        weights      = [embedding_matrix],
                        input_length = max_len,
                        trainable    = False))

    model.add(LSTM(lstm_units, return_sequences = True, recurrent_dropout = 0.2))
    model.add(Dropout(dropout))
    model.add(LSTM(lstm_units, recurrent_dropout = 0.2))

    model.add(Dropout(dropout))


    model.add(Dense(7, activation='softmax'))

    model.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['acc'])

    return model

Make it a fully LSTM based model by getting rid of the FC layers, also start with smaller LSTM units like 8, 16, 32, ...

For more improvement, you can do the following.

0) Get rid of the glove embedding and use your own learnable embedding.

1) Hyper-parameter search over the network to find the most optimal model.

There are many libraries, but I find this one very flexible. https://github.com/keras-team/keras-tuner

Just install with pip.

Here is a demo code.

from tensorflow import keras
from tensorflow.keras import layers
from kerastuner.tuners import RandomSearch


def build_model(hp):
    model = keras.Sequential()
    model.add(layers.Embedding(input_dim=hp.Int('input_dim',
                                        min_value=5000,
                                        max_value=10000,
                                        step = 1000),
                              output_dim=hp.Int('output_dim',
                                        min_value=200,
                                        max_value=800,
                                        step = 100),
                              input_length = 400))
    model.add(layers.Convolution1D(
                filters=hp.Int('filters',
                                        min_value=32,
                                        max_value=512,
                                        step = 32),
                kernel_size=hp.Int('kernel_size',
                                        min_value=3,
                                        max_value=11,
                                        step = 2),
                padding='same',
                activation='relu')),
    model.add(layers.BatchNormalization())
    model.add(layers.MaxPooling1D())
    model.add(layers.Flatten())
    model.add(layers.Dropout(0.4))
    model.add(layers.Dense(units=hp.Int('units',
                                        min_value=64,
                                        max_value=256,
                                        step=32),
                           activation='relu'))
    model.add(layers.Dropout(0.4))
    model.add(layers.Dense(7, activation='softmax'))
    model.compile(
    optimizer=keras.optimizers.Adam(
        hp.Choice('learning_rate',
                  values=[1e-2, 1e-3, 1e-4])),
    loss='categorical_crossentropy',
    metrics=['accuracy'])
    return model


tuner = RandomSearch(
    build_model,
    objective='val_accuracy',
    max_trials=5,
    executions_per_trial=3,
    directory='my_dir',
    project_name='helloworld')
tuner.search_space_summary()

## The following lines are based on your model


tuner.search(x, y,
             epochs=5,
             validation_data=(val_x, val_y))

models = tuner.get_best_models(num_models=2)

If you want to extract more meaningful features, one approach I found promising is by extracting pre-trained BERT features and then training using a CNN/LSTM.

A great repository to get started is this one - https://github.com/UKPLab/sentence-transformers

Once you get the sentence embedding from the BERT/XLNet you can use those features to train another CNN similar to the one you are using except maybe get rid of the embedding layer as it's expensive.