LSTM with multiple input features and multiple outputs

Question

Given 30 timestamps with each having 3 features, I want to predict one single output containing 4 different quantities.

I have an X_train and y_train of shape (72600, 30, 3) and (72600, 4) respectively.

where for X_train,

• 72600 represents the number of samples
• 30 represents the number of timestamps considered
• 3 represents the number of features for each timestamp

eg X_train[0] will look something like this:

[
    [1,2,3],
    [4,5,6],
    ... such 30 rows 
]

and in y_train, 4 represents the number of outputs to be predicted.

I tried the following code,

model = Sequential()
model.add(LSTM(units = 50, return_sequences = True, input_shape = (X_train.shape[1], X_train.shape[2])))
model.add(Dropout(0.2))
model.add(LSTM(units = 50, return_sequences = True))
model.add(Dropout(0.2))
model.add(LSTM(units = 50, return_sequences = True))
model.add(Dropout(0.2))
model.add(Dense(units = 4))

The output which I get from this model after passing a single sample of size (1, 30, 3) is of shape: (1, 30, 4) but I just want an output of shape (1, 4).

So how can I do that?

Answer 1

In your last LSTM layer, you will have to set the return_sequences parameter to False in order to get an 1D output:

import tensorflow as tf

model = tf.keras.Sequential()
model.add(tf.keras.layers.LSTM(units = 50, return_sequences = True, input_shape = (30, 3)))
model.add(tf.keras.layers.Dropout(0.2))
model.add(tf.keras.layers.LSTM(units = 50, return_sequences = True))
model.add(tf.keras.layers.Dropout(0.2))
model.add(tf.keras.layers.LSTM(units = 50))
model.add(tf.keras.layers.Dropout(0.2))
model.add(tf.keras.layers.Dense(units = 4))

model(tf.random.normal((1, 30, 3)))

<tf.Tensor: shape=(1, 4), dtype=float32, numpy=
array([[-1.3130311e-03,  1.0584719e-02, -6.3279571e-05, -2.3087783e-02]],
      dtype=float32)>

So, instead of returning a sequence given a sequence, your last LSTM layer returns the output state of only the last LSTM cell.

Answer 2

I figured out if I set my batch size to a small number and use early stop that I could improve my accuracy on 4 million rows of data

look_back=1
model = Sequential()
features=len(X_columns)
model.add(LSTM(units=550, activation='tanh', return_sequences=True, input_shape=(look_back, features),dtype="float32"))
model.add(Dropout(0.2))
model.add(LSTM(units = 100, activation='tanh', return_sequences = True))
model.add(Dropout(0.2))
model.add(LSTM(units = 50))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(len(y_train.columns),activation='softmax'))
model.compile(optimizer="rmsprop", loss='categorical_crossentropy', metrics=['accuracy'])

def plotHistory(history):
        plt.plot(history.history['accuracy'])
        plt.title('accuracy')
        plt.ylabel('accuracy')
        plt.xlabel('epoch')
        plt.legend(['train', 'test'], loc='upper left')
        plt.show()
        
print ("Train the model")
length=len(X_train)
path_to_file="C:\\Users\\dnishimoto\\python_files\\python-deep-learning-master\\lstm_equipment_model.h5"

batch_size=64
start_pos=0
look_back=1
early_stopping = EarlyStopping(monitor='accuracy', patience=50)
count=0
for index in range(start_pos,length,batch_size):
    if(index>0):
        file_exists = exists(path_to_file)
        if(file_exists):
            model.load_weights(path_to_file)
            #print("loaded weights")
        
        X2 = np.asarray(X_train[index-batch_size:index]).astype(np.float32)
        X2 = np.resize(X2,(X2.shape[0],look_back,X2.shape[1]))
        y2 = np.asarray(y_train[index-batch_size:index]).astype(np.float32)
        history=model.fit(X2,y2,batch_size=len(X),epochs=500,verbose=0,callbacks=[early_stopping])
        model.save_weights(path_to_file) # can give whole path to save model
        #print("saved weights")
        if (count%10000==1):
            print(index-batch_size,index)
            plotHistory(history)
        count+=1