分層 LSTM 自動編碼器 - model 未訓練

Question

我正在嘗試重構這篇關於段落分層自動編碼器的論文。

這個想法是：將一個段落分成句子，然后使用 LSTM 對每個句子進行編碼，然后將這些編碼用作另一個 LSTM 的輸入，該 LSTM 對整個段落進行編碼。

然后，使用鏡像解碼器，使用 LSTM 將編碼的段落解碼成多個句子，然后使用另一個 LSTM 對每個單詞進行解碼，頂部有一個線性層並預測單詞。

目標是嘗試預測原始段落。

我已經進行了一些預處理，現在我使用一種熱編碼將每個段落保存為(maxSentence,maxWordsPerSentence,VocabSize)的張量。

我的問題是，還有 model 不學習。 損失保持不變，似乎沒有發生任何事情。我不確定如何計算損失（我已經一起運行了一批並將其解碼為多個段落，然后計算了損失針對整個批量預測，我的火車 function 添加在下面。我不知道這是否是問題（也許我應該逐句計算損失而不是整個段落？）或者我的 model 有問題。

編碼器代碼：

class Encoder(nn.Module):
def __init__(self, input_dim, emb_dim, enc_hid_dim, dec_hid_dim, dropout):
    super().__init__()
    
    #self.embedding = nn.Embedding(input_dim, emb_dim)
    self.rnn_sent = nn.GRU(input_dim, enc_hid_dim, bidirectional = True)
    self.rnn_par = nn.GRU(enc_hid_dim*2, dec_hid_dim, bidirectional = True)

    
def forward(self, src):
    
    outputs, hidden = self.rnn_sent(src[:,0,0])
    total_out = outputs.unsqueeze(0).permute(1,0,2)
    for i in range(1,src.shape[1]):
      for j in range(src.shape[2]):
        outputs, hidden = self.rnn_sent(src[:,i,j],hidden)
      total_out = torch.cat((total_out,outputs.unsqueeze(0).permute(1,0,2)),dim=1)  

    outputs_par, hidden_par = self.rnn_par(total_out[:,0])
    
    for i in range(total_out.shape[1]):
        outputs_par, hidden_par = self.rnn_par(total_out[:,i],hidden_par)

    return outputs_par, hidden_par

解碼器代碼：

class Decoder(nn.Module):
def __init__(self, output_dim, emb_dim, enc_hid_dim, dec_hid_dim, dropout, attention):
    super().__init__()

    self.output_dim = output_dim
    self.attention = attention
    #self.embedding = nn.Embedding(output_dim, emb_dim)
    self.rnn_par = nn.GRU((enc_hid_dim * 2), dec_hid_dim*2)
    self.rnn_sen = nn.GRU(output_dim, dec_hid_dim*2)
    self.fc_out = nn.Linear(dec_hid_dim*2, output_dim)
    self.dropout = nn.Dropout(dropout)
    
def forward(self, input, hidden, encoder_outputs):
    output, hidden = self.rnn_par(encoder_outputs)
    all_par = output.unsqueeze(0).permute(1,0,2)
    for i in range(1,max_par_len):
      output,hidden = self.rnn_par(output,hidden)
      all_par = torch.cat((all_par,output.unsqueeze(0).permute(1,0,2)),dim=1)


    for i in range(max_par_len):
      output_arg = self.fc_out(all_par[:,i])
      #output_argmax = F.one_hot(output_arg.argmax(dim = 1), self.output_dim).to(torch.float)
      output_argmax = torch.softmax(output_arg,dim=1)
      output_sen, hidden_sen = self.rnn_sen(output_argmax)
      all_par_sen = output_argmax.unsqueeze(0).permute(1,0,2)
      for j in range(max_sen_len - 1):
        output_sen,hidden_sen = self.rnn_sen(output_argmax,hidden_sen)
        output_arg = self.fc_out(output_sen)
        output_argmax = torch.softmax(output_arg,dim=1)

        all_par_sen = torch.cat((all_par_sen,output_argmax.unsqueeze(0).permute(1,0,2)),dim=1)
      if i == 0:
        all_doc = all_par_sen.unsqueeze(0).permute(1,0,2,3)
      else:
        all_doc = torch.cat((all_doc,all_par_sen.unsqueeze(0).permute(1,0,2,3)),dim=1)

      i+=1
    return all_doc  ,hidden_sen

還有我的火車function：

def train(model, iterator, optimizer, criterion, clip, epoch):

model.train()

epoch_loss = 0
data = tqdm(iterator)
for i, batch in enumerate(data):
    src = batch[0].to(device)#.to(torch.long)#.reshape(batch[0].shape[0],-1)
    trg = batch[0].to(device)#.to(torch.long)#.reshape(batch[0].shape[0],-1)
    target = torch.argmax(trg,dim=3).view(-1)
    print(target)
    optimizer.zero_grad()
    output = model(src, trg).view(-1,OUTPUT_DIM)
    loss = criterion(output, target)        
    loss.backward()
    torch.nn.utils.clip_grad_norm_(model.parameters(), clip)
    optimizer.step()
    epoch_loss += loss.item()


N_EPOCHS = 20
CLIP = 1
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
criterion = nn.CrossEntropyLoss(ignore_index = vocabulary['<pad>'])
best_valid_loss = float('inf')

for epoch in range(N_EPOCHS):

    start_time = time.time()
    train_loader, valid_loader = data_loaders['train_loader'], data_loaders['test_loader']
    train_loss = train(model, train_loader, optimizer, criterion, CLIP,f'{epoch+1}/{N_EPOCHS}')
    #valid_loss = evaluate(model, valid_loader, criterion)

    end_time = time.time()

    epoch_mins, epoch_secs = epoch_time(start_time, end_time)

    print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')
    print(f'\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')