I constructed Bi-LSTM Language Model by pytorch
, and found that after 200 epochs, the model suddenly returned only meaningless tokens with Nan loss, while it returned reasonable tokens before.
Please refer to the code of model below:
# optimizer = torch.optim.Adam(model.parameters(), lr=0.009, amsgrad=False)
class BiLSTM(nn.Module):
def __init__(self, voc_size, hidn_size, emb_size=300):
super().__init__()
self.voc_size = voc_size
self.emb_size = emb_size
self.hidn_size = hidn_size
self.emb = nn.Embedding(num_embeddings=voc_size, embedding_dim=emb_size)
self.lstm = nn.LSTM(input_size=emb_size, hidden_size=hidn_size, bidirectional=True)
self.lm_out = nn.Linear(hidn_size*2, voc_size)
self.dropout = nn.Dropout(p=0.3)
def forward(self, x, prev_state):
state_h, state_c = prev_state
bs = len(x)
emb = self.emb(x)
emb = emb.permute(1,0,-1)
out, (state_h, state_c) = self.lstm(emb, (state_h[:,:bs,:].contiguous(), state_c[:,:bs,:].contiguous()))
forward_out = out[:, :, :self.hidn_size]
backward_out = out[:, :, self.hidn_size:]
concat_h = torch.cat([forward_out[:-2], backward_out[2:]], dim=2)
final_out = self.lm_out(self.dropout(concat_h.permute(1,0,2)))
return final_out.view(final_out.size()[0]*final_out.size()[1], final_out.size()[-1]), (state_h, state_c)