I converted a working pytorch training script to pytorch lightning in order to utilize multiple GPUs in training. The training on GPUs itself is working fine and I was able to get the speed boost, but after training, the model isn't learning anything and is predicting (binary) everything around 0.5. The original pytorch script works when trained on a single GPU, so it's something during this conversion that probably went wrong.
The basic setup is:
- I use a pretrained transformer, and freeze the encoder parameters,
- I connect encoder output to a multi-class classifier head and a binary classifier head,
- I'm only focusing on the binary classifier, the multi-class head is just for the model to learn a bit more.
What I have tried:
- disable auto optimization and implement pytorch optimization (zero_grad, backward, step) within
training_step
. - increase/decrease batch size
- increase learning_rate
- try training only on 1 GPU with this same script
- try 2-3 GPUs
None of them seems to be working for me and when I check the parameter values in the saved checkpoint, it looks like it's just after initialization.
Did I miss anything very important here? I only froze the transformer weights, but not the linear classifiers.
import torch
import torch.nn as nn
import pytorch_lightning as pl
from transformers import PegasusForConditionalGeneration
class MyModel(pl.LightningModule):
def __init__(self, model_name, tokenizer, num_intent_classes, pos_weight=1, input_dim=1024):
super().__init__()
self.pad_token_id = tokenizer.pad_token_id
self.transformer = PegasusForConditionalGeneration.from_pretrained(model_name)
self.domain_classifier = nn.Linear(input_dim, num_intent_classes)
self.noise_classifier = nn.Linear(input_dim, 1)
self.set_requires_grad()
self.register_buffer("pos_weight", torch.Tensor([pos_weight]))
def forward(self, x, mask=None):
enc_last_hidden_state = self.transformer.model.encoder(x, attention_mask=mask)["last_hidden_state"]
enc_summary = torch.mean(enc_last_hidden_state, dim=1)
domain_logits = self.domain_classifier(enc_summary)
noise_logits = self.noise_classifier(enc_summary)
return domain_logits, noise_logits
def set_requires_grad(self):
# We freeze all the weights in transformer model, only training the final linear layer
# so that we can combine this with the NLG model to save inference time
for n, p in self.transformer.model.named_parameters():
p.requires_grad = False
def training_step(self, batch, batch_idx):
queries, domain_labels, noise_labels = batch
attn_mask = (queries != self.pad_token_id).float()
queries, domain_labels, noise_labels = queries, domain_labels, noise_labels
domain_scores, noise_scores = self.forward(queries, attn_mask)
domain_loss, noise_loss = self.loss_function(domain_scores, domain_labels, noise_scores, noise_labels)
loss = domain_loss + noise_loss
loss_dict = {
"domain_loss": domain_loss,
"noise_loss": noise_loss,
"loss": loss
}
self.log_dict(loss_dict, prog_bar=True, on_step=True, on_epoch=True, logger=True)
return loss
def loss_function(self, domain_scores, domain_labels, noise_scores, noise_labels):
domain_loss = nn.CrossEntropyLoss()(domain_scores, domain_labels[:,0])
noise_loss = nn.BCEWithLogitsLoss(pos_weight=self.pos_weight)(noise_scores[:,0], noise_labels[:,0])
return domain_loss, noise_loss
def configure_optimizers(self):
optimizer = torch.optim.Adam(self.parameters(), lr=1e-4)
return optimizer
def train_model_ligntning(model_name,
tokenizer,
num_intent_classes,
train_loader,
valid_loader,
checkpoint_dir,
num_epochs,
pos_weight,
n_gpus):
model = MyModel(model_name, tokenizer, num_intent_classes, pos_weight=pos_weight)
trainer = pl.Trainer(logger=True,
gpus=n_gpus,
accelerator="gpu",
max_epochs=num_epochs,
check_val_every_n_epoch=1,
default_root_dir=checkpoint_dir)
trainer.fit(model=model,
train_dataloaders=train_loader,
val_dataloaders=valid_loader)
self.set_requires_grad()
in the__init__
method?self.transformer.eval()
to make the weights freeze during training. Are they different?enc_last_hidden_state
the model input, and synthesizing some random data. or (b) extend your example some data that can be downloaded by anyone attempting to debug it?enc_last_hidden_state
and just have a small linear prediction layer to test it out.