Labels-only distillation trains a student from hard and soft teacher labels, without feature alignment. It serves as a baseline for recovering capabilities in a pruned model. section 6.3.2
Listing 6.7 The training loop
def train_student(
student_model, teacher_model,dataloader,
layer_map,alpha=0.4,beta=0.4,gamma=0.2,temperature=2.0,
epochs=3,learning_rate=1e-5,experiment_name="experiment",accumulation_steps=4
):
optimizer = torch.optim.AdamW(
student_model.parameters(),
lr=learning_rate
) #1
student_model.train() #A
teacher_model.eval()
request_hidden_states = (gamma > 0) #B
loss_history = {'total': [], 'task': [], 'logits': [], 'hidden': []}
epoch_times = []
total_start_time = time.time()
for epoch in range(epochs):
epoch_start_time = time.time()
epoch_losses = {'total': [], 'task': [], 'logits': [], 'hidden': []}
progress_bar = tqdm(dataloader, desc=f"Epoch {epoch+1}/{epochs}")
accumulated_losses = {'total': 0.0, 'task': 0.0, 'logits': 0.0, 'hidden': 0.0}
accumulation_counter = 0
for batch_idx, (input_ids, attention_mask) in enumerate(progress_bar):
input_ids = input_ids.to(device)
attention_mask = attention_mask.to(device)
labels = input_ids.clone()
student_outputs = student_model(
input_ids=input_ids,
attention_mask=attention_mask,
output_hidden_states=request_hidden_states #2
) #C
with torch.no_grad(): #3
teacher_outputs = teacher_model(
input_ids=input_ids,
attention_mask=attention_mask,
output_hidden_states=request_hidden_states
) #D
student_hiddens = (
student_outputs.hidden_states[1:]
if request_hidden_states else None
) #4
teacher_hiddens = (teacher_outputs.hidden_states[1:]
if request_hidden_states else None)
loss, loss_dict = compute_compound_loss(
student_logits=student_outputs.logits,
teacher_logits=teacher_outputs.logits,
student_hiddens=student_hiddens,
teacher_hiddens=teacher_hiddens,
labels=labels,
layer_map=layer_map,
alpha=alpha, beta=beta, gamma=gamma, temperature=temperature
)
scaled_loss = loss / accumulation_steps #5
scaled_loss.backward()
for key in accumulated_losses:
accumulated_losses[key] += loss_dict[key]
accumulation_counter += 1
if (batch_idx + 1) % accumulation_steps == 0: #6
torch.nn.utils.clip_grad_norm_(student_model.parameters(), max_norm=1.0)
optimizer.step()
optimizer.zero_grad()
avg_losses = {k: v / accumulation_counter for k, v in accumulated_losses.items()}
for key in avg_losses:
epoch_losses[key].append(avg_losses[key])
progress_bar.set_postfix({
'loss': f"{avg_losses['total']:.4f}",
'task': f"{avg_losses['task']:.4f}",
'logits': f"{avg_losses['logits']:.4f}",
'hidden': f"{avg_losses['hidden']:.4f}"
})
accumulated_losses = {'total': 0.0, 'task': 0.0, 'logits': 0.0, 'hidden': 0.0}
accumulation_counter = 0
if accumulation_counter > 0: #7
torch.nn.utils.clip_grad_norm_(student_model.parameters(), max_norm=1.0)
optimizer.step()
optimizer.zero_grad()
avg_losses = {k: v / accumulation_counter for k, v in accumulated_losses.items()}
for key in avg_losses:
epoch_losses[key].append(avg_losses[key])
for key in epoch_losses:
if epoch_losses[key]:
avg_loss = np.mean(epoch_losses[key])
loss_history[key].append(avg_loss)
epoch_time = time.time() - epoch_start_time
epoch_times.append(epoch_time)
total_time = time.time() - total_start_time
loss_history['epoch_times_seconds'] = epoch_times
loss_history['total_time_seconds'] = total_time
return student_model, loss_history
The training loop exposes the loss components and controls hidden-state requests, making it a useful implementation reference for comparing distillation strategies. section 6.3.2
Knowledge recovery across data scales
With 2,000, 15,000, and 40,000 Cosmopedia samples, labels-only perplexity was 18.86, 12.65, and 11.73; capability retention was 86.74%, 90.1%, and 90.1%. section 6.3.2
