Logit-based distillation trains a student using the teacher’s soft output scores alongside hard labels. For a pruned model, label-only distillation offers a simpler, cheaper alternative to adding hidden-state alignment. section 6.3.2
Training-data scale strongly affects results: perplexity falls from 18.86 with 2,000 samples to 12.65 with 15,000 and 11.73 with 40,000. At 40,000 samples, capability retention is 90.1%; selected-layer feature alignment reaches slightly lower perplexity, 11.54, at 90% retention and about 15% more training time. section 6.3.2
Listing 6.7 The training loop
def train_student(
student_model, teacher_model,dataloader,
layer_map,alpha=0.4,beta=0.4,gamma=0.2,temperature=2.0,
epochs=3,learning_rate=1e-5,experiment_name="experiment",accumulation_steps=4
):
optimizer = torch.optim.AdamW(
student_model.parameters(),
lr=learning_rate
) #1
student_model.train() #A
teacher_model.eval()
request_hidden_states = (gamma > 0) #B
loss_history = {'total': [], 'task': [], 'logits': [], 'hidden': []}
epoch_times = []
total_start_time = time.time()
for epoch in range(epochs):
epoch_start_time = time.time()
epoch_losses = {'total': [], 'task': [], 'logits': [], 'hidden': []}
progress_bar = tqdm(dataloader, desc=f"Epoch {epoch+1}/{epochs}")
accumulated_losses = {'total': 0.0, 'task': 0.0, 'logits': 0.0, 'hidden': 0.0}
accumulation_counter = 0
for batch_idx, (input_ids, attention_mask) in enumerate(progress_bar):
input_ids = input_ids.to(device)
attention_mask = attention_mask.to(device)
labels = input_ids.clone()
student_outputs = student_model(
input_ids=input_ids,
attention_mask=attention_mask,
output_hidden_states=request_hidden_states #2
) #C
with torch.no_grad(): #3
teacher_outputs = teacher_model(
input_ids=input_ids,
attention_mask=attention_mask,
output_hidden_states=request_hidden_states
) #D
student_hiddens = (
student_outputs.hidden_states[1:]
if request_hidden_states else None
) #4
teacher_hiddens = (teacher_outputs.hidden_states[1:]
if request_hidden_states else None)
loss, loss_dict = compute_compound_loss(
student_logits=student_outputs.logits,
teacher_logits=teacher_outputs.logits,
student_hiddens=student_hiddens,
teacher_hiddens=teacher_hiddens,
labels=labels,
layer_map=layer_map,
alpha=alpha, beta=beta, gamma=gamma, temperature=temperature
)
scaled_loss = loss / accumulation_steps #5
scaled_loss.backward()
for key in accumulated_losses:
accumulated_losses[key] += loss_dict[key]
accumulation_counter += 1
if (batch_idx + 1) % accumulation_steps == 0: #6
torch.nn.utils.clip_grad_norm_(student_model.parameters(), max_norm=1.0)
optimizer.step()
optimizer.zero_grad()
avg_losses = {k: v / accumulation_counter for k, v in accumulated_losses.items()}
for key in avg_losses:
epoch_losses[key].append(avg_losses[key])
progress_bar.set_postfix({
'loss': f"{avg_losses['total']:.4f}",
'task': f"{avg_losses['task']:.4f}",
'logits': f"{avg_losses['logits']:.4f}",
'hidden': f"{avg_losses['hidden']:.4f}"
})
accumulated_losses = {'total': 0.0, 'task': 0.0, 'logits': 0.0, 'hidden': 0.0}
accumulation_counter = 0
if accumulation_counter > 0: #7
torch.nn.utils.clip_grad_norm_(student_model.parameters(), max_norm=1.0)
optimizer.step()
optimizer.zero_grad()
avg_losses = {k: v / accumulation_counter for k, v in accumulated_losses.items()}
for key in avg_losses:
epoch_losses[key].append(avg_losses[key])
for key in epoch_losses:
if epoch_losses[key]:
avg_loss = np.mean(epoch_losses[key])
loss_history[key].append(avg_loss)
epoch_time = time.time() - epoch_start_time
epoch_times.append(epoch_time)
total_time = time.time() - total_start_time
loss_history['epoch_times_seconds'] = epoch_times
loss_history['total_time_seconds'] = total_time
return student_model, loss_history
Inspect the training loop’s configurable logits-loss weight and temperature alongside task and hidden-state loss terms. section 6.3.2

