Gradient accumulation lets training use a larger effective batch without holding the entire batch in GPU memory. Gradients are collected over several batches, then the optimizer updates model parameters; dividing the loss by the accumulation steps keeps the average correct. section 6.3.2
A training loop can use this pattern:
- Process one batch and accumulate its gradients. section 6.3.2
- After the configured number of iterations, update parameters with the optimizer. section 6.3.2
- Handle a final incomplete group when the number of batches is not divisible by the accumulation steps. section 6.3.2
Effective batch size
With batches of 16 rows accumulated over four iterations, the optimizer receives information equivalent to 64 rows while each step uses memory for 16. A LoRA configuration with batch size 4 and two accumulation steps yields an effective batch of 8. section 6.3.2 section 13.3
Listing 6.7 The training loop
def train_student(
student_model, teacher_model,dataloader,
layer_map,alpha=0.4,beta=0.4,gamma=0.2,temperature=2.0,
epochs=3,learning_rate=1e-5,experiment_name="experiment",accumulation_steps=4
):
optimizer = torch.optim.AdamW(
student_model.parameters(),
lr=learning_rate
) #1
student_model.train() #A
teacher_model.eval()
request_hidden_states = (gamma > 0) #B
loss_history = {'total': [], 'task': [], 'logits': [], 'hidden': []}
epoch_times = []
total_start_time = time.time()
for epoch in range(epochs):
epoch_start_time = time.time()
epoch_losses = {'total': [], 'task': [], 'logits': [], 'hidden': []}
progress_bar = tqdm(dataloader, desc=f"Epoch {epoch+1}/{epochs}")
accumulated_losses = {'total': 0.0, 'task': 0.0, 'logits': 0.0, 'hidden': 0.0}
accumulation_counter = 0
for batch_idx, (input_ids, attention_mask) in enumerate(progress_bar):
input_ids = input_ids.to(device)
attention_mask = attention_mask.to(device)
labels = input_ids.clone()
student_outputs = student_model(
input_ids=input_ids,
attention_mask=attention_mask,
output_hidden_states=request_hidden_states #2
) #C
with torch.no_grad(): #3
teacher_outputs = teacher_model(
input_ids=input_ids,
attention_mask=attention_mask,
output_hidden_states=request_hidden_states
) #D
student_hiddens = (
student_outputs.hidden_states[1:]
if request_hidden_states else None
) #4
teacher_hiddens = (teacher_outputs.hidden_states[1:]
if request_hidden_states else None)
loss, loss_dict = compute_compound_loss(
student_logits=student_outputs.logits,
teacher_logits=teacher_outputs.logits,
student_hiddens=student_hiddens,
teacher_hiddens=teacher_hiddens,
labels=labels,
layer_map=layer_map,
alpha=alpha, beta=beta, gamma=gamma, temperature=temperature
)
scaled_loss = loss / accumulation_steps #5
scaled_loss.backward()
for key in accumulated_losses:
accumulated_losses[key] += loss_dict[key]
accumulation_counter += 1
if (batch_idx + 1) % accumulation_steps == 0: #6
torch.nn.utils.clip_grad_norm_(student_model.parameters(), max_norm=1.0)
optimizer.step()
optimizer.zero_grad()
avg_losses = {k: v / accumulation_counter for k, v in accumulated_losses.items()}
for key in avg_losses:
epoch_losses[key].append(avg_losses[key])
progress_bar.set_postfix({
'loss': f"{avg_losses['total']:.4f}",
'task': f"{avg_losses['task']:.4f}",
'logits': f"{avg_losses['logits']:.4f}",
'hidden': f"{avg_losses['hidden']:.4f}"
})
accumulated_losses = {'total': 0.0, 'task': 0.0, 'logits': 0.0, 'hidden': 0.0}
accumulation_counter = 0
if accumulation_counter > 0: #7
torch.nn.utils.clip_grad_norm_(student_model.parameters(), max_norm=1.0)
optimizer.step()
optimizer.zero_grad()
avg_losses = {k: v / accumulation_counter for k, v in accumulated_losses.items()}
for key in avg_losses:
epoch_losses[key].append(avg_losses[key])
for key in epoch_losses:
if epoch_losses[key]:
avg_loss = np.mean(epoch_losses[key])
loss_history[key].append(avg_loss)
epoch_time = time.time() - epoch_start_time
epoch_times.append(epoch_time)
total_time = time.time() - total_start_time
loss_history['epoch_times_seconds'] = epoch_times
loss_history['total_time_seconds'] = total_time
return student_model, loss_history
Training-loop listing with configurable accumulation steps and optimizer setup. section 6.3.2


