During Mixture of Experts training, Gaussian noise is added to router logits before softmax. This encourages early exploration, reducing the risk that routing collapses onto one expert before another can specialize. section 9.2.1 section 9.4.1
Listing 9.1 Trainable2ExpertMoE
class Trainable2ExpertMoE(nn.Module):
def __init__(self, original_mlp, hidden_size):
super().__init__()
target_dtype = next(original_mlp.parameters()).dtype
target_device = next(original_mlp.parameters()).device
self.expert_general = original_mlp #A
self.expert_clinical = copy.deepcopy(original_mlp) #B
self.router = nn.Linear(hidden_size, 2,
bias=False,
dtype=target_dtype,
device=target_device) #C
self.last_routing = None
def forward(self, hidden_states):
router_logits = self.router(hidden_states)
if self.training:
noise = torch.randn_like(router_logits) * 0.35 #1
router_logits = router_logits + noise
routing_weights = F.softmax(
router_logits.float(), dim=-1
).to(hidden_states.dtype) #2
out_general = self.expert_general(hidden_states)
out_clinical = self.expert_clinical(hidden_states)
# Soft routing: each token receives a weighted blend of both experts
output = out_general * routing_weights[..., 0].unsqueeze(-1)
[CA]+ out_clinical * routing_weights[..., 1].unsqueeze(-1) #3
self.last_routing = routing_weights.detach() #D
return output
Inspect the two-expert implementation: it adds 0.35-scaled random noise to router logits during training. section 9.2.1



