A custom architecture lets vLLM load a checkpoint whose model structure differs from its supported standard architectures. For a checkpoint with attention removed from six decoder blocks, the standard load fails because vLLM expects weights for components those blocks no longer contain. section 12.6.3
A plugin adapts the model to the checkpoint:
- Implement a decoder layer that uses the standard layer when attention remains and skips attention in configured blocks. section 12.6.3
- Reuse LlamaForCausalLM with the custom layer, then register the model through a Python package entry point. section 12.6.3
- Install the package and override the architecture configuration when loading the checkpoint. section 12.6.3
(EngineCore pid=4788) ERROR 10-04 18:26:16 [core.py:1366]
ValueError: Following weights were not initialized from checkpoint:
{'model.layers.21.input_layernorm.weight',
'model.layers.18.input_layernorm.weight',
'model.layers.24.input_layernorm.weight',
'model.layers.23.input_layernorm.weight',
'model.layers.22.input_layernorm.weight',
'model.layers.20.input_layernorm.weight'}
(log: ch12_vllm/logs/attn_drop_standard.log)The load error lists missing input-layer normalization weights for the six blocks without attention. section 12.6.3
Listing 12.2 A decoder layer and model class for attention-free blocks
from torch import nn
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.models.llama import (
LlamaAttention, LlamaDecoderLayer, LlamaForCausalLM, LlamaMLP)
from vllm.model_executor.models.utils import extract_layer_index
class AttnFreeAwareDecoderLayer(LlamaDecoderLayer):
def __init__(self, vllm_config, prefix="", config=None,
attn_layer_type=LlamaAttention):
config = config or vllm_config.model_config.hf_config
dropped = set(getattr(config, "dropped_attn_layers", None) or [])
if extract_layer_index(prefix) not in dropped: #1
super().__init__( #A
vllm_config=vllm_config, prefix=prefix,
config=config, attn_layer_type=attn_layer_type)
self.has_attention = True
return
nn.Module.__init__(self) #B
self.has_attention = False
self.hidden_size = config.hidden_size
self.mlp = LlamaMLP(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=self.get_quant_config(vllm_config),
bias=getattr(config, "mlp_bias", False),
prefix=f"{prefix}.mlp",
)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps)
def forward(self, positions, hidden_states, residual):
if self.has_attention:
return super().forward(positions, hidden_states, residual)
if residual is None: #2
residual = hidden_states
hidden_states = self.post_attention_layernorm(hidden_states)
else:
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
hidden_states = self.mlp(hidden_states)
return hidden_states, residual
class AttnDropLlamaForCausalLM(LlamaForCausalLM):
def __init__(self, *, vllm_config, prefix=""):
if not getattr(
vllm_config.model_config.hf_config,
"dropped_attn_layers", None):
raise ValueError(
"AttnDropLlamaForCausalLM needs config.dropped_attn_layers")
super().__init__(
vllm_config=vllm_config, prefix=prefix,
layer_type=AttnFreeAwareDecoderLayer) #3The decoder-layer implementation checks each block and builds an attention-free variant where configured. section 12.6.3
