from typing import Optional, Tuple import torch from transformers import LlamaForCausalLM class PrunedLlamaForCausalLM(LlamaForCausalLM): """ LlamaForCausalLM with physical attention layer removal. At load time, reads dropped_attn_layers from the model config and patches the forward() of each affected DecoderLayer to skip the attention sublayer entirely. The MLP sublayer and its residual connection are preserved unchanged. """ def __init__(self, config): super().__init__(config) dropped = getattr(config, "dropped_attn_layers", []) for idx in dropped: self._patch_layer(idx) def _patch_layer(self, layer_idx): """Remove attention submodules and patch forward() for one layer.""" layer = self.model.layers[layer_idx] # Release attention parameters from memory if hasattr(layer, "self_attn"): delattr(layer, "self_attn") if hasattr(layer, "input_layernorm"): delattr(layer, "input_layernorm") # Patch forward() to route hidden states directly to the MLP block def forward_no_attn( self, hidden_states: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.LongTensor] = None, past_key_value=None, use_cache: Optional[bool] = False, cache_position: Optional[torch.LongTensor] = None, position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, **kwargs, ) -> Tuple[torch.Tensor]: if isinstance(hidden_states, tuple): hidden_states = hidden_states[0] residual = hidden_states hidden_states = self.post_attention_layernorm(hidden_states) hidden_states = self.mlp(hidden_states) hidden_states = residual + hidden_states return hidden_states # Override at class level so transformers' internal forward() restore # mechanism does not overwrite the patch during model.generate() PrunedLayer = type( "PrunedDecoderLayer", (type(layer),), {"forward": forward_no_attn} ) layer.__class__ = PrunedLayer