[Model] Use context managers for encoder- and LM-only mode (#32605)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

vllm/model_executor/models/utils.py

@@ -24,7 +24,11 @@ from vllm.model_executor.model_loader.online_quantization import (
     support_quantized_model_reload_from_hp_weights,
 )
 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
-from vllm.model_executor.models.interfaces import supports_any_eagle
+from vllm.model_executor.models.interfaces import (
+    LMMissingLayer,
+    TowerMissingLayer,
+    supports_any_eagle,
+)
 from vllm.multimodal import NestedTensors
 from vllm.sequence import IntermediateTensors
 from vllm.utils.math_utils import cdiv
@@ -250,7 +254,7 @@ class AutoWeightsLoader:
         module: nn.Module,
         weights: Iterable[tuple[str, torch.Tensor]],
     ) -> Iterable[str]:
-        if isinstance(module, PPMissingLayer):
+        if isinstance(module, (LMMissingLayer, TowerMissingLayer, PPMissingLayer)):
             return
 
         # Avoid infinite recursion since this function is typically