[Model] Extend collect_children and no_init_weights contexts (#32757)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

vllm/model_executor/models/gemma3_mm.py

@@ -539,10 +539,7 @@ class Gemma3ForConditionalGeneration(
             )
 
             logit_scale = getattr(config, "logit_scale", 1.0)
-            if hasattr(self.language_model, "logits_processor"):
-                # The logits processor can be unset if we're using
-                # automatic conversion to pooling model.
-                self.language_model.logits_processor.scale *= logit_scale
+            self.language_model.logits_processor.scale *= logit_scale
 
         self.make_empty_intermediate_tensors = (
             self.language_model.make_empty_intermediate_tensors