[Renderer] Separate out RendererConfig from ModelConfig (#30145)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

tests/entrypoints/test_chat_utils.py

@@ -12,7 +12,7 @@ from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
 from vllm.assets.audio import AudioAsset
 from vllm.assets.image import ImageAsset
 from vllm.assets.video import VideoAsset
-from vllm.config import ModelConfig
+from vllm.config import ModelConfig, RendererConfig
 from vllm.entrypoints.chat_utils import (
     _try_extract_ast,
     apply_mistral_chat_template,
@@ -233,7 +233,7 @@ def test_parse_chat_messages_single_image(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -265,7 +265,7 @@ def test_parse_chat_messages_single_image_with_uuid(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -295,7 +295,7 @@ def test_parse_chat_messages_single_empty_image_with_uuid(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -328,7 +328,7 @@ def test_parse_chat_messages_single_image_with_bad_uuid_format(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -369,7 +369,7 @@ def test_parse_chat_messages_multiple_images_with_uuids(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -409,7 +409,7 @@ def test_parse_chat_messages_multiple_empty_images_with_uuids(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -451,7 +451,7 @@ def test_parse_chat_messages_mixed_empty_images_with_uuids(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -485,7 +485,7 @@ async def test_parse_chat_messages_single_image_with_uuid_async(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -516,7 +516,7 @@ async def test_parse_chat_messages_empty_image_with_uuid_async(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -554,7 +554,7 @@ async def test_parse_chat_messages_multiple_images_with_uuids_async(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -595,7 +595,7 @@ async def test_parse_chat_messages_multiple_empty_images_with_uuids_async(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -634,7 +634,7 @@ async def test_parse_chat_messages_multiple_images_with_partial_uuids_async(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -660,7 +660,7 @@ def test_parse_chat_messages_empty_system(
                 "content": [{"type": "text", "text": "Who are you?"}],
             },
         ],
-        mistral_model_config,
+        RendererConfig(model_config=mistral_model_config),
         content_format="string",
     )
     assert conversation == [
@@ -677,7 +677,7 @@ def test_parse_chat_messages_empty_system(
                 "content": [{"type": "text", "text": "Who are you?"}],
             },
         ],
-        mistral_model_config,
+        RendererConfig(model_config=mistral_model_config),
         content_format="openai",
     )
     assert conversation == [
@@ -701,7 +701,7 @@ async def test_parse_chat_messages_single_image_async(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -730,7 +730,7 @@ def test_parse_chat_messages_multiple_images(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -758,7 +758,7 @@ def test_parse_chat_messages_empty_pil_image_with_uuid(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -786,7 +786,7 @@ def test_parse_chat_messages_empty_image_embeds_with_uuid(
                 ],
             }
         ],
-        phi3v_model_config_image_embeds,
+        RendererConfig(model_config=phi3v_model_config_image_embeds),
         content_format="string",
     )
 
@@ -818,7 +818,7 @@ def test_parse_chat_messages_empty_audio_embeds_with_uuid(
                 ],
             }
         ],
-        audio_embeds_model_config,
+        RendererConfig(model_config=audio_embeds_model_config),
         content_format="string",
     )
 
@@ -858,7 +858,7 @@ def test_parse_chat_messages_audio_embeds_with_string(
                 ],
             }
         ],
-        audio_embeds_model_config,
+        RendererConfig(model_config=audio_embeds_model_config),
         content_format="string",
     )
 
@@ -900,7 +900,7 @@ async def test_parse_chat_messages_audio_embeds_async(
                 ],
             }
         ],
-        audio_embeds_model_config,
+        RendererConfig(model_config=audio_embeds_model_config),
         content_format="string",
     )
 
@@ -1108,7 +1108,7 @@ async def test_parse_chat_messages_empty_image_embeds_with_uuid_async(
                 ],
             }
         ],
-        phi3v_model_config_image_embeds,
+        RendererConfig(model_config=phi3v_model_config_image_embeds),
         content_format="string",
     )
 
@@ -1144,7 +1144,7 @@ async def test_parse_chat_messages_multiple_images_async(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -1176,7 +1176,7 @@ def test_parse_chat_messages_placeholder_already_in_prompt(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
     assert conversation == [
@@ -1208,7 +1208,7 @@ def test_parse_chat_messages_placeholder_one_already_in_prompt(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -1245,7 +1245,7 @@ def test_parse_chat_messages_multiple_images_across_messages(
                 ],
             },
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -1289,7 +1289,7 @@ def test_parse_chat_messages_multiple_images_with_uuids_across_messages(
                 ],
             },
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -1314,7 +1314,7 @@ def test_parse_chat_messages_context_text_format(
             {"role": "assistant", "content": "Some stuff."},
             {"role": "user", "content": "What about this one?"},
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="openai",
     )
 
@@ -1367,7 +1367,7 @@ def test_parse_chat_messages_rejects_too_many_images_in_one_message(
                         ],
                     }
                 ],
-                phi3v_model_config,
+                RendererConfig(model_config=phi3v_model_config),
                 content_format="string",
             )
 
@@ -1410,7 +1410,7 @@ def test_parse_chat_messages_rejects_too_many_images_across_messages(
                         ],
                     },
                 ],
-                phi3v_model_config,
+                RendererConfig(model_config=phi3v_model_config),
                 content_format="string",
             )
 
@@ -1430,7 +1430,7 @@ def test_parse_chat_messages_multiple_images_uncommon_input(
                 ],
             }
         ],
-        phi3v_model_config,
+        RendererConfig(model_config=phi3v_model_config),
         content_format="string",
     )
 
@@ -1464,7 +1464,7 @@ def test_parse_chat_messages_multiple_images_interleave(
                 ],
             }
         ],
-        phi3v_model_config_mm_interleaved,
+        RendererConfig(model_config=phi3v_model_config_mm_interleaved),
         content_format="string",
     )
 
@@ -1500,7 +1500,7 @@ async def test_parse_chat_messages_multiple_images_interleave_async(
                 ],
             }
         ],
-        phi3v_model_config_mm_interleaved,
+        RendererConfig(model_config=phi3v_model_config_mm_interleaved),
         content_format="string",
     )
 
@@ -1545,7 +1545,7 @@ async def test_parse_chat_messages_multiple_images_with_uuids_interleave_async(
                 ],
             }
         ],
-        phi3v_model_config_mm_interleaved,
+        RendererConfig(model_config=phi3v_model_config_mm_interleaved),
         content_format="string",
     )
 
@@ -1583,7 +1583,7 @@ def test_parse_chat_messages_multiple_images_multiple_messages_interleave(
                 ],
             },
         ],
-        phi3v_model_config_mm_interleaved,
+        RendererConfig(model_config=phi3v_model_config_mm_interleaved),
         content_format="string",
     )
 
@@ -1631,7 +1631,7 @@ def test_parse_chat_messages_multiple_images_with_uuids_multiple_messages_interl
                 ],
             },
         ],
-        phi3v_model_config_mm_interleaved,
+        RendererConfig(model_config=phi3v_model_config_mm_interleaved),
         content_format="string",
     )
 
@@ -1675,7 +1675,7 @@ def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(
                 ],
             },
         ],
-        qwen25omni_model_config_mm_interleaved,
+        RendererConfig(model_config=qwen25omni_model_config_mm_interleaved),
         content_format="string",
     )
 
@@ -1743,7 +1743,7 @@ def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interl
                 ],
             },
         ],
-        qwen25omni_model_config_mm_interleaved,
+        RendererConfig(model_config=qwen25omni_model_config_mm_interleaved),
         content_format="string",
     )
 
@@ -1813,7 +1813,7 @@ def test_parse_chat_messages_multiple_modals_with_uuids_multiple_empty_media_mes
                 ],
             },
         ],
-        qwen25omni_model_config_mm_interleaved,
+        RendererConfig(model_config=qwen25omni_model_config_mm_interleaved),
         content_format="string",
     )
 
@@ -1879,7 +1879,7 @@ def test_parse_chat_messages_multiple_modals_with_partial_uuids_multiple_message
                 ],
             },
         ],
-        qwen25omni_model_config_mm_interleaved,
+        RendererConfig(model_config=qwen25omni_model_config_mm_interleaved),
         content_format="string",
     )
 
@@ -1927,7 +1927,7 @@ def test_parse_chat_messages_multiple_images_interleave_with_placeholders(
                     ],
                 }
             ],
-            phi3v_model_config_mm_interleaved,
+            RendererConfig(model_config=phi3v_model_config_mm_interleaved),
             content_format="string",
         )
 
@@ -1945,24 +1945,11 @@ def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):
     model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
     model_info.check_available_online(on_fail="skip")
 
-    model_config = ModelConfig(
-        model,
-        tokenizer=model_info.tokenizer or model,
-        tokenizer_mode=model_info.tokenizer_mode,
-        revision=model_info.revision,
-        trust_remote_code=model_info.trust_remote_code,
-        hf_overrides=model_info.hf_overrides,
-        skip_tokenizer_init=model_info.require_embed_inputs,
-        enable_prompt_embeds=model_info.require_embed_inputs,
-        enable_mm_embeds=model_info.require_embed_inputs,
-        enforce_eager=model_info.enforce_eager,
-        dtype=model_info.dtype,
-    )
+    renderer_config = model_info.build_renderer_config(model)
 
-    # Build the tokenizer
     tokenizer = get_tokenizer(
-        model,
-        trust_remote_code=model_config.trust_remote_code,
+        renderer_config.tokenizer,
+        trust_remote_code=renderer_config.trust_remote_code,
     )
 
     tools = (
@@ -1985,7 +1972,7 @@ def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):
         tokenizer,
         chat_template=None,
         tools=tools,
-        model_config=model_config,
+        model_config=renderer_config.model_config,
     )
     assert isinstance(chat_template, str)
 
@@ -2047,24 +2034,11 @@ def test_resolve_hf_chat_template_kwargs(sample_json_schema, model, expected_kwa
         "enable_thinking": True,
     }
 
-    model_config = ModelConfig(
-        model,
-        tokenizer=model_info.tokenizer or model,
-        tokenizer_mode=model_info.tokenizer_mode,
-        revision=model_info.revision,
-        trust_remote_code=model_info.trust_remote_code,
-        hf_overrides=model_info.hf_overrides,
-        skip_tokenizer_init=model_info.require_embed_inputs,
-        enable_prompt_embeds=model_info.require_embed_inputs,
-        enable_mm_embeds=model_info.require_embed_inputs,
-        enforce_eager=model_info.enforce_eager,
-        dtype=model_info.dtype,
-    )
+    renderer_config = model_info.build_renderer_config(model)
 
-    # Build the tokenizer
     tokenizer = get_tokenizer(
-        model,
-        trust_remote_code=model_config.trust_remote_code,
+        renderer_config.tokenizer,
+        trust_remote_code=renderer_config.trust_remote_code,
     )
 
     # Test detecting the tokenizer's chat_template
@@ -2072,7 +2046,7 @@ def test_resolve_hf_chat_template_kwargs(sample_json_schema, model, expected_kwa
         tokenizer,
         chat_template=None,
         tools=tools,
-        model_config=model_config,
+        model_config=renderer_config.model_config,
     )
     with pytest.raises(
         ValueError, match="Found unexpected chat template kwargs from request"
@@ -2143,23 +2117,11 @@ def test_resolve_content_format_hf_defined(model, expected_format):
     model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
     model_info.check_available_online(on_fail="skip")
 
-    model_config = ModelConfig(
-        model,
-        tokenizer=model_info.tokenizer or model,
-        tokenizer_mode=model_info.tokenizer_mode,
-        revision=model_info.revision,
-        trust_remote_code=model_info.trust_remote_code,
-        hf_overrides=model_info.hf_overrides,
-        skip_tokenizer_init=model_info.require_embed_inputs,
-        enable_prompt_embeds=model_info.require_embed_inputs,
-        enable_mm_embeds=model_info.require_embed_inputs,
-        enforce_eager=model_info.enforce_eager,
-        dtype=model_info.dtype,
-    )
+    renderer_config = model_info.build_renderer_config(model)
 
     tokenizer = get_tokenizer(
-        model,
-        trust_remote_code=model_config.trust_remote_code,
+        renderer_config.tokenizer,
+        trust_remote_code=renderer_config.trust_remote_code,
     )
 
     # Test detecting the tokenizer's chat_template
@@ -2167,7 +2129,7 @@ def test_resolve_content_format_hf_defined(model, expected_format):
         tokenizer,
         chat_template=None,
         tools=None,
-        model_config=model_config,
+        model_config=renderer_config.model_config,
     )
     assert isinstance(chat_template, str)
 
@@ -2181,7 +2143,7 @@ def test_resolve_content_format_hf_defined(model, expected_format):
         None,
         "auto",
         tokenizer,
-        model_config=model_config,
+        renderer_config=renderer_config,
     )
 
     assert resolved_format == expected_format
@@ -2203,23 +2165,11 @@ def test_resolve_content_format_fallbacks(model, expected_format):
     model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
     model_info.check_available_online(on_fail="skip")
 
-    model_config = ModelConfig(
-        model,
-        tokenizer=model_info.tokenizer or model,
-        tokenizer_mode=model_info.tokenizer_mode,
-        revision=model_info.revision,
-        trust_remote_code=model_info.trust_remote_code,
-        hf_overrides=model_info.hf_overrides,
-        skip_tokenizer_init=model_info.require_embed_inputs,
-        enable_prompt_embeds=model_info.require_embed_inputs,
-        enable_mm_embeds=model_info.require_embed_inputs,
-        enforce_eager=model_info.enforce_eager,
-        dtype=model_info.dtype,
-    )
+    renderer_config = model_info.build_renderer_config(model)
 
     tokenizer = get_tokenizer(
-        model_config.tokenizer,
-        trust_remote_code=model_config.trust_remote_code,
+        renderer_config.tokenizer,
+        trust_remote_code=renderer_config.trust_remote_code,
     )
 
     # Test detecting the tokenizer's chat_template
@@ -2227,7 +2177,7 @@ def test_resolve_content_format_fallbacks(model, expected_format):
         tokenizer,
         chat_template=None,
         tools=None,
-        model_config=model_config,
+        model_config=renderer_config.model_config,
     )
     assert isinstance(chat_template, str)
 
@@ -2241,7 +2191,7 @@ def test_resolve_content_format_fallbacks(model, expected_format):
         None,
         "auto",
         tokenizer,
-        model_config=model_config,
+        renderer_config=renderer_config,
     )
 
     assert resolved_format == expected_format
@@ -2272,15 +2222,13 @@ def test_resolve_content_format_fallbacks(model, expected_format):
     ],
 )
 def test_resolve_content_format_examples(template_path, expected_format):
-    model_config = ModelConfig(
-        PHI3V_MODEL_ID,  # Dummy
-        tokenizer=PHI3V_MODEL_ID,  # Dummy
-        trust_remote_code=True,
-    )
+    model = PHI3V_MODEL_ID  # Dummy
+    model_config = ModelConfig(model, trust_remote_code=True)
+    renderer_config = RendererConfig(model_config=model_config, tokenizer=model)
 
     dummy_tokenizer = get_tokenizer(
-        PHI3V_MODEL_ID,  # Dummy
-        trust_remote_code=model_config.trust_remote_code,
+        renderer_config.tokenizer,
+        trust_remote_code=renderer_config.trust_remote_code,
     )
     dummy_tokenizer.chat_template = None
 
@@ -2297,7 +2245,7 @@ def test_resolve_content_format_examples(template_path, expected_format):
         None,
         "auto",
         dummy_tokenizer,
-        model_config=model_config,
+        renderer_config=renderer_config,
     )
 
     assert resolved_format == expected_format
@@ -2332,7 +2280,7 @@ def test_parse_chat_messages_include_thinking_chunk(mistral_model_config):
 
     conversation_with_thinking, _, _ = parse_chat_messages(
         messages,
-        mistral_model_config,
+        RendererConfig(model_config=mistral_model_config),
         content_format="openai",
     )
 
@@ -2432,7 +2380,7 @@ def test_parse_chat_messages_single_empty_audio_with_uuid(
                 ],
             }
         ],
-        qwen2_audio_model_config,
+        RendererConfig(model_config=qwen2_audio_model_config),
         content_format="string",
     )
 
@@ -2466,7 +2414,7 @@ async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
                 ],
             }
         ],
-        qwen2_audio_model_config,
+        RendererConfig(model_config=qwen2_audio_model_config),
         content_format="string",
     )