Revert "[Renderer] Separate out RendererConfig from ModelConfig (#30145)" (#30199)

tests/entrypoints/test_chat_utils.py

@@ -12,7 +12,7 @@ from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
 from vllm.assets.audio import AudioAsset
 from vllm.assets.image import ImageAsset
 from vllm.assets.video import VideoAsset
-from vllm.config import ModelConfig, RendererConfig
+from vllm.config import ModelConfig
 from vllm.entrypoints.chat_utils import (
     _try_extract_ast,
     apply_mistral_chat_template,
@@ -233,7 +233,7 @@ def test_parse_chat_messages_single_image(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -265,7 +265,7 @@ def test_parse_chat_messages_single_image_with_uuid(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -295,7 +295,7 @@ def test_parse_chat_messages_single_empty_image_with_uuid(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -328,7 +328,7 @@ def test_parse_chat_messages_single_image_with_bad_uuid_format(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -369,7 +369,7 @@ def test_parse_chat_messages_multiple_images_with_uuids(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -409,7 +409,7 @@ def test_parse_chat_messages_multiple_empty_images_with_uuids(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -451,7 +451,7 @@ def test_parse_chat_messages_mixed_empty_images_with_uuids(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -485,7 +485,7 @@ async def test_parse_chat_messages_single_image_with_uuid_async(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -516,7 +516,7 @@ async def test_parse_chat_messages_empty_image_with_uuid_async(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -554,7 +554,7 @@ async def test_parse_chat_messages_multiple_images_with_uuids_async(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -595,7 +595,7 @@ async def test_parse_chat_messages_multiple_empty_images_with_uuids_async(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -634,7 +634,7 @@ async def test_parse_chat_messages_multiple_images_with_partial_uuids_async(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -660,7 +660,7 @@ def test_parse_chat_messages_empty_system(
                 "content": [{"type": "text", "text": "Who are you?"}],
             },
         ],
-        RendererConfig(model_config=mistral_model_config),
+        mistral_model_config,
         content_format="string",
     )
     assert conversation == [
@@ -677,7 +677,7 @@ def test_parse_chat_messages_empty_system(
                 "content": [{"type": "text", "text": "Who are you?"}],
             },
         ],
-        RendererConfig(model_config=mistral_model_config),
+        mistral_model_config,
         content_format="openai",
     )
     assert conversation == [
@@ -701,7 +701,7 @@ async def test_parse_chat_messages_single_image_async(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -730,7 +730,7 @@ def test_parse_chat_messages_multiple_images(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -758,7 +758,7 @@ def test_parse_chat_messages_empty_pil_image_with_uuid(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -786,7 +786,7 @@ def test_parse_chat_messages_empty_image_embeds_with_uuid(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config_image_embeds),
+        phi3v_model_config_image_embeds,
         content_format="string",
     )
 
@@ -818,7 +818,7 @@ def test_parse_chat_messages_empty_audio_embeds_with_uuid(
                 ],
             }
         ],
-        RendererConfig(model_config=audio_embeds_model_config),
+        audio_embeds_model_config,
         content_format="string",
     )
 
@@ -858,7 +858,7 @@ def test_parse_chat_messages_audio_embeds_with_string(
                 ],
             }
         ],
-        RendererConfig(model_config=audio_embeds_model_config),
+        audio_embeds_model_config,
         content_format="string",
     )
 
@@ -900,7 +900,7 @@ async def test_parse_chat_messages_audio_embeds_async(
                 ],
             }
         ],
-        RendererConfig(model_config=audio_embeds_model_config),
+        audio_embeds_model_config,
         content_format="string",
     )
 
@@ -1108,7 +1108,7 @@ async def test_parse_chat_messages_empty_image_embeds_with_uuid_async(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config_image_embeds),
+        phi3v_model_config_image_embeds,
         content_format="string",
     )
 
@@ -1144,7 +1144,7 @@ async def test_parse_chat_messages_multiple_images_async(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -1176,7 +1176,7 @@ def test_parse_chat_messages_placeholder_already_in_prompt(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
     assert conversation == [
@@ -1208,7 +1208,7 @@ def test_parse_chat_messages_placeholder_one_already_in_prompt(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -1245,7 +1245,7 @@ def test_parse_chat_messages_multiple_images_across_messages(
                 ],
             },
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -1289,7 +1289,7 @@ def test_parse_chat_messages_multiple_images_with_uuids_across_messages(
                 ],
             },
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -1314,7 +1314,7 @@ def test_parse_chat_messages_context_text_format(
             {"role": "assistant", "content": "Some stuff."},
             {"role": "user", "content": "What about this one?"},
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="openai",
     )
 
@@ -1367,7 +1367,7 @@ def test_parse_chat_messages_rejects_too_many_images_in_one_message(
                         ],
                     }
                 ],
-                RendererConfig(model_config=phi3v_model_config),
+                phi3v_model_config,
                 content_format="string",
             )
 
@@ -1410,7 +1410,7 @@ def test_parse_chat_messages_rejects_too_many_images_across_messages(
                         ],
                     },
                 ],
-                RendererConfig(model_config=phi3v_model_config),
+                phi3v_model_config,
                 content_format="string",
             )
 
@@ -1430,7 +1430,7 @@ def test_parse_chat_messages_multiple_images_uncommon_input(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config),
+        phi3v_model_config,
         content_format="string",
     )
 
@@ -1464,7 +1464,7 @@ def test_parse_chat_messages_multiple_images_interleave(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config_mm_interleaved),
+        phi3v_model_config_mm_interleaved,
         content_format="string",
     )
 
@@ -1500,7 +1500,7 @@ async def test_parse_chat_messages_multiple_images_interleave_async(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config_mm_interleaved),
+        phi3v_model_config_mm_interleaved,
         content_format="string",
     )
 
@@ -1545,7 +1545,7 @@ async def test_parse_chat_messages_multiple_images_with_uuids_interleave_async(
                 ],
             }
         ],
-        RendererConfig(model_config=phi3v_model_config_mm_interleaved),
+        phi3v_model_config_mm_interleaved,
         content_format="string",
     )
 
@@ -1583,7 +1583,7 @@ def test_parse_chat_messages_multiple_images_multiple_messages_interleave(
                 ],
             },
         ],
-        RendererConfig(model_config=phi3v_model_config_mm_interleaved),
+        phi3v_model_config_mm_interleaved,
         content_format="string",
     )
 
@@ -1631,7 +1631,7 @@ def test_parse_chat_messages_multiple_images_with_uuids_multiple_messages_interl
                 ],
             },
         ],
-        RendererConfig(model_config=phi3v_model_config_mm_interleaved),
+        phi3v_model_config_mm_interleaved,
         content_format="string",
     )
 
@@ -1675,7 +1675,7 @@ def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(
                 ],
             },
         ],
-        RendererConfig(model_config=qwen25omni_model_config_mm_interleaved),
+        qwen25omni_model_config_mm_interleaved,
         content_format="string",
     )
 
@@ -1743,7 +1743,7 @@ def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interl
                 ],
             },
         ],
-        RendererConfig(model_config=qwen25omni_model_config_mm_interleaved),
+        qwen25omni_model_config_mm_interleaved,
         content_format="string",
     )
 
@@ -1813,7 +1813,7 @@ def test_parse_chat_messages_multiple_modals_with_uuids_multiple_empty_media_mes
                 ],
             },
         ],
-        RendererConfig(model_config=qwen25omni_model_config_mm_interleaved),
+        qwen25omni_model_config_mm_interleaved,
         content_format="string",
     )
 
@@ -1879,7 +1879,7 @@ def test_parse_chat_messages_multiple_modals_with_partial_uuids_multiple_message
                 ],
             },
         ],
-        RendererConfig(model_config=qwen25omni_model_config_mm_interleaved),
+        qwen25omni_model_config_mm_interleaved,
         content_format="string",
     )
 
@@ -1927,7 +1927,7 @@ def test_parse_chat_messages_multiple_images_interleave_with_placeholders(
                     ],
                 }
             ],
-            RendererConfig(model_config=phi3v_model_config_mm_interleaved),
+            phi3v_model_config_mm_interleaved,
             content_format="string",
         )
 
@@ -1945,11 +1945,24 @@ def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):
     model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
     model_info.check_available_online(on_fail="skip")
 
-    renderer_config = model_info.build_renderer_config(model)
+    model_config = ModelConfig(
+        model,
+        tokenizer=model_info.tokenizer or model,
+        tokenizer_mode=model_info.tokenizer_mode,
+        revision=model_info.revision,
+        trust_remote_code=model_info.trust_remote_code,
+        hf_overrides=model_info.hf_overrides,
+        skip_tokenizer_init=model_info.require_embed_inputs,
+        enable_prompt_embeds=model_info.require_embed_inputs,
+        enable_mm_embeds=model_info.require_embed_inputs,
+        enforce_eager=model_info.enforce_eager,
+        dtype=model_info.dtype,
+    )
 
+    # Build the tokenizer
     tokenizer = get_tokenizer(
-        renderer_config.tokenizer,
-        trust_remote_code=renderer_config.trust_remote_code,
+        model,
+        trust_remote_code=model_config.trust_remote_code,
     )
 
     tools = (
@@ -1972,7 +1985,7 @@ def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):
         tokenizer,
         chat_template=None,
         tools=tools,
-        model_config=renderer_config.model_config,
+        model_config=model_config,
     )
     assert isinstance(chat_template, str)
 
@@ -2034,11 +2047,24 @@ def test_resolve_hf_chat_template_kwargs(sample_json_schema, model, expected_kwa
         "enable_thinking": True,
     }
 
-    renderer_config = model_info.build_renderer_config(model)
+    model_config = ModelConfig(
+        model,
+        tokenizer=model_info.tokenizer or model,
+        tokenizer_mode=model_info.tokenizer_mode,
+        revision=model_info.revision,
+        trust_remote_code=model_info.trust_remote_code,
+        hf_overrides=model_info.hf_overrides,
+        skip_tokenizer_init=model_info.require_embed_inputs,
+        enable_prompt_embeds=model_info.require_embed_inputs,
+        enable_mm_embeds=model_info.require_embed_inputs,
+        enforce_eager=model_info.enforce_eager,
+        dtype=model_info.dtype,
+    )
 
+    # Build the tokenizer
     tokenizer = get_tokenizer(
-        renderer_config.tokenizer,
-        trust_remote_code=renderer_config.trust_remote_code,
+        model,
+        trust_remote_code=model_config.trust_remote_code,
     )
 
     # Test detecting the tokenizer's chat_template
@@ -2046,7 +2072,7 @@ def test_resolve_hf_chat_template_kwargs(sample_json_schema, model, expected_kwa
         tokenizer,
         chat_template=None,
         tools=tools,
-        model_config=renderer_config.model_config,
+        model_config=model_config,
     )
     with pytest.raises(
         ValueError, match="Found unexpected chat template kwargs from request"
@@ -2117,11 +2143,23 @@ def test_resolve_content_format_hf_defined(model, expected_format):
     model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
     model_info.check_available_online(on_fail="skip")
 
-    renderer_config = model_info.build_renderer_config(model)
+    model_config = ModelConfig(
+        model,
+        tokenizer=model_info.tokenizer or model,
+        tokenizer_mode=model_info.tokenizer_mode,
+        revision=model_info.revision,
+        trust_remote_code=model_info.trust_remote_code,
+        hf_overrides=model_info.hf_overrides,
+        skip_tokenizer_init=model_info.require_embed_inputs,
+        enable_prompt_embeds=model_info.require_embed_inputs,
+        enable_mm_embeds=model_info.require_embed_inputs,
+        enforce_eager=model_info.enforce_eager,
+        dtype=model_info.dtype,
+    )
 
     tokenizer = get_tokenizer(
-        renderer_config.tokenizer,
-        trust_remote_code=renderer_config.trust_remote_code,
+        model,
+        trust_remote_code=model_config.trust_remote_code,
     )
 
     # Test detecting the tokenizer's chat_template
@@ -2129,7 +2167,7 @@ def test_resolve_content_format_hf_defined(model, expected_format):
         tokenizer,
         chat_template=None,
         tools=None,
-        model_config=renderer_config.model_config,
+        model_config=model_config,
     )
     assert isinstance(chat_template, str)
 
@@ -2143,7 +2181,7 @@ def test_resolve_content_format_hf_defined(model, expected_format):
         None,
         "auto",
         tokenizer,
-        renderer_config=renderer_config,
+        model_config=model_config,
     )
 
     assert resolved_format == expected_format
@@ -2165,11 +2203,23 @@ def test_resolve_content_format_fallbacks(model, expected_format):
     model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
     model_info.check_available_online(on_fail="skip")
 
-    renderer_config = model_info.build_renderer_config(model)
+    model_config = ModelConfig(
+        model,
+        tokenizer=model_info.tokenizer or model,
+        tokenizer_mode=model_info.tokenizer_mode,
+        revision=model_info.revision,
+        trust_remote_code=model_info.trust_remote_code,
+        hf_overrides=model_info.hf_overrides,
+        skip_tokenizer_init=model_info.require_embed_inputs,
+        enable_prompt_embeds=model_info.require_embed_inputs,
+        enable_mm_embeds=model_info.require_embed_inputs,
+        enforce_eager=model_info.enforce_eager,
+        dtype=model_info.dtype,
+    )
 
     tokenizer = get_tokenizer(
-        renderer_config.tokenizer,
-        trust_remote_code=renderer_config.trust_remote_code,
+        model_config.tokenizer,
+        trust_remote_code=model_config.trust_remote_code,
     )
 
     # Test detecting the tokenizer's chat_template
@@ -2177,7 +2227,7 @@ def test_resolve_content_format_fallbacks(model, expected_format):
         tokenizer,
         chat_template=None,
         tools=None,
-        model_config=renderer_config.model_config,
+        model_config=model_config,
     )
     assert isinstance(chat_template, str)
 
@@ -2191,7 +2241,7 @@ def test_resolve_content_format_fallbacks(model, expected_format):
         None,
         "auto",
         tokenizer,
-        renderer_config=renderer_config,
+        model_config=model_config,
     )
 
     assert resolved_format == expected_format
@@ -2222,13 +2272,15 @@ def test_resolve_content_format_fallbacks(model, expected_format):
     ],
 )
 def test_resolve_content_format_examples(template_path, expected_format):
-    model = PHI3V_MODEL_ID  # Dummy
-    model_config = ModelConfig(model, trust_remote_code=True)
-    renderer_config = RendererConfig(model_config=model_config, tokenizer=model)
+    model_config = ModelConfig(
+        PHI3V_MODEL_ID,  # Dummy
+        tokenizer=PHI3V_MODEL_ID,  # Dummy
+        trust_remote_code=True,
+    )
 
     dummy_tokenizer = get_tokenizer(
-        renderer_config.tokenizer,
-        trust_remote_code=renderer_config.trust_remote_code,
+        PHI3V_MODEL_ID,  # Dummy
+        trust_remote_code=model_config.trust_remote_code,
     )
     dummy_tokenizer.chat_template = None
 
@@ -2245,7 +2297,7 @@ def test_resolve_content_format_examples(template_path, expected_format):
         None,
         "auto",
         dummy_tokenizer,
-        renderer_config=renderer_config,
+        model_config=model_config,
     )
 
     assert resolved_format == expected_format
@@ -2280,7 +2332,7 @@ def test_parse_chat_messages_include_thinking_chunk(mistral_model_config):
 
     conversation_with_thinking, _, _ = parse_chat_messages(
         messages,
-        RendererConfig(model_config=mistral_model_config),
+        mistral_model_config,
         content_format="openai",
     )
 
@@ -2380,7 +2432,7 @@ def test_parse_chat_messages_single_empty_audio_with_uuid(
                 ],
             }
         ],
-        RendererConfig(model_config=qwen2_audio_model_config),
+        qwen2_audio_model_config,
         content_format="string",
     )
 
@@ -2414,7 +2466,7 @@ async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
                 ],
             }
         ],
-        RendererConfig(model_config=qwen2_audio_model_config),
+        qwen2_audio_model_config,
         content_format="string",
     )