[Model] Add Qwen2-Audio model support (#9248)

Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
2024-10-24 01:54:22 +08:00
parent 150b779081
commit fc6c274626
7 changed files with 515 additions and 17 deletions
--- a/vllm/entrypoints/chat_utils.py
+++ b/vllm/entrypoints/chat_utils.py
@@ -196,7 +196,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
        elif modality == "audio":
            if model_type == "ultravox":
                return "<|reserved_special_token_0|>"
-            raise TypeError(f"Unknown {modality} model type: {model_type}")
+            if model_type == "qwen2_audio":
+                return (f"Audio {current_count}: "
+                        f"<|audio_bos|><|AUDIO|><|audio_eos|>")
+            raise TypeError(f"Unknown model type: {model_type}")
        elif modality == "video":
            if model_type == "qwen2_vl":
                return "<|vision_start|><|video_pad|><|vision_end|>"