[Doc] Update usage of --limit-mm-per-prompt (#34148)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -18,11 +18,11 @@ from vllm.assets.image import ImageAsset
|
||||
# # Mistral format
|
||||
# vllm serve mistralai/Mistral-Small-3.1-24B-Instruct-2503 \
|
||||
# --tokenizer-mode mistral --config-format mistral --load-format mistral \
|
||||
# --limit-mm-per-prompt '{"image":4}' --max-model-len 16384
|
||||
# --limit-mm-per-prompt.image 4 --max-model-len 16384
|
||||
#
|
||||
# # HF format
|
||||
# vllm serve mistralai/Mistral-Small-3.1-24B-Instruct-2503 \
|
||||
# --limit-mm-per-prompt '{"image":4}' --max-model-len 16384
|
||||
# --limit-mm-per-prompt.image 4 --max-model-len 16384
|
||||
# ```
|
||||
#
|
||||
# - Client:
|
||||
|
||||
@@ -10,7 +10,7 @@ vllm serve llava-hf/llava-1.5-7b-hf
|
||||
|
||||
(multi-image inference with Phi-3.5-vision-instruct)
|
||||
vllm serve microsoft/Phi-3.5-vision-instruct --runner generate \
|
||||
--trust-remote-code --max-model-len 4096 --limit-mm-per-prompt '{"image":2}'
|
||||
--trust-remote-code --max-model-len 4096 --limit-mm-per-prompt.image 2
|
||||
|
||||
(audio inference with Ultravox)
|
||||
vllm serve fixie-ai/ultravox-v0_5-llama-3_2-1b \
|
||||
|
||||
@@ -7,7 +7,7 @@ NOTE:
|
||||
vllm serve muziyongshixin/Qwen2.5-VL-7B-for-VideoCls \
|
||||
--runner pooling \
|
||||
--max-model-len 5000 \
|
||||
--limit-mm-per-prompt '{"video": 1}' \
|
||||
--limit-mm-per-prompt.video 1 \
|
||||
--hf-overrides '{"text_config": {"architectures": ["Qwen2_5_VLForSequenceClassification"]}}'
|
||||
"""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user