biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Model Runner V2] Support Streaming Inputs (#37028)

Browse Source 
Signed-off-by: Santino Ramos <elsantinoramos@gmail.com>
This commit is contained in:
Santino Ramos
2026-03-20 13:42:25 -07:00
committed by GitHub
parent 8bc6b5cdb0
commit 85f671b8e1
7 changed files with 263 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/model_executor/models/whisper_causal.py
View File

@@ -150,8 +150,10 @@ def create_whisper_attention_backend_with_block_pooling(
new_common_attn_metadata.query_start_loc *= block_pool_size
new_common_attn_metadata.query_start_loc_cpu *= block_pool_size
new_common_attn_metadata.seq_lens *= block_pool_size
new_common_attn_metadata._seq_lens_cpu *= block_pool_size
new_common_attn_metadata._num_computed_tokens_cpu *= block_pool_size
if new_common_attn_metadata._seq_lens_cpu is not None:
new_common_attn_metadata._seq_lens_cpu *= block_pool_size
if new_common_attn_metadata._num_computed_tokens_cpu is not None:
new_common_attn_metadata._num_computed_tokens_cpu *= block_pool_size
new_common_attn_metadata.num_actual_tokens *= block_pool_size
new_common_attn_metadata.max_query_len *= block_pool_size
new_common_attn_metadata.max_seq_len *= block_pool_size