biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Bugfix][Model] Jamba assertions and no chunked prefill by default for Jamba (#6784)

Browse Source
This commit is contained in:
tomeras91
2024-07-27 06:45:31 +03:00
committed by GitHub
parent 3c3012398e
commit ed94e4f427
2 changed files with 10 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/engine/arg_utils.py
View File

@@ -754,10 +754,14 @@ class EngineArgs:
use_sliding_window = (model_config.get_sliding_window()
is not None)
use_spec_decode = self.speculative_model is not None
has_seqlen_agnostic_layers = (
model_config.contains_seqlen_agnostic_layers(
parallel_config))
if (is_gpu and not use_sliding_window and not use_spec_decode
and not self.enable_lora
and not self.enable_prompt_adapter
and not self.enable_prefix_caching):
and not self.enable_prefix_caching
and not has_seqlen_agnostic_layers):
self.enable_chunked_prefill = True
logger.warning(
"Chunked prefill is enabled by default for models with "