biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Core] Combine async postprocessor and multi-step (#7921)

Browse Source
This commit is contained in:
Alexander Matveev
2024-08-29 14:18:26 -04:00
committed by GitHub
parent f205c09854
commit 3f60f2244e
8 changed files with 215 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
vllm/sequence.py
View File

@@ -1295,6 +1295,7 @@ class ExecuteModelRequest(
last_sampled_token_ids: Optional[torch.Tensor] = None
# Async callback
async_callback: Optional[Callable] = None
use_async_and_multi_step: bool = False
@property
def is_first_multi_step(self) -> bool:
@@ -1341,4 +1342,5 @@ class ExecuteModelRequest(
finished_requests_ids=self.finished_requests_ids,
last_sampled_token_ids=self.last_sampled_token_ids.clone()
if self.last_sampled_token_ids is not None else None,
async_callback=self.async_callback)
async_callback=self.async_callback,
use_async_and_multi_step=self.use_async_and_multi_step)