[Speculative decoding 6/9] Integrate speculative decoding with LLMEngine (#3894)

2024-04-16 13:09:21 -07:00
parent 69e1d2fb69
commit e95cd87959
31 changed files with 1347 additions and 407 deletions
--- a/vllm/executor/ray_gpu_executor.py
+++ b/vllm/executor/ray_gpu_executor.py
@@ -242,7 +242,8 @@ class RayGPUExecutor(ExecutorBase):
                      seq_group_metadata_list: List[SequenceGroupMetadata],
                      blocks_to_swap_in: Dict[int, int],
                      blocks_to_swap_out: Dict[int, int],
-                      blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
+                      blocks_to_copy: Dict[int, List[int]],
+                      num_lookahead_slots: int = 0) -> SamplerOutput:
        all_outputs = self._run_workers(
            "execute_model",
            driver_kwargs={