[3/N] Refactor scheduler for chunked prefill scheduling (#3550)

2024-04-04 06:13:49 +09:00
parent c64cf38673
commit 3dcb3e8b98
5 changed files with 1021 additions and 256 deletions
--- a/vllm/engine/llm_engine.py
+++ b/vllm/engine/llm_engine.py
@@ -728,7 +728,7 @@ class LLMEngine:
        time_per_output_tokens = []
        time_e2e_requests = []
        if scheduler_outputs is not None:
-            prompt_run = scheduler_outputs.prompt_run
+            prompt_run = scheduler_outputs.num_prefill_groups > 0

            # Number of Tokens.
            if prompt_run: