[Core] Refactoring sampler and support prompt logprob for chunked prefill (#4309)

2024-04-26 22:02:02 +09:00
parent a88081bf76
commit 603ad84815
18 changed files with 859 additions and 630 deletions
--- a/vllm/engine/async_llm_engine.py
+++ b/vllm/engine/async_llm_engine.py
@@ -219,7 +219,7 @@ class _AsyncLLMEngine(LLMEngine):

        request_outputs = self._process_model_outputs(
            output, scheduler_outputs.scheduled_seq_groups,
-            scheduler_outputs.ignored_seq_groups)
+            scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)

        # Log stats.
        if self.log_stats: