Prefix Cache Aware Scheduling [1/n] (#10128)

Signed-off-by: rickyx <rickyx@anyscale.com>
2024-11-22 21:15:55 -08:00
parent 7c25fe45a6
commit 4634a89d18
13 changed files with 962 additions and 236 deletions
--- a/vllm/sequence.py
+++ b/vllm/sequence.py
@@ -579,6 +579,9 @@ class Sequence:
            return 1
        return self.data.get_num_uncomputed_tokens()

+    def get_num_computed_tokens(self) -> int:
+        return self.data.get_num_computed_tokens()
+
    def is_prefill(self) -> bool:
        return self.data.stage == SequenceStage.PREFILL