[feat] Add per-block extra_keys to KV events (#33304)
Signed-off-by: zhongdaor-nv <zhongdaor@nvidia.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -37,6 +37,12 @@ class BlockStored(KVCacheEvent):
|
||||
medium: str | None
|
||||
lora_name: str | None
|
||||
|
||||
extra_keys: list[tuple[Any, ...] | None] | None = None
|
||||
"""Extra keys used in block hash computation, one entry per block in
|
||||
block_hashes. Each entry contains MM identifiers, LoRA name, cache_salt,
|
||||
prompt embeddings data, etc. for that specific block.
|
||||
"""
|
||||
|
||||
|
||||
class BlockRemoved(KVCacheEvent):
|
||||
block_hashes: list[ExternalBlockHash]
|
||||
|
||||
Reference in New Issue
Block a user