[KVConnector] Keep KVTransferParams as a dict (#18033)
This commit is contained in:
@@ -3,7 +3,6 @@
|
||||
import enum
|
||||
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1 import KVTransferParams
|
||||
from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import is_list_of
|
||||
@@ -62,14 +61,10 @@ class Request:
|
||||
self.num_encoder_inputs = len(self.mm_inputs)
|
||||
self.has_encoder_inputs = self.num_encoder_inputs > 0
|
||||
|
||||
# P/D: KV transfer parameters (raw and parsed).
|
||||
raw_params = (None if sampling_params.extra_args is None
|
||||
else sampling_params.extra_args.get(
|
||||
"kv_transfer_params", None))
|
||||
self.raw_kv_transfer_params: Optional[dict[str, Any]] = raw_params
|
||||
# Each connector parses the raw dictionary and sets this
|
||||
# attr the first time that the request is processed.
|
||||
self.kv_transfer_params: Optional[KVTransferParams] = None
|
||||
# P/D: Connector-specific KV transfer parameters.
|
||||
kv_params = (None if sampling_params.extra_args is None else
|
||||
sampling_params.extra_args.get("kv_transfer_params"))
|
||||
self.kv_transfer_params: Optional[dict[str, Any]] = kv_params
|
||||
|
||||
# Sanity check
|
||||
assert len(self.mm_inputs) == len(self.mm_positions)
|
||||
|
||||
Reference in New Issue
Block a user