biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Bugfix] More type hint fixes for py 3.8 (#4039)

Browse Source
This commit is contained in:
Dylan Hawk
2024-04-12 21:07:04 -07:00
committed by GitHub
parent 546e721168
commit 5c2e66e487
4 changed files with 8 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/worker/worker_base.py
View File

@@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
from typing import Dict, List
from typing import Dict, List, Tuple
from vllm.lora.request import LoRARequest
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -18,14 +18,14 @@ class WorkerBase(ABC):
raise NotImplementedError
@abstractmethod
def determine_num_available_blocks(self) -> tuple[int, int]:
def determine_num_available_blocks(self) -> Tuple[int, int]:
"""Determine the number of available blocks for the GPU KV cache and
swappable CPU KV cache.
The implementation may run profiling or other heuristics to determine
the size of caches.
Returns a tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
Returns a Tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
are blocks that are "active" on the device and can be appended to.
num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be
appended to.