biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[hardware][misc] introduce platform abstraction (#6080)

Browse Source
This commit is contained in:
youkaichao
2024-07-02 20:12:22 -07:00
committed by GitHub
parent 9d6a8daa87
commit 482045ee77
16 changed files with 113 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/quantization/utils.py
View File

@@ -1,7 +1,7 @@
import torch
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
from vllm.utils import get_device_capability_stateless
from vllm.platforms import current_platform
def is_quant_method_supported(quant_method: str) -> bool:
@@ -9,7 +9,7 @@ def is_quant_method_supported(quant_method: str) -> bool:
if not torch.cuda.is_available():
return False
capability = get_device_capability_stateless()
capability = current_platform.get_device_capability()
capability = capability[0] * 10 + capability[1]
return (capability >=
QUANTIZATION_METHODS[quant_method].get_min_capability())