biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Hardware][XPU] AWQ/GPTQ support for xpu backend (#10107)

Browse Source 
Signed-off-by: yan ma <yan.ma@intel.com>
This commit is contained in:
Yan Ma
2024-11-19 02:18:05 +08:00
committed by GitHub
parent 281cc4b3cd
commit 6b2d25efc7
7 changed files with 146 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
tests/quantization/test_ipex_quant.py
View File

@@ -1,5 +1,5 @@
"""Test model set-up and inference for quantized HF models supported
on the CPU backend using IPEX (including AWQ).
on the CPU/GPU backend using IPEX (including AWQ/GPTQ).
Validating the configuration and printing results for manual checking.
@@ -11,13 +11,15 @@ import pytest
from vllm.platforms import current_platform
MODELS = [
"casperhansen/llama-3-8b-instruct-awq",
"AMead10/Llama-3.2-1B-Instruct-AWQ",
"shuyuej/Llama-3.2-1B-Instruct-GPTQ", # with g_idx
]
DTYPE = ["bfloat16"]
@pytest.mark.skipif(not current_platform.is_cpu(),
reason="only supports the CPU backend.")
@pytest.mark.skipif(not current_platform.is_cpu()
and not current_platform.is_xpu(),
reason="only supports Intel CPU/XPU backend.")
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", DTYPE)
def test_ipex_quant(vllm_runner, model, dtype):