biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Bugfix][CI/Build][Hardware][AMD] Fix AMD tests, add HF cache, update CK FA, add partially supported model notes (#6543)

Browse Source
This commit is contained in:
Matt Wong
2024-07-20 11:39:07 -05:00
committed by GitHub
parent 683e3cb9c4
commit 06d6c5fe9f
12 changed files with 116 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
tests/basic_correctness/test_cpu_offload.py
View File

@@ -1,8 +1,13 @@
from vllm.utils import is_hip
from ..utils import compare_two_settings
def test_cpu_offload():
compare_two_settings("meta-llama/Llama-2-7b-hf", [],
["--cpu-offload-gb", "4"])
compare_two_settings("nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t",
[], ["--cpu-offload-gb", "1"])
if not is_hip():
# compressed-tensors quantization is currently not supported in ROCm.
compare_two_settings(
"nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t", [],
["--cpu-offload-gb", "1"])