biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Reduce GPU memory utilization to make sure OOM doesn't happen (#153)

Browse Source
This commit is contained in:
Zhuohan Li
2023-06-18 17:33:50 +08:00
committed by GitHub
parent bec7b2dc26
commit bf5f121c02
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
vllm/engine/arg_utils.py
View File

@@ -21,7 +21,7 @@ class EngineArgs:
tensor_parallel_size: int = 1 tensor_parallel_size: int = 1
block_size: int = 16 block_size: int = 16
swap_space: int = 4 # GiB swap_space: int = 4 # GiB
gpu_memory_utilization: float = 0.95 gpu_memory_utilization: float = 0.90
max_num_batched_tokens: int = 2560 max_num_batched_tokens: int = 2560
max_num_seqs: int = 256 max_num_seqs: int = 256
disable_log_stats: bool = False disable_log_stats: bool = False