[Core] Consolidate GB constant and enable float GB arguments (#7416)

This commit is contained in:
Cyrus Leung
2024-08-13 05:14:14 +08:00
committed by GitHub
parent 6aa33cb2dd
commit 4ddc4743d7
6 changed files with 21 additions and 21 deletions

View File

@@ -58,8 +58,8 @@ class EngineArgs:
enable_prefix_caching: bool = False
disable_sliding_window: bool = False
use_v2_block_manager: bool = False
swap_space: int = 4 # GiB
cpu_offload_gb: int = 0 # GiB
swap_space: float = 4 # GiB
cpu_offload_gb: float = 0 # GiB
gpu_memory_utilization: float = 0.90
max_num_batched_tokens: Optional[int] = None
max_num_seqs: int = 256
@@ -321,7 +321,7 @@ class EngineArgs:
default=EngineArgs.seed,
help='Random seed for operations.')
parser.add_argument('--swap-space',
type=int,
type=float,
default=EngineArgs.swap_space,
help='CPU swap space size (GiB) per GPU.')
parser.add_argument(