biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[bugfix] fix the default value of llm_int8_threshold in BitsAndBytesConfig (#10657)

Browse Source
This commit is contained in:
yansh97
2024-11-27 13:55:23 +08:00
committed by GitHub
parent 1bf905ddaa
commit cfb3bf25fb
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
vllm/model_executor/layers/quantization/bitsandbytes.py
View File

@@ -26,7 +26,7 @@ class BitsAndBytesConfig(QuantizationConfig):
llm_int8_enable_fp32_cpu_offload: bool = False,
llm_int8_has_fp16_weight: bool = False,
llm_int8_skip_modules: Optional[List[str]] = None,
llm_int8_threshold: float = 0.0,
llm_int8_threshold: float = 6.0,
) -> None:
self.load_in_8bit = load_in_8bit
@@ -103,7 +103,7 @@ class BitsAndBytesConfig(QuantizationConfig):
["llm_int8_skip_modules"],
default_value=[])
llm_int8_threshold = get_safe_value(config, ["llm_int8_threshold"],
default_value=0.0)
default_value=6.0)
return cls(
load_in_8bit=load_in_8bit,