biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Set torch_dtype in TransformersModel (#13088)

Browse Source 
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-02-11 15:51:19 +00:00
committed by GitHub
parent 75e6e14516
commit ad9776353e
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
vllm/model_executor/models/transformers.py
View File

@@ -143,6 +143,7 @@ class TransformersModel(nn.Module):
self.model: PreTrainedModel = AutoModel.from_config( self.model: PreTrainedModel = AutoModel.from_config(
self.config, self.config,
attn_implementation="vllm", attn_implementation="vllm",
torch_dtype=vllm_config.model_config.dtype,
trust_remote_code=vllm_config.model_config.trust_remote_code, trust_remote_code=vllm_config.model_config.trust_remote_code,
) )
prefix = self.model.base_model_prefix prefix = self.model.base_model_prefix