biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Add trust-remote-code flag to handle remote tokenizers (#364)

Browse Source
This commit is contained in:
codethazine
2023-07-07 20:04:58 +02:00
committed by GitHub
parent be54f8e5c4
commit a945fcc2ae
5 changed files with 39 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
vllm/engine/llm_engine.py
View File

@@ -62,6 +62,7 @@ class LLMEngine:
f"model={model_config.model!r}, "
f"tokenizer={model_config.tokenizer!r}, "
f"tokenizer_mode={model_config.tokenizer_mode}, "
f"trust_remote_code={model_config.trust_remote_code}, "
f"dtype={model_config.dtype}, "
f"use_dummy_weights={model_config.use_dummy_weights}, "
f"download_dir={model_config.download_dir!r}, "
@@ -78,7 +79,9 @@ class LLMEngine:
self._verify_args()
self.tokenizer = get_tokenizer(
model_config.tokenizer, tokenizer_mode=model_config.tokenizer_mode)
model_config.tokenizer,
tokenizer_mode=model_config.tokenizer_mode,
trust_remote_code=model_config.trust_remote_code)
self.seq_counter = Counter()
# Create the parallel GPU workers.