[Core] Support dynamically loading Lora adapter from HuggingFace (#6234)

Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>
This commit is contained in:
Jiaxin Shan
2024-07-22 15:42:40 -07:00
committed by GitHub
parent 69d5ae38dc
commit 42c7f66a38
11 changed files with 201 additions and 18 deletions

View File

@@ -43,7 +43,7 @@ class PromptAdapterPath:
@dataclass
class LoRAModulePath:
name: str
local_path: str
path: str
AnyRequest = Union[ChatCompletionRequest, CompletionRequest, DetokenizeRequest,
@@ -83,7 +83,7 @@ class OpenAIServing:
LoRARequest(
lora_name=lora.name,
lora_int_id=i,
lora_local_path=lora.local_path,
lora_path=lora.path,
) for i, lora in enumerate(lora_modules, start=1)
]