Add docstrings for LLMServer and related classes and examples (#142)

This commit is contained in:
Zhuohan Li
2023-06-07 18:25:20 +08:00
committed by GitHub
parent e38074b1e6
commit 4298374265
10 changed files with 212 additions and 18 deletions

View File

@@ -9,6 +9,7 @@ from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
@dataclass
class ServerArgs:
"""Arguments for CacheFlow servers."""
model: str
download_dir: Optional[str] = None
use_np_weights: bool = False
@@ -117,6 +118,7 @@ class ServerArgs:
@dataclass
class AsyncServerArgs(ServerArgs):
"""Arguments for asynchronous CacheFlow servers."""
server_use_ray: bool = False
@staticmethod