[platforms] absorb worker cls difference into platforms folder (#10555)

Signed-off-by: youkaichao <youkaichao@gmail.com>
Co-authored-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
youkaichao
2024-11-21 21:00:32 -08:00
committed by GitHub
parent 446c7806b2
commit a111d0151f
21 changed files with 272 additions and 282 deletions

View File

@@ -191,6 +191,7 @@ class EngineArgs:
override_neuron_config: Optional[Dict[str, Any]] = None
override_pooler_config: Optional[PoolerConfig] = None
compilation_config: Optional[CompilationConfig] = None
worker_cls: str = "auto"
def __post_init__(self):
if not self.tokenizer:
@@ -887,6 +888,12 @@ class EngineArgs:
'compilers, using -O without space is also '
'supported. -O3 is equivalent to -O 3.')
parser.add_argument(
'--worker-cls',
type=str,
default="auto",
help='The worker class to use for distributed execution.')
return parser
@classmethod
@@ -999,7 +1006,9 @@ class EngineArgs:
self.tokenizer_pool_extra_config,
),
ray_workers_use_nsight=self.ray_workers_use_nsight,
distributed_executor_backend=self.distributed_executor_backend)
distributed_executor_backend=self.distributed_executor_backend,
worker_cls=self.worker_cls,
)
max_model_len = model_config.max_model_len
use_long_context = max_model_len > 32768