[Feature] Support async scheduling + PP (#32359)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -411,9 +411,9 @@ class MultiprocExecutor(Executor):
|
||||
|
||||
@cached_property
|
||||
def max_concurrent_batches(self) -> int:
|
||||
if self.scheduler_config.async_scheduling:
|
||||
return 2
|
||||
return self.parallel_config.pipeline_parallel_size
|
||||
# PP requires PP-size concurrent batches to fill the pipeline.
|
||||
pp_size = self.parallel_config.pipeline_parallel_size
|
||||
return 2 if pp_size <= 1 and self.scheduler_config.async_scheduling else pp_size
|
||||
|
||||
def _get_output_rank(self) -> int:
|
||||
# Only returns ModelRunnerOutput from TP rank=0 and PP rank=-1
|
||||
|
||||
Reference in New Issue
Block a user