biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[V1] [2/n] Logging and Metrics - OutputProcessor Abstraction (#11973)

Browse Source 
Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com>
This commit is contained in:
Robert Shaw
2025-01-12 23:54:10 -05:00
committed by GitHub
parent d14e98d924
commit 619ae268c3
8 changed files with 449 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
vllm/v1/engine/llm_engine.py
View File

@@ -18,7 +18,7 @@ from vllm.transformers_utils.tokenizer_group import (
BaseTokenizerGroup, init_tokenizer_from_configs)
from vllm.usage.usage_lib import UsageContext
from vllm.v1.engine.core_client import EngineCoreClient
from vllm.v1.engine.detokenizer import Detokenizer
from vllm.v1.engine.output_processor import OutputProcessor
from vllm.v1.engine.processor import Processor
from vllm.v1.executor.abstract import Executor
@@ -60,13 +60,9 @@ class LLMEngine:
input_registry=input_registry,
mm_registry=mm_registry)
# Detokenizer (converts EngineCoreOutputs --> RequestOutput)
self.detokenizer = Detokenizer(
tokenizer_name=vllm_config.model_config.tokenizer,
tokenizer_mode=vllm_config.model_config.tokenizer_mode,
trust_remote_code=vllm_config.model_config.trust_remote_code,
revision=vllm_config.model_config.tokenizer_revision,
)
# OutputProcessor (convert EngineCoreOutputs --> RequestOutput).
self.output_processor = OutputProcessor(self.tokenizer,
log_stats=False)
# EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs)
self.engine_core = EngineCoreClient.make_client(
@@ -103,10 +99,10 @@ class LLMEngine:
multiprocess_mode=enable_multiprocessing)
def get_num_unfinished_requests(self) -> int:
return self.detokenizer.get_num_unfinished_requests()
return self.output_processor.get_num_unfinished_requests()
def has_unfinished_requests(self) -> bool:
return self.detokenizer.has_unfinished_requests()
return self.output_processor.has_unfinished_requests()
@classmethod
def validate_outputs(cls, outputs, output_type):
@@ -116,7 +112,7 @@ class LLMEngine:
"""Remove request_ids from EngineCore and Detokenizer."""
self.engine_core.abort_requests(request_ids)
self.detokenizer.abort_requests(request_ids)
self.output_processor.abort_requests(request_ids)
def add_request(
self,
@@ -137,8 +133,8 @@ class LLMEngine:
prompt_adapter_request,
priority)
# 2) Add the request to Detokenizer.
self.detokenizer.add_request(request)
# 2) Make a new RequestState and queue.
self.output_processor.add_request(request)
# 3) Add the request to EngineCore.
self.engine_core.add_request(request)
@@ -148,15 +144,14 @@ class LLMEngine:
# 1) Get EngineCoreOutput from the EngineCore.
outputs = self.engine_core.get_output()
# 2) Detokenizer the EngineCoreOutput.
request_outputs, requests_to_abort = self.detokenizer.step(
# 2) Process EngineCoreOutputs.
processed_outputs = self.output_processor.process_outputs(
outputs.outputs)
# 3) Abort requests that finished due to stopping criteria.
if requests_to_abort:
self.abort_request(requests_to_abort)
# 3) Abort any reqs that finished due to stop strings.
self.engine_core.abort_requests(processed_outputs.reqs_to_abort)
return request_outputs
return processed_outputs.request_outputs
def get_model_config(self):
return self.model_config