[Core] [Frontend] Priority scheduling for embeddings and in the OpenAI-API (#8965)

vllm/entrypoints/openai/protocol.py

@@ -279,6 +279,12 @@ class ChatCompletionRequest(OpenAIBaseModel):
         description=(
             "If specified, will override the default whitespace pattern "
             "for guided json decoding."))
+    priority: int = Field(
+        default=0,
+        description=(
+            "The priority of the request (lower means earlier handling; "
+            "default: 0). Any priority other than 0 will raise an error "
+            "if the served model does not use priority scheduling."))
 
     # doc: end-chat-completion-extra-params
 
@@ -552,6 +558,12 @@ class CompletionRequest(OpenAIBaseModel):
         description=(
             "If specified, will override the default whitespace pattern "
             "for guided json decoding."))
+    priority: int = Field(
+        default=0,
+        description=(
+            "The priority of the request (lower means earlier handling; "
+            "default: 0). Any priority other than 0 will raise an error "
+            "if the served model does not use priority scheduling."))
 
     # doc: end-completion-extra-params
 
@@ -665,6 +677,16 @@ class EmbeddingRequest(OpenAIBaseModel):
 
     # doc: end-embedding-pooling-params
 
+    # doc: begin-embedding-extra-params
+    priority: int = Field(
+        default=0,
+        description=(
+            "The priority of the request (lower means earlier handling; "
+            "default: 0). Any priority other than 0 will raise an error "
+            "if the served model does not use priority scheduling."))
+
+    # doc: end-embedding-extra-params
+
     def to_pooling_params(self):
         return PoolingParams(additional_data=self.additional_data)
 

vllm/entrypoints/openai/serving_chat.py

@@ -235,6 +235,7 @@ class OpenAIServingChat(OpenAIServing):
                 lora_request=lora_request,
                 trace_headers=trace_headers,
                 prompt_adapter_request=prompt_adapter_request,
+                priority=request.priority,
             )
         except ValueError as e:
             # TODO: Use a vllm-specific Validation Error

vllm/entrypoints/openai/serving_completion.py

@@ -148,6 +148,7 @@ class OpenAIServingCompletion(OpenAIServing):
                     lora_request=lora_request,
                     prompt_adapter_request=prompt_adapter_request,
                     trace_headers=trace_headers,
+                    priority=request.priority,
                 )
 
                 generators.append(generator)

vllm/entrypoints/openai/serving_embedding.py

@@ -148,6 +148,7 @@ class OpenAIServingEmbedding(OpenAIServing):
                     pooling_params,
                     request_id_item,
                     lora_request=lora_request,
+                    priority=request.priority,
                 )
 
                 generators.append(generator)