[CI][MCP][Harmony] Heavy refactoring Harmony & MCP response tests and stabilizing with deterministic test infrastructure (#33949)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
@@ -1,7 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BASE_TEST_ENV = {
|
||||
# The day vLLM said "hello world" on arxiv 🚀
|
||||
"VLLM_SYSTEM_START_DATE": "2023-09-12",
|
||||
}
|
||||
DEFAULT_MAX_RETRIES = 3
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pairs_of_event_types() -> dict[str, str]:
|
||||
@@ -28,3 +43,159 @@ def pairs_of_event_types() -> dict[str, str]:
|
||||
}
|
||||
# fmt: on
|
||||
return event_pairs
|
||||
|
||||
|
||||
async def retry_for_tool_call(
|
||||
client,
|
||||
*,
|
||||
model: str,
|
||||
expected_tool_type: str,
|
||||
max_retries: int = DEFAULT_MAX_RETRIES,
|
||||
**create_kwargs: Any,
|
||||
):
|
||||
"""Call ``client.responses.create`` up to *max_retries* times, returning
|
||||
the first response that contains an output item of *expected_tool_type*.
|
||||
|
||||
Returns the **last** response if none match so the caller's assertions
|
||||
fire with a clear diagnostic.
|
||||
"""
|
||||
last_response = None
|
||||
for attempt in range(max_retries):
|
||||
response = await client.responses.create(model=model, **create_kwargs)
|
||||
last_response = response
|
||||
if any(
|
||||
getattr(item, "type", None) == expected_tool_type
|
||||
for item in response.output
|
||||
):
|
||||
return response
|
||||
assert last_response is not None
|
||||
return last_response
|
||||
|
||||
|
||||
async def retry_streaming_for(
|
||||
client,
|
||||
*,
|
||||
model: str,
|
||||
validate_events: Callable[[list], bool],
|
||||
max_retries: int = DEFAULT_MAX_RETRIES,
|
||||
**create_kwargs: Any,
|
||||
) -> list:
|
||||
"""Call ``client.responses.create(stream=True)`` up to *max_retries*
|
||||
times, returning the first event list where *validate_events* returns
|
||||
``True``.
|
||||
"""
|
||||
last_events: list = []
|
||||
for attempt in range(max_retries):
|
||||
stream = await client.responses.create(
|
||||
model=model, stream=True, **create_kwargs
|
||||
)
|
||||
events: list = []
|
||||
async for event in stream:
|
||||
events.append(event)
|
||||
last_events = events
|
||||
if validate_events(events):
|
||||
return events
|
||||
return last_events
|
||||
|
||||
|
||||
def has_output_type(response, type_name: str) -> bool:
|
||||
"""Return True if *response* has at least one output item of *type_name*."""
|
||||
return any(getattr(item, "type", None) == type_name for item in response.output)
|
||||
|
||||
|
||||
def events_contain_type(events: list, type_substring: str) -> bool:
|
||||
"""Return True if any event's type contains *type_substring*."""
|
||||
return any(type_substring in getattr(e, "type", "") for e in events)
|
||||
|
||||
|
||||
def validate_streaming_event_stack(
|
||||
events: list, pairs_of_event_types: dict[str, str]
|
||||
) -> None:
|
||||
"""Validate that streaming events are properly nested/paired."""
|
||||
stack: list[str] = []
|
||||
for event in events:
|
||||
etype = event.type
|
||||
if etype == "response.created":
|
||||
stack.append(etype)
|
||||
elif etype == "response.completed":
|
||||
assert stack and stack[-1] == pairs_of_event_types[etype], (
|
||||
f"Unexpected stack top for {etype}: "
|
||||
f"got {stack[-1] if stack else '<empty>'}"
|
||||
)
|
||||
stack.pop()
|
||||
elif etype.endswith("added") or etype == "response.mcp_call.in_progress":
|
||||
stack.append(etype)
|
||||
elif etype.endswith("delta"):
|
||||
if stack and stack[-1] == etype:
|
||||
continue
|
||||
stack.append(etype)
|
||||
elif etype.endswith("done") or etype == "response.mcp_call.completed":
|
||||
assert etype in pairs_of_event_types, f"Unknown done event: {etype}"
|
||||
expected_start = pairs_of_event_types[etype]
|
||||
assert stack and stack[-1] == expected_start, (
|
||||
f"Stack mismatch for {etype}: "
|
||||
f"expected {expected_start}, "
|
||||
f"got {stack[-1] if stack else '<empty>'}"
|
||||
)
|
||||
stack.pop()
|
||||
assert len(stack) == 0, f"Unclosed events on stack: {stack}"
|
||||
|
||||
|
||||
def log_response_diagnostics(
|
||||
response,
|
||||
*,
|
||||
label: str = "Response Diagnostics",
|
||||
) -> dict[str, Any]:
|
||||
"""Extract and log diagnostic info from a Responses API response.
|
||||
|
||||
Logs reasoning, tool-call attempts, MCP items, and output types so
|
||||
that CI output (``pytest -s`` or ``--log-cli-level=INFO``) gives
|
||||
full visibility into model behaviour even on passing runs.
|
||||
|
||||
Returns the extracted data so callers can make additional assertions
|
||||
if needed.
|
||||
"""
|
||||
reasoning_texts = [
|
||||
text
|
||||
for item in response.output
|
||||
if getattr(item, "type", None) == "reasoning"
|
||||
for content in getattr(item, "content", [])
|
||||
if (text := getattr(content, "text", None))
|
||||
]
|
||||
|
||||
tool_call_attempts = [
|
||||
{
|
||||
"recipient": msg.get("recipient"),
|
||||
"channel": msg.get("channel"),
|
||||
}
|
||||
for msg in response.output_messages
|
||||
if (msg.get("recipient") or "").startswith("python")
|
||||
]
|
||||
|
||||
mcp_items = [
|
||||
{
|
||||
"name": getattr(item, "name", None),
|
||||
"status": getattr(item, "status", None),
|
||||
}
|
||||
for item in response.output
|
||||
if getattr(item, "type", None) == "mcp_call"
|
||||
]
|
||||
|
||||
output_types = [getattr(o, "type", None) for o in response.output]
|
||||
|
||||
diagnostics = {
|
||||
"model_attempted_tool_calls": bool(tool_call_attempts),
|
||||
"tool_call_attempts": tool_call_attempts,
|
||||
"mcp_items": mcp_items,
|
||||
"reasoning": reasoning_texts,
|
||||
"output_text": response.output_text,
|
||||
"output_types": output_types,
|
||||
}
|
||||
|
||||
logger.info(
|
||||
"\n====== %s ======\n%s\n==============================",
|
||||
label,
|
||||
json.dumps(diagnostics, indent=2, default=str),
|
||||
)
|
||||
|
||||
return diagnostics
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Integration tests for MCP tool support in the Responses API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
@@ -10,11 +12,31 @@ from openai_harmony import ToolDescription, ToolNamespaceConfig
|
||||
from vllm.entrypoints.mcp.tool_server import MCPToolServer
|
||||
|
||||
from ....utils import RemoteOpenAIServer
|
||||
from .conftest import (
|
||||
BASE_TEST_ENV,
|
||||
events_contain_type,
|
||||
log_response_diagnostics,
|
||||
retry_for_tool_call,
|
||||
retry_streaming_for,
|
||||
validate_streaming_event_stack,
|
||||
)
|
||||
|
||||
MODEL_NAME = "openai/gpt-oss-20b"
|
||||
|
||||
_BASE_SERVER_ARGS = [
|
||||
"--enforce-eager",
|
||||
"--tool-server",
|
||||
"demo",
|
||||
"--max_model_len",
|
||||
"5000",
|
||||
]
|
||||
|
||||
def test_get_tool_description():
|
||||
_PYTHON_TOOL_INSTRUCTION = (
|
||||
"You must use the Python tool to execute code. Never simulate execution."
|
||||
)
|
||||
|
||||
|
||||
class TestMCPToolServerUnit:
|
||||
"""Test MCPToolServer.get_tool_description filtering logic.
|
||||
|
||||
Note: The wildcard "*" is normalized to None by
|
||||
@@ -22,283 +44,240 @@ def test_get_tool_description():
|
||||
so we only test None and specific tool filtering here.
|
||||
See test_serving_responses.py for "*" normalization tests.
|
||||
"""
|
||||
pytest.importorskip("mcp")
|
||||
|
||||
server = MCPToolServer()
|
||||
tool1 = ToolDescription.new(
|
||||
name="tool1", description="First", parameters={"type": "object"}
|
||||
)
|
||||
tool2 = ToolDescription.new(
|
||||
name="tool2", description="Second", parameters={"type": "object"}
|
||||
)
|
||||
tool3 = ToolDescription.new(
|
||||
name="tool3", description="Third", parameters={"type": "object"}
|
||||
)
|
||||
def test_get_tool_description(self):
|
||||
pytest.importorskip("mcp")
|
||||
|
||||
server.harmony_tool_descriptions = {
|
||||
"test_server": ToolNamespaceConfig(
|
||||
name="test_server", description="test", tools=[tool1, tool2, tool3]
|
||||
server = MCPToolServer()
|
||||
tool1 = ToolDescription.new(
|
||||
name="tool1", description="First", parameters={"type": "object"}
|
||||
)
|
||||
tool2 = ToolDescription.new(
|
||||
name="tool2", description="Second", parameters={"type": "object"}
|
||||
)
|
||||
tool3 = ToolDescription.new(
|
||||
name="tool3", description="Third", parameters={"type": "object"}
|
||||
)
|
||||
}
|
||||
|
||||
# Nonexistent server
|
||||
assert server.get_tool_description("nonexistent") is None
|
||||
server.harmony_tool_descriptions = {
|
||||
"test_server": ToolNamespaceConfig(
|
||||
name="test_server",
|
||||
description="test",
|
||||
tools=[tool1, tool2, tool3],
|
||||
)
|
||||
}
|
||||
|
||||
# None (no filter) - returns all tools
|
||||
result = server.get_tool_description("test_server", allowed_tools=None)
|
||||
assert len(result.tools) == 3
|
||||
# Nonexistent server
|
||||
assert server.get_tool_description("nonexistent") is None
|
||||
|
||||
# Filter to specific tools
|
||||
result = server.get_tool_description(
|
||||
"test_server", allowed_tools=["tool1", "tool3"]
|
||||
)
|
||||
assert len(result.tools) == 2
|
||||
assert result.tools[0].name == "tool1"
|
||||
assert result.tools[1].name == "tool3"
|
||||
# None (no filter) - returns all tools
|
||||
result = server.get_tool_description("test_server", allowed_tools=None)
|
||||
assert len(result.tools) == 3
|
||||
|
||||
# Single tool
|
||||
result = server.get_tool_description(
|
||||
"test_server",
|
||||
allowed_tools=["tool2"],
|
||||
)
|
||||
assert len(result.tools) == 1
|
||||
assert result.tools[0].name == "tool2"
|
||||
# Filter to specific tools
|
||||
result = server.get_tool_description(
|
||||
"test_server", allowed_tools=["tool1", "tool3"]
|
||||
)
|
||||
assert len(result.tools) == 2
|
||||
assert result.tools[0].name == "tool1"
|
||||
assert result.tools[1].name == "tool3"
|
||||
|
||||
# No matching tools - returns None
|
||||
result = server.get_tool_description("test_server", allowed_tools=["nonexistent"])
|
||||
assert result is None
|
||||
# Single tool
|
||||
result = server.get_tool_description("test_server", allowed_tools=["tool2"])
|
||||
assert len(result.tools) == 1
|
||||
assert result.tools[0].name == "tool2"
|
||||
|
||||
# Empty list - returns None
|
||||
assert server.get_tool_description("test_server", allowed_tools=[]) is None
|
||||
# No matching tools - returns None
|
||||
result = server.get_tool_description(
|
||||
"test_server", allowed_tools=["nonexistent"]
|
||||
)
|
||||
assert result is None
|
||||
|
||||
# Empty list - returns None
|
||||
assert server.get_tool_description("test_server", allowed_tools=[]) is None
|
||||
|
||||
def test_builtin_tools_consistency(self):
|
||||
"""MCP_BUILTIN_TOOLS must match _BUILTIN_TOOL_TO_MCP_SERVER_LABEL values."""
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
_BUILTIN_TOOL_TO_MCP_SERVER_LABEL,
|
||||
MCP_BUILTIN_TOOLS,
|
||||
)
|
||||
|
||||
assert set(_BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values()) == MCP_BUILTIN_TOOLS, (
|
||||
f"MCP_BUILTIN_TOOLS {MCP_BUILTIN_TOOLS} does not match "
|
||||
f"_BUILTIN_TOOL_TO_MCP_SERVER_LABEL values "
|
||||
f"{set(_BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values())}"
|
||||
)
|
||||
|
||||
|
||||
class TestMCPEnabled:
|
||||
"""Tests that require MCP tools to be enabled via environment variable."""
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def monkeypatch_class(self):
|
||||
from _pytest.monkeypatch import MonkeyPatch
|
||||
|
||||
mpatch = MonkeyPatch()
|
||||
yield mpatch
|
||||
mpatch.undo()
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def mcp_enabled_server(self, monkeypatch_class: pytest.MonkeyPatch):
|
||||
args = ["--enforce-eager", "--tool-server", "demo"]
|
||||
|
||||
with monkeypatch_class.context() as m:
|
||||
m.setenv("VLLM_ENABLE_RESPONSES_API_STORE", "1")
|
||||
m.setenv("PYTHON_EXECUTION_BACKEND", "dangerously_use_uv")
|
||||
m.setenv(
|
||||
"VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS", "code_interpreter,container"
|
||||
)
|
||||
# Helps the model follow instructions better
|
||||
m.setenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "1")
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
def mcp_enabled_server(self):
|
||||
env_dict = {
|
||||
**BASE_TEST_ENV,
|
||||
"VLLM_ENABLE_RESPONSES_API_STORE": "1",
|
||||
"PYTHON_EXECUTION_BACKEND": "dangerously_use_uv",
|
||||
"VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS": ("code_interpreter,container"),
|
||||
"VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS": "1",
|
||||
}
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME, list(_BASE_SERVER_ARGS), env_dict=env_dict
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def mcp_enabled_client(self, mcp_enabled_server):
|
||||
async def client(self, mcp_enabled_server):
|
||||
async with mcp_enabled_server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
@staticmethod
|
||||
def _mcp_tools_payload(*, allowed_tools: list[str] | None = None) -> list[dict]:
|
||||
tool: dict = {
|
||||
"type": "mcp",
|
||||
"server_label": "code_interpreter",
|
||||
"server_url": "http://localhost:8888",
|
||||
}
|
||||
if allowed_tools is not None:
|
||||
tool["allowed_tools"] = allowed_tools
|
||||
return [tool]
|
||||
|
||||
@staticmethod
|
||||
def _python_exec_input(code: str = "") -> str:
|
||||
if not code:
|
||||
code = "import random; print(random.randint(1, 1000000))"
|
||||
return f"Execute the following code: {code}"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_mcp_tool_env_flag_enabled(
|
||||
self, mcp_enabled_client: OpenAI, model_name: str
|
||||
):
|
||||
response = await mcp_enabled_client.responses.create(
|
||||
async def test_mcp_tool_env_flag_enabled(self, client: OpenAI, model_name: str):
|
||||
response = await retry_for_tool_call(
|
||||
client,
|
||||
model=model_name,
|
||||
input=(
|
||||
"Execute the following code: "
|
||||
"import random; print(random.randint(1, 1000000))"
|
||||
),
|
||||
instructions=(
|
||||
"You must use the Python tool to execute code. "
|
||||
"Never simulate execution."
|
||||
),
|
||||
tools=[
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_label": "code_interpreter",
|
||||
# URL unused for DemoToolServer
|
||||
"server_url": "http://localhost:8888",
|
||||
}
|
||||
],
|
||||
expected_tool_type="mcp_call",
|
||||
input=self._python_exec_input(),
|
||||
instructions=_PYTHON_TOOL_INSTRUCTION,
|
||||
tools=self._mcp_tools_payload(),
|
||||
temperature=0.0,
|
||||
extra_body={"enable_response_messages": True},
|
||||
)
|
||||
assert response is not None
|
||||
|
||||
assert response.status == "completed"
|
||||
# Verify output messages: Tool calls and responses on analysis channel
|
||||
log_response_diagnostics(response, label="MCP Enabled")
|
||||
|
||||
tool_call_found = False
|
||||
tool_response_found = False
|
||||
for message in response.output_messages:
|
||||
recipient = message.get("recipient")
|
||||
if recipient and recipient.startswith("python"):
|
||||
tool_call_found = True
|
||||
assert message.get("channel") == "analysis", (
|
||||
"Tool call should be on analysis channel"
|
||||
)
|
||||
assert message.get("channel") == "analysis"
|
||||
author = message.get("author", {})
|
||||
if (
|
||||
author.get("role") == "tool"
|
||||
and author.get("name")
|
||||
and author.get("name").startswith("python")
|
||||
if author.get("role") == "tool" and (author.get("name") or "").startswith(
|
||||
"python"
|
||||
):
|
||||
tool_response_found = True
|
||||
assert message.get("channel") == "analysis", (
|
||||
"Tool response should be on analysis channel"
|
||||
)
|
||||
assert message.get("channel") == "analysis"
|
||||
|
||||
assert tool_call_found, "Should have found at least one Python tool call"
|
||||
assert tool_response_found, (
|
||||
"Should have found at least one Python tool response"
|
||||
assert tool_call_found, (
|
||||
f"No Python tool call found. "
|
||||
f"Output types: "
|
||||
f"{[getattr(o, 'type', None) for o in response.output]}"
|
||||
)
|
||||
for message in response.input_messages:
|
||||
assert message.get("author").get("role") != "developer", (
|
||||
"No developer messages should be present with valid mcp tool"
|
||||
)
|
||||
assert tool_response_found, "No Python tool response found"
|
||||
|
||||
for message in response.input_messages:
|
||||
assert message.get("author", {}).get("role") != "developer"
|
||||
|
||||
@pytest.mark.flaky(reruns=3)
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_mcp_tool_with_allowed_tools_star(
|
||||
self, mcp_enabled_client: OpenAI, model_name: str
|
||||
self, client: OpenAI, model_name: str
|
||||
):
|
||||
"""Test MCP tool with allowed_tools=['*'] to select all available
|
||||
tools.
|
||||
|
||||
This E2E test verifies that the "*" wildcard works end-to-end.
|
||||
See test_serving_responses.py for detailed unit tests of "*"
|
||||
normalization.
|
||||
"""
|
||||
response = await mcp_enabled_client.responses.create(
|
||||
response = await retry_for_tool_call(
|
||||
client,
|
||||
model=model_name,
|
||||
input=(
|
||||
"Execute the following code: "
|
||||
"import random; print(random.randint(1, 1000000))"
|
||||
),
|
||||
instructions=(
|
||||
"You must use the Python tool to execute code. "
|
||||
"Never simulate execution."
|
||||
),
|
||||
tools=[
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_label": "code_interpreter",
|
||||
"server_url": "http://localhost:8888",
|
||||
# Using "*" to allow all tools from this MCP server
|
||||
"allowed_tools": ["*"],
|
||||
}
|
||||
],
|
||||
expected_tool_type="mcp_call",
|
||||
input=self._python_exec_input(),
|
||||
instructions=_PYTHON_TOOL_INSTRUCTION,
|
||||
tools=self._mcp_tools_payload(allowed_tools=["*"]),
|
||||
temperature=0.0,
|
||||
extra_body={"enable_response_messages": True},
|
||||
)
|
||||
assert response is not None
|
||||
|
||||
assert response.status == "completed"
|
||||
# Verify tool calls work with allowed_tools=["*"]
|
||||
tool_call_found = False
|
||||
for message in response.output_messages:
|
||||
recipient = message.get("recipient")
|
||||
if recipient and recipient.startswith("python"):
|
||||
tool_call_found = True
|
||||
break
|
||||
log_response_diagnostics(response, label="MCP Allowed Tools *")
|
||||
|
||||
tool_call_found = any(
|
||||
(msg.get("recipient") or "").startswith("python")
|
||||
for msg in response.output_messages
|
||||
)
|
||||
assert tool_call_found, (
|
||||
"Should have found at least one Python tool call with '*'"
|
||||
f"No Python tool call with '*'. "
|
||||
f"Output types: "
|
||||
f"{[getattr(o, 'type', None) for o in response.output]}"
|
||||
)
|
||||
|
||||
@pytest.mark.flaky(reruns=3)
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_mcp_tool_calling_streaming_types(
|
||||
self,
|
||||
pairs_of_event_types: dict[str, str],
|
||||
mcp_enabled_client: OpenAI,
|
||||
client: OpenAI,
|
||||
model_name: str,
|
||||
):
|
||||
tools = [
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_label": "code_interpreter",
|
||||
}
|
||||
]
|
||||
input_text = "What is 123 * 456? Use python to calculate the result."
|
||||
def _has_mcp_events(events: list) -> bool:
|
||||
return events_contain_type(events, "mcp_call")
|
||||
|
||||
stream_response = await mcp_enabled_client.responses.create(
|
||||
events = await retry_streaming_for(
|
||||
client,
|
||||
model=model_name,
|
||||
input=input_text,
|
||||
tools=tools,
|
||||
stream=True,
|
||||
instructions=(
|
||||
"You must use the Python tool to execute code. "
|
||||
"Never simulate execution."
|
||||
),
|
||||
validate_events=_has_mcp_events,
|
||||
input=("What is 123 * 456? Use Python to calculate the result."),
|
||||
tools=[{"type": "mcp", "server_label": "code_interpreter"}],
|
||||
instructions=_PYTHON_TOOL_INSTRUCTION,
|
||||
temperature=0.0,
|
||||
)
|
||||
|
||||
stack_of_event_types = []
|
||||
saw_mcp_type = False
|
||||
async for event in stream_response:
|
||||
if event.type == "response.created":
|
||||
stack_of_event_types.append(event.type)
|
||||
elif event.type == "response.completed":
|
||||
assert stack_of_event_types[-1] == pairs_of_event_types[event.type]
|
||||
stack_of_event_types.pop()
|
||||
elif (
|
||||
event.type.endswith("added")
|
||||
or event.type == "response.mcp_call.in_progress"
|
||||
):
|
||||
stack_of_event_types.append(event.type)
|
||||
elif event.type.endswith("delta"):
|
||||
if stack_of_event_types[-1] == event.type:
|
||||
continue
|
||||
stack_of_event_types.append(event.type)
|
||||
elif (
|
||||
event.type.endswith("done")
|
||||
or event.type == "response.mcp_call.completed"
|
||||
):
|
||||
assert stack_of_event_types[-1] == pairs_of_event_types[event.type]
|
||||
if "mcp_call" in event.type:
|
||||
saw_mcp_type = True
|
||||
stack_of_event_types.pop()
|
||||
validate_streaming_event_stack(events, pairs_of_event_types)
|
||||
|
||||
assert len(stack_of_event_types) == 0
|
||||
assert saw_mcp_type, "Should have seen at least one mcp call"
|
||||
assert events_contain_type(events, "mcp_call"), (
|
||||
f"No mcp_call events after retries. "
|
||||
f"Event types: {sorted({e.type for e in events})}"
|
||||
)
|
||||
|
||||
|
||||
class TestMCPDisabled:
|
||||
"""Tests that verify behavior when MCP tools are disabled."""
|
||||
"""Tests that MCP tools are not executed when the env flag is unset."""
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def monkeypatch_class(self):
|
||||
from _pytest.monkeypatch import MonkeyPatch
|
||||
|
||||
mpatch = MonkeyPatch()
|
||||
yield mpatch
|
||||
mpatch.undo()
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def mcp_disabled_server(self, monkeypatch_class: pytest.MonkeyPatch):
|
||||
args = ["--enforce-eager", "--tool-server", "demo"]
|
||||
|
||||
with monkeypatch_class.context() as m:
|
||||
m.setenv("VLLM_ENABLE_RESPONSES_API_STORE", "1")
|
||||
m.setenv("PYTHON_EXECUTION_BACKEND", "dangerously_use_uv")
|
||||
# Helps the model follow instructions better
|
||||
m.setenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "1")
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
def mcp_disabled_server(self):
|
||||
env_dict = {
|
||||
**BASE_TEST_ENV,
|
||||
"VLLM_ENABLE_RESPONSES_API_STORE": "1",
|
||||
"PYTHON_EXECUTION_BACKEND": "dangerously_use_uv",
|
||||
"VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS": "1",
|
||||
}
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME, list(_BASE_SERVER_ARGS), env_dict=env_dict
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def mcp_disabled_client(self, mcp_disabled_server):
|
||||
async def client(self, mcp_disabled_server):
|
||||
async with mcp_disabled_server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_mcp_tool_env_flag_disabled(
|
||||
self, mcp_disabled_client: OpenAI, model_name: str
|
||||
async def test_mcp_disabled_server_does_not_execute(
|
||||
self, client: OpenAI, model_name: str
|
||||
):
|
||||
response = await mcp_disabled_client.responses.create(
|
||||
"""When MCP is disabled the model may still attempt tool calls
|
||||
(tool descriptions can remain in the prompt), but the server
|
||||
must NOT execute them."""
|
||||
response = await client.responses.create(
|
||||
model=model_name,
|
||||
input=(
|
||||
"Execute the following code if the tool is present: "
|
||||
@@ -308,38 +287,35 @@ class TestMCPDisabled:
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_label": "code_interpreter",
|
||||
# URL unused for DemoToolServer
|
||||
"server_url": "http://localhost:8888",
|
||||
}
|
||||
],
|
||||
temperature=0.0,
|
||||
extra_body={"enable_response_messages": True},
|
||||
)
|
||||
assert response is not None
|
||||
assert response.status == "completed"
|
||||
# Verify output messages: No tool calls and responses
|
||||
tool_call_found = False
|
||||
tool_response_found = False
|
||||
|
||||
log_response_diagnostics(response, label="MCP Disabled")
|
||||
|
||||
# Server must not have executed any tool calls
|
||||
for message in response.output_messages:
|
||||
recipient = message.get("recipient")
|
||||
if recipient and recipient.startswith("python"):
|
||||
tool_call_found = True
|
||||
assert message.get("channel") == "analysis", (
|
||||
"Tool call should be on analysis channel"
|
||||
)
|
||||
author = message.get("author", {})
|
||||
if (
|
||||
assert not (
|
||||
author.get("role") == "tool"
|
||||
and author.get("name")
|
||||
and author.get("name").startswith("python")
|
||||
):
|
||||
tool_response_found = True
|
||||
assert message.get("channel") == "analysis", (
|
||||
"Tool response should be on analysis channel"
|
||||
and (author.get("name") or "").startswith("python")
|
||||
), (
|
||||
"Server executed a python tool call even though MCP is "
|
||||
f"disabled. Message: {message}"
|
||||
)
|
||||
|
||||
# No completed mcp_call output items
|
||||
for item in response.output:
|
||||
if getattr(item, "type", None) == "mcp_call":
|
||||
assert getattr(item, "status", None) != "completed", (
|
||||
"MCP call should not be completed when MCP is disabled"
|
||||
)
|
||||
|
||||
assert not tool_call_found, "Should not have a python call"
|
||||
assert not tool_response_found, "Should not have a tool response"
|
||||
# No developer messages injected
|
||||
for message in response.input_messages:
|
||||
assert message.get("author").get("role") != "developer", (
|
||||
"No developer messages should be present without a valid tool"
|
||||
)
|
||||
assert message.get("author", {}).get("role") != "developer"
|
||||
|
||||
@@ -3,15 +3,29 @@
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from openai import OpenAI
|
||||
|
||||
from ....utils import RemoteOpenAIServer
|
||||
from .conftest import (
|
||||
BASE_TEST_ENV,
|
||||
has_output_type,
|
||||
log_response_diagnostics,
|
||||
retry_for_tool_call,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-8B"
|
||||
|
||||
_PYTHON_TOOL_INSTRUCTION = (
|
||||
"You must use the Python tool to execute code. "
|
||||
"Never simulate execution. You must print the final answer."
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
@@ -32,12 +46,12 @@ def server():
|
||||
"--tool-server",
|
||||
"demo",
|
||||
]
|
||||
env_dict = dict(
|
||||
VLLM_ENABLE_RESPONSES_API_STORE="1",
|
||||
VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT="1",
|
||||
PYTHON_EXECUTION_BACKEND="dangerously_use_uv",
|
||||
)
|
||||
|
||||
env_dict = {
|
||||
**BASE_TEST_ENV,
|
||||
"VLLM_ENABLE_RESPONSES_API_STORE": "1",
|
||||
"VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT": "1",
|
||||
"PYTHON_EXECUTION_BACKEND": "dangerously_use_uv",
|
||||
}
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
@@ -54,6 +68,7 @@ async def test_basic(client: OpenAI, model_name: str):
|
||||
response = await client.responses.create(
|
||||
model=model_name,
|
||||
input="What is 123 * 456?",
|
||||
temperature=0.0,
|
||||
)
|
||||
assert response is not None
|
||||
print("response: ", response)
|
||||
@@ -99,10 +114,15 @@ async def test_reasoning_and_function_items(client: OpenAI, model_name: str):
|
||||
)
|
||||
assert response is not None
|
||||
assert response.status == "completed"
|
||||
# make sure we get a reasoning and text output
|
||||
assert response.output[0].type == "reasoning"
|
||||
assert response.output[1].type == "message"
|
||||
assert type(response.output[1].content[0].text) is str
|
||||
|
||||
output_types = [getattr(o, "type", None) for o in response.output]
|
||||
assert "reasoning" in output_types, (
|
||||
f"Expected reasoning in output, got: {output_types}"
|
||||
)
|
||||
assert "message" in output_types, f"Expected message in output, got: {output_types}"
|
||||
|
||||
msg = next(o for o in response.output if o.type == "message")
|
||||
assert type(msg.content[0].text) is str
|
||||
|
||||
|
||||
def get_horoscope(sign):
|
||||
@@ -110,10 +130,10 @@ def get_horoscope(sign):
|
||||
|
||||
|
||||
def call_function(name, args):
|
||||
logger.info("Calling function %s with args %s", name, args)
|
||||
if name == "get_horoscope":
|
||||
return get_horoscope(**args)
|
||||
else:
|
||||
raise ValueError(f"Unknown function: {name}")
|
||||
raise ValueError(f"Unknown function: {name}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -136,61 +156,111 @@ async def test_function_call_first_turn(client: OpenAI, model_name: str):
|
||||
}
|
||||
]
|
||||
|
||||
response = await client.responses.create(
|
||||
response = await retry_for_tool_call(
|
||||
client,
|
||||
model=model_name,
|
||||
expected_tool_type="function_call",
|
||||
input="What is the horoscope for Aquarius today?",
|
||||
tools=tools,
|
||||
temperature=0.0,
|
||||
)
|
||||
assert response is not None
|
||||
assert response.status == "completed"
|
||||
assert len(response.output) == 2
|
||||
assert response.output[0].type == "reasoning"
|
||||
assert response.output[1].type == "function_call"
|
||||
|
||||
function_call = response.output[1]
|
||||
output_types = [getattr(o, "type", None) for o in response.output]
|
||||
assert "reasoning" in output_types, (
|
||||
f"Expected reasoning in output, got: {output_types}"
|
||||
)
|
||||
assert has_output_type(response, "function_call"), (
|
||||
f"Expected function_call in output, got: {output_types}"
|
||||
)
|
||||
|
||||
function_call = next(o for o in response.output if o.type == "function_call")
|
||||
assert function_call.name == "get_horoscope"
|
||||
assert function_call.call_id is not None
|
||||
|
||||
args = json.loads(function_call.arguments)
|
||||
assert "sign" in args
|
||||
|
||||
# the multi turn function call is tested above in
|
||||
# test_reasoning_and_function_items
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_mcp_tool_call(client: OpenAI, model_name: str):
|
||||
response = await client.responses.create(
|
||||
"""MCP tool calling with code_interpreter.
|
||||
|
||||
The model may make one or more tool calls before producing a final
|
||||
message. We validate server invariants (mcp_call items have correct
|
||||
fields) with hard assertions. Output indices are never hardcoded
|
||||
since the model can produce multiple tool-call rounds.
|
||||
"""
|
||||
# MCP + container init + code execution can be slow
|
||||
client_with_timeout = client.with_options(timeout=client.timeout * 3)
|
||||
|
||||
response = await retry_for_tool_call(
|
||||
client_with_timeout,
|
||||
model=model_name,
|
||||
input="What is 123 * 456? Use python to calculate the result.",
|
||||
expected_tool_type="mcp_call",
|
||||
input=(
|
||||
"What is 123 * 456? Use python to calculate the result. "
|
||||
"Print the result with print()."
|
||||
),
|
||||
tools=[{"type": "code_interpreter", "container": {"type": "auto"}}],
|
||||
extra_body={"enable_response_messages": True},
|
||||
instructions=_PYTHON_TOOL_INSTRUCTION,
|
||||
temperature=0.0,
|
||||
extra_body={"enable_response_messages": True},
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
assert response.status == "completed"
|
||||
|
||||
# The model may produce multiple reasoning/mcp_call rounds before the
|
||||
# final message, so validate structurally rather than by exact index.
|
||||
output_types = [o.type for o in response.output]
|
||||
assert "reasoning" in output_types
|
||||
mcp_calls = [o for o in response.output if o.type == "mcp_call"]
|
||||
assert len(mcp_calls) >= 1
|
||||
assert type(mcp_calls[0].arguments) is str
|
||||
assert type(mcp_calls[0].output) is str
|
||||
output_types = [getattr(o, "type", None) for o in response.output]
|
||||
log_response_diagnostics(response, label="test_mcp_tool_call")
|
||||
|
||||
# The final output should be a message containing the correct answer
|
||||
assert response.output[-1].type == "message"
|
||||
assert any(s in response.output[-1].content[0].text for s in ("56088", "56,088"))
|
||||
assert response.status == "completed", (
|
||||
f"Response status={response.status} "
|
||||
f"(details={getattr(response, 'incomplete_details', None)}). "
|
||||
f"Output types: {output_types}."
|
||||
)
|
||||
|
||||
# Test raw input_messages / output_messages
|
||||
assert len(response.input_messages) == 1
|
||||
assert len(response.output_messages) >= 3
|
||||
assert "reasoning" in output_types, (
|
||||
f"Expected reasoning in output, got: {output_types}"
|
||||
)
|
||||
assert "mcp_call" in output_types, (
|
||||
f"Expected mcp_call in output, got: {output_types}"
|
||||
)
|
||||
|
||||
# Every mcp_call item must have well-typed fields
|
||||
for item in response.output:
|
||||
if getattr(item, "type", None) == "mcp_call":
|
||||
assert type(item.arguments) is str, (
|
||||
f"mcp_call.arguments should be str, got {type(item.arguments)}"
|
||||
)
|
||||
assert type(item.output) is str, (
|
||||
f"mcp_call.output should be str, got {type(item.output)}"
|
||||
)
|
||||
|
||||
# The model may make 1+ tool-call rounds but must still produce
|
||||
# a final message for a trivial calculation like 123 * 456.
|
||||
message_outputs = [
|
||||
o for o in response.output if getattr(o, "type", None) == "message"
|
||||
]
|
||||
assert message_outputs, (
|
||||
f"Model did not produce a final message. Output types: {output_types}"
|
||||
)
|
||||
|
||||
final_message = message_outputs[-1]
|
||||
assert any(s in final_message.content[0].text for s in ("56088", "56,088")), (
|
||||
f"Expected 56088 in final message, got: {final_message.content[0].text!r}"
|
||||
)
|
||||
|
||||
# Validate raw input_messages / output_messages
|
||||
assert len(response.input_messages) >= 1, "Expected at least 1 input message"
|
||||
assert len(response.output_messages) >= 1, "Expected at least 1 output message"
|
||||
assert any(
|
||||
s in response.output_messages[-1]["message"] for s in ("56088", "56,088")
|
||||
any(s in str(msg) for s in ("56088", "56,088"))
|
||||
for msg in response.output_messages
|
||||
), (
|
||||
f"Expected 56088 in at least one output_message, "
|
||||
f"got {len(response.output_messages)} messages"
|
||||
)
|
||||
|
||||
|
||||
@@ -202,6 +272,7 @@ async def test_max_tokens(client: OpenAI, model_name: str):
|
||||
input="What is the first paragraph of Moby Dick?",
|
||||
reasoning={"effort": "low"},
|
||||
max_output_tokens=30,
|
||||
temperature=0.0,
|
||||
)
|
||||
assert response is not None
|
||||
assert response.status == "incomplete"
|
||||
|
||||
@@ -12,13 +12,15 @@ MODEL_NAME = "Qwen/Qwen3-8B"
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = ["--reasoning-parser", "qwen3", "--max_model_len", "5000"]
|
||||
env_dict = dict(
|
||||
VLLM_ENABLE_RESPONSES_API_STORE="1",
|
||||
# uncomment for tool calling
|
||||
# PYTHON_EXECUTION_BACKEND="dangerously_use_uv",
|
||||
)
|
||||
from .conftest import BASE_TEST_ENV
|
||||
|
||||
args = ["--reasoning-parser", "qwen3", "--max_model_len", "5000"]
|
||||
env_dict = {
|
||||
**BASE_TEST_ENV,
|
||||
"VLLM_ENABLE_RESPONSES_API_STORE": "1",
|
||||
# uncomment for tool calling
|
||||
# PYTHON_EXECUTION_BACKEND: "dangerously_use_uv",
|
||||
}
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
Reference in New Issue
Block a user