Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -14,7 +14,6 @@ def include_stop_str_in_output(request):
|
||||
|
||||
|
||||
class _DummyDetokenizer(BaseIncrementalDetokenizer):
|
||||
|
||||
def __init__(self, request: EngineCoreRequest):
|
||||
super().__init__(request)
|
||||
|
||||
@@ -27,7 +26,8 @@ def _make_request(stop, include_stop_str_in_output: bool, min_tokens: int = 0):
|
||||
params = SamplingParams(
|
||||
stop=stop,
|
||||
include_stop_str_in_output=include_stop_str_in_output,
|
||||
min_tokens=min_tokens)
|
||||
min_tokens=min_tokens,
|
||||
)
|
||||
# Keep other fields minimal for unit test purposes.
|
||||
req = EngineCoreRequest(
|
||||
request_id="test",
|
||||
@@ -44,26 +44,25 @@ def _make_request(stop, include_stop_str_in_output: bool, min_tokens: int = 0):
|
||||
return req
|
||||
|
||||
|
||||
def test_stop_string_while_stop_token_terminates(
|
||||
include_stop_str_in_output: bool):
|
||||
def test_stop_string_while_stop_token_terminates(include_stop_str_in_output: bool):
|
||||
"""
|
||||
This test verifies that the detokenizer correctly handles the case where
|
||||
the generated token sequence contains both:
|
||||
- a stop token
|
||||
- an <eos> token
|
||||
|
||||
|
||||
The detokenizer should respect the stop string and truncate the output
|
||||
accordingly.
|
||||
|
||||
|
||||
Imagine the following sequence:
|
||||
- "abcdeZ" is generated, where "Z" is the <eos> token.
|
||||
- "cd" is the stop string.
|
||||
|
||||
|
||||
If include_stop_str_in_output=False, the detokenizer should truncate the
|
||||
output to "ab" because the stop string "cd" is excluded.
|
||||
If include_stop_str_in_output=True, the detokenizer should include the stop
|
||||
string "cd" in the output, resulting in "abcd".
|
||||
|
||||
|
||||
|
||||
This verifies the behavioral change introduced in BaseIncrementalDetokenizer
|
||||
where stop-string evaluation occurs before the early-return on
|
||||
@@ -78,8 +77,9 @@ def test_stop_string_while_stop_token_terminates(
|
||||
token_ids = [ord(c) for c in generated_text]
|
||||
|
||||
# Create a request with the stop string and initialize the detokenizer.
|
||||
req = _make_request(stop=[stop_string],
|
||||
include_stop_str_in_output=include_stop_str_in_output)
|
||||
req = _make_request(
|
||||
stop=[stop_string], include_stop_str_in_output=include_stop_str_in_output
|
||||
)
|
||||
detok = _DummyDetokenizer(req)
|
||||
|
||||
# Simulate that the last token ('Z') is a stop token (stop_terminated=True).
|
||||
@@ -99,5 +99,4 @@ def test_stop_string_while_stop_token_terminates(
|
||||
|
||||
# get_next_output_text should return the full text when finished=True.
|
||||
# (Buffering only applies during streaming when finished=False.)
|
||||
assert detok.get_next_output_text(finished=True,
|
||||
delta=False) == expected_text
|
||||
assert detok.get_next_output_text(finished=True, delta=False) == expected_text
|
||||
|
||||
Reference in New Issue
Block a user