[Attention] Update tests to remove deprecated env vars (#30563)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
@@ -38,53 +38,48 @@ def test_ngram_max_len(num_speculative_tokens: int):
|
||||
def test_eagle_max_len(
|
||||
monkeypatch: pytest.MonkeyPatch, num_speculative_tokens: int, attn_backend: str
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ATTENTION_BACKEND", attn_backend)
|
||||
|
||||
if attn_backend == "TRITON_ATTN" and not current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"TRITON_ATTN does not support "
|
||||
"multi-token eagle spec decode on current platform"
|
||||
)
|
||||
|
||||
if attn_backend == "ROCM_AITER_FA" and current_platform.is_rocm():
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
|
||||
llm = LLM(
|
||||
model="meta-llama/Meta-Llama-3-8B-Instruct",
|
||||
enforce_eager=True, # For faster initialization.
|
||||
speculative_config={
|
||||
"method": "eagle",
|
||||
"model": "yuhuili/EAGLE-LLaMA3-Instruct-8B",
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
"max_model_len": 80,
|
||||
},
|
||||
max_model_len=200,
|
||||
if attn_backend == "TRITON_ATTN" and not current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"TRITON_ATTN does not support "
|
||||
"multi-token eagle spec decode on current platform"
|
||||
)
|
||||
sampling_params = SamplingParams(max_tokens=200, ignore_eos=True)
|
||||
outputs = llm.generate(_PROMPTS, sampling_params)
|
||||
for o in outputs:
|
||||
assert o.outputs[0].finish_reason == "length", (
|
||||
"This test is only meaningful if the output "
|
||||
"is truncated due to max length"
|
||||
)
|
||||
|
||||
sampling_params = SamplingParams(
|
||||
max_tokens=200,
|
||||
structured_outputs=StructuredOutputsParams(
|
||||
regex="^" + "a b c d e " * 15 + "$"
|
||||
),
|
||||
if attn_backend == "ROCM_AITER_FA" and current_platform.is_rocm():
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
|
||||
llm = LLM(
|
||||
model="meta-llama/Meta-Llama-3-8B-Instruct",
|
||||
enforce_eager=True, # For faster initialization.
|
||||
speculative_config={
|
||||
"method": "eagle",
|
||||
"model": "yuhuili/EAGLE-LLaMA3-Instruct-8B",
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
"max_model_len": 80,
|
||||
},
|
||||
max_model_len=200,
|
||||
attention_config={"backend": attn_backend},
|
||||
)
|
||||
sampling_params = SamplingParams(max_tokens=200, ignore_eos=True)
|
||||
outputs = llm.generate(_PROMPTS, sampling_params)
|
||||
for o in outputs:
|
||||
assert o.outputs[0].finish_reason == "length", (
|
||||
"This test is only meaningful if the output is truncated due to max length"
|
||||
)
|
||||
output = llm.generate(_PROMPTS, sampling_params)
|
||||
for o in output:
|
||||
assert o.prompt_token_ids is not None
|
||||
assert (
|
||||
len(o.prompt_token_ids)
|
||||
< 80
|
||||
< len(o.prompt_token_ids) + len(o.outputs[0].token_ids)
|
||||
<= 200
|
||||
), (
|
||||
"This test is only meaningful if the output "
|
||||
"is longer than the eagle max length"
|
||||
)
|
||||
assert o.outputs[0].text == "a b c d e " * 15
|
||||
|
||||
sampling_params = SamplingParams(
|
||||
max_tokens=200,
|
||||
structured_outputs=StructuredOutputsParams(regex="^" + "a b c d e " * 15 + "$"),
|
||||
)
|
||||
output = llm.generate(_PROMPTS, sampling_params)
|
||||
for o in output:
|
||||
assert o.prompt_token_ids is not None
|
||||
assert (
|
||||
len(o.prompt_token_ids)
|
||||
< 80
|
||||
< len(o.prompt_token_ids) + len(o.outputs[0].token_ids)
|
||||
<= 200
|
||||
), (
|
||||
"This test is only meaningful if the output "
|
||||
"is longer than the eagle max length"
|
||||
)
|
||||
assert o.outputs[0].text == "a b c d e " * 15
|
||||
|
||||
Reference in New Issue
Block a user