implement Structural Tag with Guidance backend (#17333)

Signed-off-by: Michal Moskal <michal@moskal.me>
This commit is contained in:
Michał Moskal
2025-04-28 19:21:32 -07:00
committed by GitHub
parent 506475de5f
commit 86d9fc29cb
2 changed files with 32 additions and 10 deletions

View File

@@ -435,13 +435,10 @@ Given the previous instructions, what is the weather in New York City?
"""
# Change this once other backends support structural_tag
if guided_decoding_backend.startswith("xgrammar"):
outputs = llm.generate(prompts=prompt,
sampling_params=sampling_params,
use_tqdm=True)
assert outputs is not None
else:
outputs = []
outputs = llm.generate(prompts=prompt,
sampling_params=sampling_params,
use_tqdm=True)
assert outputs is not None
for output in outputs:
assert output is not None