[TPU] Increase block size and reset block shapes (#16458)

This commit is contained in:
Jevin Jiang
2025-05-06 10:55:04 -07:00
committed by GitHub
parent 6115b11582
commit 621ca2c0ab
5 changed files with 35 additions and 11 deletions

View File

@@ -22,7 +22,8 @@ def main():
# In real workloads, `enforace_eager` should be `False`.
llm = LLM(model="Qwen/Qwen2-1.5B-Instruct",
max_num_batched_tokens=64,
max_num_seqs=4)
max_num_seqs=4,
max_model_len=128)
outputs = llm.generate(prompts, sampling_params)
print("-" * 50)
for output, answer in zip(outputs, answers):