[CI] Fix neuron CI and run offline tests (#11779)

Signed-off-by: Liangfu Chen <liangfc@amazon.com>
This commit is contained in:
Liangfu Chen
2025-01-06 21:36:10 -08:00
committed by GitHub
parent 0f3f3c86ec
commit 898cdf033e
3 changed files with 35 additions and 37 deletions

View File

@@ -1,12 +1,5 @@
import os
from vllm import LLM, SamplingParams
# creates XLA hlo graphs for all the context length buckets.
os.environ['NEURON_CONTEXT_LENGTH_BUCKETS'] = "128,512,1024,2048"
# creates XLA hlo graphs for all the token gen buckets.
os.environ['NEURON_TOKEN_GEN_BUCKETS'] = "128,512,1024,2048"
# Sample prompts.
prompts = [
"Hello, my name is",
@@ -26,8 +19,8 @@ llm = LLM(
# Currently, this is a known limitation in continuous batching support
# in transformers-neuronx.
# TODO(liangfu): Support paged-attention in transformers-neuronx.
max_model_len=2048,
block_size=2048,
max_model_len=1024,
block_size=1024,
# The device can be automatically detected when AWS Neuron SDK is installed.
# The device argument can be either unspecified for automated detection,
# or explicitly assigned.