2025-06-06 20:10:24 -07:00
|
|
|
# Environment config
|
|
|
|
|
TEST_NAME=llama8b
|
2025-08-05 16:54:56 -07:00
|
|
|
CONTAINER_NAME=tpu-test
|
2025-06-06 20:10:24 -07:00
|
|
|
|
|
|
|
|
# vllm config
|
|
|
|
|
MODEL=meta-llama/Llama-3.1-8B-Instruct
|
2025-06-23 17:18:10 -07:00
|
|
|
MAX_NUM_SEQS=256
|
|
|
|
|
MAX_NUM_BATCHED_TOKENS=1024
|
2025-06-06 20:10:24 -07:00
|
|
|
TENSOR_PARALLEL_SIZE=1
|
|
|
|
|
MAX_MODEL_LEN=2048
|
|
|
|
|
DOWNLOAD_DIR=/mnt/disks/persist
|
|
|
|
|
EXPECTED_THROUGHPUT=8.0
|
|
|
|
|
INPUT_LEN=1800
|
|
|
|
|
OUTPUT_LEN=128
|
