feat: auto-warmup in build_and_run.sh

After the container starts, the script waits for the API to come up,
then sends a warmup request to trigger all JIT compilation (Triton,
TileLang, CuTeDSL). This way the first real inference request is fast.

Also added tqdm for expert weight loading:
  Loading Native NVFP4 Expert Weights: 50%|██████████░░| 480/960
This commit is contained in:
2026-05-16 06:11:38 +00:00
parent 5d975d00d9
commit f792537719
2 changed files with 47 additions and 0 deletions

View File

@@ -20,3 +20,23 @@ docker compose up -d
sed -i -E "s/ARG CACHE_BUSTER=.*/ARG CACHE_BUSTER=\${TIMESTAMP}/" Dockerfile
echo "=== Done. Container: $(docker compose ps -q) ==="
echo ""
echo "=== Waiting for API to come up, then sending warmup request... ==="
echo "=== (This triggers Triton/TileLang/CuTeDSL JIT compilation) ==="
for i in $(seq 1 120); do
if curl -s http://localhost:8000/health > /dev/null 2>&1; then
echo "=== API is up. Sending warmup request (expect 2-3 min)... ==="
START=$(date +%s)
curl -s -X POST http://localhost:8000/v1/completions \
-H "Content-Type: application/json" \
-d '{"model": "/model", "prompt": "The capital of France is", "max_tokens": 1}' \
2>&1 | head -5
END=$(date +%s)
ELAPSED=$((END - START))
echo ""
echo "=== Warmup complete (${ELAPSED}s). Model is ready for inference. ==="
break
fi
sleep 5
done

27
warmup.sh Executable file
View File

@@ -0,0 +1,27 @@
#!/bin/bash
# Wait for vLLM API to be ready, then send a warmup request
# to compile all JIT kernels (Triton, TileLang, CuTeDSL)
# before the first real inference request.
echo "=== Warmup: waiting for vLLM API... ==="
# Wait for the API to respond
for i in $(seq 1 120); do
if curl -s http://localhost:8000/health > /dev/null 2>&1; then
echo "=== Warmup: API is up, sending warmup request... ==="
break
fi
sleep 5
done
# Send a warmup request to trigger all JIT compilation
# This will be slow (2-3 min) but subsequent requests will be fast
START=$(date +%s)
curl -s -X POST http://localhost:8000/v1/completions \
-H "Content-Type: application/json" \
-d '{"model": "/model", "prompt": "The capital of France is", "max_tokens": 1}' \
> /dev/null 2>&1
END=$(date +%s)
ELAPSED=$((END - START))
echo "=== Warmup complete (${ELAPSED}s). Model is ready. ==="