49 lines
2.1 KiB
Docker
49 lines
2.1 KiB
Docker
# Kimi K2.6 DFlash source-patched image for 8x MI300X (gfx942)
|
|
#
|
|
# Base: vllm/vllm-openai-rocm:nightly
|
|
# When a date-pinned tag becomes available (e.g. :2026-04-21), switch to it
|
|
# and record the vLLM version (v0.19.2rc1.dev21 at time of writing).
|
|
#
|
|
# This image bakes the DFlash ROCm patches at build time so the launcher
|
|
# no longer needs to run patch_dflash_rocm.py at container startup.
|
|
# The patches are idempotent — running the script again inside this image
|
|
# is a safe no-op.
|
|
|
|
FROM vllm/vllm-openai-rocm:nightly
|
|
|
|
# --- ROCm / AITER / vLLM environment defaults for gfx942 ---
|
|
ENV PYTORCH_ROCM_ARCH=gfx942 \
|
|
AITER_ROCM_ARCH=gfx942 \
|
|
GPU_ARCHS=gfx942 \
|
|
VLLM_ROCM_USE_AITER=1 \
|
|
VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 \
|
|
VLLM_ROCM_USE_AITER_RMSNORM=0 \
|
|
HSA_ENABLE_SDMA=0 \
|
|
HSA_NO_SCRATCH_RECLAIM=1 \
|
|
OMP_NUM_THREADS=1
|
|
|
|
# --- Copy and apply DFlash patches ---
|
|
COPY patches/patch_dflash_rocm.py /tmp/patch_dflash_rocm.py
|
|
RUN python3 /tmp/patch_dflash_rocm.py && rm /tmp/patch_dflash_rocm.py
|
|
|
|
# --- Pre-download DFlash draft models ---
|
|
# These are needed for speculative decoding and must be local paths.
|
|
# Baking them into the image avoids runtime downloads/mounts.
|
|
# Pass HF_TOKEN build arg if the models are gated.
|
|
ARG HF_TOKEN=
|
|
RUN bash -c 'if [ ! -d "/opt/draft-models/Kimi-K2.5-DFlash" ]; then \
|
|
pip install --no-cache-dir huggingface_hub && \
|
|
python3 -c "from huggingface_hub import snapshot_download; snapshot_download(\"z-lab/Kimi-K2.5-DFlash\", local_dir=\"/opt/draft-models/Kimi-K2.5-DFlash\")" && \
|
|
rm -rf /root/.cache/huggingface; \
|
|
fi'
|
|
|
|
# Patch tool and reasoning parsers for Eagle
|
|
#COPY kimi_k2_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/kimi_k2_tool_parser.py
|
|
|
|
#COPY kimi_k2_reasoning_parser.py /usr/local/lib/python3.12/dist-packages/vllm/reasoning/kimi_k2_reasoning_parser.py
|
|
|
|
# Patch serving layer: flush reasoning→content on finish_reason=length
|
|
#COPY serving.py /usr/local/lib/python3.12/dist-packages/vllm/entrypoints/openai/chat_completion/serving.py
|
|
|
|
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
|