34 lines
1.2 KiB
Docker
34 lines
1.2 KiB
Docker
#FROM vllm/vllm-openai:v0.19.0-cu130
|
|
#vllm says version 0.20.2rc1.dev9+g01d4d1ad3
|
|
FROM vllm/vllm-openai:nightly
|
|
#FROM vllm/vllm-openai:glm51-cu130
|
|
|
|
# Install LMCache for KV cache offloading / sharing across nodes
|
|
# Build with system CUDA 13.0 for Blackwell (B200)
|
|
RUN apt-get update && apt-get install -y git \
|
|
libcusolver-dev-13-0 \
|
|
libcusparse-dev-13-0 \
|
|
libcublas-dev-13-0 \
|
|
libcurand-dev-13-0 \
|
|
libcufft-dev-13-0 \
|
|
libnvjitlink-dev-13-0 && \
|
|
git clone https://github.com/biondizzle/LMCache.git /tmp/lmcache && \
|
|
cd /tmp/lmcache && \
|
|
git checkout dream-build && \
|
|
CUDA_HOME=/usr/local/cuda \
|
|
TORCH_CUDA_ARCH_LIST="10.0" \
|
|
pip install --no-cache-dir --no-build-isolation . && \
|
|
rm -rf /tmp/lmcache && export CACHE_BUSTER=4
|
|
|
|
|
|
# Make sure we have patch to make MTP work on GLM
|
|
COPY indexer.py /usr/local/lib/python3.12/dist-packages/vllm/v1/attention/backends/mla/indexer.py
|
|
|
|
# Make sure we have the latest up to date chat template
|
|
COPY glm_5.1_chat_template.jinja /opt/chat_template.jinja
|
|
|
|
# GLM 5.1 LMCache config
|
|
COPY lmcache-config-glm-51.yaml /opt/lmcache-config-glm-51.yaml
|
|
|
|
# DEEPSEEK v4 LMCache config
|
|
COPY lmcache-config-dsv4.yaml /opt/lmcache-config-dsv4.yaml |