54 lines
2.5 KiB
Docker
54 lines
2.5 KiB
Docker
#FROM vllm/vllm-openai:v0.19.0-cu130
|
|
FROM vllm/vllm-openai:cu130-nightly-x86_64
|
|
|
|
# Fix the broken ass nightly build that forgot to include pandas
|
|
RUN pip install --no-cache-dir pandas
|
|
|
|
# Install LMCache for KV cache offloading / sharing across nodes
|
|
# Build with system CUDA 13.0 for Blackwell (B200)
|
|
RUN apt-get update && apt-get install -y git \
|
|
libcusolver-dev-13-0 \
|
|
libcusparse-dev-13-0 \
|
|
libcublas-dev-13-0 \
|
|
libcurand-dev-13-0 \
|
|
libcufft-dev-13-0 \
|
|
libnvjitlink-dev-13-0 && \
|
|
git clone https://github.com/biondizzle/LMCache.git /tmp/lmcache && \
|
|
cd /tmp/lmcache && \
|
|
git checkout feat/redis-ttl && \
|
|
CUDA_HOME=/usr/local/cuda \
|
|
TORCH_CUDA_ARCH_LIST="10.0" \
|
|
pip install --no-cache-dir --no-build-isolation . && \
|
|
rm -rf /tmp/lmcache && export CACHE_BUSTER=1
|
|
|
|
# Copy over nemotron reasonong parser
|
|
COPY ./super_v3_reasoning_parser.py /opt/super_v3_reasoning_parser.py
|
|
|
|
# Copy over deepseek tool call parser with MTP fixes
|
|
COPY deepseekv32_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/deepseekv32_tool_parser.py
|
|
|
|
# Copy over minimax tool call parser with kwargs fixes
|
|
COPY minimax_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/minimax_tool_parser.py
|
|
|
|
# Copy over minimax parsers with kwargs fixes
|
|
COPY minimax_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/minimax_tool_parser.py
|
|
COPY minimax_m2_parser.py /usr/local/lib/python3.12/dist-packages/vllm/parser/minimax_m2_parser.py
|
|
|
|
# Copy vLLM shim that intercepts --model to download custom weights from URLs
|
|
COPY vllm_shim_module.py /opt/vllm-shim/vllm_shim_module.py
|
|
|
|
# Shadow `python -m vllm.*` invocations via PYTHONPATH
|
|
# The shim masquerades as the vllm package so python -m vllm/entrypoints/openai/api_server
|
|
# hits our interceptor first, which downloads weights then execs the real vLLM
|
|
RUN mkdir -p /opt/vllm-shim/vllm/entrypoints/openai \
|
|
/opt/vllm-shim/vllm/entrypoints/cli && \
|
|
cp /opt/vllm-shim/vllm_shim_module.py /opt/vllm-shim/vllm/__main__.py && \
|
|
cp /opt/vllm-shim/vllm_shim_module.py /opt/vllm-shim/vllm/entrypoints/openai/api_server.py && \
|
|
cp /opt/vllm-shim/vllm_shim_module.py /opt/vllm-shim/vllm/entrypoints/cli/main.py && \
|
|
touch /opt/vllm-shim/vllm/__init__.py \
|
|
/opt/vllm-shim/vllm/entrypoints/__init__.py \
|
|
/opt/vllm-shim/vllm/entrypoints/openai/__init__.py \
|
|
/opt/vllm-shim/vllm/entrypoints/cli/__init__.py
|
|
|
|
ENV PYTHONPATH="/opt/vllm-shim:${PYTHONPATH}"
|