diff --git a/docker-compose.yml b/docker-compose.yml index 886c56f..c12f298 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,7 @@ services: - -- environment: - HF_TOKEN=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 + - OMP_NUM_THREADS=64 command: - /model - --trust-remote-code @@ -21,9 +22,7 @@ services: - --compilation-config={"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]} - --attention_config.use_fp4_indexer_cache=True - --tokenizer-mode=deepseek_v4 - - --tool-call-parser=deepseek_v4 - - --enable-auto-tool-choice - - --reasoning-parser=deepseek_v4 + - --speculative_config={"method":"mtp","num_speculative_tokens":2} - --host=0.0.0.0 - --port=8000 @@ -43,4 +42,5 @@ services: - /root/nvidia-meeting/DeepSeek-V4-Pro-NVFP4:/model:ro - /root/nvidia-meeting/deepseek-v4-quant/patches/deepseek_v4.py:/patches/deepseek_v4.py:ro - /root/nvidia-meeting/deepseek-v4-quant/patches:/patches:ro + - /srv/vllmcache:/root/.cache/vllm/deep_gemm/cache network_mode: host