Fix OOM: add --max-model-len=876544 + revert CPU dummy weight

The CPU dummy weight broke torch.mm(compressor.weight.T) which expects
GPU tensors. Instead, reduce max_model_len to fit KV cache within
available memory (876544 instead of 1048576).

docker-compose.yml

@@ -23,6 +23,7 @@ services:
       - --reasoning-parser=deepseek_v4
       - --moe-backend=cutedsl
       - --gpu-memory-utilization=0.9
+      - --max-model-len=876544
       - --host=0.0.0.0
       - --port=8000
     deploy:

vllm/cutedsl_quant_method.py

@@ -112,12 +112,11 @@ class CuTeDSLNvfp4Method(LinearMethodBase):
             torch.cuda.empty_cache()
 
         # Replace weight with dummy BF16 (needed by vLLM module introspection)
-        # Replace weight with a minimal CPU dummy (vLLM introspection expects
-        # layer.weight to exist, but our kernel never uses it). Keeping it on
-        # GPU would waste ~5-8 GiB across all layers.
+        # Replace weight with a GPU dummy (some vLLM code paths like
+        # torch.mm(compressor.weight.T) expect weight on GPU).
         layer.weight = torch.nn.Parameter(
             torch.zeros(out_features, in_features, dtype=torch.bfloat16,
-                        device="cpu"),
+                        device=device),
             requires_grad=False,
         )
 

vllm/kernels/linear/nvfp4/cutedsl.py

@@ -99,13 +99,11 @@ class CuTeDSLNvFp4LinearKernel(NvFp4LinearKernel):
         layer._cutedsl_runner_id = register_runner(runner)
         layer._cutedsl_out_features = out_features
 
-        # Replace weight with a minimal CPU dummy (vLLM introspection expects
-        # layer.weight to exist, but our kernel never uses it). Keeping it on
-        # GPU would waste ~5-8 GiB across all layers (e.g. q_b_proj alone
-        # is 192 MiB of zeros).
+        # Replace weight with a GPU dummy (some vLLM code paths like
+        # torch.mm(compressor.weight.T) expect weight on GPU).
         layer.weight = torch.nn.Parameter(
             torch.zeros(out_features, in_features, dtype=torch.bfloat16,
-                        device="cpu"),
+                        device=device),
             requires_grad=False,
         )