[bugfix] Fix static asymmetric quantization case (#10334)

Signed-off-by: Daniël de Kok <me@danieldk.eu> Signed-off-by: luka <luka@neuralmagic.com> Co-authored-by: Daniël de Kok <me@danieldk.eu>
2024-11-14 20:35:11 -05:00
parent 972112d82f
commit bf2ddc6610
5 changed files with 58 additions and 15 deletions
--- a/vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+++ b/vllm/model_executor/layers/quantization/utils/w8a8_utils.py
@@ -211,13 +211,16 @@ def apply_int8_linear(
                                               symmetric=symmetric)

    if x_zp is not None:
+        # Currently, static is always per-tensor and dynamic is per-token
+        static = input_zero_point is not None
+        azp = None if static else x_zp
        return ops.cutlass_scaled_mm_azp(x_q,
                                         weight,
                                         scale_a=x_scale,
                                         scale_b=weight_scale,
                                         out_dtype=input.dtype,
                                         azp_adj=azp_adj,
-                                         azp=x_zp,
+                                         azp=azp,
                                         bias=bias)
    return ops.cutlass_scaled_mm(x_q,
                                 weight,