[Speculative Decoding 2/2 ] Integrate typical acceptance sampler into Spec Decode Worker (#5348)

2024-07-01 00:33:05 -07:00
parent 614aa51203
commit 80ca1e6a3a
14 changed files with 480 additions and 208 deletions
--- a/vllm/engine/metrics.py
+++ b/vllm/engine/metrics.py
@@ -457,4 +457,4 @@ class PrometheusStatLogger(StatLoggerBase):

 class RayPrometheusStatLogger(PrometheusStatLogger):
    """RayPrometheusStatLogger uses Ray metrics instead."""
-    _metrics_cls = RayMetrics
+    _metrics_cls = RayMetrics