[Speculative decoding 6/9] Integrate speculative decoding with LLMEngine (#3894)

2024-04-16 13:09:21 -07:00
parent 69e1d2fb69
commit e95cd87959
31 changed files with 1347 additions and 407 deletions
--- a/tests/spec_decode/utils.py
+++ b/tests/spec_decode/utils.py
@@ -212,7 +212,7 @@ def create_sampler_output_list(
                    SequenceOutput(
                        output_token=token_id,
                        parent_seq_id=seq_ids[seq_index],
-                        logprobs={token_id: 0},
+                        logprobs={token_id: Logprob(0)},
                    )
                ],
                prompt_logprobs=None,