[BugFix] Fix MLPSpeculator handling of num_speculative_tokens (#5876)

This commit is contained in:
Nick Hill
2024-06-27 10:59:33 -07:00
committed by GitHub
parent 3fd02bda51
commit 691e29ecf3
3 changed files with 18 additions and 10 deletions

View File

@@ -35,6 +35,7 @@ class MLPSpeculatorConfig(PretrainedConfig):
candidate tree.
For each candidate branch in the tree, head n produces topk[n]
additional sub-branches.
NOTE: This parameter is currently unused.
n_candidates: int
number of child candidates to create per sequence
"""
@@ -47,4 +48,6 @@ class MLPSpeculatorConfig(PretrainedConfig):
self.n_predict = n_predict
self.top_k_tokens_per_head = top_k_tokens_per_head
self.n_candidates = n_candidates
self.num_lookahead_tokens = n_predict
super().__init__(**kwargs)