[Spec Decode] Add hidden states extraction system (#33736)
Signed-off-by: Fynn Schmitt-Ulms <fschmitt@redhat.com>
This commit is contained in:
committed by
GitHub
parent
d1a6e96d9e
commit
9433acb8df
@@ -108,7 +108,7 @@ class _HfExamplesInfo:
|
||||
|
||||
use_original_num_layers: bool = False
|
||||
"""
|
||||
If True, use the original number of layers from the model config
|
||||
If True, use the original number of layers from the model config
|
||||
instead of minimal layers for testing.
|
||||
"""
|
||||
|
||||
@@ -1156,6 +1156,10 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
speculative_model="LGAI-EXAONE/K-EXAONE-236B-A23B",
|
||||
min_transformers_version="5.1.0",
|
||||
),
|
||||
"ExtractHiddenStatesModel": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-8B",
|
||||
speculative_method="extract_hidden_states",
|
||||
),
|
||||
"Glm4MoeMTPModel": _HfExamplesInfo(
|
||||
"zai-org/GLM-4.5",
|
||||
speculative_model="zai-org/GLM-4.5",
|
||||
|
||||
Reference in New Issue
Block a user