[Spec Decode] Add hidden states extraction system (#33736)

Signed-off-by: Fynn Schmitt-Ulms <fschmitt@redhat.com>
This commit is contained in:
Fynn Schmitt-Ulms
2026-03-02 14:29:09 -05:00
committed by GitHub
parent d1a6e96d9e
commit 9433acb8df
16 changed files with 2102 additions and 38 deletions

View File

@@ -108,7 +108,7 @@ class _HfExamplesInfo:
use_original_num_layers: bool = False
"""
If True, use the original number of layers from the model config
If True, use the original number of layers from the model config
instead of minimal layers for testing.
"""
@@ -1156,6 +1156,10 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
speculative_model="LGAI-EXAONE/K-EXAONE-236B-A23B",
min_transformers_version="5.1.0",
),
"ExtractHiddenStatesModel": _HfExamplesInfo(
"Qwen/Qwen3-8B",
speculative_method="extract_hidden_states",
),
"Glm4MoeMTPModel": _HfExamplesInfo(
"zai-org/GLM-4.5",
speculative_model="zai-org/GLM-4.5",