biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Add GLM-ASR multimodal support (#31436)

Browse Source 
Signed-off-by: baonudesifeizhai <baonudesifeizhai@gmail.com>
Signed-off-by: baonudesifeizhai <85092850+baonudesifeizhai@users.noreply.github.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
baonudesifeizhai
2025-12-31 10:12:24 -05:00
committed by GitHub
parent cf16342d43
commit d722e9e614
8 changed files with 764 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
docs/models/supported_models.md
View File

@@ -774,10 +774,11 @@ Speech2Text models trained specifically for Automatic Speech Recognition.
| Architecture | Models | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
|--------------|--------|-------------------|----------------------|---------------------------|
| `WhisperForConditionalGeneration` | Whisper | `openai/whisper-small`, `openai/whisper-large-v3-turbo`, etc. | | |
| `VoxtralForConditionalGeneration` | Voxtral (Mistral format) | `mistralai/Voxtral-Mini-3B-2507`, `mistralai/Voxtral-Small-24B-2507`, etc. | ✅︎ | ✅︎ |
| `Gemma3nForConditionalGeneration` | Gemma3n | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | |
| `GlmAsrForConditionalGeneration` | GLM-ASR | `zai-org/GLM-ASR-Nano-2512` | ✅︎ | ✅︎ |
| `GraniteSpeechForConditionalGeneration` | Granite Speech | `ibm-granite/granite-speech-3.3-2b`, `ibm-granite/granite-speech-3.3-8b`, etc. | ✅︎ | ✅︎ |
| `VoxtralForConditionalGeneration` | Voxtral (Mistral format) | `mistralai/Voxtral-Mini-3B-2507`, `mistralai/Voxtral-Small-24B-2507`, etc. | ✅︎ | ✅︎ |
| `WhisperForConditionalGeneration` | Whisper | `openai/whisper-small`, `openai/whisper-large-v3-turbo`, etc. | | |
!!! note
`VoxtralForConditionalGeneration` requires `mistral-common[audio]` to be installed.

29
examples/offline_inference/audio_language.py
View File

@@ -358,6 +358,34 @@ def run_voxtral(question: str, audio_count: int) -> ModelRequestData:
)
# GLM-ASR
def run_glmasr(question: str, audio_count: int) -> ModelRequestData:
model_name = "zai-org/GLM-ASR-Nano-2512"
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
# GLM-ASR uses <|pad|> token for audio
audio_placeholder = "<|pad|>" * audio_count
messages = [{"role": "user", "content": f"{audio_placeholder}{question}"}]
prompt = tokenizer.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
engine_args = EngineArgs(
model=model_name,
trust_remote_code=True,
max_model_len=4096,
max_num_seqs=2,
limit_mm_per_prompt={"audio": audio_count},
)
return ModelRequestData(
engine_args=engine_args,
prompt=prompt,
)
# Whisper
def run_whisper(question: str, audio_count: int) -> ModelRequestData:
assert audio_count == 1, "Whisper only support single audio input per prompt"
@@ -381,6 +409,7 @@ def run_whisper(question: str, audio_count: int) -> ModelRequestData:
model_example_map = {
"audioflamingo3": run_audioflamingo3,
"gemma3n": run_gemma3n,
"glmasr": run_glmasr,
"granite_speech": run_granite_speech,
"midashenglm": run_midashenglm,
"minicpmo": run_minicpmo,

14
tests/models/multimodal/processing/test_common.py
View File

@@ -84,6 +84,19 @@ def qwen3_vl_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
return mm_data
def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
"""
Patch the multimodal data for GLM-ASR model.
GLM-ASR requires text and audio to match 1:1, so we limit audio to 1.
"""
if "audio" in mm_data:
audio = mm_data["audio"]
if isinstance(audio, list) and len(audio) > 1:
# Limit to single audio to match text requirement
mm_data["audio"] = [audio[0]]
return mm_data
# For some multimodal models, tokenizer will always add bos_token
# at the beginning of prompt by default, causing hf_processor outputs
# incorrect token ids. So we need use `add_special_tokens=False` here
@@ -108,6 +121,7 @@ MM_DATA_PATCHES = {
"ernie4_5_moe_vl": qwen3_vl_patch_mm_data,
"glm4v": glm4_1v_patch_mm_data,
"glm4v_moe": glm4_1v_patch_mm_data,
"glmasr": glmasr_patch_mm_data,
"qwen3_vl": qwen3_vl_patch_mm_data,
"qwen3_vl_moe": qwen3_vl_patch_mm_data,
}

5
tests/models/registry.py
View File

@@ -655,6 +655,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
"Gemma3ForConditionalGeneration": _HfExamplesInfo("google/gemma-3-4b-it"),
"Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"),
"GlmAsrForConditionalGeneration": _HfExamplesInfo(
"zai-org/GLM-ASR-Nano-2512",
trust_remote_code=True,
min_transformers_version="5.0",
),
"GraniteSpeechForConditionalGeneration": _HfExamplesInfo(
"ibm-granite/granite-speech-3.3-2b"
),

545
vllm/model_executor/models/glmasr.py Normal file
View File

@@ -0,0 +1,545 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal, TypeAlias, cast
import numpy as np
import torch
import torch.nn as nn
from transformers import BatchFeature
from transformers.models.glmasr import GlmAsrConfig, GlmAsrEncoder, GlmAsrProcessor
from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs.data import PromptType
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalDataDict,
MultiModalFieldConfig,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import (
DictEmbeddingItems,
ModalityData,
ModalityDataItems,
MultiModalDataItems,
MultiModalDataParser,
)
from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.profiling import BaseDummyInputsBuilder
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .audioflamingo3 import (
AudioFlamingo3MultiModalDataParser,
AudioFlamingo3MultiModalProcessor,
AudioFlamingo3ProcessingInfo,
)
from .audioflamingo3 import (
_audioflamingo3_field_config as _glmasr_field_config,
)
from .glmasr_utils import (
DEFAULT_CONV_PARAMS,
DEFAULT_MAX_AUDIO_LEN_S,
DEFAULT_MERGE_FACTOR,
_flatten_audio_features_by_length,
_get_audio_output_lengths_for_tower,
_get_num_features_for_item,
_group_audio_embeddings,
_normalize_chunk_counts,
)
from .interfaces import (
MultiModalEmbeddings,
SupportsLoRA,
SupportsMultiModal,
SupportsPP,
SupportsTranscription,
)
from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
from .whisper import ISO639_1_SUPPORTED_LANGS
class GlmAsrFeatureInputs(TensorSchema):
"""
Dimensions:
- num_chunks: Number of audio chunks (flattened)
- nmb: Number of mel bins
- num_audios: Number of original audio files
"""
type: Literal["audio_features"]
input_features: Annotated[
torch.Tensor | list[torch.Tensor],
TensorShape("num_chunks", "nmb", "chunk_length", dynamic_dims={"chunk_length"}),
]
feature_attention_mask: Annotated[
torch.Tensor | list[torch.Tensor],
TensorShape("num_chunks", "chunk_length", dynamic_dims={"chunk_length"}),
]
chunk_counts: Annotated[
torch.Tensor | list[torch.Tensor],
TensorShape("num_audios"),
]
class GlmAsrEmbeddingInputs(TensorSchema):
"""
Dimensions:
- bn: Batch size
- naf: Number of audio features
- hs: Hidden size (must match the hidden size of language model
backbone)
"""
type: Literal["audio_embeds"] = "audio_embeds"
audio_embeds: Annotated[
list[torch.Tensor],
TensorShape("bn", "naf", "hs", dynamic_dims={"naf"}),
]
GlmAsrInputs: TypeAlias = GlmAsrFeatureInputs | GlmAsrEmbeddingInputs
class GlmAsrMultiModalProjector(nn.Module):
def __init__(
self,
config: GlmAsrConfig,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.linear_1 = ColumnParallelLinear(
input_size=config.audio_config.intermediate_size,
output_size=config.text_config.hidden_size * 2,
quant_config=quant_config,
prefix=f"{prefix}.linear_1",
)
self.act = get_act_fn(config.projector_hidden_act)
self.linear_2 = RowParallelLinear(
input_size=config.text_config.hidden_size * 2,
output_size=config.text_config.hidden_size,
quant_config=quant_config,
prefix=f"{prefix}.linear_2",
)
def forward(self, audio_features: torch.Tensor) -> torch.Tensor:
hidden_states, _ = self.linear_1(audio_features)
hidden_states = self.act(hidden_states)
hidden_states, _ = self.linear_2(hidden_states)
return hidden_states
class GlmAsrProcessingInfo(AudioFlamingo3ProcessingInfo):
def get_hf_config(self) -> GlmAsrConfig:
return self.ctx.get_hf_config(GlmAsrConfig)
def get_hf_processor(self, **kwargs: object) -> GlmAsrProcessor:
return self.ctx.get_hf_processor(GlmAsrProcessor, **kwargs)
def get_feature_extractor(self, **kwargs: object) -> WhisperFeatureExtractor:
# Reuse parent implementation, but add type annotation and assertion
feature_extractor = super().get_feature_extractor(**kwargs)
assert isinstance(feature_extractor, WhisperFeatureExtractor)
return feature_extractor
class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_audios = mm_counts.get("audio", 0)
hf_processor = self.info.get_hf_processor()
return hf_processor.audio_token * num_audios
def get_dummy_mm_data(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio") if mm_options else None
max_audio_len = getattr(
self.info.get_hf_processor(), "max_audio_len", DEFAULT_MAX_AUDIO_LEN_S
)
audio_len = int(max_audio_len * sampling_rate)
return {
"audio": self._get_dummy_audios(
length=audio_len, num_audios=num_audios, overrides=audio_overrides
)
}
class GlmAsrMultiModalDataParser(AudioFlamingo3MultiModalDataParser):
def _parse_audio_data(
self,
data: dict[str, torch.Tensor] | ModalityData[Any],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
return DictEmbeddingItems(
data,
modality="audio",
required_fields={"audio_embeds"},
fields_factory=_glmasr_field_config,
)
return super()._parse_audio_data(data)
class GlmAsrMultiModalProcessor(AudioFlamingo3MultiModalProcessor):
def _get_data_parser(self) -> MultiModalDataParser:
feature_extractor = self.info.get_feature_extractor()
return GlmAsrMultiModalDataParser(target_sr=feature_extractor.sampling_rate)
def _calculate_chunk_counts(
self,
audio_list: list[Any],
feature_extractor: WhisperFeatureExtractor,
processor: GlmAsrProcessor,
) -> list[int]:
"""Calculate chunk counts for each audio."""
sampling_rate = feature_extractor.sampling_rate
chunk_length = feature_extractor.chunk_length
max_audio_len = getattr(processor, "max_audio_len", DEFAULT_MAX_AUDIO_LEN_S)
window_size = int(sampling_rate * chunk_length)
max_windows = int(max_audio_len // chunk_length)
chunk_counts = []
for audio in audio_list:
n_samples = len(audio) if isinstance(audio, list) else audio.shape[0]
n_chunks = max(1, (n_samples + window_size - 1) // window_size)
chunk_counts.append(min(n_chunks, max_windows))
return chunk_counts
def _call_hf_processor(
self,
prompt: str,
mm_data: dict[str, object],
mm_kwargs: Mapping[str, Any],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
# Normalize input: handle deprecated key and list conversion.
if "audios" in mm_data:
mm_data["audio"] = mm_data.pop("audios")
audio = mm_data.get("audio", [])
audio_list = [audio] if audio and not isinstance(audio, list) else audio
# Early return for text-only.
if not audio_list:
prompt_ids = self.info.get_tokenizer().encode(prompt)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
# Get processor for chunk counts calculation
processor = self.info.get_hf_processor(**mm_kwargs)
# Call parent method (it will handle sampling_rate)
outputs = super()._call_hf_processor(
prompt=prompt,
mm_data=mm_data,
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
# Postprocess: rename mask and add chunk counts.
if "input_features_mask" in outputs:
outputs["feature_attention_mask"] = outputs.pop("input_features_mask")
# Override chunk counts calculation with GLM-ASR specific logic
chunk_counts = self._calculate_chunk_counts(
audio_list, processor.feature_extractor, processor
)
outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
return outputs
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
return _glmasr_field_config(hf_inputs)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab()
config = self.info.get_hf_config()
audio_token = getattr(processor, "audio_token", "<|pad|>")
audio_token_id = vocab.get(audio_token)
if audio_token_id is None:
audio_token_id = processor.audio_token_id
merge_factor = getattr(config, "merge_factor", DEFAULT_MERGE_FACTOR)
out_mm_data = out_mm_kwargs.get_data()
feature_attention_mask = out_mm_data.get("feature_attention_mask")
chunk_counts = out_mm_data.get("chunk_counts")
def get_replacement_glmasr(item_idx: int):
conv_params = getattr(config, "conv_params", DEFAULT_CONV_PARAMS)
audio_embeds = out_mm_data.get("audio_embeds")
num_features = _get_num_features_for_item(
feature_attention_mask,
chunk_counts,
item_idx,
audio_embeds,
merge_factor,
conv_params,
)
if num_features == 0:
raise ValueError("Audio is too short")
audio_tokens = [audio_token_id] * int(num_features)
return PromptUpdateDetails.select_token_id(
audio_tokens,
embed_token_id=audio_token_id,
)
return [
PromptReplacement(
modality="audio",
target=audio_token,
replacement=get_replacement_glmasr,
)
]
@MULTIMODAL_REGISTRY.register_processor(
GlmAsrMultiModalProcessor,
info=GlmAsrProcessingInfo,
dummy_inputs=GlmAsrDummyInputsBuilder,
)
class GlmAsrForConditionalGeneration(
nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, SupportsTranscription
):
supported_languages = ISO639_1_SUPPORTED_LANGS
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config
self.config = config
self.multimodal_config = multimodal_config
self.audio_tower = GlmAsrEncoder(config.audio_config)
self.multi_modal_projector = GlmAsrMultiModalProjector(
config,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "multi_modal_projector"),
)
self.quant_config = quant_config
self.language_model = init_vllm_registered_model(
vllm_config=vllm_config,
hf_config=config.text_config,
prefix=maybe_prefix(prefix, "language_model"),
architectures=["LlamaForCausalLM"],
)
self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("audio"):
return "<|begin_of_audio|><|pad|><|end_of_audio|>"
raise ValueError("Only audio modality is supported")
def get_mm_mapping(self) -> MultiModelKeys:
return MultiModelKeys.from_string_field(
language_model="language_model.",
connector="multi_modal_projector.",
tower_model="audio_tower.",
)
def _parse_and_validate_audio_input(self, **kwargs: object) -> GlmAsrInputs | None:
audio_embeds = kwargs.pop("audio_embeds", None)
if audio_embeds is not None:
return GlmAsrEmbeddingInputs(type="audio_embeds", audio_embeds=audio_embeds)
input_features = kwargs.pop("input_features", None)
if input_features is None:
return None
return GlmAsrFeatureInputs(
type="audio_features",
input_features=input_features,
feature_attention_mask=kwargs.pop("feature_attention_mask", None),
chunk_counts=kwargs.pop("chunk_counts", None),
)
def _process_audio_input(
self, audio_input: GlmAsrInputs
) -> torch.Tensor | tuple[torch.Tensor, ...]:
if audio_input["type"] == "audio_embeds":
return tuple(audio_input["audio_embeds"])
input_features = audio_input["input_features"]
feature_attention_mask = audio_input["feature_attention_mask"]
if isinstance(input_features, list):
input_features = torch.cat(input_features, dim=0)
feature_attention_mask = torch.cat(feature_attention_mask, dim=0)
num_chunks = input_features.shape[0]
chunk_counts = _normalize_chunk_counts(
audio_input.get("chunk_counts"), num_chunks=num_chunks
)
audio_hidden_states = self.audio_tower(input_features).last_hidden_state
audio_hidden_states = audio_hidden_states.reshape(
num_chunks,
-1,
self.config.audio_config.intermediate_size,
)
audio_features = self.multi_modal_projector(audio_hidden_states)
merge_factor = getattr(self.config, "merge_factor", DEFAULT_MERGE_FACTOR)
conv_params = getattr(self.config, "conv_params", DEFAULT_CONV_PARAMS)
audio_output_lengths = _get_audio_output_lengths_for_tower(
self.audio_tower,
feature_attention_mask.sum(-1),
merge_factor,
conv_params,
)
masked_audio_features = _flatten_audio_features_by_length(
audio_features, audio_output_lengths
)
chunk_embeddings = torch.split(
masked_audio_features, audio_output_lengths.flatten().tolist()
)
return _group_audio_embeddings(chunk_embeddings, chunk_counts)
def get_language_model(self) -> torch.nn.Module:
return self.language_model
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
audio_input = self._parse_and_validate_audio_input(**kwargs)
if audio_input is None:
return []
masked_audio_features = self._process_audio_input(audio_input)
return masked_audio_features
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
inputs_embeds = None
hidden_states = self.language_model.model(
input_ids,
positions,
intermediate_tensors,
inputs_embeds=inputs_embeds,
)
return hidden_states
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
skip_prefixes = ["audio_tower.embed_positions"]
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
return loader.load_weights(weights)
@classmethod
def _get_audio_token(cls, model_config: ModelConfig) -> str:
"""Get the audio token from processor.
Similar to get_placeholder_str but returns single token.
"""
processor = cached_processor_from_config(model_config)
return getattr(processor, "audio_token", "<|pad|>")
@classmethod
def get_speech_to_text_config(
cls, model_config: ModelConfig, task_type: str
) -> SpeechToTextConfig:
processor = cached_processor_from_config(model_config)
feature_extractor = processor.feature_extractor
max_audio_clip_s = getattr(processor, "max_audio_len", DEFAULT_MAX_AUDIO_LEN_S)
return SpeechToTextConfig(
max_audio_clip_s=max_audio_clip_s,
sample_rate=feature_extractor.sampling_rate,
)
@classmethod
def get_generation_prompt(
cls,
audio: np.ndarray,
model_config: ModelConfig,
stt_config: SpeechToTextConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
to_language: str | None,
) -> PromptType:
"""Get the generation prompt to be used for transcription requests."""
tokenizer = cached_tokenizer_from_config(model_config)
audio_token = cls._get_audio_token(model_config)
if task_type == "translate":
full_lang_name_to = cls.supported_languages.get(to_language, to_language)
user_content = f"{audio_token}translate the speech to {full_lang_name_to}"
elif task_type == "transcribe":
user_content = (
f"{audio_token}can you transcribe the speech into a written format?"
)
else:
raise ValueError(f"Unsupported task type {task_type}")
messages = [{"role": "user", "content": user_content}]
prompt = tokenizer.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
prompt_token_ids = tokenizer.encode(prompt)
prompt_dict = {
"prompt_token_ids": prompt_token_ids,
"multi_modal_data": {"audio": audio},
}
return cast(PromptType, prompt_dict)

165
vllm/model_executor/models/glmasr_utils.py Normal file
View File

@@ -0,0 +1,165 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from typing import cast
import torch
import torch.nn as nn
DEFAULT_MAX_AUDIO_LEN_S = 655
DEFAULT_MERGE_FACTOR = 4
# Default convolution parameters: (padding, kernel_size, stride)
# These correspond to the two conv layers in GlmAsrEncoder
DEFAULT_CONV_PARAMS = [(1, 3, 1), (1, 3, 2)]
def _calculate_conv_output_length(
input_length: torch.Tensor, padding: int, kernel_size: int, stride: int
) -> torch.Tensor:
"""Calculate Conv1d output length using standard formula."""
# Standard formula: floor((input + 2*padding - kernel_size) / stride) + 1
return (input_length + 2 * padding - kernel_size) // stride + 1
def _as_list_chunk_counts(
chunk_counts: torch.Tensor | list[int] | list[torch.Tensor],
) -> list[int]:
if isinstance(chunk_counts, torch.Tensor):
return chunk_counts.tolist()
if chunk_counts and isinstance(chunk_counts[0], torch.Tensor):
tensor_counts = cast(list[torch.Tensor], chunk_counts)
return [int(c.item()) for c in tensor_counts]
return [int(c) for c in chunk_counts]
def _normalize_chunk_counts(
chunk_counts: torch.Tensor | list[int] | list[torch.Tensor] | None,
num_chunks: int,
) -> list[int]:
if chunk_counts is None:
return [1] * num_chunks
return _as_list_chunk_counts(chunk_counts)
def _get_audio_output_lengths_from_lengths(
audio_lengths: torch.Tensor,
merge_factor: int,
conv_params: list[tuple[int, int, int]],
) -> torch.Tensor:
for padding, kernel_size, stride in conv_params:
audio_lengths = _calculate_conv_output_length(
audio_lengths, padding, kernel_size, stride
)
return (audio_lengths - merge_factor) // merge_factor + 1
def _get_audio_output_lengths_from_mask(
mask: torch.Tensor,
merge_factor: int,
conv_params: list[tuple[int, int, int]],
) -> torch.Tensor:
audio_lengths = mask.sum(-1)
return _get_audio_output_lengths_from_lengths(
audio_lengths, merge_factor, conv_params
)
def _get_audio_output_lengths_for_tower(
audio_tower: nn.Module,
audio_lengths: torch.Tensor,
merge_factor: int,
conv_params: list[tuple[int, int, int]],
) -> torch.Tensor:
if hasattr(audio_tower, "_get_feat_extract_output_lengths"):
_, audio_output_lengths = audio_tower._get_feat_extract_output_lengths(
audio_lengths
)
return audio_output_lengths
return _get_audio_output_lengths_from_lengths(
audio_lengths, merge_factor, conv_params
)
def _flatten_audio_features_by_length(
audio_features: torch.Tensor,
audio_output_lengths: torch.Tensor,
) -> torch.Tensor:
num_chunks, max_audio_tokens, embed_dim = audio_features.shape
audio_output_lengths = audio_output_lengths.unsqueeze(1)
audio_features_mask = (
torch.arange(max_audio_tokens)
.expand(num_chunks, max_audio_tokens)
.to(audio_output_lengths.device)
< audio_output_lengths
)
return audio_features[audio_features_mask].view(-1, embed_dim)
def _group_audio_embeddings(
chunk_embeddings: Sequence[torch.Tensor],
chunk_counts: Sequence[int],
) -> tuple[torch.Tensor, ...]:
grouped_embeddings = []
current_idx = 0
for count in chunk_counts:
audio_chunks = chunk_embeddings[current_idx : current_idx + count]
grouped_embeddings.append(torch.cat(audio_chunks, dim=0))
current_idx += count
return tuple(grouped_embeddings)
def _normalize_to_tensor(mask: torch.Tensor | list[torch.Tensor]) -> torch.Tensor:
"""Convert mask to tensor, handling both list and tensor formats."""
if isinstance(mask, list):
return (
torch.stack(mask)
if mask and isinstance(mask[0], torch.Tensor)
else torch.tensor(mask)
)
return mask
def _extract_mask_for_item(
feature_attention_mask: torch.Tensor | list[torch.Tensor],
chunk_counts: torch.Tensor | list[int] | None,
item_idx: int,
) -> torch.Tensor:
"""Extract attention mask for a specific audio item."""
if chunk_counts is None:
# Single item per audio
mask = feature_attention_mask[item_idx]
if isinstance(feature_attention_mask, torch.Tensor):
return mask.unsqueeze(0)
return _normalize_to_tensor(mask)
# Multiple chunks per audio: calculate slice indices
counts = _as_list_chunk_counts(chunk_counts)
start_idx = sum(counts[:item_idx])
end_idx = start_idx + counts[item_idx]
# Extract slice
if isinstance(feature_attention_mask, torch.Tensor):
return feature_attention_mask[start_idx:end_idx]
mask_slice = feature_attention_mask[start_idx:end_idx]
return _normalize_to_tensor(mask_slice)
def _get_num_features_for_item(
feature_attention_mask: torch.Tensor | None,
chunk_counts: torch.Tensor | list[int] | None,
item_idx: int,
audio_embeds: list[torch.Tensor] | None,
merge_factor: int,
conv_params: list[tuple[int, int, int]],
) -> int:
"""Get number of features for a specific audio item."""
if feature_attention_mask is not None:
mask = _extract_mask_for_item(feature_attention_mask, chunk_counts, item_idx)
audio_output_lengths = _get_audio_output_lengths_from_mask(
mask, merge_factor, conv_params
)
return audio_output_lengths.sum().item()
if audio_embeds is not None:
return audio_embeds[item_idx].shape[0]
raise ValueError("Either feature_attention_mask or audio_embeds must be provided")

1
vllm/model_executor/models/registry.py
View File

@@ -304,6 +304,7 @@ _MULTIMODAL_MODELS = {
"gemma3n_mm",
"Gemma3nForConditionalGeneration",
),
"GlmAsrForConditionalGeneration": ("glmasr", "GlmAsrForConditionalGeneration"),
"GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
"Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"), # noqa: E501
"Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"), # noqa: E501

2
vllm/transformers_utils/config.py
View File

@@ -136,6 +136,7 @@ class HFConfigParser(ConfigParserBase):
model,
revision=revision,
code_revision=code_revision,
trust_remote_code=trust_remote_code,
token=_get_hf_token(),
**kwargs,
)
@@ -157,6 +158,7 @@ class HFConfigParser(ConfigParserBase):
model,
revision=revision,
code_revision=code_revision,
trust_remote_code=trust_remote_code,
token=_get_hf_token(),
**kwargs,
)