biondizzle/vllm-with-lmcache
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

latest build of glm fixes

Browse Source
This commit is contained in:
biondizzle
2026-05-05 08:19:47 +00:00
parent a0b1bfb7af
commit f79691e0ec
2 changed files with 133 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
Dockerfile
View File

@@ -1,6 +1,6 @@
#FROM vllm/vllm-openai:v0.19.0-cu130
#FROM vllm/vllm-openai:cu130-nightly-x86_64
FROM vllm/vllm-openai:glm51-cu130
FROM vllm/vllm-openai:nightly
#FROM vllm/vllm-openai:glm51-cu130
# Install LMCache for KV cache offloading / sharing across nodes
# Build with system CUDA 13.0 for Blackwell (B200)
@@ -17,26 +17,29 @@ RUN apt-get update && apt-get install -y git \
CUDA_HOME=/usr/local/cuda \
TORCH_CUDA_ARCH_LIST="10.0" \
pip install --no-cache-dir --no-build-isolation . && \
rm -rf /tmp/lmcache && export CACHE_BUSTER=1
rm -rf /tmp/lmcache && export CACHE_BUSTER=2
# Copy over nemotron reasonong parser
COPY ./super_v3_reasoning_parser.py /opt/super_v3_reasoning_parser.py
#COPY ./super_v3_reasoning_parser.py /opt/super_v3_reasoning_parser.py
# Copy over deepseek tool call parser with MTP fixes
COPY deepseekv32_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/deepseekv32_tool_parser.py
#COPY deepseekv32_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/deepseekv32_tool_parser.py
# Copy over minimax tool call parser with kwargs fixes
COPY minimax_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/minimax_tool_parser.py
#COPY minimax_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/minimax_tool_parser.py
# Copy over minimax parsers with kwargs fixes
COPY minimax_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/minimax_tool_parser.py
COPY minimax_m2_parser.py /usr/local/lib/python3.12/dist-packages/vllm/parser/minimax_m2_parser.py
#COPY minimax_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/minimax_tool_parser.py
#COPY minimax_m2_parser.py /usr/local/lib/python3.12/dist-packages/vllm/parser/minimax_m2_parser.py
# Patch tool parser for GLM regex fix
COPY glm4_moe_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/glm4_moe_tool_parser.py
COPY utils.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/utils.py
#COPY glm4_moe_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/glm4_moe_tool_parser.py
#COPY utils.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/utils.py
# Patch hf renderer to force string content format for GLM models
# This fixes the issue where tool response content is dropped
COPY hf.py /usr/local/lib/python3.12/dist-packages/vllm/renderers/hf.py
#COPY hf.py /usr/local/lib/python3.12/dist-packages/vllm/renderers/hf.py
# Make sure we have the latest up to date chat template
COPY glm_5.1_chat_template.jinja /opt/chat_template.jinja

119
glm_5.1_chat_template.jinja Normal file
View File

@@ -0,0 +1,119 @@
[gMASK]<sop>
{%- if tools -%}
{%- macro tool_to_json(tool) -%}
{%- set ns_tool = namespace(first=true) -%}
{{ '{' -}}
{%- for k, v in tool.items() -%}
{%- if k != 'defer_loading' and k != 'strict' -%}
{%- if not ns_tool.first -%}{{- ', ' -}}{%- endif -%}
{%- set ns_tool.first = false -%}
"{{ k }}": {{ v | tojson(ensure_ascii=False) }}
{%- endif -%}
{%- endfor -%}
{{- '}' -}}
{%- endmacro -%}
<|system|>
# Tools
You may call one or more functions to assist with the user query.
You are provided with function signatures within <tools></tools> XML tags:
<tools>
{% for tool in tools %}
{%- if 'function' in tool -%}
{%- set tool = tool['function'] -%}
{%- endif -%}
{% if tool.defer_loading is not defined or not tool.defer_loading %}
{{ tool_to_json(tool) }}
{% endif %}
{% endfor %}
</tools>
For each function call, output the function name and arguments within the following XML format:
<tool_call>{function-name}<arg_key>{arg-key-1}</arg_key><arg_value>{arg-value-1}</arg_value><arg_key>{arg-key-2}</arg_key><arg_value>{arg-value-2}</arg_value>...</tool_call>{%- endif -%}
{%- macro visible_text(content) -%}
{%- if content is string -%}
{{- content }}
{%- elif content is iterable and content is not mapping -%}
{%- for item in content -%}
{%- if item is mapping and item.type == 'text' -%}
{{- item.text }}
{%- elif item is string -%}
{{- item }}
{%- endif -%}
{%- endfor -%}
{%- else -%}
{{- content }}
{%- endif -%}
{%- endmacro -%}
{%- set ns = namespace(last_user_index=-1, thinking_indices='') -%}
{%- for m in messages %}
{%- if m.role == 'user' %}
{%- set ns.last_user_index = loop.index0 -%}
{%- elif m.role == 'assistant' %}
{%- if m.reasoning_content is string %}
{%- set ns.thinking_indices = ns.thinking_indices ~ ',' ~ ns.last_user_index ~ ',' -%}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- set ns.has_thinking = false -%}
{%- for m in messages -%}
{%- if m.role == 'user' -%}<|user|>{{ visible_text(m.content) }}{% set ns.has_thinking = (',' ~ loop.index0 ~ ',') in ns.thinking_indices -%}
{%- elif m.role == 'assistant' -%}
<|assistant|>
{%- set content = visible_text(m.content) %}
{%- if m.reasoning_content is string %}
{%- set reasoning_content = m.reasoning_content %}
{%- elif '</think>' in content %}
{%- set reasoning_content = content.split('</think>')[0].split('<think>')[-1] %}
{%- set content = content.split('</think>')[-1] %}
{%- elif loop.index0 > ns.last_user_index and not (enable_thinking is defined and not enable_thinking) %}
{%- set reasoning_content = '' %}
{%- elif loop.index0 < ns.last_user_index and ns.has_thinking %}
{%- set reasoning_content = '' %}
{%- endif %}
{%- if ((clear_thinking is defined and not clear_thinking) or loop.index0 > ns.last_user_index) and reasoning_content is defined -%}
{{ '<think>' + reasoning_content + '</think>'}}
{%- else -%}
{{ '</think>' }}
{%- endif -%}
{%- if content.strip() -%}
{{ content.strip() }}
{%- endif -%}
{% if m.tool_calls %}
{% for tc in m.tool_calls %}
{%- if tc.function %}
{%- set tc = tc.function %}
{%- endif %}
{{- '<tool_call>' + tc.name -}}
{% set _args = tc.arguments %}{% for k, v in _args.items() %}<arg_key>{{ k }}</arg_key><arg_value>{{ v | tojson(ensure_ascii=False) if v is not string else v }}</arg_value>{% endfor %}</tool_call>{% endfor %}
{% endif %}
{%- elif m.role == 'tool' -%}
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
{{- '<|observation|>' -}}
{%- endif %}
{%- if m.content is string -%}
{{- '<tool_response>' + m.content + '</tool_response>' -}}
{%- elif m.content is iterable and m.content is not mapping and m.content and m.content.0.type == "tool_reference" -%}
{{- '<tool_response><tools>\n' -}}
{% for tr in m.content %}
{%- for tool in tools -%}
{%- if 'function' in tool -%}
{%- set tool = tool['function'] -%}
{%- endif -%}
{%- if tool.name == tr.name -%}
{{- tool_to_json(tool) + '\n' -}}
{%- endif -%}
{%- endfor -%}
{%- endfor -%}
{{- '</tools></tool_response>' -}}
{%- else -%}
{{- '<tool_response>' + visible_text(m.content) + '</tool_response>' -}}
{% endif -%}
{%- elif m.role == 'system' -%}
<|system|>{{ visible_text(m.content) }}
{%- endif -%}
{%- endfor -%}
{%- if add_generation_prompt -%}
<|assistant|>{{- '</think>' if (enable_thinking is defined and not enable_thinking) else '<think>' -}}
{%- endif -%}