diff --git a/Dockerfile b/Dockerfile index c837367..4dc4ecd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ #FROM vllm/vllm-openai:v0.19.0-cu130 -#FROM vllm/vllm-openai:cu130-nightly-x86_64 -FROM vllm/vllm-openai:glm51-cu130 +FROM vllm/vllm-openai:nightly +#FROM vllm/vllm-openai:glm51-cu130 # Install LMCache for KV cache offloading / sharing across nodes # Build with system CUDA 13.0 for Blackwell (B200) @@ -17,26 +17,29 @@ RUN apt-get update && apt-get install -y git \ CUDA_HOME=/usr/local/cuda \ TORCH_CUDA_ARCH_LIST="10.0" \ pip install --no-cache-dir --no-build-isolation . && \ - rm -rf /tmp/lmcache && export CACHE_BUSTER=1 + rm -rf /tmp/lmcache && export CACHE_BUSTER=2 # Copy over nemotron reasonong parser -COPY ./super_v3_reasoning_parser.py /opt/super_v3_reasoning_parser.py +#COPY ./super_v3_reasoning_parser.py /opt/super_v3_reasoning_parser.py # Copy over deepseek tool call parser with MTP fixes -COPY deepseekv32_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/deepseekv32_tool_parser.py +#COPY deepseekv32_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/deepseekv32_tool_parser.py # Copy over minimax tool call parser with kwargs fixes -COPY minimax_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/minimax_tool_parser.py +#COPY minimax_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/minimax_tool_parser.py # Copy over minimax parsers with kwargs fixes -COPY minimax_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/minimax_tool_parser.py -COPY minimax_m2_parser.py /usr/local/lib/python3.12/dist-packages/vllm/parser/minimax_m2_parser.py +#COPY minimax_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/minimax_tool_parser.py +#COPY minimax_m2_parser.py /usr/local/lib/python3.12/dist-packages/vllm/parser/minimax_m2_parser.py # Patch tool parser for GLM regex fix -COPY glm4_moe_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/glm4_moe_tool_parser.py -COPY utils.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/utils.py +#COPY glm4_moe_tool_parser.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/glm4_moe_tool_parser.py +#COPY utils.py /usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/utils.py # Patch hf renderer to force string content format for GLM models # This fixes the issue where tool response content is dropped -COPY hf.py /usr/local/lib/python3.12/dist-packages/vllm/renderers/hf.py \ No newline at end of file +#COPY hf.py /usr/local/lib/python3.12/dist-packages/vllm/renderers/hf.py + +# Make sure we have the latest up to date chat template +COPY glm_5.1_chat_template.jinja /opt/chat_template.jinja \ No newline at end of file diff --git a/glm_5.1_chat_template.jinja b/glm_5.1_chat_template.jinja new file mode 100644 index 0000000..e230020 --- /dev/null +++ b/glm_5.1_chat_template.jinja @@ -0,0 +1,119 @@ +[gMASK] +{%- if tools -%} +{%- macro tool_to_json(tool) -%} + {%- set ns_tool = namespace(first=true) -%} + {{ '{' -}} + {%- for k, v in tool.items() -%} + {%- if k != 'defer_loading' and k != 'strict' -%} + {%- if not ns_tool.first -%}{{- ', ' -}}{%- endif -%} + {%- set ns_tool.first = false -%} + "{{ k }}": {{ v | tojson(ensure_ascii=False) }} + {%- endif -%} + {%- endfor -%} + {{- '}' -}} +{%- endmacro -%} +<|system|> +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{% for tool in tools %} +{%- if 'function' in tool -%} + {%- set tool = tool['function'] -%} +{%- endif -%} +{% if tool.defer_loading is not defined or not tool.defer_loading %} +{{ tool_to_json(tool) }} +{% endif %} +{% endfor %} + + +For each function call, output the function name and arguments within the following XML format: +{function-name}{arg-key-1}{arg-value-1}{arg-key-2}{arg-value-2}...{%- endif -%} +{%- macro visible_text(content) -%} + {%- if content is string -%} + {{- content }} + {%- elif content is iterable and content is not mapping -%} + {%- for item in content -%} + {%- if item is mapping and item.type == 'text' -%} + {{- item.text }} + {%- elif item is string -%} + {{- item }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- content }} + {%- endif -%} +{%- endmacro -%} +{%- set ns = namespace(last_user_index=-1, thinking_indices='') -%} +{%- for m in messages %} + {%- if m.role == 'user' %} + {%- set ns.last_user_index = loop.index0 -%} + {%- elif m.role == 'assistant' %} + {%- if m.reasoning_content is string %} + {%- set ns.thinking_indices = ns.thinking_indices ~ ',' ~ ns.last_user_index ~ ',' -%} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- set ns.has_thinking = false -%} +{%- for m in messages -%} +{%- if m.role == 'user' -%}<|user|>{{ visible_text(m.content) }}{% set ns.has_thinking = (',' ~ loop.index0 ~ ',') in ns.thinking_indices -%} +{%- elif m.role == 'assistant' -%} +<|assistant|> +{%- set content = visible_text(m.content) %} +{%- if m.reasoning_content is string %} + {%- set reasoning_content = m.reasoning_content %} +{%- elif '' in content %} + {%- set reasoning_content = content.split('')[0].split('')[-1] %} + {%- set content = content.split('')[-1] %} +{%- elif loop.index0 > ns.last_user_index and not (enable_thinking is defined and not enable_thinking) %} + {%- set reasoning_content = '' %} +{%- elif loop.index0 < ns.last_user_index and ns.has_thinking %} + {%- set reasoning_content = '' %} +{%- endif %} +{%- if ((clear_thinking is defined and not clear_thinking) or loop.index0 > ns.last_user_index) and reasoning_content is defined -%} +{{ '' + reasoning_content + ''}} +{%- else -%} +{{ '' }} +{%- endif -%} +{%- if content.strip() -%} +{{ content.strip() }} +{%- endif -%} +{% if m.tool_calls %} +{% for tc in m.tool_calls %} +{%- if tc.function %} + {%- set tc = tc.function %} +{%- endif %} +{{- '' + tc.name -}} +{% set _args = tc.arguments %}{% for k, v in _args.items() %}{{ k }}{{ v | tojson(ensure_ascii=False) if v is not string else v }}{% endfor %}{% endfor %} +{% endif %} +{%- elif m.role == 'tool' -%} +{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|observation|>' -}} +{%- endif %} +{%- if m.content is string -%} + {{- '' + m.content + '' -}} +{%- elif m.content is iterable and m.content is not mapping and m.content and m.content.0.type == "tool_reference" -%} + {{- '\n' -}} + {% for tr in m.content %} + {%- for tool in tools -%} + {%- if 'function' in tool -%} + {%- set tool = tool['function'] -%} + {%- endif -%} + {%- if tool.name == tr.name -%} + {{- tool_to_json(tool) + '\n' -}} + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + {{- '' -}} +{%- else -%} + {{- '' + visible_text(m.content) + '' -}} +{% endif -%} +{%- elif m.role == 'system' -%} +<|system|>{{ visible_text(m.content) }} +{%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%} + <|assistant|>{{- '' if (enable_thinking is defined and not enable_thinking) else '' -}} +{%- endif -%} \ No newline at end of file