biondizzle/smollm3-3b-vllm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add in lora

Browse Source
This commit is contained in:
biondizzle
2026-04-10 14:01:42 +00:00
parent 5029ab3b40
commit a5bb29930f
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
docker-compose.yaml
View File

@@ -12,8 +12,10 @@ services:
- --chat-template-content-format=string - --chat-template-content-format=string
- --chat-template=/root/chat_template.jinja - --chat-template=/root/chat_template.jinja
- --enable-auto-tool-choice - --enable-auto-tool-choice
- --tool-call-parser=hermes - --tool-call-parser=/root/smol_tool_parser.py
- --reasoning-parser=deepseek_r1 - --reasoning-parser=deepseek_r1
- --enable-lora
- --lora-modules=smollm-toolcall=/root/loras/better-tool-call
#- --max-model-len=131072 #- --max-model-len=131072
#- --hf-overrides={"rope_scaling":{"type":"yarn","factor":2.0,"original_max_position_embeddings":65536}} #- --hf-overrides={"rope_scaling":{"type":"yarn","factor":2.0,"original_max_position_embeddings":65536}}
deploy: deploy:
@@ -31,5 +33,6 @@ services:
volumes: volumes:
- /srv:/root/.cache/huggingface - /srv:/root/.cache/huggingface
- ./chat_template.jinja:/root/chat_template.jinja - ./chat_template.jinja:/root/chat_template.jinja
- ./hermes_tool_parser.py:/usr/local/lib/python3.12/dist-packages/vllm/tool_parsers/hermes_tool_parser.py - ./smol_tool_parser.py:/root/smol_tool_parser.py
- ./loras:/root/loras
network_mode: host network_mode: host