diff --git a/vllm/v1/structured_output/backend_guidance.py b/vllm/v1/structured_output/backend_guidance.py
index 6d1e4b7f7..6063a2dc2 100644
--- a/vllm/v1/structured_output/backend_guidance.py
+++ b/vllm/v1/structured_output/backend_guidance.py
@@ -93,7 +93,7 @@ class GuidanceBackend(StructuredOutputBackend):
         )
 
         self.ll_tokenizer = llguidance_hf.from_tokenizer(
-            self.tokenizer, self.vocab_size
+            self.tokenizer, max(self.vocab_size, len(self.tokenizer))
         )
 
     def compile_grammar(