Allow user to define whitespace pattern for outlines (#4305)

2024-05-01 05:48:39 +02:00
parent a822eb3413
commit c3845d82dc
4 changed files with 21 additions and 8 deletions
--- a/tests/entrypoints/test_guided_processors.py
+++ b/tests/entrypoints/test_guided_processors.py
@@ -57,7 +57,9 @@ def test_guided_logits_processors():
    """Basic unit test for RegexLogitsProcessor and JSONLogitsProcessor."""
    tokenizer = AutoTokenizer.from_pretrained('HuggingFaceH4/zephyr-7b-beta')
    regex_LP = RegexLogitsProcessor(TEST_REGEX, tokenizer)
-    json_LP = JSONLogitsProcessor(TEST_SCHEMA, tokenizer)
+    json_LP = JSONLogitsProcessor(TEST_SCHEMA,
+                                  tokenizer,
+                                  whitespace_pattern=None)

    regex_LP.init_state()
    token_ids = tokenizer.encode(