[Bugfix] Add fix for JSON whitespace (#4189)
Co-authored-by: Ubuntu <ubuntu@ip-172-31-13-147.ec2.internal>
This commit is contained in:
@@ -131,6 +131,11 @@ class CFGLogitsProcessor(BaseLogitsProcessor):
|
||||
fsm = CFGFSM(cfg, tokenizer)
|
||||
self.fsm = fsm
|
||||
|
||||
def init_state(self):
|
||||
"""Initialize state with a CFGFSM copy."""
|
||||
super().init_state()
|
||||
self.fsm = self.fsm.copy()
|
||||
|
||||
|
||||
@lru_cache
|
||||
def _adapt_tokenizer(tokenizer: PreTrainedTokenizerBase):
|
||||
|
||||
Reference in New Issue
Block a user