Remove all_special_tokens_extended from tokenizer code (#29686)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
2025-11-28 20:26:51 +00:00
parent 8d9338fae4
commit fecae12cd7
6 changed files with 40 additions and 65 deletions
--- a/tests/tokenization/test_tokenizer_registry.py
+++ b/tests/tokenization/test_tokenizer_registry.py
@@ -15,10 +15,6 @@ class TestTokenizer(TokenizerBase):
    def from_pretrained(cls, *args, **kwargs) -> "TestTokenizer":
        return TestTokenizer()

-    @property
-    def all_special_tokens_extended(self) -> list[str]:
-        raise NotImplementedError()
-
    @property
    def all_special_tokens(self) -> list[str]:
        raise NotImplementedError()