[2/N] handling placeholders in merged multi-modal processor (#10485)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
2024-11-23 13:25:09 +08:00
parent 4634a89d18
commit c8acd80548
5 changed files with 979 additions and 151 deletions
--- a/tests/multimodal/test_utils.py
+++ b/tests/multimodal/test_utils.py
@@ -139,7 +139,8 @@ def test_repeat_and_pad_placeholder_tokens(model):
            2,
            "<image><image><image>",
            [32000, 32000, 32000],
-            [{ "offset": 0, "length": 2 }]),
+            [{ "offset": 0, "length": 2 }],
+        ),
        (
            "<image><image>",
            [3, 2],