[3/N] Support and implement merged input processor for LLaVA model (#10676)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
Cyrus Leung
@@ -3,50 +3,15 @@ from typing import cast
|
||||
import pytest
|
||||
from transformers import BatchFeature
|
||||
|
||||
from vllm.multimodal.processing import (PromptReplacement, find_text_matches,
|
||||
find_token_matches, iter_token_matches,
|
||||
iter_token_runs, replace_text_matches)
|
||||
from vllm.multimodal.processing import (PromptReplacement, _PlaceholderInfo,
|
||||
find_text_matches, find_token_matches,
|
||||
iter_placeholders, iter_token_matches,
|
||||
replace_text_matches,
|
||||
replace_token_matches)
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.utils import full_groupby
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("token_ids", "expected"),
|
||||
[
|
||||
([], []),
|
||||
(
|
||||
[32000, 32000, 32000],
|
||||
[{ "token_id": 32000, "start_idx": 0, "length": 3 }],
|
||||
),
|
||||
(
|
||||
[9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
|
||||
[
|
||||
{ "token_id": 9833, "start_idx": 0, "length": 1 },
|
||||
{ "token_id": 28747, "start_idx": 1, "length": 1 },
|
||||
{ "token_id": 32000, "start_idx": 2, "length": 3 },
|
||||
{ "token_id": 9833, "start_idx": 5, "length": 1 },
|
||||
{ "token_id": 28747, "start_idx": 6, "length": 1 },
|
||||
{ "token_id": 32000, "start_idx": 7, "length": 2 },
|
||||
{ "token_id": 918, "start_idx": 9, "length": 1 },
|
||||
],
|
||||
),
|
||||
],
|
||||
)
|
||||
# yapf: enable
|
||||
def test_iter_token_runs(token_ids, expected):
|
||||
result = list(iter_token_runs(token_ids))
|
||||
|
||||
# Only displayed on error
|
||||
print("result:", result)
|
||||
|
||||
# Manually constructed results
|
||||
assert [item._asdict() for item in result] == expected
|
||||
|
||||
# Invariants
|
||||
assert sum(run_info.length for run_info in result) == len(token_ids)
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("token_ids", "match_ids", "expected"),
|
||||
@@ -170,13 +135,11 @@ def test_find_token_matches(prompt, target_by_key, expected_by_key):
|
||||
# Should not be used since there is nothing to convert to token IDs
|
||||
mock_tokenizer = cast(AnyTokenizer, object())
|
||||
|
||||
result = find_token_matches(
|
||||
prompt,
|
||||
[
|
||||
PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
|
||||
for key, target in target_by_key.items()
|
||||
],
|
||||
)
|
||||
prompt_repls = [
|
||||
PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
|
||||
for key, target in target_by_key.items()
|
||||
]
|
||||
result = find_token_matches(prompt, prompt_repls)
|
||||
|
||||
# Only displayed on error
|
||||
print("result:", result)
|
||||
@@ -279,13 +242,11 @@ def test_find_text_matches(prompt, target_by_key, expected_by_key):
|
||||
# Should not be used since there is nothing to convert to text
|
||||
mock_tokenizer = cast(AnyTokenizer, object())
|
||||
|
||||
result = find_text_matches(
|
||||
prompt,
|
||||
[
|
||||
PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
|
||||
for key, target in target_by_key.items()
|
||||
],
|
||||
)
|
||||
prompt_repls = [
|
||||
PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
|
||||
for key, target in target_by_key.items()
|
||||
]
|
||||
result = find_text_matches(prompt, prompt_repls)
|
||||
|
||||
# Only displayed on error
|
||||
print("result:", result)
|
||||
@@ -303,7 +264,7 @@ def test_find_text_matches(prompt, target_by_key, expected_by_key):
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("prompt", "target_by_key", "repl_by_key", "expected_by_mm_count"),
|
||||
("prompt", "target_by_key", "repl_by_key"),
|
||||
[
|
||||
(
|
||||
"Image:<image>Image:<image><image>!",
|
||||
@@ -322,49 +283,201 @@ def test_find_text_matches(prompt, target_by_key, expected_by_key):
|
||||
# Test multiple repl_count
|
||||
"pattern_3": ("?", 2),
|
||||
},
|
||||
{
|
||||
# Test no replacement
|
||||
0: "Image:<image>Image:<image><image>!",
|
||||
# Test single replacement
|
||||
1: "<image><image>Image:<image><image>??",
|
||||
# Test repeated replacement
|
||||
2: "<image><image><image><image><image>??",
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("mm_count", "expected"),
|
||||
[
|
||||
(0, "Image:<image>Image:<image><image>!"),
|
||||
(1, "<image><image>Image:<image><image>??"),
|
||||
(2, "<image><image><image><image><image>??"),
|
||||
]
|
||||
)
|
||||
# yapf: enable
|
||||
def test_find_replace_text(
|
||||
prompt,
|
||||
target_by_key,
|
||||
repl_by_key,
|
||||
expected_by_mm_count,
|
||||
mm_count,
|
||||
expected,
|
||||
):
|
||||
# Should not be used since there is nothing to convert to text
|
||||
mock_tokenizer = cast(AnyTokenizer, object())
|
||||
|
||||
matches = find_text_matches(
|
||||
prompt_repls = [
|
||||
PromptReplacement(target, *repl_by_key[key]).bind(key, mock_tokenizer)
|
||||
for key, target in target_by_key.items()
|
||||
]
|
||||
matches = find_text_matches(prompt, prompt_repls)
|
||||
|
||||
result = replace_text_matches(
|
||||
prompt,
|
||||
[
|
||||
PromptReplacement(target, *repl_by_key[key]) \
|
||||
.bind(key, mock_tokenizer)
|
||||
for key, target in target_by_key.items()
|
||||
],
|
||||
matches,
|
||||
{key: list(range(mm_count))
|
||||
for key in repl_by_key},
|
||||
BatchFeature(),
|
||||
)
|
||||
result_by_mm_count = {
|
||||
mm_count: replace_text_matches(
|
||||
prompt,
|
||||
matches,
|
||||
{key: list(range(mm_count))
|
||||
for key in repl_by_key},
|
||||
BatchFeature(),
|
||||
)
|
||||
for mm_count in expected_by_mm_count
|
||||
}
|
||||
|
||||
# Only displayed on error
|
||||
print("matches:", matches)
|
||||
print("result_by_mm_count:", result_by_mm_count)
|
||||
print("result:", result)
|
||||
|
||||
# Manually constructed results
|
||||
assert result_by_mm_count == expected_by_mm_count
|
||||
assert result == expected
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
("prompt", "target_by_key", "repl_by_key"),
|
||||
[
|
||||
# Tokenized test cases of `test_find_replace_text`
|
||||
# using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
|
||||
(
|
||||
[1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
|
||||
{
|
||||
# We use `<image>` before `Image:` to test matches that
|
||||
# occur out of order
|
||||
"pattern_1": [32000],
|
||||
"pattern_2": [9833, 28747],
|
||||
"pattern_3": [918],
|
||||
},
|
||||
{
|
||||
# Test whether target is confused with repl_unit
|
||||
"pattern_1": ([32000, 32000], 1),
|
||||
# Test empty repl_unit
|
||||
"pattern_2": ([], 1),
|
||||
# Test multiple repl_count
|
||||
"pattern_3": ([1550], 2),
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("mm_count", "expected"),
|
||||
[
|
||||
(0, [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918]),
|
||||
(1, [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 1550]),
|
||||
(2, [1, 32000, 32000, 32000, 32000, 32000, 1550, 1550]),
|
||||
]
|
||||
)
|
||||
# yapf: enable
|
||||
def test_find_replace_tokens(
|
||||
prompt,
|
||||
target_by_key,
|
||||
repl_by_key,
|
||||
mm_count,
|
||||
expected,
|
||||
):
|
||||
# Should not be used since there is nothing to convert to tokens
|
||||
mock_tokenizer = cast(AnyTokenizer, object())
|
||||
|
||||
prompt_repls = [
|
||||
PromptReplacement(target, *repl_by_key[key]).bind(key, mock_tokenizer)
|
||||
for key, target in target_by_key.items()
|
||||
]
|
||||
matches = find_token_matches(prompt, prompt_repls)
|
||||
|
||||
result = replace_token_matches(
|
||||
prompt,
|
||||
matches,
|
||||
{key: list(range(mm_count))
|
||||
for key in repl_by_key},
|
||||
BatchFeature(),
|
||||
)
|
||||
|
||||
# Only displayed on error
|
||||
print("matches:", matches)
|
||||
print("result:", result)
|
||||
|
||||
# Manually constructed results
|
||||
assert result == expected
|
||||
|
||||
|
||||
# yapf: disable
|
||||
@pytest.mark.parametrize(
|
||||
"repl_by_key",
|
||||
[
|
||||
{
|
||||
"pattern_1": ([32000, 32000], 1),
|
||||
"pattern_2": ([], 1),
|
||||
"pattern_3": ([1550], 2),
|
||||
},
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("prompt", "expected"),
|
||||
[
|
||||
(
|
||||
[1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
|
||||
[
|
||||
_PlaceholderInfo(
|
||||
modality="pattern_1",
|
||||
start_idx=6,
|
||||
unit=[32000, 32000],
|
||||
unit_count=1,
|
||||
),
|
||||
],
|
||||
),
|
||||
(
|
||||
[1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 1550],
|
||||
[
|
||||
_PlaceholderInfo(
|
||||
modality="pattern_1",
|
||||
start_idx=1,
|
||||
unit=[32000, 32000],
|
||||
unit_count=1,
|
||||
),
|
||||
_PlaceholderInfo(
|
||||
modality="pattern_1",
|
||||
start_idx=5,
|
||||
unit=[32000, 32000],
|
||||
unit_count=1,
|
||||
),
|
||||
_PlaceholderInfo(
|
||||
modality="pattern_3",
|
||||
start_idx=7,
|
||||
unit=[1550],
|
||||
unit_count=2,
|
||||
),
|
||||
],
|
||||
),
|
||||
(
|
||||
[1, 32000, 32000, 32000, 32000, 32000, 1550, 1550],
|
||||
[
|
||||
_PlaceholderInfo(
|
||||
modality="pattern_1",
|
||||
start_idx=1,
|
||||
unit=[32000, 32000],
|
||||
unit_count=2,
|
||||
),
|
||||
_PlaceholderInfo(
|
||||
modality="pattern_3",
|
||||
start_idx=6,
|
||||
unit=[1550],
|
||||
unit_count=2,
|
||||
),
|
||||
],
|
||||
),
|
||||
]
|
||||
)
|
||||
def test_iter_placeholders(
|
||||
repl_by_key,
|
||||
prompt,
|
||||
expected,
|
||||
):
|
||||
# Should not be used since there is nothing to convert to tokens
|
||||
mock_tokenizer = cast(AnyTokenizer, object())
|
||||
|
||||
prompt_repls = [
|
||||
PromptReplacement([], *repl).bind(key, mock_tokenizer)
|
||||
for key, repl in repl_by_key.items()
|
||||
]
|
||||
|
||||
result = list(iter_placeholders(prompt_repls, prompt))
|
||||
|
||||
# Only displayed on error
|
||||
print("result:", result)
|
||||
|
||||
# Manually constructed results
|
||||
assert result == expected
|
||||
|
||||
Reference in New Issue
Block a user