[Feature] Add vision language model support. (#3042)

2024-03-25 14:16:30 -07:00
parent f408d05c52
commit 64172a976c
28 changed files with 936 additions and 94 deletions
--- a/tests/spec_decode/utils.py
+++ b/tests/spec_decode/utils.py
@@ -109,7 +109,7 @@ def create_worker(cls: type,
    )

    (model_config, cache_config, parallel_config, scheduler_config,
-     device_config, _) = engine_args.create_engine_configs()
+     device_config, _, _) = engine_args.create_engine_configs()

    distributed_init_method = get_distributed_init_method(
        get_ip(), get_open_port())