Raise error for long prompt (#273)

2023-06-30 18:48:49 -07:00
parent 598dc4b79a
commit dafd924c1f
5 changed files with 42 additions and 11 deletions
--- a/vllm/engine/arg_utils.py
+++ b/vllm/engine/arg_utils.py
@@ -123,8 +123,12 @@ class EngineArgs:
        parallel_config = ParallelConfig(self.pipeline_parallel_size,
                                         self.tensor_parallel_size,
                                         self.worker_use_ray)
+        max_seq_len = min(
+            self.max_num_batched_tokens, 
+            getattr(model_config.hf_config, "max_position_embeddings", 
+                    float("inf")))
        scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
-                                           self.max_num_seqs)
+                                           self.max_num_seqs, max_seq_len)
        return model_config, cache_config, parallel_config, scheduler_config