[Frontend] Use new Renderer for Completions and Tokenize API (#32863)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

tests/models/language/pooling_mteb_test/mteb_embed_utils.py

@@ -187,7 +187,10 @@ def mteb_test_embed_models(
         head_dtype = model_config.head_dtype
 
         # Test embedding_size, isnan and whether to use normalize
-        vllm_outputs = vllm_model.embed(example_prompts, truncate_prompt_tokens=-1)
+        vllm_outputs = vllm_model.embed(
+            example_prompts,
+            tokenization_kwargs=dict(truncate_prompt_tokens=-1),
+        )
         outputs_tensor = torch.tensor(vllm_outputs)
         assert not torch.any(torch.isnan(outputs_tensor))
         embedding_size = model_config.embedding_size

tests/models/language/pooling_mteb_test/mteb_score_utils.py

@@ -79,9 +79,9 @@ class VllmMtebCrossEncoder(MtebCrossEncoderMixin):
         outputs = self.llm.score(
             queries,
             corpus,
-            truncate_prompt_tokens=-1,
             use_tqdm=False,
             chat_template=self.chat_template,
+            tokenization_kwargs={"truncate_prompt_tokens": -1},
         )
         scores = np.array(outputs)
         scores = scores[np.argsort(r)]