[Bugfix] fix output parsing error for trtllm backend (#4137)

Co-authored-by: Roger Wang <ywang@roblox.com>
2024-04-17 19:07:23 +08:00
parent 8438e0569e
commit fe3b5bbc23
1 changed files with 1 additions and 1 deletions
--- a/benchmarks/backend_request_func.py
+++ b/benchmarks/backend_request_func.py
@@ -135,6 +135,7 @@ async def async_request_trt_llm(
                                              "data:")

                        data = json.loads(chunk)
+                        output.generated_text += data["text_output"]
                        timestamp = time.perf_counter()
                        # First token
                        if ttft == 0.0:
@@ -149,7 +150,6 @@ async def async_request_trt_llm(
                        most_recent_timestamp = timestamp

                    output.latency = most_recent_timestamp - st
-                    output.generated_text = json.loads(data)["text_output"]
                    output.success = True

                else: