biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Add script for benchmarking serving throughput (#145)

Browse Source
This commit is contained in:
Woosuk Kwon
2023-06-14 19:55:38 -07:00
committed by GitHub
parent da5ddcd544
commit 311490a720
10 changed files with 421 additions and 415 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
examples/simple_fastapi_client.py
View File

@@ -1,12 +1,14 @@
import argparse
import json
import requests
from typing import Iterable, List
def clear_line(n:int = 1) -> None:
import requests
def clear_line(n: int = 1) -> None:
LINE_UP = '\033[1A'
LINE_CLEAR = '\x1b[2K'
for i in range(n):
for _ in range(n):
print(LINE_UP, end=LINE_CLEAR, flush=True)
@@ -53,7 +55,7 @@ if __name__ == "__main__":
n = args.n
stream = args.stream
print(f"Prompt: {prompt}\n", flush=True)
print(f"Prompt: {prompt!r}\n", flush=True)
response = post_http_request(prompt, api_url, n, stream)
if stream:
@@ -63,8 +65,8 @@ if __name__ == "__main__":
num_printed_lines = 0
for i, line in enumerate(h):
num_printed_lines += 1
print(f"Beam candidate {i}: {line}", flush=True)
print(f"Beam candidate {i}: {line!r}", flush=True)
else:
output = get_response(response)
for i, line in enumerate(output):
print(f"Beam candidate {i}: {line}", flush=True)
print(f"Beam candidate {i}: {line!r}", flush=True)