Add docstrings for LLM (#137)

benchmarks/benchmark_latency.py

@@ -30,7 +30,6 @@ def main(args: argparse.Namespace):
         max_tokens=args.output_len,
     )
     print(sampling_params)
-    dummy_prompts = [""] * args.batch_size
     dummy_prompt_token_ids = [[0] * args.input_len] * args.batch_size
 
     def run_to_completion(profile: bool = False):
@@ -38,7 +37,8 @@ def main(args: argparse.Namespace):
             torch.cuda.cudart().cudaProfilerStart()
         start_time = time.time()
 
-        llm.generate(dummy_prompts, sampling_params, dummy_prompt_token_ids,
+        llm.generate(prompt_token_ids=dummy_prompt_token_ids,
+                     sampling_params=sampling_params,
                      use_tqdm=False)
 
         end_time = time.time()

benchmarks/benchmark_throughput.py

@@ -72,9 +72,9 @@ def main(args: argparse.Namespace):
         )
         # FIXME(woosuk): Do not use internal method.
         llm._add_request(
-            prompt="",
-            sampling_params=sampling_params,
+            prompt=None,
             prompt_token_ids=prompt_token_ids,
+            sampling_params=sampling_params,
         )
 
     start = time.time()
@@ -85,7 +85,9 @@ def main(args: argparse.Namespace):
         len(prompt_token_ids) + output_len
         for prompt_token_ids, output_len in requests
     )
-    print(f"Throughput: {total_num_tokens / (end - start):.2f} tokens/s")
+    elapsed_time = end - start
+    print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
+          f"{total_num_tokens / elapsed_time:.2f} tokens/s")
 
 
 if __name__ == "__main__":