Refactor scheduler (#658)

2023-08-02 16:42:01 -07:00
parent e8ddc08ec8
commit 55fe8a81ec
4 changed files with 205 additions and 144 deletions
--- a/examples/llm_engine_example.py
+++ b/examples/llm_engine_example.py
@@ -28,7 +28,7 @@ def main(args: argparse.Namespace):
    # Run the engine by calling `engine.step()` manually.
    request_id = 0
    while True:
-        # To test iteration-level scheduling, we add one request at each step.
+        # To test continuous batching, we add one request at each step.
        if test_prompts:
            prompt, sampling_params = test_prompts.pop(0)
            engine.add_request(str(request_id), prompt, sampling_params)