Add an option to use dummy model weights (#33)

2023-04-08 23:36:12 -07:00
parent c267b1a02c
commit ee88a7e5f3
9 changed files with 36 additions and 8 deletions
--- a/cacheflow/http_frontend/fastapi_frontend.py
+++ b/cacheflow/http_frontend/fastapi_frontend.py
@@ -47,6 +47,7 @@ class FastAPIFrontend:
        self.server = remote_server_class.remote(
            model=model,
            model_path=model_path,
+            use_dummy_weights=False,
            pipeline_parallel_size=pipeline_parallel_size,
            tensor_parallel_size=tensor_parallel_size,
            block_size=block_size,