diff --git a/vllm/transformers_utils/runai_utils.py b/vllm/transformers_utils/runai_utils.py
index 041056720..7e6af2602 100644
--- a/vllm/transformers_utils/runai_utils.py
+++ b/vllm/transformers_utils/runai_utils.py
@@ -64,9 +64,7 @@ class ObjectStorageModel:
             "model_streamer",
             hashlib.sha256(str(url).encode()).hexdigest()[:8],
         )
-        if os.path.exists(dir_name):
-            shutil.rmtree(dir_name)
-        os.makedirs(dir_name)
+        os.makedirs(dir_name, exist_ok=True)
         self.dir = dir_name
         logger.debug("Init object storage, model cache path is: %s", dir_name)