[Tests] Disable retries and use context manager for openai client (#7565)

tests/async_engine/test_openapi_server_ray.py

@@ -1,5 +1,6 @@
 import openai  # use the official client for correctness check
 import pytest
+import pytest_asyncio
 
 from ..utils import VLLM_PATH, RemoteOpenAIServer
 
@@ -31,9 +32,10 @@ def server():
         yield remote_server
 
 
-@pytest.fixture(scope="module")
-def client(server):
-    return server.get_async_client()
+@pytest_asyncio.fixture
+async def client(server):
+    async with server.get_async_client() as async_client:
+        yield async_client
 
 
 @pytest.mark.asyncio