[CI][Tracing] Fix race condition by adding server readiness check (#34364)
Attempt to resolve #34284: "Metrics Tracing (2GPU)" fails with a segmentation fault. Signed-off-by: emricksini-h <emrick.birivoutin@hcompany.ai>
This commit is contained in:
@@ -107,6 +107,22 @@ class FakeTraceService(TraceServiceServicer):
|
||||
self.evt.clear()
|
||||
|
||||
|
||||
def _wait_for_server_ready(address: str, timeout: float = 5.0) -> bool:
|
||||
"""Wait for the gRPC server to be ready to accept connections."""
|
||||
import socket
|
||||
import time
|
||||
|
||||
host, port = address.rsplit(":", 1)
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with socket.create_connection((host, int(port)), timeout=0.5):
|
||||
return True
|
||||
except (OSError, ConnectionRefusedError):
|
||||
time.sleep(0.1)
|
||||
return False
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def trace_service() -> Generator[FakeTraceService, None, None]:
|
||||
"""Fixture to set up a fake gRPC trace service."""
|
||||
@@ -116,6 +132,13 @@ def trace_service() -> Generator[FakeTraceService, None, None]:
|
||||
server.add_insecure_port(FAKE_TRACE_SERVER_ADDRESS)
|
||||
server.start()
|
||||
|
||||
# Wait for the server to be ready to accept connections
|
||||
if not _wait_for_server_ready(FAKE_TRACE_SERVER_ADDRESS):
|
||||
server.stop(grace=None)
|
||||
raise RuntimeError(
|
||||
f"Fake trace server failed to start on {FAKE_TRACE_SERVER_ADDRESS}"
|
||||
)
|
||||
|
||||
yield service
|
||||
|
||||
server.stop(grace=None)
|
||||
|
||||
Reference in New Issue
Block a user