From 911b51b69f92b7cbccc54453991a64eac72f2366 Mon Sep 17 00:00:00 2001
From: Micah Williamson <micah.williamson@amd.com>
Date: Tue, 27 Jan 2026 21:32:31 -0600
Subject: [PATCH] [ROCm][CI] Add TORCH_NCCL_BLOCKING_WAIT For Distributed Tests
 (A100) (#32891)

Signed-off-by: Micah Williamson <micah.williamson@amd.com>
---
 .buildkite/test-amd.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml
index 6638e576a..a14dcd030 100644
--- a/.buildkite/test-amd.yaml
+++ b/.buildkite/test-amd.yaml
@@ -1509,6 +1509,9 @@ steps:
   source_file_dependencies:
   - vllm/
   commands:
+  # Work around HIP bug tracked here: https://github.com/ROCm/hip/issues/3876
+  # TODO: Remove when the bug is fixed in a future ROCm release
+  - export TORCH_NCCL_BLOCKING_WAIT=1
   # NOTE: don't test llama model here, it seems hf implementation is buggy
   # see https://github.com/vllm-project/vllm/pull/5689 for details
   - pytest -v -s distributed/test_custom_all_reduce.py