From 5b77bc81a9b191b32ec7e31209add63d4de539cd Mon Sep 17 00:00:00 2001 From: Akash Verma Date: Thu, 27 Aug 2026 22:52:03 +0000 Subject: [PATCH 1/7] [ROCm][CI] Re-enable ROCm CI on new design with ciflow/rocm tag support Renames integration_test_features_rocm.yaml to integration_test_rocm.yaml and re-enables the ROCm matrix in set-matrix.yaml, which was temporarily disabled while ROCm jobs were red. Key changes: - Adds ciflow/rocm/* tag trigger to integration_test_rocm.yaml so ROCm jobs can be triggered on demand from PRs via tag, in addition to the existing push-to-main trigger. - Restores the ROCm matrix entry in set-matrix.yaml (runner: linux.rocm.gpu.ecosystem.mi350.8, ROCm 10.0, rocm10.0 nightly index). - Fixes the matrix selection logic so that when gpu-arch=rocm is explicitly requested (e.g. from a ciflow/rocm tag run via workflow_call), FULL_MATRIX is used rather than falling through to CUDA-only. Previously IS_MAIN_PUSH/IS_SCHEDULE were the only conditions that included ROCm, which meant tag-triggered workflow_call runs produced an empty matrix. Test Plan: Push a ciflow/rocm/* tag to trigger the ROCm workflow and verify jobs land on linux.rocm.gpu.ecosystem.mi350.8 with ROCm 10.0. Co-authored-by: AI assistant Co-authored-by: Cursor --- ...s_rocm.yaml => integration_test_rocm.yaml} | 2 + .github/workflows/set-matrix.yaml | 65 ++++++++----------- 2 files changed, 30 insertions(+), 37 deletions(-) rename .github/workflows/{integration_test_features_rocm.yaml => integration_test_rocm.yaml} (91%) diff --git a/.github/workflows/integration_test_features_rocm.yaml b/.github/workflows/integration_test_rocm.yaml similarity index 91% rename from .github/workflows/integration_test_features_rocm.yaml rename to .github/workflows/integration_test_rocm.yaml index a2ecd8cec3..f4c4cb4323 100644 --- a/.github/workflows/integration_test_features_rocm.yaml +++ b/.github/workflows/integration_test_rocm.yaml @@ -3,6 +3,8 @@ name: Integration Tests - Features (ROCm) on: push: branches: [ main ] + tags: + - ciflow/rocm/* paths-ignore: - 'torchtitan/experiments/**' diff --git a/.github/workflows/set-matrix.yaml b/.github/workflows/set-matrix.yaml index d15de4a2f0..d2ba386e74 100644 --- a/.github/workflows/set-matrix.yaml +++ b/.github/workflows/set-matrix.yaml @@ -42,7 +42,7 @@ jobs: IS_SCHEDULE: ${{ github.event_name == 'schedule' }} # Default CUDA & ROCm runners - DEFAULT_ROCM_RUNNER: linux.rocm.gpu.ecosystem.gfx950.8 + DEFAULT_ROCM_RUNNER: linux.rocm.gpu.ecosystem.mi350.8 DEFAULT_CUDA_RUNNER: linux.g5.48xlarge.nvidia.gpu # Input CUDA & ROCm runners @@ -78,46 +78,38 @@ jobs: EOF )" - # Temporarily disable ROCm CI while the ROCm jobs are red. - # Re-enable by removing this override and uncommenting the original - # ROCm matrix block below. - # ROCm remains disabled. When enabled, it should run from a separate - # AMD workflow triggered only by pushes to main. - ROCM_MATRIX="" - FULL_MATRIX="$CUDA_MATRIX" - - # Original ROCm matrix definition: - # if [[ "${{ inputs.is-experimental }}" == "true" ]]; then - # ROCM_MATRIX="" - # FULL_MATRIX="$CUDA_MATRIX" - # else - # ROCM_MATRIX="$(cat < matrix.json < matrix.json < matrix.filtered.json + jq -c --arg arch "$GPU_ARCH_INPUT" '{include: [.include[] | select(.["gpu-arch-type"] == $arch)]}' matrix.json > matrix.filtered.json mv matrix.filtered.json matrix.json ;; *) - echo "::error::Invalid gpu-arch '${GPU_ARCH}'. Expected one of: all, cuda, rocm." >&2 + echo "::error::Invalid gpu-arch '${GPU_ARCH_INPUT}'. Expected one of: all, cuda, rocm." >&2 exit 1 ;; esac From f64a06834ce32833de1cd1e46a7d1db443aec220 Mon Sep 17 00:00:00 2001 From: Akash Verma Date: Thu, 27 Aug 2026 22:55:08 +0000 Subject: [PATCH 2/7] [ROCm][CI] Add ROCm test status badge to README Adds an AMD GPU (ROCm) row to the Test status table pointing to the integration_test_rocm.yaml workflow. Co-authored-by: AI assistant Co-authored-by: Cursor --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8a7503c78a..530e82c474 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ We look forward to your contributions! | --- | --- | --- | | CPU | - | [![CPU Unit Test](https://github.com/pytorch/torchtitan/actions/workflows/unit_test_cpu.yaml/badge.svg?branch=main)](https://github.com/pytorch/torchtitan/actions/workflows/unit_test_cpu.yaml?query=branch%3Amain) | | NVIDIA GPU | [![Integration Tests](https://github.com/pytorch/torchtitan/actions/workflows/integration_test.yaml/badge.svg?branch=main)](https://github.com/pytorch/torchtitan/actions/workflows/integration_test.yaml?query=branch%3Amain) [![H100 Tests](https://github.com/pytorch/torchtitan/actions/workflows/integration_test_h100.yaml/badge.svg?branch=main)](https://github.com/pytorch/torchtitan/actions/workflows/integration_test_h100.yaml?query=branch%3Amain) | [![GPU Unit Tests](https://github.com/pytorch/torchtitan/actions/workflows/unit_test_gpu.yaml/badge.svg?branch=main)](https://github.com/pytorch/torchtitan/actions/workflows/unit_test_gpu.yaml?query=branch%3Amain) | +| AMD GPU (ROCm) | [![ROCm Integration Tests](https://github.com/pytorch/torchtitan/actions/workflows/integration_test_rocm.yaml/badge.svg?branch=main)](https://github.com/pytorch/torchtitan/actions/workflows/integration_test_rocm.yaml?query=branch%3Amain) | - | ## Llama 3.1 training From c16978e656fc4d47942e39be655878a38dae8e9f Mon Sep 17 00:00:00 2001 From: Akash Verma Date: Thu, 27 Aug 2026 22:59:53 +0000 Subject: [PATCH 3/7] [ROCm][CI] Add ciflow/rocm to probot push tags Registers ciflow/rocm with pytorch-probot so it can push the ciflow/rocm tag to trigger ROCm CI jobs on demand. Co-authored-by: AI assistant Co-authored-by: Cursor --- .github/pytorch-probot.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/pytorch-probot.yml b/.github/pytorch-probot.yml index 3f40dcd327..779c9b268d 100644 --- a/.github/pytorch-probot.yml +++ b/.github/pytorch-probot.yml @@ -1,5 +1,6 @@ ciflow_push_tags: - ciflow/b200 + - ciflow/rocm - ciflow/h100.8 - ciflow/rl labeler_config: labeler.yml From 95b510ec31ad93b890abcc3f4cb281db1ea00fdb Mon Sep 17 00:00:00 2001 From: Akash Verma Date: Thu, 27 Aug 2026 23:20:06 +0000 Subject: [PATCH 4/7] [ROCm][CI] Fix workflow name and job name in integration_test_rocm.yaml Renames the workflow from "Integration Tests - Features (ROCm)" to "Integration Tests (ROCm)" and the job from "features" to "rocm", since the workflow runs both features and models test suites. This fixes the confusing "features / 8 GPU Integration ..." prefix shown for both jobs in the PR checks UI. Co-authored-by: AI assistant Co-authored-by: Cursor --- .github/workflows/integration_test_rocm.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_test_rocm.yaml b/.github/workflows/integration_test_rocm.yaml index f4c4cb4323..cc7b17c7c7 100644 --- a/.github/workflows/integration_test_rocm.yaml +++ b/.github/workflows/integration_test_rocm.yaml @@ -1,4 +1,4 @@ -name: Integration Tests - Features (ROCm) +name: Integration Tests (ROCm) on: push: @@ -13,7 +13,7 @@ permissions: contents: read jobs: - features: + rocm: uses: ./.github/workflows/integration_test.yaml with: gpu-arch: rocm From 138563765ebc411e93b1bb984ed02b38f0005438 Mon Sep 17 00:00:00 2001 From: Akash Verma Date: Fri, 28 Aug 2026 19:24:58 +0000 Subject: [PATCH 5/7] trigger CI From fa3d1f2ec8cbe3d3eebdbc7e1eec880870a3c4a6 Mon Sep 17 00:00:00 2001 From: Akash Verma Date: Tue, 1 Sep 2026 16:29:14 +0000 Subject: [PATCH 6/7] trigger CI From ac7cbed3a9a6fab2f5097e1689eae3c993ada2af Mon Sep 17 00:00:00 2001 From: Rishi Sinha <60558850+rishisinhanj@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:02:54 -0400 Subject: [PATCH 7/7] [ROCm][CI] Guard HIPBLASLT_TENSILE_LIBPATH and disable expandable_segments (#7) --- .github/workflows/integration_test.yaml | 12 ++++++++++-- run_train.sh | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 9bb2d9a032..509a0e266d 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -104,7 +104,11 @@ jobs: USE_CPP=0 python -m pip install --pre torchao --index-url ${{ matrix.index-url }} if [[ "${{ matrix.gpu-arch-type }}" == "rocm" ]]; then - export HIPBLASLT_TENSILE_LIBPATH="$(python -c 'import os, torch; print(os.path.join(os.path.dirname(torch.__file__), "lib", "hipblaslt", "library"))')" + HIPBLASLT_LIB_DIR="$(python -c 'import os, torch; print(os.path.join(os.path.dirname(torch.__file__), "lib", "hipblaslt", "library"))')" + if [ -d "${HIPBLASLT_LIB_DIR}" ]; then + export HIPBLASLT_TENSILE_LIBPATH="${HIPBLASLT_LIB_DIR}" + fi + export PYTORCH_ALLOC_CONF="expandable_segments:False" fi sudo mkdir -p "$RUNNER_ARTIFACT_DIR" @@ -171,7 +175,11 @@ jobs: USE_CPP=0 python -m pip install --pre torchao --index-url ${{ matrix.runner_config['index-url'] }} if [[ "${{ matrix.runner_config['gpu-arch-type'] }}" == "rocm" ]]; then - export HIPBLASLT_TENSILE_LIBPATH="$(python -c 'import os, torch; print(os.path.join(os.path.dirname(torch.__file__), "lib", "hipblaslt", "library"))')" + HIPBLASLT_LIB_DIR="$(python -c 'import os, torch; print(os.path.join(os.path.dirname(torch.__file__), "lib", "hipblaslt", "library"))')" + if [ -d "${HIPBLASLT_LIB_DIR}" ]; then + export HIPBLASLT_TENSILE_LIBPATH="${HIPBLASLT_LIB_DIR}" + fi + export PYTORCH_ALLOC_CONF="expandable_segments:False" fi sudo mkdir -p "$RUNNER_ARTIFACT_DIR" diff --git a/run_train.sh b/run_train.sh index df3ab6f7c6..094c73fb88 100755 --- a/run_train.sh +++ b/run_train.sh @@ -38,7 +38,7 @@ if [ "$COMM_MODE" = "fake_backend" ]; then NGPU="${NGPU}" LOCAL_RANK=0 python3 -m torchtitan.train --module ${MODULE} --config ${CONFIG} --comm.mode=fake_backend --training.steps 1 "$@" else # Normal training with torchrun - PYTORCH_ALLOC_CONF="expandable_segments:True" \ + PYTORCH_ALLOC_CONF="${PYTORCH_ALLOC_CONF:-expandable_segments:True}" \ TORCHFT_LIGHTHOUSE=${TORCHFT_LIGHTHOUSE} \ torchrun --nproc_per_node=${NGPU} --rdzv_backend c10d --rdzv_endpoint="localhost:0" \ --local-ranks-filter ${LOG_RANK} --role rank --tee 3 \