From 0c7a255aad52a2bad5cb1d085e28da095450a0bf Mon Sep 17 00:00:00 2001 From: Matthias Gehre Date: Thu, 9 Apr 2026 09:22:11 -0600 Subject: [PATCH 1/6] Fix: make upstream tag fetch resilient to failures git fetch upstream --tags and git describe can fail if the upstream repo is unreachable or no tags are reachable from HEAD. Use || to avoid aborting the workflow. Signed-off-by: Matthias Gehre --- .github/workflows/build-rocm-wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-rocm-wheels.yml b/.github/workflows/build-rocm-wheels.yml index 37753e75db61..b57ad25af464 100644 --- a/.github/workflows/build-rocm-wheels.yml +++ b/.github/workflows/build-rocm-wheels.yml @@ -37,8 +37,8 @@ jobs: - name: Fetch upstream tags for versioning run: | git remote add upstream https://github.com/vllm-project/vllm.git || true - git fetch upstream --tags - echo "Git describe: $(git describe --tags --always)" + git fetch upstream --tags || echo "Warning: could not fetch upstream tags" + git describe --tags --always || echo "Warning: git describe failed" - name: Free disk space uses: jlumbroso/free-disk-space@main From b4eeb2774cec3346f07079e2ec6844ed0f3ec501 Mon Sep 17 00:00:00 2001 From: Matthias Gehre Date: Thu, 9 Apr 2026 09:53:19 -0600 Subject: [PATCH 2/6] Adapt wheel workflow for gfx11: trigger on gfx11, build-only Trigger on push to gfx11 instead of main/matthias.awq_gemv. Remove create-release and publish-to-gh-pages jobs. Wheel is available as a GitHub Actions artifact. Signed-off-by: Matthias Gehre --- .github/workflows/build-rocm-wheels.yml | 197 +----------------------- 1 file changed, 1 insertion(+), 196 deletions(-) diff --git a/.github/workflows/build-rocm-wheels.yml b/.github/workflows/build-rocm-wheels.yml index b57ad25af464..d596cae47c99 100644 --- a/.github/workflows/build-rocm-wheels.yml +++ b/.github/workflows/build-rocm-wheels.yml @@ -2,12 +2,8 @@ name: Build ROCm Wheels on: push: - tags: - - 'v*' branches: - - main - - 'matthias.awq_gemv' - - 'release/*' + - gfx11 workflow_dispatch: inputs: pytorch_index: @@ -184,194 +180,3 @@ jobs: path: dist/*.whl retention-days: 90 - - name: Generate wheel index - run: | - python scripts/generate-wheel-index.py \ - --wheels-dir dist \ - --output-dir gh-pages \ - --base-url "https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/wheels" - - - name: Upload index artifact - uses: actions/upload-artifact@v4 - with: - name: wheel-index - path: gh-pages/ - retention-days: 90 - - create-release: - needs: build-wheel - runs-on: ubuntu-latest - permissions: - contents: write - outputs: - release_tag: ${{ steps.create-tag.outputs.tag }} - - steps: - - name: Download wheel artifact - uses: actions/download-artifact@v4 - with: - name: vllm-rocm-wheel - path: dist/ - - - name: Determine release tag - id: create-tag - run: | - # shellcheck disable=SC2193 # github.ref is substituted by Actions - if [[ "${{ github.ref }}" == refs/tags/* ]]; then - TAG="${{ github.ref_name }}" - else - WHEEL=$(find dist -name '*.whl' -print -quit) - BASENAME=$(basename "$WHEEL" .whl) - # Extract version from wheel name: vllm--cp312-... - VERSION=$(echo "$BASENAME" | sed 's/^vllm-//; s/-cp[0-9].*//') - TAG="wheels/rocm-${VERSION}" - fi - echo "tag=$TAG" >> "$GITHUB_OUTPUT" - echo "Release tag: $TAG" - - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ steps.create-tag.outputs.tag }} - target_commitish: ${{ github.sha }} - files: dist/*.whl - prerelease: ${{ !startsWith(github.ref, 'refs/tags/') }} - body: | - ## vLLM ROCm Wheel - - Built for ROCm architectures: `${{ env.PYTORCH_ROCM_ARCH }}` - - ### Installation - ```bash - pip install torch --extra-index-url https://rocm.nightlies.amd.com/v2/gfx1151 - pip install vllm --extra-index-url https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/wheels - ``` - - publish-to-gh-pages: - needs: create-release - runs-on: ubuntu-latest - permissions: - contents: write - pages: write - - steps: - - name: Download wheel artifact - uses: actions/download-artifact@v4 - with: - name: vllm-rocm-wheel - path: new-wheels/ - - - name: Generate index pointing to GitHub Releases - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - BASE_URL: "https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/wheels" - RELEASE_TAG: ${{ needs.create-release.outputs.release_tag }} - run: | - pip install packaging - - cat > /tmp/generate_index.py << 'PYEOF' - import os, json, subprocess - from pathlib import Path - from datetime import datetime - from urllib.parse import quote - - REPO = os.environ["REPO"] - BASE_URL = os.environ["BASE_URL"] - RELEASE_TAG = os.environ.get("RELEASE_TAG", "latest") - - PACKAGE_INDEX_TEMPLATE = """ - - {package_name} - - {items} - - - """ - - def parse_wheel_name(filename): - base = filename.removesuffix(".whl") - parts = base.split("-") - for i, part in enumerate(parts): - if part[0].isdigit(): - return "-".join(parts[:i]).replace("_", "-") - return parts[0].replace("_", "-") - - # Collect wheels from all GitHub Releases - result = subprocess.run( - ["gh", "release", "list", "--repo", REPO, "--limit", "100", "--json", "tagName"], - capture_output=True, text=True - ) - releases = json.loads(result.stdout) if result.returncode == 0 else [] - print(f"Found {len(releases)} releases") - - wheels = {} # pkg_name -> [(whl_name, download_url)] - for rel in releases: - tag = rel["tagName"] - result = subprocess.run( - ["gh", "release", "view", tag, "--repo", REPO, "--json", "assets"], - capture_output=True, text=True - ) - if result.returncode != 0: - continue - assets = json.loads(result.stdout).get("assets", []) - for asset in assets: - name = asset["name"] - url = asset["url"] - if name.endswith(".whl"): - pkg = parse_wheel_name(name) - wheels.setdefault(pkg, []).append((name, url)) - - # Include new wheels from this build if not already found in releases - new_dir = Path("new-wheels") - if new_dir.exists(): - for whl in new_dir.glob("*.whl"): - pkg = parse_wheel_name(whl.name) - release_names = {w[0] for w in wheels.get(pkg, [])} - if whl.name not in release_names: - url = f"https://github.com/{REPO}/releases/download/{quote(RELEASE_TAG, safe='')}/{quote(whl.name, safe='')}" - wheels.setdefault(pkg, []).append((whl.name, url)) - - print(f"Total: {sum(len(v) for v in wheels.values())} wheels in {len(wheels)} packages") - - out = Path("gh-pages/wheels") - out.mkdir(parents=True, exist_ok=True) - - for pkg_name, whl_list in wheels.items(): - pkg_dir = out / pkg_name - pkg_dir.mkdir(exist_ok=True) - links = [] - for whl_name, url in sorted(whl_list, key=lambda x: x[0], reverse=True): - links.append(f' {whl_name}
') - (pkg_dir / "index.html").write_text( - PACKAGE_INDEX_TEMPLATE.format(package_name=pkg_name, items="\n".join(links)) - ) - - top_index = f""" - - ROCm Wheels - -

ROCm Wheels

-

Pre-built wheels for AMD ROCm (gfx11 / Strix Halo / RDNA3+)

-

Generated: {datetime.now().isoformat()}

-

Installation

- 
pip install torch --extra-index-url https://rocm.nightlies.amd.com/v2/gfx1151
-          pip install vllm --extra-index-url {BASE_URL}
-

Available Packages

- """ - for p in sorted(wheels): - top_index += f' {p}
\n' - top_index += " \n" - (out / "index.html").write_text(top_index) - print("Index generated successfully") - PYEOF - - python /tmp/generate_index.py - - - name: Deploy to gh-pages - uses: peaceiris/actions-gh-pages@v4 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./gh-pages/wheels - destination_dir: wheels - keep_files: false From 695453a1f244b7fd01ab9189ad1beeb29ef68411 Mon Sep 17 00:00:00 2001 From: Matthias Gehre Date: Thu, 9 Apr 2026 16:20:15 -0600 Subject: [PATCH 3/6] Reduce default ROCM_ARCH to gfx1150;gfx1151 Signed-off-by: Matthias Gehre --- .github/workflows/build-rocm-wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-rocm-wheels.yml b/.github/workflows/build-rocm-wheels.yml index d596cae47c99..61d71f4b7dd4 100644 --- a/.github/workflows/build-rocm-wheels.yml +++ b/.github/workflows/build-rocm-wheels.yml @@ -12,12 +12,12 @@ on: required: false rocm_arch: description: 'ROCm architecture (e.g., gfx1151, gfx1150;gfx1151)' - default: 'gfx1150;gfx1151;gfx1152;gfx1153' + default: 'gfx1150;gfx1151' required: false env: PYTORCH_INDEX_URL: ${{ github.event.inputs.pytorch_index || 'https://rocm.nightlies.amd.com/v2/gfx1151' }} - PYTORCH_ROCM_ARCH: ${{ github.event.inputs.rocm_arch || 'gfx1150;gfx1151;gfx1152;gfx1153' }} + PYTORCH_ROCM_ARCH: ${{ github.event.inputs.rocm_arch || 'gfx1150;gfx1151' }} jobs: build-wheel: From 372b22588fa65e2ae8df994e24d7156f67d60de1 Mon Sep 17 00:00:00 2001 From: Matthias Gehre Date: Thu, 9 Apr 2026 18:03:52 -0600 Subject: [PATCH 4/6] Add PR trigger and S3 wheel upload for gfx11 branch - Trigger workflow on PRs targeting gfx11 (build-only) - On push to gfx11, upload wheel to S3 via OIDC + boto3 - S3 upload gated on ROCm org to skip on forks Signed-off-by: Matthias Gehre --- .github/workflows/build-rocm-wheels.yml | 32 ++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-rocm-wheels.yml b/.github/workflows/build-rocm-wheels.yml index 61d71f4b7dd4..88192c973be2 100644 --- a/.github/workflows/build-rocm-wheels.yml +++ b/.github/workflows/build-rocm-wheels.yml @@ -4,6 +4,9 @@ on: push: branches: - gfx11 + pull_request: + branches: + - gfx11 workflow_dispatch: inputs: pytorch_index: @@ -22,7 +25,10 @@ env: jobs: build-wheel: runs-on: ubuntu-latest - + permissions: + id-token: write + contents: read + steps: - name: Checkout code uses: actions/checkout@v4 @@ -180,3 +186,27 @@ jobs: path: dist/*.whl retention-days: 90 + - name: Configure AWS credentials via OIDC + if: github.event_name == 'push' && github.repository_owner == 'ROCm' + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::317668459450:role/therock-aig-embd-gfx11-wheels-s3-oidc + aws-region: us-east-1 + + - name: Upload wheel to S3 + if: github.event_name == 'push' && github.repository_owner == 'ROCm' + run: | + pip install boto3 + python - <<'PYEOF' + import boto3, glob + + s3 = boto3.client("s3") + bucket = "aig-embd-gfx11-wheels" + + for whl in glob.glob("dist/*.whl"): + key = whl.split("/")[-1] + print(f"Uploading {key} to s3://{bucket}/{key}") + s3.upload_file(whl, bucket, key) + print(f" Done") + PYEOF + From b12094062fd17f6d246f0f2bea29a815a0092646 Mon Sep 17 00:00:00 2001 From: Matthias Gehre Date: Fri, 10 Apr 2026 00:54:54 -0600 Subject: [PATCH 5/6] Fix AWS credentials: use us-east-2, pin action to v6.0.0 Signed-off-by: Matthias Gehre --- .github/workflows/build-rocm-wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-rocm-wheels.yml b/.github/workflows/build-rocm-wheels.yml index 88192c973be2..cbf567e53d81 100644 --- a/.github/workflows/build-rocm-wheels.yml +++ b/.github/workflows/build-rocm-wheels.yml @@ -188,10 +188,10 @@ jobs: - name: Configure AWS credentials via OIDC if: github.event_name == 'push' && github.repository_owner == 'ROCm' - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: role-to-assume: arn:aws:iam::317668459450:role/therock-aig-embd-gfx11-wheels-s3-oidc - aws-region: us-east-1 + aws-region: us-east-2 - name: Upload wheel to S3 if: github.event_name == 'push' && github.repository_owner == 'ROCm' From 654c66b6a1b2a23b5fca313ec350d0fd2624a62e Mon Sep 17 00:00:00 2001 From: Matthias Gehre Date: Fri, 10 Apr 2026 01:01:47 -0600 Subject: [PATCH 6/6] Enable FLASH_ATTN backend with upstream flash-attn CK on ROCm The FLASH_ATTN backend in vLLM V1 was tightly coupled to vllm_flash_attn (CUDA-only). On ROCm, fa_utils.py imported upstream flash_attn_varlen_func but the forward path passed vllm-specific kwargs (out, fa_version, scheduler_metadata, etc.) that the upstream API doesn't accept, and get_flash_attn_version() returned None causing an assertion failure. Changes: - Replace raw upstream import with a wrapper in fa_utils.py that translates vLLM's calling convention to the upstream _wrapped_flash_attn_varlen_forward API, handling seqused_k -> cu_seqlens_k conversion for paged KV cache - Return FA version 2 on ROCm when upstream flash-attn is available - Set block_size to MultipleOf(128) on ROCm (CK kernel requirement) The Triton AMD backend (FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE) does NOT support paged attention, so FLASH_ATTN as LLM backend requires the CK backend (FLASH_ATTENTION_TRITON_AMD_ENABLE unset). Validated on gfx1151 with Qwen2.5-1.5B-Instruct: correct text generation with paged KV cache, concurrent requests, and multi-token decode. Signed-off-by: Matthias Gehre --- vllm/v1/attention/backends/fa_utils.py | 95 +++++++++++++++++++++++- vllm/v1/attention/backends/flash_attn.py | 4 + 2 files changed, 96 insertions(+), 3 deletions(-) diff --git a/vllm/v1/attention/backends/fa_utils.py b/vllm/v1/attention/backends/fa_utils.py index a4423b301d69..9942926d3514 100644 --- a/vllm/v1/attention/backends/fa_utils.py +++ b/vllm/v1/attention/backends/fa_utils.py @@ -31,7 +31,96 @@ get_scheduler_metadata = xpu_ops.get_scheduler_metadata # type: ignore[assignment] elif current_platform.is_rocm(): try: - from flash_attn import flash_attn_varlen_func # type: ignore[no-redef] + import torch + from flash_attn.flash_attn_interface import ( + _wrapped_flash_attn_varlen_forward, + ) + + def flash_attn_varlen_func( # type: ignore[no-redef,misc] + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens_q: torch.Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + softmax_scale: float, + causal: bool = True, + window_size: "list[int] | None" = None, + block_table: "torch.Tensor | None" = None, + softcap: float = 0.0, + alibi_slopes: "torch.Tensor | None" = None, + out: "torch.Tensor | None" = None, + seqused_k: "torch.Tensor | None" = None, + cu_seqlens_k: "torch.Tensor | None" = None, + return_softmax_lse: bool = False, + # vLLM-specific kwargs not supported by upstream, ignored: + scheduler_metadata: Any = None, + fa_version: Any = None, + q_descale: Any = None, + k_descale: Any = None, + v_descale: Any = None, + num_splits: Any = None, + s_aux: Any = None, + ) -> "torch.Tensor | tuple[torch.Tensor, torch.Tensor]": + """Wrapper that adapts vLLM's calling convention to upstream + flash-attn 2.x API on ROCm. + + The upstream _wrapped_flash_attn_varlen_forward accepts both + cu_seqlens_k and seqused_k. When using paged KV cache + (block_table is set), seqused_k gives per-sequence K lengths + and cu_seqlens_k can be a dummy tensor. + """ + if window_size is None: + window_size_left = -1 + window_size_right = -1 + else: + window_size_left = window_size[0] + window_size_right = window_size[1] + + # Build cu_seqlens_k if not provided. + # For paged KV cache, cu_seqlens_k must be the cumulative + # sum of actual sequence lengths (seqused_k), NOT zeros. + if cu_seqlens_k is None: + if seqused_k is not None: + cu_seqlens_k = torch.zeros( + seqused_k.shape[0] + 1, + dtype=torch.int32, + device=q.device, + ) + cu_seqlens_k[1:] = seqused_k.cumsum(0) + else: + batch_size = cu_seqlens_q.shape[0] - 1 + cu_seqlens_k = torch.zeros( + batch_size + 1, dtype=torch.int32, device=q.device + ) + + out_padded, softmax_lse, _, _ = _wrapped_flash_attn_varlen_forward( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + 0.0, # dropout_p + softmax_scale, + causal, + window_size_left=window_size_left, + window_size_right=window_size_right, + softcap=softcap, + alibi_slopes=alibi_slopes, + return_softmax=False, + block_table=block_table, + seqused_k=seqused_k, + ) + if out is not None: + out.copy_(out_padded) + result = out + else: + result = out_padded + if return_softmax_lse: + return result, softmax_lse + return result # Mark that upstream flash-attn is available on ROCm _ROCM_FLASH_ATTN_AVAILABLE = True @@ -59,8 +148,8 @@ def get_flash_attn_version( if current_platform.is_xpu(): return 2 if current_platform.is_rocm(): - # ROCm doesn't use vllm_flash_attn; return None to skip fa_version arg - return None + # ROCm uses upstream flash-attn which is FA2-compatible + return 2 if _ROCM_FLASH_ATTN_AVAILABLE else None try: from vllm.vllm_flash_attn.flash_attn_interface import ( fa_version_unsupported_reason, diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index 245995be2642..7e6b06a0a63f 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -43,6 +43,7 @@ from vllm.config.cache import CacheDType from vllm.distributed.parallel_state import get_dcp_group from vllm.logger import init_logger +from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability from vllm.utils.math_utils import cdiv, round_up from vllm.v1.attention.backend import ( @@ -70,6 +71,9 @@ class FlashAttentionBackend(AttentionBackend): @staticmethod def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + if current_platform.is_rocm(): + # CK flash-attn requires block_size divisible by 128 + return [MultipleOf(128)] vllm_config = get_current_vllm_config() model_config = vllm_config.model_config cache_config = vllm_config.cache_config