feat: replace Git LFS pushes with direct Batch API migration

.env.example

@@ -6,5 +6,14 @@ GHMLFS_TARGET_ORGANIZATION=
 GHMLFS_TARGET_HOSTNAME=
 GHMLFS_TARGET_TOKEN=
 GHMLFS_WORKERS=
+GHMLFS_SEARCH_DEPTH=1
+GHMLFS_BATCH_SIZE=100
+GHMLFS_UPLOAD_PARALLEL=8
+GHMLFS_CHECK_HASHES=false
+GHMLFS_DRY_RUN=false
+GHMLFS_NO_FINAL_CHECK=false
+GHMLFS_RETRY_MAX=3
+GHMLFS_RETRY_DELAY=1s
+GHMLFS_STATE_DIR=.lfs-migrate
 GHMLFS_WORK_DIR=
-GHMLFS_FILE=${GHMLFS_SOURCE_ORGANIZATION}_lfs.csv
+GHMLFS_FILE=

.github/scripts/seed-live-lfs-fixtures.sh

@@ -0,0 +1,167 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+set +x
+
+organization="${LIVE_LFS_SOURCE_ORGANIZATION:-}"
+prefix="${LIVE_LFS_FIXTURE_PREFIX:-ghmlfs-live-fixture}"
+repository_count="${LIVE_LFS_FIXTURE_REPOSITORIES:-3}"
+object_count="${LIVE_LFS_FIXTURE_OBJECTS:-12}"
+dry_run="${LIVE_LFS_FIXTURE_DRY_RUN:-false}"
+
+if [[ -z "$organization" ]]; then
+  printf 'Required environment variable is not set: LIVE_LFS_SOURCE_ORGANIZATION\n' >&2
+  exit 2
+fi
+if [[ ! "$organization" =~ ^[A-Za-z0-9][A-Za-z0-9-]*$ ]]; then
+  printf 'Invalid source organization: %s\n' "$organization" >&2
+  exit 2
+fi
+if [[ ! "$prefix" =~ ^[a-z0-9][a-z0-9.-]*$ || ${#prefix} -gt 80 ]]; then
+  printf 'Invalid fixture repository prefix: %s\n' "$prefix" >&2
+  exit 2
+fi
+if [[ ! "$repository_count" =~ ^[0-9]+$ || "$repository_count" -lt 1 || "$repository_count" -gt 10 ]]; then
+  printf 'LIVE_LFS_FIXTURE_REPOSITORIES must be between 1 and 10.\n' >&2
+  exit 2
+fi
+if [[ ! "$object_count" =~ ^[0-9]+$ || "$object_count" -lt 1 || "$object_count" -gt 100 ]]; then
+  printf 'LIVE_LFS_FIXTURE_OBJECTS must be between 1 and 100.\n' >&2
+  exit 2
+fi
+if [[ "$dry_run" != true && "$dry_run" != false ]]; then
+  printf 'LIVE_LFS_FIXTURE_DRY_RUN must be true or false.\n' >&2
+  exit 2
+fi
+
+repository_names=()
+for ((repository_index = 1; repository_index <= repository_count; repository_index++)); do
+  repository_names+=("$(printf '%s-%02d' "$prefix" "$repository_index")")
+done
+
+if [[ "$dry_run" == true ]]; then
+  printf '%s\n' "${repository_names[@]}"
+  exit 0
+fi
+
+if [[ -z "${GH_TOKEN:-}" ]]; then
+  printf 'Required environment variable is not set: GH_TOKEN\n' >&2
+  exit 2
+fi
+for command in gh git; do
+  if ! command -v "$command" >/dev/null 2>&1; then
+    printf 'Required command is not available: %s\n' "$command" >&2
+    exit 2
+  fi
+done
+if ! git lfs version >/dev/null 2>&1; then
+  printf 'Git LFS is required.\n' >&2
+  exit 2
+fi
+
+work_dir="$(mktemp -d "${TMPDIR:-/tmp}/gh-migrate-lfs-fixtures.XXXXXX")"
+cleanup() {
+  status=$?
+  trap - EXIT
+  rm -rf "$work_dir"
+  exit "$status"
+}
+trap cleanup EXIT
+
+askpass="$work_dir/git-askpass.sh"
+cat >"$askpass" <<'EOF'
+#!/usr/bin/env bash
+case "$1" in
+  *Username*) printf '%s\n' 'x-access-token' ;;
+  *) printf '%s\n' "$GHMLFS_GIT_TOKEN" ;;
+esac
+EOF
+chmod 700 "$askpass"
+
+git_with_auth() {
+  GIT_ASKPASS="$askpass" \
+  GIT_TERMINAL_PROMPT=0 \
+  GHMLFS_GIT_TOKEN="$GH_TOKEN" \
+    git "$@"
+}
+
+for repository_index in "${!repository_names[@]}"; do
+  repository="${repository_names[$repository_index]}"
+  repository_url="https://github.com/$organization/$repository.git"
+  created=false
+
+  if ! GH_TOKEN="$GH_TOKEN" gh api --silent "repos/$organization/$repository" 2>/dev/null; then
+    printf 'Creating private fixture %s/%s\n' "$organization" "$repository" >&2
+    GH_TOKEN="$GH_TOKEN" gh api \
+      --silent \
+      --method POST \
+      "orgs/$organization/repos" \
+      -f "name=$repository" \
+      -f 'description=Deterministic Git LFS fixture managed by gh-migrate-lfs' \
+      -F private=true \
+      -F has_issues=false \
+      -F has_projects=false \
+      -F has_wiki=false
+    created=true
+  else
+    printf 'Refreshing fixture %s/%s\n' "$organization" "$repository" >&2
+  fi
+
+  repository_dir="$work_dir/$repository"
+  git_with_auth clone "$repository_url" "$repository_dir" >&2
+
+  marker="$repository_dir/.github/gh-migrate-lfs-fixture"
+  if [[ "$created" == false ]] && ! grep -Fxq 'managed-by=gh-migrate-lfs' "$marker" 2>/dev/null; then
+    if git -C "$repository_dir" rev-parse --verify HEAD >/dev/null 2>&1; then
+      printf 'Refusing to modify unmarked repository: %s/%s\n' "$organization" "$repository" >&2
+      exit 1
+    fi
+  fi
+
+  git -C "$repository_dir" checkout -B main >&2
+  git -C "$repository_dir" lfs install --local >&2
+  rm -rf "$repository_dir/fixtures"
+  mkdir -p "$repository_dir/.github"
+  printf 'managed-by=gh-migrate-lfs\nfixture-version=1\n' >"$marker"
+  printf 'fixtures/** filter=lfs diff=lfs merge=lfs -text\n' >"$repository_dir/.gitattributes"
+
+  for ((object_index = 1; object_index <= object_count; object_index++)); do
+    set_index=$(((object_index - 1) % 3 + 1))
+    case "$set_index" in
+      1) size_kib=64 ;;
+      2) size_kib=128 ;;
+      3) size_kib=256 ;;
+    esac
+    fixture_dir="$repository_dir/fixtures/set-$(printf '%02d' "$set_index")"
+    fixture_path="$fixture_dir/object-$(printf '%03d' "$object_index").bin"
+    mkdir -p "$fixture_dir"
+    seed="$organization/$repository/object-$object_index"
+    printf '%s' "$seed" >"$fixture_path"
+    seed_size=$(wc -c <"$fixture_path" | tr -d '[:space:]')
+    remaining=$((size_kib * 1024 - seed_size))
+    full_blocks=$((remaining / 1024))
+    tail_bytes=$((remaining % 1024))
+    if [[ "$full_blocks" -gt 0 ]]; then
+      dd if=/dev/zero bs=1024 count="$full_blocks" >>"$fixture_path" 2>/dev/null
+    fi
+    if [[ "$tail_bytes" -gt 0 ]]; then
+      dd if=/dev/zero bs=1 count="$tail_bytes" >>"$fixture_path" 2>/dev/null
+    fi
+  done
+
+  git -C "$repository_dir" config user.name 'gh-migrate-lfs fixture automation'
+  git -C "$repository_dir" config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+  git -C "$repository_dir" add .gitattributes .github/gh-migrate-lfs-fixture fixtures
+  if ! git -C "$repository_dir" diff --cached --quiet; then
+    git -C "$repository_dir" commit -m 'Refresh deterministic LFS fixtures' >&2
+    git_with_auth -C "$repository_dir" push --set-upstream origin main >&2
+  fi
+
+  tracked_objects=$(git -C "$repository_dir" lfs ls-files --all --name-only | wc -l | tr -d '[:space:]')
+  if [[ "$tracked_objects" -ne "$object_count" ]]; then
+    printf '%s/%s has %s LFS objects, expected %s.\n' \
+      "$organization" "$repository" "$tracked_objects" "$object_count" >&2
+    exit 1
+  fi
+  printf '%s\n' "$repository"
+done

.github/workflows/build.yml

@@ -7,7 +7,6 @@ on:
       - "*.md"
   pull_request:
     branches:
-
       - main
     paths-ignore:
       - "*.md"
@@ -22,10 +21,22 @@ jobs:
     env:
       CI: true
     steps:
-      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b #v4.1.4
-      - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 #v5.0.1
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
         with:
-          go-version: 1.21
-      - run: go get -v -t -d ./...
+          go-version-file: .go-version
+          cache: true
+      - run: go test -race ./...
+      - run: go vet ./...
+      - name: Check for dead code and static issues
+        run: go run honnef.co/go/tools/cmd/staticcheck@v0.8.1 ./...
+      - name: Check for duplicate code
+        shell: bash
+        run: |
+          duplicates="$(go run github.com/mibk/dupl@v1.1.0 -plumbing -t 100 .)"
+          if [[ -n "$duplicates" ]]; then
+            printf '%s\n' "$duplicates"
+            exit 1
+          fi
       - run: go build -v .
 

.github/workflows/live-transfer.yml

@@ -0,0 +1,113 @@
+name: live LFS transfer
+
+on:
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+
+concurrency:
+  group: live-lfs-transfer
+  cancel-in-progress: false
+
+jobs:
+  live-transfer:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    environment: live-lfs
+    steps:
+      - name: Check configuration
+        shell: bash
+        env:
+          SOURCE_APP_CLIENT_ID: ${{ vars.LIVE_LFS_SOURCE_APP_CLIENT_ID }}
+          TARGET_APP_CLIENT_ID: ${{ vars.LIVE_LFS_TARGET_APP_CLIENT_ID }}
+          SOURCE_ORGANIZATION: ${{ vars.LIVE_LFS_SOURCE_ORGANIZATION }}
+          SOURCE_REPOSITORIES: ${{ vars.LIVE_LFS_SOURCE_REPOSITORIES }}
+          TARGET_ORGANIZATION: ${{ vars.LIVE_LFS_TARGET_ORGANIZATION }}
+        run: |
+          for variable in SOURCE_APP_CLIENT_ID TARGET_APP_CLIENT_ID SOURCE_ORGANIZATION SOURCE_REPOSITORIES TARGET_ORGANIZATION; do
+            if [[ -z "${!variable}" ]]; then
+              printf 'Required live-lfs variable is not set: %s\n' "$variable" >&2
+              exit 1
+            fi
+          done
+
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          persist-credentials: false
+
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: .go-version
+          cache: true
+
+      - name: Create source installation token
+        id: source-token
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          client-id: ${{ vars.LIVE_LFS_SOURCE_APP_CLIENT_ID }}
+          private-key: ${{ secrets.LIVE_LFS_SOURCE_APP_PRIVATE_KEY }}
+          owner: ${{ vars.LIVE_LFS_SOURCE_ORGANIZATION }}
+          repositories: ${{ vars.LIVE_LFS_SOURCE_REPOSITORIES }}
+          permission-contents: read
+
+      - name: Create target installation token
+        id: target-token
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          client-id: ${{ vars.LIVE_LFS_TARGET_APP_CLIENT_ID }}
+          private-key: ${{ secrets.LIVE_LFS_TARGET_APP_PRIVATE_KEY }}
+          owner: ${{ vars.LIVE_LFS_TARGET_ORGANIZATION }}
+          permission-administration: write
+          permission-contents: write
+
+      - name: Verify live transfer
+        shell: bash
+        env:
+          LIVE_LFS_SOURCE_ORGANIZATION: ${{ vars.LIVE_LFS_SOURCE_ORGANIZATION }}
+          LIVE_LFS_SOURCE_REPOSITORIES: ${{ vars.LIVE_LFS_SOURCE_REPOSITORIES }}
+          LIVE_LFS_SOURCE_TOKEN: ${{ steps.source-token.outputs.token }}
+          LIVE_LFS_TARGET_ORGANIZATION: ${{ vars.LIVE_LFS_TARGET_ORGANIZATION }}
+          LIVE_LFS_TARGET_TOKEN: ${{ steps.target-token.outputs.token }}
+          LIVE_LFS_WORKERS: 3
+          LIVE_LFS_UPLOAD_PARALLEL: 4
+          LIVE_LFS_BATCH_SIZE: 5
+        run: bash scripts/live-transfer-test.sh
+
+      - name: Clean up target fallback
+        if: always()
+        shell: bash
+        env:
+          TARGET_ORGANIZATION: ${{ vars.LIVE_LFS_TARGET_ORGANIZATION }}
+          TARGET_TOKEN: ${{ steps.target-token.outputs.token }}
+        run: |
+          target_file=live-transfer-artifacts/target-repositories.txt
+          if [[ ! -f "$target_file" || -z "$TARGET_TOKEN" ]]; then
+            exit 0
+          fi
+          while IFS= read -r target_repository; do
+            if [[ ! "$target_repository" =~ ^ghmlfs-live-[a-z0-9.-]+$ ]]; then
+              printf 'Refusing to delete unexpected repository name: %s\n' "$target_repository" >&2
+              exit 1
+            fi
+            if error=$(GH_TOKEN="$TARGET_TOKEN" gh api \
+              --silent \
+              --method DELETE \
+              "repos/$TARGET_ORGANIZATION/$target_repository" 2>&1); then
+              printf '%s\tdeleted by workflow fallback\n' "$target_repository" \
+                >>live-transfer-artifacts/cleanup-status.txt
+            elif [[ "$error" != *"HTTP 404"* ]]; then
+              printf '%s\n' "$error" >&2
+              exit 1
+            fi
+          done <"$target_file"
+
+      - name: Upload live-transfer reports
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: live-lfs-transfer-${{ github.run_id }}-${{ github.run_attempt }}
+          path: live-transfer-artifacts/
+          if-no-files-found: warn
+          retention-days: 14

.github/workflows/release.yml

@@ -10,8 +10,16 @@ jobs:
   release:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b #v4.1.4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
-      - uses: cli/gh-extension-precompile@v2
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
         with:
-          go_version: 1.23
+          go-version-file: .go-version
+          cache: true
+
+      - run: go test ./...
+      - run: go vet ./...
+
+      - uses: cli/gh-extension-precompile@76961aa3bd1123d0a6fd42d0a41aca0696937c39 # v2.2.0
+        with:
+          go_version: 1.27.0

.go-version

@@ -1 +1 @@
-1.23.3
+1.27.0