ci-testing.yml

name: CI Testing

# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
on: # Trigger the workflow on push or pull request, but only for the main branch
  push:
    branches: [main]
  pull_request:
    branches: [main]
    types: [opened, reopened, ready_for_review, synchronize]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

defaults:
  run:
    shell: bash

jobs:
  torch-support:
    runs-on: ubuntu-latest
    timeout-minutes: 5
    # Keep the supported PyTorch versions in one place and share them with the
    # test matrix so CI fails early when the policy and workflow drift apart.
    outputs:
      latest_spec: ${{ steps.policy.outputs.latest_spec }}
      previous_spec: ${{ steps.policy.outputs.previous_spec }}
      minimum_spec: ${{ steps.policy.outputs.minimum_spec }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Validate the PyTorch support policy
        run: python3 .github/scripts/torch_support.py check
      - name: Export the supported PyTorch versions
        id: policy
        run: python3 .github/scripts/torch_support.py export

  pytester:
    needs: torch-support
    name: pytester (${{ matrix.os }}, ${{ matrix.python-version }}, torch ${{ matrix.torch-spec || needs.torch-support.outputs.latest_spec }})
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: ["ubuntu-22.04", "macos-14", "windows-2022"]
        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
        exclude:
          - { os: "windows-2022", python-version: "3.13" }
          - { os: "windows-2022", python-version: "3.14" }
        # every combination tests the `latest` PyTorch from .github/torch-support.json; these entries
        # merge into existing combinations (they add no jobs), switching two of them to `previous` and
        # to the declared `minimum` — the floor sits on 3.10 because old torch has no newer wheels
        include:
          - os: "ubuntu-22.04"
            python-version: "3.12"
            torch-spec: "${{ needs.torch-support.outputs.previous_spec }}"
          - os: "ubuntu-22.04"
            python-version: "3.10"
            torch-spec: "${{ needs.torch-support.outputs.minimum_spec }}"

    # Timeout: https://stackoverflow.com/a/59076067/4521646
    timeout-minutes: 60
    env:
      UV_TORCH_BACKEND: "cpu"
      # Supply-chain guard: skip PyPI releases newer than this. See https://docs.astral.sh/uv/reference/settings/#exclude-newer
      UV_EXCLUDE_NEWER: "2 days"

    steps:
      # FFmpeg is required for Video Deserializer tests
      - name: Setup FFmpeg (shared libs, Linux)
        if: runner.os == 'Linux'
        run: sudo apt-get update && sudo apt-get install -y ffmpeg

      - name: Setup FFmpeg (shared libs, macOS)
        if: runner.os == 'macOS'
        run: brew install ffmpeg

      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Install uv and setup python ${{ matrix.python-version }}
        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
        with:
          activate-environment: true
          python-version: ${{ matrix.python-version }}
          enable-cache: true

      - name: Install package & dependencies
        env:
          # the supported PyTorch this job tests, from .github/torch-support.json
          TORCH_SPEC: ${{ matrix.torch-spec || needs.torch-support.outputs.latest_spec }}
        run: |
          test -n "${TORCH_SPEC}" || { echo "::error::TORCH_SPEC is empty, the torch-support job exported no spec"; exit 1; }
          uv pip install -U lightning-sdk
          uv pip install -e ".[extras]" -r requirements/test.txt "torch==${TORCH_SPEC}" -U -q
          uv pip list
          python -c "import torch; print('torch', torch.__version__)"

      # torchcodec has no build for the oldest supported torch (it starts at 2.5), and uv installs a
      # newer one that only fails on import; dropping it lets the video tests skip themselves instead
      - name: Drop the video stack on the minimum-PyTorch job
        if: matrix.torch-spec == needs.torch-support.outputs.minimum_spec
        run: uv pip uninstall torchcodec

      - name: Run fast tests in parallel
        run: |
          pytest tests \
              --ignore=tests/processing \
              --ignore=tests/raw \
              -n 2 --dist=loadgroup --cov=litdata --durations=0 --timeout=120 --capture=no --verbose

      - name: Run processing tests sequentially
        run: |
          # note that the listed test should match ignored in the previous step
          pytest \
            tests/processing tests/raw \
            --cov=litdata --cov-append --durations=0 --timeout=120 --capture=no --verbose

      - name: Statistics
        continue-on-error: true
        run: |
          coverage report
          coverage xml

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: ./coverage.xml
          flags: unittests,${{ matrix.os }},${{ matrix.python-version }}
          env_vars: OS,PYTHON
          name: codecov-umbrella
          fail_ci_if_error: false

      - name: Minimize uv cache
        run: uv cache prune --ci

  testing-guardian:
    runs-on: ubuntu-latest
    needs: pytester
    if: always()
    steps:
      - run: echo "${{ needs.pytester.result }}"
      - name: failing...
        if: needs.pytester.result == 'failure'
        run: exit 1
      - name: cancelled or skipped...
        if: contains(fromJSON('["cancelled", "skipped"]'), needs.pytester.result)
        timeout-minutes: 1
        run: sleep 90