openbench/.github/workflows/ci.yml at main · deepinfra/openbench · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
name: CI

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

# Cancel in-progress runs when a new run is queued on the same branch
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

# Set environment variables for consistency
env:
  UV_CACHE_TTL_SECONDS: 604800  # 1 week
  UV_HTTP_TIMEOUT: 600  # 10 minutes

jobs:
  # Single job for linting and type checking to reduce overhead
  quality-checks:
    name: Quality Checks (Python ${{ matrix.python-version }})
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]  # Use latest for quality checks
    steps:
      - uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
          enable-cache: true
          cache-dependency-glob: |
            **/pyproject.toml
            **/uv.lock

      - name: Set up Python ${{ matrix.python-version }}
        run: uv python install ${{ matrix.python-version }}

      - name: Install dependencies
        run: uv sync --group dev

      - name: Cache pre-commit
        uses: actions/cache@v4
        with:
          path: ~/.cache/pre-commit
          key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }}
          restore-keys: |
            pre-commit-${{ runner.os }}-

      - name: Run pre-commit hooks (all files)
        run: uv run pre-commit run --all-files --show-diff-on-failure --color always

  test:
    name: Tests (Python ${{ matrix.python-version }})
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10", "3.11", "3.12", "3.13"]
    steps:
      - uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
          enable-cache: true
          cache-dependency-glob: |
            **/pyproject.toml
            **/uv.lock

      - name: Set up Python ${{ matrix.python-version }}
        run: uv python install ${{ matrix.python-version }}

      - name: Install dependencies
        run: uv sync --group dev

      - name: Run unit tests with coverage
        run: |
          uv run pytest -m "not integration" --cov=openbench --cov-report=term-missing

  integration-test:
    name: Integration Tests
    runs-on: ubuntu-latest
    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
    steps:
      - uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
          enable-cache: true
          cache-dependency-glob: |
            **/pyproject.toml
            **/uv.lock

      - name: Set up Python
        run: uv python install 3.12

      - name: Install dependencies
        run: uv sync --group dev

      - name: Run integration tests
        env:
          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
        run: |
          uv run pytest -m integration -v
        if: env.GROQ_API_KEY != ''

  # Security scanning with pip-audit
  security:
    name: Security Scan
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
          enable-cache: true

      - name: Set up Python
        run: uv python install 3.12

      - name: Install dependencies
        run: uv sync --group dev

      - name: Run security audit
        run: |
          uv pip install pip-audit
          uv run pip-audit --desc
        continue-on-error: true  # Don't fail the build on vulnerabilities

  # All checks must pass
  all-checks:
    name: All Checks Pass
    runs-on: ubuntu-latest
    needs: [quality-checks, test, integration-test, security]
    if: always()
    steps:
      - name: Verify all checks passed
        run: |
          # Quality checks and test must always pass
          if [[ "${{ needs.quality-checks.result }}" != "success" ||
                "${{ needs.test.result }}" != "success" ]]; then
            echo "One or more required checks failed"
            exit 1
          fi

          # Integration test can be skipped (for external PRs) or must succeed
          if [[ "${{ needs.integration-test.result }}" != "success" &&
                "${{ needs.integration-test.result }}" != "skipped" ]]; then
            echo "Integration tests failed"
            exit 1
          fi

          # Security is continue-on-error, so we don't check it
          echo "All required checks passed!"