python-sdk/.github/workflows/conformance.yml at main · modelcontextprotocol/python-sdk · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
name: Conformance Tests

on:
  push:
    branches: [main]
  pull_request:
  workflow_dispatch:

concurrency:
  group: conformance-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  # Pinned conformance harness package spec (passed verbatim to `npx --yes`).
  # Bump deliberately and reconcile both
  # .github/actions/conformance/expected-failures*.yml files in the same change.
  CONFORMANCE_PKG: "@modelcontextprotocol/conformance@0.2.0-alpha.10"

jobs:
  server-conformance:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false
      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
        with:
          enable-cache: true
          version: 0.9.5
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: 24
      - run: uv sync --frozen --all-extras --package mcp-everything-server
      - name: Run server conformance (active suite)
        run: >-
          ./.github/actions/conformance/run-server.sh
          --suite active
          --expected-failures ./.github/actions/conformance/expected-failures.yml
          --output-dir conformance-results/server-active
      - name: Run server conformance (draft suite)
        run: >-
          ./.github/actions/conformance/run-server.sh
          --suite draft
          --expected-failures ./.github/actions/conformance/expected-failures.yml
          --output-dir conformance-results/server-draft
      - name: Run server conformance (2026-07-28 wire, all suite)
        run: >-
          ./.github/actions/conformance/run-server.sh
          --suite all
          --spec-version 2026-07-28
          --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml
          --output-dir conformance-results/server-2026-07-28
      - name: Run server conformance (all suite, extension scenarios)
        # A bare `--suite all` (no --spec-version) selects every scenario
        # shipped with the pinned harness — including the extension-tagged
        # tasks-* scenarios and pending-listed ones like server-sse-polling,
        # which no other leg reaches (extension scenarios never match a
        # --spec-version filter, and the pending list keeps them out of the
        # active suite). Running the full set keeps unimplemented surfaces
        # visible as baselined known failures in expected-failures.yml instead
        # of silent exclusions, and stays robust to scenarios moving between
        # harness suite lists across pin bumps. `--suite pending` would cover
        # the same union slightly faster; the full set is preferred for the
        # self-contained run and for parity with typescript-sdk's CI.
        run: >-
          ./.github/actions/conformance/run-server.sh
          --suite all
          --expected-failures ./.github/actions/conformance/expected-failures.yml
          --output-dir conformance-results/server-all
      - name: Upload conformance results
        # The log has only summary counts; per-check data is in checks.json.
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: server-conformance-results
          path: conformance-results/
          if-no-files-found: ignore

  client-conformance:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          persist-credentials: false
      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
        with:
          enable-cache: true
          version: 0.9.5
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: 24
      # --compile-bytecode: without it, ~40 concurrently spawned interpreters
      # race to byte-compile site-packages during the timing-sensitive window.
      - run: uv sync --frozen --all-extras --package mcp --compile-bytecode
      - name: Pre-compile bytecode (editable sources)
        run: uv run --frozen python -m compileall -q src .github/actions/conformance
      - name: Run client conformance (all suite)
        # The harness runs all scenarios via unbounded Promise.all; with 40
        # scenarios on a 2-core runner the slowest one (sse-retry, which has a
        # real-time SSE reconnect wait) needs more than the 30s default budget.
        # `.venv/bin/python` (not `uv run`) avoids lockfile re-checks in ~40
        # concurrent spawns; run-client.sh re-runs unexpected failures solo.
        run: >-
          ./.github/actions/conformance/run-client.sh
          --command '.venv/bin/python .github/actions/conformance/client.py'
          --suite all
          --timeout 60000
          --expected-failures ./.github/actions/conformance/expected-failures.yml
          --output-dir conformance-results/client-all
      - name: Run client conformance (2026-07-28 wire, all suite)
        run: >-
          ./.github/actions/conformance/run-client.sh
          --command '.venv/bin/python .github/actions/conformance/client.py'
          --suite all
          --timeout 60000
          --spec-version 2026-07-28
          --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml
          --output-dir conformance-results/client-2026-07-28
      - name: Upload conformance results
        # The log has only summary counts; per-check data is in checks.json.
        # Also on FLAKE_RESCUED: rescued-flake evidence is otherwise discarded.
        if: failure() || hashFiles('conformance-results/**/FLAKE_RESCUED') != ''
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: client-conformance-results
          path: conformance-results/
          if-no-files-found: ignore