diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 042a2eb56e6..dc4147aa3d0 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -27,28 +27,11 @@ on:
versionOverrideArg:
required: false
type: string
- # Triggers downloading the built nugets, and running the tests
- # from the archive outside the repo
- requiresNugets:
+ # JSON object of test properties (e.g. {"requiresNugets": true})
+ properties:
required: false
- type: boolean
- default: false
- # Installs the sdks used for test via tests/workloads.proj
- # Useful for tests that can't use system dotnet
- requiresTestSdk:
- required: false
- type: boolean
- default: false
- # Controls whether to set CLI E2E environment variables (GH_TOKEN, GITHUB_PR_NUMBER, GITHUB_PR_HEAD_SHA)
- requiresCliArchive:
- required: false
- type: boolean
- default: false
- # Controls whether to install Playwright browsers during project build
- enablePlaywrightInstall:
- required: false
- type: boolean
- default: false
+ type: string
+ default: '{}'
os:
required: false
type: string
@@ -107,7 +90,7 @@ jobs:
- name: Set up .NET Core
uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0
# Used for running tests outside the repo
- if: ${{ inputs.requiresNugets }}
+ if: ${{ fromJson(inputs.properties).requiresNugets == true }}
with:
dotnet-version: |
8.x
@@ -136,14 +119,14 @@ jobs:
run: docker info
- name: Download built nugets
- if: ${{ inputs.requiresNugets }}
+ if: ${{ fromJson(inputs.properties).requiresNugets == true }}
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
pattern: built-nugets
path: ${{ github.workspace }}/artifacts/packages
- name: Copy nugets to the correct location
- if: ${{ inputs.requiresNugets }}
+ if: ${{ fromJson(inputs.properties).requiresNugets == true }}
shell: pwsh
run: |
$packageRoot = "${{ github.workspace }}/artifacts/packages"
@@ -159,7 +142,7 @@ jobs:
}
- name: Compute RID for arch-specific packages
- if: ${{ inputs.requiresNugets }}
+ if: ${{ fromJson(inputs.properties).requiresNugets == true }}
id: compute_rid
shell: pwsh
run: |
@@ -179,7 +162,7 @@ jobs:
"rid=$rid" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
- name: Download arch-specific nugets
- if: ${{ inputs.requiresNugets }}
+ if: ${{ fromJson(inputs.properties).requiresNugets == true }}
continue-on-error: true
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
@@ -187,7 +170,7 @@ jobs:
path: ${{ github.workspace }}/arch-specific
- name: Merge arch-specific nugets into package feed
- if: ${{ inputs.requiresNugets }}
+ if: ${{ fromJson(inputs.properties).requiresNugets == true }}
shell: pwsh
run: |
$dest = "${{ github.workspace }}/artifacts/packages/Debug/Shipping"
@@ -207,7 +190,7 @@ jobs:
}
- name: Install sdk for nuget based testing
- if: ${{ inputs.requiresTestSdk }}
+ if: ${{ fromJson(inputs.properties).requiresTestSdk == true }}
run: >
${{ env.DOTNET_SCRIPT }} build ${{ github.workspace }}/tests/workloads.proj
/p:SkipPackageCheckForTemplatesTesting=true
@@ -243,24 +226,24 @@ jobs:
echo "TEST_ASSEMBLY_NAME=$filenameWithoutExtension" >> $env:GITHUB_ENV
- name: Build test project
- if: ${{ ! inputs.requiresNugets }}
+ if: ${{ fromJson(inputs.properties).requiresNugets != true }}
run: >
${{ env.BUILD_SCRIPT }}
-restore
-ci
-build
-projects ${{ env.TEST_PROJECT_PATH }}
- ${{ !inputs.enablePlaywrightInstall && '/p:InstallBrowsersForPlaywright=false' || '' }}
+ ${{ fromJson(inputs.properties).enablePlaywrightInstall != true && '/p:InstallBrowsersForPlaywright=false' || '' }}
${{ inputs.versionOverrideArg }}
- name: Build and archive test project
- if: ${{ inputs.requiresNugets }}
+ if: ${{ fromJson(inputs.properties).requiresNugets == true }}
run: >
${{ env.BUILD_SCRIPT }} -restore -ci -build -projects ${{ env.TEST_PROJECT_PATH }}
/p:PrepareForHelix=true
/p:ArchiveTests=true
/bl:${{ github.workspace }}/artifacts/log/Debug/BuildAndArchive.binlog
- ${{ !inputs.enablePlaywrightInstall && '/p:InstallBrowsersForPlaywright=false' || '' }}
+ ${{ fromJson(inputs.properties).enablePlaywrightInstall != true && '/p:InstallBrowsersForPlaywright=false' || '' }}
${{ inputs.versionOverrideArg }}
# Workaround for bug in Azure Functions Worker SDK. See https://github.com/Azure/azure-functions-dotnet-worker/issues/2969.
@@ -272,7 +255,7 @@ jobs:
${{ env.DOTNET_SCRIPT }} build ${{ github.workspace }}/playground/AzureFunctionsEndToEnd/AzureFunctionsEndToEnd.Functions/AzureFunctionsEndToEnd.Functions.csproj /p:SkipUnstableEmulators=true
- name: Unpack tests
- if: ${{ inputs.requiresNugets }}
+ if: ${{ fromJson(inputs.properties).requiresNugets == true }}
shell: pwsh
run: |
# Find all zip files in the given path
@@ -322,20 +305,20 @@ jobs:
run: ${{ env.DOTNET_SCRIPT }} build-server shutdown
- name: Run nuget dependent tests (Linux/macOS)
- if: ${{ inputs.requiresNugets && runner.os != 'Windows' }}
+ if: ${{ fromJson(inputs.properties).requiresNugets == true && runner.os != 'Windows' }}
working-directory: ${{ github.workspace }}/run-tests/
env:
ASPIRE__TEST__DCPLOGBASEPATH: ${{ github.workspace }}/testresults/dcp
BUILT_NUGETS_PATH: ${{ github.workspace }}/artifacts/packages/Debug/Shipping
NUGET_PACKAGES: ${{ github.workspace }}/nuget-cache
- PLAYWRIGHT_INSTALLED: ${{ !inputs.enablePlaywrightInstall && 'false' || 'true' }}
+ PLAYWRIGHT_INSTALLED: ${{ fromJson(inputs.properties).enablePlaywrightInstall != true && 'false' || 'true' }}
TEST_LOG_PATH: ${{ github.workspace }}/artifacts/log/test-logs
TestsRunningOutsideOfRepo: true
TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: 'netaspireci.azurecr.io'
# PR metadata and token for CLI E2E tests that download artifacts from a PR
- GITHUB_PR_NUMBER: ${{ inputs.requiresCliArchive && github.event.pull_request.number || '' }}
- GITHUB_PR_HEAD_SHA: ${{ inputs.requiresCliArchive && github.event.pull_request.head.sha || '' }}
- GH_TOKEN: ${{ inputs.requiresCliArchive && github.token || '' }}
+ GITHUB_PR_NUMBER: ${{ fromJson(inputs.properties).requiresCliArchive == true && github.event.pull_request.number || '' }}
+ GITHUB_PR_HEAD_SHA: ${{ fromJson(inputs.properties).requiresCliArchive == true && github.event.pull_request.head.sha || '' }}
+ GH_TOKEN: ${{ fromJson(inputs.properties).requiresCliArchive == true && github.token || '' }}
run: |
# Start heartbeat monitor in background (60s interval to reduce log noise)
${{ github.workspace }}/${{ env.DOTNET_SCRIPT }} ${{ github.workspace }}/tools/scripts/Heartbeat.cs 60 &
@@ -362,23 +345,23 @@ jobs:
exit $TEST_EXIT_CODE
- name: Run nuget dependent tests (Windows)
- if: ${{ inputs.requiresNugets && runner.os == 'Windows' }}
+ if: ${{ fromJson(inputs.properties).requiresNugets == true && runner.os == 'Windows' }}
working-directory: ${{ github.workspace }}/run-tests/
shell: pwsh
env:
ASPIRE__TEST__DCPLOGBASEPATH: ${{ github.workspace }}/testresults/dcp
BUILT_NUGETS_PATH: ${{ github.workspace }}/artifacts/packages/Debug/Shipping
NUGET_PACKAGES: ${{ github.workspace }}/nuget-cache
- PLAYWRIGHT_INSTALLED: ${{ !inputs.enablePlaywrightInstall && 'false' || 'true' }}
+ PLAYWRIGHT_INSTALLED: ${{ fromJson(inputs.properties).enablePlaywrightInstall != true && 'false' || 'true' }}
TEST_LOG_PATH: ${{ github.workspace }}/artifacts/log/test-logs
TestsRunningOutsideOfRepo: true
TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: 'netaspireci.azurecr.io'
# Prevent VBCSCompiler from starting during tests. See #15832
UseSharedCompilation: false
# PR metadata and token for CLI E2E tests that download artifacts from a PR
- GITHUB_PR_NUMBER: ${{ inputs.requiresCliArchive && github.event.pull_request.number || '' }}
- GITHUB_PR_HEAD_SHA: ${{ inputs.requiresCliArchive && github.event.pull_request.head.sha || '' }}
- GH_TOKEN: ${{ inputs.requiresCliArchive && github.token || '' }}
+ GITHUB_PR_NUMBER: ${{ fromJson(inputs.properties).requiresCliArchive == true && github.event.pull_request.number || '' }}
+ GITHUB_PR_HEAD_SHA: ${{ fromJson(inputs.properties).requiresCliArchive == true && github.event.pull_request.head.sha || '' }}
+ GH_TOKEN: ${{ fromJson(inputs.properties).requiresCliArchive == true && github.token || '' }}
run: |
# Start heartbeat monitor in background (output goes to console directly)
# Use 60s interval on Windows to reduce overhead on constrained 2-core runners
@@ -414,7 +397,7 @@ jobs:
}
- name: Run tests (Linux/macOS)
- if: ${{ ! inputs.requiresNugets && runner.os != 'Windows' }}
+ if: ${{ fromJson(inputs.properties).requiresNugets != true && runner.os != 'Windows' }}
id: run-tests-unix
env:
ASPIRE__TEST__DCPLOGBASEPATH: ${{ github.workspace }}/testresults/dcp
@@ -422,12 +405,12 @@ jobs:
# In this step, we are not using Arcade, but want to make sure that MSBuild is able to evaluate correctly.
# So, we manually set NUGET_PACKAGES
NUGET_PACKAGES: ${{ github.workspace }}/.packages
- PLAYWRIGHT_INSTALLED: ${{ !inputs.enablePlaywrightInstall && 'false' || 'true' }}
+ PLAYWRIGHT_INSTALLED: ${{ fromJson(inputs.properties).enablePlaywrightInstall != true && 'false' || 'true' }}
TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: 'netaspireci.azurecr.io'
# PR metadata and token for CLI E2E tests that download artifacts from a PR
- GITHUB_PR_NUMBER: ${{ inputs.requiresCliArchive && github.event.pull_request.number || '' }}
- GITHUB_PR_HEAD_SHA: ${{ inputs.requiresCliArchive && github.event.pull_request.head.sha || '' }}
- GH_TOKEN: ${{ inputs.requiresCliArchive && github.token || '' }}
+ GITHUB_PR_NUMBER: ${{ fromJson(inputs.properties).requiresCliArchive == true && github.event.pull_request.number || '' }}
+ GITHUB_PR_HEAD_SHA: ${{ fromJson(inputs.properties).requiresCliArchive == true && github.event.pull_request.head.sha || '' }}
+ GH_TOKEN: ${{ fromJson(inputs.properties).requiresCliArchive == true && github.token || '' }}
run: |
# Start heartbeat monitor in background (60s interval to reduce log noise)
${{ env.DOTNET_SCRIPT }} ${{ github.workspace }}/tools/scripts/Heartbeat.cs 60 &
@@ -460,13 +443,13 @@ jobs:
exit $TEST_EXIT_CODE
- name: Run tests (Windows)
- if: ${{ ! inputs.requiresNugets && runner.os == 'Windows' }}
+ if: ${{ fromJson(inputs.properties).requiresNugets != true && runner.os == 'Windows' }}
id: run-tests-windows
shell: pwsh
env:
ASPIRE__TEST__DCPLOGBASEPATH: ${{ github.workspace }}/testresults/dcp
NUGET_PACKAGES: ${{ github.workspace }}/.packages
- PLAYWRIGHT_INSTALLED: ${{ !inputs.enablePlaywrightInstall && 'false' || 'true' }}
+ PLAYWRIGHT_INSTALLED: ${{ fromJson(inputs.properties).enablePlaywrightInstall != true && 'false' || 'true' }}
TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: 'netaspireci.azurecr.io'
# Prevent the Roslyn compiler server (VBCSCompiler) from starting during tests.
# On 2-core runners, VBCSCompiler consumes ~150% CPU / 700MB RAM and starves the
@@ -475,9 +458,9 @@ jobs:
# See https://github.com/microsoft/aspire/issues/15832
UseSharedCompilation: false
# PR metadata and token for CLI E2E tests that download artifacts from a PR
- GITHUB_PR_NUMBER: ${{ inputs.requiresCliArchive && github.event.pull_request.number || '' }}
- GITHUB_PR_HEAD_SHA: ${{ inputs.requiresCliArchive && github.event.pull_request.head.sha || '' }}
- GH_TOKEN: ${{ inputs.requiresCliArchive && github.token || '' }}
+ GITHUB_PR_NUMBER: ${{ fromJson(inputs.properties).requiresCliArchive == true && github.event.pull_request.number || '' }}
+ GITHUB_PR_HEAD_SHA: ${{ fromJson(inputs.properties).requiresCliArchive == true && github.event.pull_request.head.sha || '' }}
+ GH_TOKEN: ${{ fromJson(inputs.properties).requiresCliArchive == true && github.token || '' }}
run: |
# Start heartbeat monitor in background (output goes to console directly)
# Use 60s interval on Windows to reduce overhead on constrained 2-core runners
diff --git a/.github/workflows/specialized-test-runner.yml b/.github/workflows/specialized-test-runner.yml
index 641862ddf44..b86fe955f35 100644
--- a/.github/workflows/specialized-test-runner.yml
+++ b/.github/workflows/specialized-test-runner.yml
@@ -36,7 +36,7 @@ jobs:
runs-on: ubuntu-latest
if: ${{ github.repository_owner == 'microsoft' }}
outputs:
- runsheet: ${{ steps.generate_tests_matrix.outputs.runsheet }}
+ runsheet: ${{ steps.inject_properties.outputs.runsheet }}
requiresNugets: ${{ steps.check_nugets.outputs.requiresNugets }}
requiresCliArchive: ${{ steps.check_cli_archive.outputs.requiresCliArchive }}
steps:
@@ -59,7 +59,7 @@ jobs:
/p:BeforeBuildPropsPath=${{ github.workspace }}/artifacts/BeforeBuildProps.props
- name: Generate test runsheet
- id: generate_tests_matrix
+ id: generate_tests_matrix_raw
env:
GITHUB_EVENT_NAME: ${{ github.event_name }}
run: >
@@ -74,11 +74,22 @@ jobs:
/p:BeforeBuildPropsPath=${{ github.workspace }}/artifacts/BeforeBuildProps.props
/bl:${{ github.workspace }}/artifacts/log/Release/runsheet.binlog
+ - name: Inject enablePlaywrightInstall into runsheet properties
+ id: inject_properties
+ run: |
+ RUNSHEET='${{ steps.generate_tests_matrix_raw.outputs.runsheet }}'
+ if [ "$RUNSHEET" = "[]" ] || [ -z "$RUNSHEET" ]; then
+ echo "runsheet=[]" >> $GITHUB_OUTPUT
+ else
+ UPDATED=$(echo "$RUNSHEET" | jq -c --argjson epw ${{ inputs.enablePlaywrightInstall }} '[.[] | .properties.enablePlaywrightInstall = $epw]')
+ echo "runsheet=$UPDATED" >> $GITHUB_OUTPUT
+ fi
+
- name: Check if any test requires NuGets
id: check_nugets
run: |
- RUNSHEET='${{ steps.generate_tests_matrix.outputs.runsheet }}'
- if echo "$RUNSHEET" | jq -e 'any(.[]; .requiresNugets == true)' > /dev/null 2>&1; then
+ RUNSHEET='${{ steps.inject_properties.outputs.runsheet }}'
+ if echo "$RUNSHEET" | jq -e 'any(.[]; .properties.requiresNugets == true)' > /dev/null 2>&1; then
echo "requiresNugets=true" >> $GITHUB_OUTPUT
else
echo "requiresNugets=false" >> $GITHUB_OUTPUT
@@ -87,8 +98,8 @@ jobs:
- name: Check if any test requires CLI archives
id: check_cli_archive
run: |
- RUNSHEET='${{ steps.generate_tests_matrix.outputs.runsheet }}'
- if echo "$RUNSHEET" | jq -e 'any(.[]; .requiresCliArchive == true)' > /dev/null 2>&1; then
+ RUNSHEET='${{ steps.inject_properties.outputs.runsheet }}'
+ if echo "$RUNSHEET" | jq -e 'any(.[]; .properties.requiresCliArchive == true)' > /dev/null 2>&1; then
echo "requiresCliArchive=true" >> $GITHUB_OUTPUT
else
echo "requiresCliArchive=false" >> $GITHUB_OUTPUT
@@ -129,15 +140,12 @@ jobs:
uses: ./.github/workflows/run-tests.yml
with:
testShortName: ${{ matrix.tests.project }}
- requiresNugets: ${{ matrix.tests.requiresNugets == true }}
- requiresTestSdk: ${{ matrix.tests.requiresTestSdk == true }}
+ properties: ${{ toJson(matrix.tests.properties) }}
os: ${{ matrix.tests.os }}
testSessionTimeout: ${{ matrix.tests.testSessionTimeout }}
testHangTimeout: ${{ matrix.tests.testHangTimeout }}
- enablePlaywrightInstall: ${{ inputs.enablePlaywrightInstall }}
extraTestArgs: ${{ inputs.extraTestArgs }}
ignoreTestFailures: ${{ inputs.ignoreTestFailures }}
- requiresCliArchive: ${{ matrix.tests.requiresCliArchive == true }}
results:
if: ${{ always() && github.repository_owner == 'microsoft' }}
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 4b48f831324..cf46abaaf0d 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -98,8 +98,7 @@ jobs:
testHangTimeout: ${{ matrix.testHangTimeout }}
extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}"
versionOverrideArg: ${{ inputs.versionOverrideArg }}
- requiresNugets: ${{ matrix.requiresNugets }}
- requiresTestSdk: ${{ matrix.requiresTestSdk }}
+ properties: ${{ toJson(matrix.properties) }}
tests_no_nugets_overflow:
name: ${{ matrix.shortname }}
@@ -117,8 +116,7 @@ jobs:
testHangTimeout: ${{ matrix.testHangTimeout }}
extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}"
versionOverrideArg: ${{ inputs.versionOverrideArg }}
- requiresNugets: ${{ matrix.requiresNugets }}
- requiresTestSdk: ${{ matrix.requiresTestSdk }}
+ properties: ${{ toJson(matrix.properties) }}
# Split requires-nugets tests by OS so each group depends only on
# the CLI archive build for its platform, avoiding cross-OS blocking.
@@ -138,8 +136,7 @@ jobs:
testHangTimeout: ${{ matrix.testHangTimeout }}
extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}"
versionOverrideArg: ${{ inputs.versionOverrideArg }}
- requiresNugets: ${{ matrix.requiresNugets }}
- requiresTestSdk: ${{ matrix.requiresTestSdk }}
+ properties: ${{ toJson(matrix.properties) }}
tests_requires_nugets_windows:
name: ${{ matrix.shortname }}
@@ -157,8 +154,7 @@ jobs:
testHangTimeout: ${{ matrix.testHangTimeout }}
extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}"
versionOverrideArg: ${{ inputs.versionOverrideArg }}
- requiresNugets: ${{ matrix.requiresNugets }}
- requiresTestSdk: ${{ matrix.requiresTestSdk }}
+ properties: ${{ toJson(matrix.properties) }}
tests_requires_nugets_macos:
name: ${{ matrix.shortname }}
@@ -176,8 +172,7 @@ jobs:
testHangTimeout: ${{ matrix.testHangTimeout }}
extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}"
versionOverrideArg: ${{ inputs.versionOverrideArg }}
- requiresNugets: ${{ matrix.requiresNugets }}
- requiresTestSdk: ${{ matrix.requiresTestSdk }}
+ properties: ${{ toJson(matrix.properties) }}
tests_requires_cli_archive:
name: ${{ matrix.shortname }}
@@ -195,9 +190,7 @@ jobs:
testHangTimeout: ${{ matrix.testHangTimeout }}
extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}"
versionOverrideArg: ${{ inputs.versionOverrideArg }}
- requiresNugets: ${{ matrix.requiresNugets }}
- requiresTestSdk: ${{ matrix.requiresTestSdk }}
- requiresCliArchive: true
+ properties: ${{ toJson(matrix.properties) }}
polyglot_validation:
name: Polyglot SDK Validation
diff --git a/docs/ci/TestingOnCI.md b/docs/ci/TestingOnCI.md
index f9da01e4c08..f774c732fdf 100644
--- a/docs/ci/TestingOnCI.md
+++ b/docs/ci/TestingOnCI.md
@@ -56,8 +56,7 @@ This invokes `eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.
- Writes a `.tests-metadata.json` file to `artifacts/helix/` containing:
- `projectName`, `shortName`, `testProjectPath`
- `supportedOSes` array (e.g., `["windows", "linux", "macos"]`)
- - `requiresNugets`, `requiresTestSdk`, `requiresCliArchive` flags
- - `enablePlaywrightInstall` flag
+ - `properties` object with boolean flags (defined in `eng/testing/CITestsProperties.props`): `requiresNugets`, `requiresTestSdk`, `requiresCliArchive`, `enablePlaywrightInstall`
- `testSessionTimeout`, `testHangTimeout` values
- `uncollectedTestsSessionTimeout`, `uncollectedTestsHangTimeout` values
- `splitTests` flag
@@ -85,7 +84,7 @@ After all projects build, `eng/AfterSolutionBuild.targets` runs `eng/scripts/bui
- **Regular tests**: One entry per project
- **Partition-based splits**: One entry per partition + one for `uncollected:*`
- **Class-based splits**: One entry per test class
-6. Outputs `artifacts/canonical-test-matrix.json` in canonical format (flat array with `requiresNugets`, `requiresCliArchive` booleans per entry)
+6. Outputs `artifacts/canonical-test-matrix.json` in canonical format (entries with a `properties` sub-object containing boolean flags like `requiresNugets`, `requiresCliArchive`)
**Canonical format:**
```json
@@ -96,8 +95,12 @@ After all projects build, `eng/AfterSolutionBuild.targets` runs `eng/scripts/bui
"shortname": "Templates-StarterTests",
"testProjectPath": "tests/Aspire.Templates.Tests/...",
"supportedOSes": ["windows", "linux", "macos"],
- "requiresNugets": true,
- "requiresTestSdk": true,
+ "properties": {
+ "requiresNugets": true,
+ "requiresTestSdk": true,
+ "requiresCliArchive": false,
+ "enablePlaywrightInstall": false
+ },
"testSessionTimeout": "20m",
"testHangTimeout": "10m",
"extraTestArgs": "--filter-class \"...\""
@@ -107,7 +110,12 @@ After all projects build, `eng/AfterSolutionBuild.targets` runs `eng/scripts/bui
"shortname": "Hosting-Docker",
"testProjectPath": "tests/Aspire.Hosting.Tests/...",
"supportedOSes": ["linux"],
- "requiresNugets": false,
+ "properties": {
+ "requiresNugets": false,
+ "requiresTestSdk": false,
+ "requiresCliArchive": false,
+ "enablePlaywrightInstall": false
+ },
"testSessionTimeout": "30m",
"extraTestArgs": "--filter-trait \"Partition=Docker\"",
"runners": { "macos": "macos-latest-xlarge" }
@@ -123,7 +131,7 @@ Each CI platform has a thin script that transforms the canonical matrix:
**GitHub Actions** (`eng/scripts/expand-test-matrix-github.ps1`):
- Expands each entry for every OS in its `supportedOSes` array
- Maps OS names to GitHub runners (`linux` → `ubuntu-latest`, etc.)
-- Preserves dependency metadata such as `requiresNugets`, `requiresCliArchive`, and custom runner overrides on each expanded entry
+- Preserves dependency metadata within the `properties` sub-object (including `requiresNugets`, `requiresCliArchive`), and custom runner overrides on each expanded entry
- Applies overflow splitting for the `no_nugets` category (threshold: 250 entries) to stay under the GitHub Actions 256-job-per-matrix limit
- Outputs a single `all_tests` matrix, which `.github/workflows/tests.yml` further splits by dependency type and OS using `eng/scripts/split-test-matrix-by-deps.ps1`
@@ -275,6 +283,49 @@ For tests that require Playwright browser automation:
This flag is tracked in the test metadata and controls whether Playwright browsers are installed during the test build step.
+## CI Test Property Registry
+
+All boolean test properties (such as `RequiresNugets`, `RequiresTestSdk`, `RequiresCliArchive`, `EnablePlaywrightInstall`) are defined in a single source of truth:
+
+**`eng/testing/CITestsProperties.props`**
+
+```xml
+
+
+
+
+
+
+```
+
+Each `CITestsProperty` item has three attributes:
+
+| Attribute | Description |
+|-----------|-------------|
+| `Include` | The JSON key name used in metadata files and workflow properties (camelCase) |
+| `MSBuildProp` | The MSBuild property name read from test project `.csproj` files (PascalCase) |
+| `Default` | The default value when the property is not set in a test project |
+
+### How it flows through the system
+
+1. **MSBuild targets** (`TestEnumerationRunsheetBuilder.targets`, `SpecializedTestRunsheetBuilderBase.targets`) import this file and iterate `@(CITestsProperty)` to dynamically resolve property values and emit the `"properties"` JSON object — no hardcoded property names in the targets.
+2. **PowerShell scripts** (`build-test-matrix.ps1`) parse the `.props` XML at startup to build the defaults dictionary and copy properties generically — no per-property code blocks.
+3. **GitHub Actions workflows** (`run-tests.yml`) read properties from the opaque `properties` JSON string with bespoke `if:` conditions for each property that controls unique workflow behavior.
+
+### Adding a new boolean test property
+
+1. Add one line to `eng/testing/CITestsProperties.props`:
+ ```xml
+
+ ```
+2. Set the MSBuild property in the test project's `.csproj`:
+ ```xml
+
+ true
+
+ ```
+3. Add behavioral logic in the GitHub Actions workflow YAML (e.g., `if:` conditions in `run-tests.yml`) to act on the new property.
+
## Custom GitHub Actions Runners
By default, tests run on `ubuntu-latest`, `windows-latest`, and `macos-latest`. To override the runner for a specific OS (e.g., to use larger runners or specific OS versions), set the corresponding property in the test project's `.csproj`:
@@ -313,10 +364,11 @@ During enumeration, these files are generated in `artifacts/`:
| `helix/.tests-partitions.json` | Partition/class list for split projects |
| `canonical-test-matrix.json` | Canonical matrix (platform-agnostic) |
-## Scripts Reference
+## Scripts and Configuration Reference
-| Script | Purpose |
-|--------|---------|
+| File | Purpose |
+|------|---------|
+| `eng/testing/CITestsProperties.props` | Single source of truth for CI test property names, defaults, and MSBuild mappings |
| `eng/scripts/build-test-matrix.ps1` | Generates canonical matrix from metadata files |
| `eng/scripts/expand-test-matrix-github.ps1` | Expands canonical matrix for GitHub Actions |
| `eng/scripts/split-test-projects-for-ci.ps1` | Discovers test partitions/classes for splitting |
diff --git a/eng/SpecializedTestRunsheetBuilderBase.targets b/eng/SpecializedTestRunsheetBuilderBase.targets
index b471d24afb1..73dbd4edfa8 100644
--- a/eng/SpecializedTestRunsheetBuilderBase.targets
+++ b/eng/SpecializedTestRunsheetBuilderBase.targets
@@ -1,4 +1,5 @@
+
<_TestCommand>$([System.String]::Copy($(_TestCommand)).Replace("\", "/").Replace('"', '\"'))
+
+
+
+
+ <_ResolvedCITestsProperty Include="@(CITestsProperty)">
+ true
+ %(CITestsProperty.Default)
+
+
+
+
+ <_PropertiesJson>@(_ResolvedCITestsProperty -> '"%(Identity)": %(Value)', ', ')
- <_TestRunsheetWindows>{ "label": "w: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "$(_GithubActionsRunnerWindows)", "command": "./eng/build.ps1 $(_TestCommand)", "requiresNugets": $(_RequiresNugets), "requiresTestSdk": $(_RequiresTestSdk), "requiresCliArchive": $(_RequiresCliArchive), "testSessionTimeout": "$(TestSessionTimeout)", "testHangTimeout": "$(TestHangTimeout)" }
- <_TestRunsheetLinux>{ "label": "l: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "$(_GithubActionsRunnerLinux)", "command": "./eng/build.sh $(_TestCommand)", "requiresNugets": $(_RequiresNugets), "requiresTestSdk": $(_RequiresTestSdk), "requiresCliArchive": $(_RequiresCliArchive), "testSessionTimeout": "$(TestSessionTimeout)", "testHangTimeout": "$(TestHangTimeout)" }
- <_TestRunsheetMacOS>{ "label": "m: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "$(_GithubActionsRunnerMacOS)", "command": "./eng/build.sh $(_TestCommand)", "requiresNugets": $(_RequiresNugets), "requiresTestSdk": $(_RequiresTestSdk), "requiresCliArchive": $(_RequiresCliArchive), "testSessionTimeout": "$(TestSessionTimeout)", "testHangTimeout": "$(TestHangTimeout)" }
+ <_TestRunsheetWindows>{ "label": "w: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "$(_GithubActionsRunnerWindows)", "command": "./eng/build.ps1 $(_TestCommand)", "properties": {$(_PropertiesJson)}, "testSessionTimeout": "$(TestSessionTimeout)", "testHangTimeout": "$(TestHangTimeout)" }
+ <_TestRunsheetLinux>{ "label": "l: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "$(_GithubActionsRunnerLinux)", "command": "./eng/build.sh $(_TestCommand)", "properties": {$(_PropertiesJson)}, "testSessionTimeout": "$(TestSessionTimeout)", "testHangTimeout": "$(TestHangTimeout)" }
+ <_TestRunsheetMacOS>{ "label": "m: $(_TestRunsheet)", "project": "$(_TestRunsheet)", "os": "$(_GithubActionsRunnerMacOS)", "command": "./eng/build.sh $(_TestCommand)", "properties": {$(_PropertiesJson)}, "testSessionTimeout": "$(TestSessionTimeout)", "testHangTimeout": "$(TestHangTimeout)" }
+
<_RelativeProjectPath Condition="$(_RelativeProjectPath.StartsWith('/')) or $(_RelativeProjectPath.StartsWith('\\'))">$(_RelativeProjectPath.Substring(1))
-
- <_RequiresNugets>false
- <_RequiresNugets Condition="'$(RequiresNugets)' == 'true'">true
-
-
- <_RequiresTestSdk>false
- <_RequiresTestSdk Condition="'$(RequiresTestSdk)' == 'true'">true
-
-
- <_RequiresCliArchive>false
- <_RequiresCliArchive Condition="'$(RequiresCliArchive)' == 'true'">true
-
-
- <_EnablePlaywrightInstall>false
- <_EnablePlaywrightInstall Condition="'$(EnablePlaywrightInstall)' == 'true'">true
-
<_SupportedOSesJson />
<_SupportedOSesJson Condition="'$(RunOnGithubActionsWindows)' == 'true'">$(_SupportedOSesJson)"windows",
@@ -96,6 +81,19 @@
<_RunnersJsonParts Condition="'$(_RunnersJsonParts)' != ''">$(_RunnersJsonParts.TrimEnd(','))
<_RunnersJsonLine Condition="'$(_RunnersJsonParts)' != ''">,
"runners": {$(_RunnersJsonParts)}
+
+
+
+
+ <_ResolvedCITestsProperty Include="@(CITestsProperty)">
+ true
+ %(CITestsProperty.Default)
+
+
+
+
+ <_PropertiesJsonInner>@(_ResolvedCITestsProperty -> ' "%(Identity)": %(Value)', ',
+')
<_MetadataJson>{
@@ -103,10 +101,9 @@
"shortName": "$(_ShortName)",
"testClassNamesPrefix": "$(MSBuildProjectName)",
"testProjectPath": "$(_RelativeProjectPath)",
- "requiresNugets": "$(_RequiresNugets.ToLowerInvariant())",
- "requiresTestSdk": "$(_RequiresTestSdk.ToLowerInvariant())",
- "requiresCliArchive": "$(_RequiresCliArchive.ToLowerInvariant())",
- "enablePlaywrightInstall": "$(_EnablePlaywrightInstall.ToLowerInvariant())",
+ "properties": {
+$(_PropertiesJsonInner)
+ },
"splitTests": "$(SplitTestsOnCI)",
"supportedOSes": [$(_SupportedOSesJson)],
"testSessionTimeout": "$(_TestSessionTimeout)",
@@ -125,7 +122,7 @@
-
+
diff --git a/eng/scripts/build-test-matrix.ps1 b/eng/scripts/build-test-matrix.ps1
index 9c6b67a75cb..862a046014b 100644
--- a/eng/scripts/build-test-matrix.ps1
+++ b/eng/scripts/build-test-matrix.ps1
@@ -27,7 +27,7 @@
Output format:
{
- "tests": [ { entry with supportedOSes array and requiresNugets boolean }, ... ]
+ "tests": [ { entry with supportedOSes array and properties sub-object }, ... ]
}
#>
@@ -43,16 +43,30 @@ param(
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version Latest
+# Load CI test property definitions from the single source of truth
+$script:CITestsPropertiesPath = Join-Path $PSScriptRoot '../testing/CITestsProperties.props'
+[xml]$ciTestsPropsXml = Get-Content $script:CITestsPropertiesPath
+$script:ciTestsPropertyDefs = @{}
+foreach ($item in $ciTestsPropsXml.Project.ItemGroup.CITestsProperty) {
+ $script:ciTestsPropertyDefs[$item.Include] = @{
+ Default = ($item.Default -eq 'true')
+ }
+}
+
# Default values applied to all entries
$script:defaults = @{
extraTestArgs = ''
- requiresNugets = $false
- requiresTestSdk = $false
+ properties = [ordered]@{}
testSessionTimeout = '20m'
testHangTimeout = '10m'
supportedOSes = @('windows', 'linux', 'macos')
}
+# Populate default property values from CITestsProperties.props
+foreach ($propName in $script:ciTestsPropertyDefs.Keys) {
+ $script:defaults.properties[$propName] = $script:ciTestsPropertyDefs[$propName].Default
+}
+
Write-Host "Building canonical test matrix"
Write-Host "Artifacts directory: $ArtifactsDir"
@@ -62,6 +76,23 @@ function ConvertTo-Boolean {
return ($Value -eq 'true' -or $Value -eq $true)
}
+# Helper function to copy CI test properties from metadata into an entry's properties sub-object.
+# Reads from the 'properties' sub-object (new format) or top-level keys (legacy format).
+function Copy-CITestProperties {
+ param(
+ [Parameter(Mandatory=$true)]$Entry,
+ [Parameter(Mandatory=$true)]$Metadata
+ )
+
+ $propsSource = if ($Metadata.PSObject.Properties['properties'] -and $Metadata.properties) { $Metadata.properties } else { $Metadata }
+ $Entry['properties'] = [ordered]@{}
+ foreach ($propName in $script:ciTestsPropertyDefs.Keys) {
+ if ($propsSource.PSObject.Properties[$propName]) {
+ $Entry['properties'][$propName] = $propsSource.$propName
+ }
+ }
+}
+
# Helper function to apply defaults and normalize an entry
function Complete-EntryWithDefaults {
param([Parameter(Mandatory=$true)]$Entry)
@@ -74,10 +105,32 @@ function Complete-EntryWithDefaults {
$Entry['supportedOSes'] = $script:defaults.supportedOSes
}
- # Normalize boolean values
- $Entry['requiresNugets'] = if ($Entry.Contains('requiresNugets')) { ConvertTo-Boolean $Entry['requiresNugets'] } else { $false }
- $Entry['requiresTestSdk'] = if ($Entry.Contains('requiresTestSdk')) { ConvertTo-Boolean $Entry['requiresTestSdk'] } else { $false }
- $Entry['requiresCliArchive'] = if ($Entry.Contains('requiresCliArchive')) { ConvertTo-Boolean $Entry['requiresCliArchive'] } else { $false }
+ # Ensure properties sub-object exists
+ if (-not $Entry.Contains('properties') -or -not $Entry['properties']) {
+ $Entry['properties'] = [ordered]@{}
+ }
+
+ # If properties is a PSCustomObject (from JSON), convert to ordered hashtable
+ $props = $Entry['properties']
+ if ($props -is [PSCustomObject]) {
+ $converted = [ordered]@{}
+ foreach ($p in $props.PSObject.Properties) {
+ $converted[$p.Name] = $p.Value
+ }
+ $props = $converted
+ $Entry['properties'] = $props
+ }
+
+ # Normalize boolean values within properties using CITestsProperties.props definitions
+ foreach ($propName in $script:ciTestsPropertyDefs.Keys) {
+ $propDefault = $script:ciTestsPropertyDefs[$propName].Default
+ $props[$propName] = if ($props.Contains($propName)) { ConvertTo-Boolean $props[$propName] } else { $propDefault }
+ }
+
+ # Remove any legacy top-level boolean keys (property names that belong in the properties sub-object)
+ foreach ($key in $script:ciTestsPropertyDefs.Keys) {
+ if ($Entry.Contains($key)) { $Entry.Remove($key) }
+ }
return $Entry
}
@@ -103,10 +156,9 @@ function New-RegularTestEntry {
if ($Metadata) {
if ($Metadata.PSObject.Properties['testSessionTimeout']) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout }
if ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout }
- if ($Metadata.PSObject.Properties['requiresNugets']) { $entry['requiresNugets'] = $Metadata.requiresNugets }
- if ($Metadata.PSObject.Properties['requiresTestSdk']) { $entry['requiresTestSdk'] = $Metadata.requiresTestSdk }
- if ($Metadata.PSObject.Properties['requiresCliArchive']) { $entry['requiresCliArchive'] = $Metadata.requiresCliArchive }
if ($Metadata.PSObject.Properties['extraTestArgs'] -and $Metadata.extraTestArgs) { $entry['extraTestArgs'] = $Metadata.extraTestArgs }
+
+ Copy-CITestProperties -Entry $entry -Metadata $Metadata
}
# Add supported OSes
@@ -163,9 +215,7 @@ function New-CollectionTestEntry {
if ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout }
}
- if ($Metadata.PSObject.Properties['requiresNugets']) { $entry['requiresNugets'] = $Metadata.requiresNugets }
- if ($Metadata.PSObject.Properties['requiresTestSdk']) { $entry['requiresTestSdk'] = $Metadata.requiresTestSdk }
- if ($Metadata.PSObject.Properties['requiresCliArchive']) { $entry['requiresCliArchive'] = $Metadata.requiresCliArchive }
+ Copy-CITestProperties -Entry $entry -Metadata $Metadata
# Add test filter for collection-based splitting
if ($IsUncollected) {
@@ -213,9 +263,8 @@ function New-ClassTestEntry {
if ($Metadata.PSObject.Properties['testSessionTimeout']) { $entry['testSessionTimeout'] = $Metadata.testSessionTimeout }
if ($Metadata.PSObject.Properties['testHangTimeout']) { $entry['testHangTimeout'] = $Metadata.testHangTimeout }
- if ($Metadata.PSObject.Properties['requiresNugets']) { $entry['requiresNugets'] = $Metadata.requiresNugets }
- if ($Metadata.PSObject.Properties['requiresTestSdk']) { $entry['requiresTestSdk'] = $Metadata.requiresTestSdk }
- if ($Metadata.PSObject.Properties['requiresCliArchive']) { $entry['requiresCliArchive'] = $Metadata.requiresCliArchive }
+
+ Copy-CITestProperties -Entry $entry -Metadata $Metadata
# Add test filter for class-based splitting
$entry['extraTestArgs'] = "--filter-class `"$ClassName`""
@@ -321,8 +370,8 @@ Write-Host "Generated $($matrixEntries.Count) total matrix entries"
$sortedEntries = @($matrixEntries | Sort-Object -Property projectName, name)
-$requiresNugetsCount = @($sortedEntries | Where-Object { $_.requiresNugets -eq $true }).Count
-$noNugetsCount = @($sortedEntries | Where-Object { $_.requiresNugets -ne $true }).Count
+$requiresNugetsCount = @($sortedEntries | Where-Object { $_.properties.requiresNugets -eq $true }).Count
+$noNugetsCount = @($sortedEntries | Where-Object { $_.properties.requiresNugets -ne $true }).Count
Write-Host " - Requiring NuGets: $requiresNugetsCount"
Write-Host " - Not requiring NuGets: $noNugetsCount"
diff --git a/eng/scripts/split-test-matrix-by-deps.ps1 b/eng/scripts/split-test-matrix-by-deps.ps1
index 306ded912d4..d271809d1ec 100644
--- a/eng/scripts/split-test-matrix-by-deps.ps1
+++ b/eng/scripts/split-test-matrix-by-deps.ps1
@@ -74,15 +74,25 @@ if ($matrix.include -and $matrix.include.Count -gt 0) {
Write-Host "Input matrix: $($allEntries.Count) total entries"
+# Helper to safely read a boolean flag from the properties sub-object
+function Get-PropertyFlag {
+ param($Entry, [string]$Name)
+ if ($Entry.PSObject.Properties.Name -contains 'properties' -and $Entry.properties -and
+ $Entry.properties.PSObject.Properties.Name -contains $Name) {
+ return $Entry.properties.$Name -eq $true
+ }
+ return $false
+}
+
# Split into categories based on dependency requirements
-$cliArchiveEntries = @($allEntries | Where-Object { $_.PSObject.Properties.Name -contains 'requiresCliArchive' -and $_.requiresCliArchive -eq $true })
+$cliArchiveEntries = @($allEntries | Where-Object { Get-PropertyFlag $_ 'requiresCliArchive' })
$nugetEntries = @($allEntries | Where-Object {
- ($_.PSObject.Properties.Name -contains 'requiresNugets' -and $_.requiresNugets -eq $true) -and
- -not ($_.PSObject.Properties.Name -contains 'requiresCliArchive' -and $_.requiresCliArchive -eq $true)
+ (Get-PropertyFlag $_ 'requiresNugets') -and
+ -not (Get-PropertyFlag $_ 'requiresCliArchive')
})
$noNugetEntries = @($allEntries | Where-Object {
- -not ($_.PSObject.Properties.Name -contains 'requiresNugets' -and $_.requiresNugets -eq $true) -and
- -not ($_.PSObject.Properties.Name -contains 'requiresCliArchive' -and $_.requiresCliArchive -eq $true)
+ -not (Get-PropertyFlag $_ 'requiresNugets') -and
+ -not (Get-PropertyFlag $_ 'requiresCliArchive')
})
Write-Host " - No nugets: $($noNugetEntries.Count)"
diff --git a/eng/testing/CITestsProperties.props b/eng/testing/CITestsProperties.props
new file mode 100644
index 00000000000..b64698c5158
--- /dev/null
+++ b/eng/testing/CITestsProperties.props
@@ -0,0 +1,22 @@
+
+
+
+
+
+
+
+
+
diff --git a/tests/Infrastructure.Tests/PowerShellScripts/BuildTestMatrixTests.cs b/tests/Infrastructure.Tests/PowerShellScripts/BuildTestMatrixTests.cs
index cf107e1d20d..329fb7d5bff 100644
--- a/tests/Infrastructure.Tests/PowerShellScripts/BuildTestMatrixTests.cs
+++ b/tests/Infrastructure.Tests/PowerShellScripts/BuildTestMatrixTests.cs
@@ -51,7 +51,7 @@ public async Task GeneratesMatrixFromSingleProject()
Assert.Equal("MyProject", entry.ProjectName);
Assert.Equal("MyProj", entry.Name);
Assert.Equal("regular", entry.Type);
- Assert.False(entry.RequiresNugets);
+ Assert.False(entry.Properties.GetValueOrDefault("requiresNugets"));
}
[Fact]
@@ -250,8 +250,8 @@ public async Task PreservesRequiresNugetsProperty()
var matrix = ParseCanonicalMatrix(outputFile);
Assert.Equal(2, matrix.Tests.Length);
- Assert.Contains(matrix.Tests, e => e.ProjectName == "NeedsNugets" && e.RequiresNugets == true);
- Assert.Contains(matrix.Tests, e => e.ProjectName == "NoNugets" && e.RequiresNugets == false);
+ Assert.Contains(matrix.Tests, e => e.ProjectName == "NeedsNugets" && e.Properties.GetValueOrDefault("requiresNugets") == true);
+ Assert.Contains(matrix.Tests, e => e.ProjectName == "NoNugets" && e.Properties.GetValueOrDefault("requiresNugets") == false);
}
[Fact]
@@ -373,7 +373,7 @@ public async Task PassesRequiresTestSdkProperty()
var matrix = ParseCanonicalMatrix(outputFile);
var entry = Assert.Single(matrix.Tests);
- Assert.True(entry.RequiresTestSdk);
+ Assert.True(entry.Properties.GetValueOrDefault("requiresTestSdk"));
}
[Fact]
@@ -571,6 +571,172 @@ public async Task PassesThroughRunnersForClassEntries()
Assert.Equal("macos-latest-xlarge", entry.Runners["macos"]);
}
+ [Fact]
+ [RequiresTools(["pwsh"])]
+ public async Task AllCITestsPropertiesAppearInOutputWithDefaults()
+ {
+ // Verifies that every property defined in CITestsProperties.props
+ // appears in the canonical matrix output with its default value
+ // when the input metadata doesn't set any properties to true.
+ var expectedProperties = ReadCITestsPropertyNames();
+
+ var artifactsDir = Path.Combine(_tempDir.Path, "artifacts");
+ Directory.CreateDirectory(artifactsDir);
+
+ TestDataBuilder.CreateTestsMetadataJson(
+ Path.Combine(artifactsDir, "DefaultProps.tests-metadata.json"),
+ projectName: "DefaultProps",
+ testProjectPath: "tests/DefaultProps/DefaultProps.csproj");
+
+ var outputFile = Path.Combine(_tempDir.Path, "matrix.json");
+
+ var result = await RunScript(artifactsDir, outputFile);
+
+ result.EnsureSuccessful();
+
+ var matrix = ParseCanonicalMatrix(outputFile);
+ var entry = Assert.Single(matrix.Tests);
+
+ foreach (var propName in expectedProperties)
+ {
+ Assert.True(entry.Properties.ContainsKey(propName),
+ $"Expected property '{propName}' from CITestsProperties.props to be present in matrix output, but it was missing.");
+ Assert.False(entry.Properties[propName],
+ $"Expected property '{propName}' to have default value 'false', but it was 'true'.");
+ }
+ }
+
+ [Fact]
+ [RequiresTools(["pwsh"])]
+ public async Task DefaultsAreAppliedWhenPropertiesAreMissingFromMetadata()
+ {
+ // Create metadata JSON manually with a partial properties object
+ // (only requiresNugets=true, everything else omitted) to verify
+ // that the script fills in defaults from CITestsProperties.props.
+ var expectedProperties = ReadCITestsPropertyNames();
+
+ var artifactsDir = Path.Combine(_tempDir.Path, "artifacts");
+ Directory.CreateDirectory(artifactsDir);
+
+ var partialMetadata = """
+ {
+ "projectName": "PartialProps",
+ "testProjectPath": "tests/PartialProps/PartialProps.csproj",
+ "shortName": "PartialProps",
+ "splitTests": "false",
+ "properties": {
+ "requiresNugets": true
+ },
+ "supportedOSes": ["windows", "linux"]
+ }
+ """;
+ File.WriteAllText(
+ Path.Combine(artifactsDir, "PartialProps.tests-metadata.json"),
+ partialMetadata);
+
+ var outputFile = Path.Combine(_tempDir.Path, "matrix.json");
+
+ var result = await RunScript(artifactsDir, outputFile);
+
+ result.EnsureSuccessful();
+
+ var matrix = ParseCanonicalMatrix(outputFile);
+ var entry = Assert.Single(matrix.Tests);
+
+ // requiresNugets should be true (from input)
+ Assert.True(entry.Properties["requiresNugets"]);
+
+ // All other properties should be present with their default value (false)
+ foreach (var propName in expectedProperties.Where(p => p != "requiresNugets"))
+ {
+ Assert.True(entry.Properties.ContainsKey(propName),
+ $"Expected property '{propName}' to be filled in by defaults, but it was missing.");
+ Assert.False(entry.Properties[propName],
+ $"Expected property '{propName}' default to be 'false', but it was 'true'.");
+ }
+ }
+
+ [Fact]
+ public void CITestsPropertiesPropsFileIsValidAndComplete()
+ {
+ // Validates that CITestsProperties.props is well-formed XML
+ // and contains the expected property definitions.
+ var propsPath = Path.Combine(FindRepoRoot(), "eng", "testing", "CITestsProperties.props");
+ Assert.True(File.Exists(propsPath), $"CITestsProperties.props not found at {propsPath}");
+
+ var doc = new System.Xml.XmlDocument();
+ doc.Load(propsPath);
+
+ var items = doc.SelectNodes("/Project/ItemGroup/CITestsProperty");
+ Assert.NotNull(items);
+ Assert.True(items.Count > 0, "CITestsProperties.props should define at least one CITestsProperty item.");
+
+ foreach (System.Xml.XmlElement item in items)
+ {
+ var include = item.GetAttribute("Include");
+ var msbuildProp = item.GetAttribute("MSBuildProp");
+ var defaultVal = item.GetAttribute("Default");
+
+ Assert.False(string.IsNullOrWhiteSpace(include),
+ "Each CITestsProperty must have an Include attribute (JSON key name).");
+ Assert.False(string.IsNullOrWhiteSpace(msbuildProp),
+ $"CITestsProperty '{include}' must have an MSBuildProp attribute.");
+ Assert.False(string.IsNullOrWhiteSpace(defaultVal),
+ $"CITestsProperty '{include}' must have a Default attribute.");
+
+ // JSON keys should be camelCase (start with lowercase)
+ Assert.True(char.IsLower(include[0]),
+ $"CITestsProperty Include '{include}' should be camelCase (start with lowercase).");
+
+ // MSBuild properties should be PascalCase (start with uppercase)
+ Assert.True(char.IsUpper(msbuildProp[0]),
+ $"CITestsProperty MSBuildProp '{msbuildProp}' should be PascalCase (start with uppercase).");
+ }
+ }
+
+ [Fact]
+ [RequiresTools(["pwsh"])]
+ public async Task SplitTestEntriesInheritAllCITestsProperties()
+ {
+ // Verifies that partition-based split test entries also get
+ // all properties from CITestsProperties.props.
+ var expectedProperties = ReadCITestsPropertyNames();
+
+ var artifactsDir = Path.Combine(_tempDir.Path, "artifacts");
+ Directory.CreateDirectory(artifactsDir);
+
+ TestDataBuilder.CreateSplitTestsMetadataJson(
+ Path.Combine(artifactsDir, "SplitProps.tests-metadata.json"),
+ projectName: "SplitProps",
+ testProjectPath: "tests/SplitProps/SplitProps.csproj",
+ shortName: "SplitP",
+ requiresNugets: true);
+
+ TestDataBuilder.CreateTestsPartitionsJson(
+ Path.Combine(artifactsDir, "SplitProps.tests-partitions.json"),
+ "PartA");
+
+ var outputFile = Path.Combine(_tempDir.Path, "matrix.json");
+
+ var result = await RunScript(artifactsDir, outputFile);
+
+ result.EnsureSuccessful();
+
+ var matrix = ParseCanonicalMatrix(outputFile);
+ Assert.True(matrix.Tests.Length >= 2, "Expected at least 2 entries (partition + uncollected).");
+
+ foreach (var entry in matrix.Tests)
+ {
+ foreach (var propName in expectedProperties)
+ {
+ Assert.True(entry.Properties.ContainsKey(propName),
+ $"Split entry '{entry.Name}' is missing property '{propName}'.");
+ }
+ Assert.True(entry.Properties["requiresNugets"],
+ $"Split entry '{entry.Name}' should have requiresNugets=true.");
+ }
+ }
+
private async Task RunScript(string artifactsDir, string outputFile)
{
using var cmd = new PowerShellCommand(_scriptPath, _output)
@@ -601,4 +767,25 @@ private static string FindRepoRoot()
}
throw new InvalidOperationException("Could not find repository root");
}
+
+ ///
+ /// Reads the CITestsProperties.props XML file and returns
+ /// the list of property names (Include attributes).
+ ///
+ private static HashSet ReadCITestsPropertyNames()
+ {
+ var propsPath = Path.Combine(FindRepoRoot(), "eng", "testing", "CITestsProperties.props");
+ var doc = new System.Xml.XmlDocument();
+ doc.Load(propsPath);
+
+ var items = doc.SelectNodes("/Project/ItemGroup/CITestsProperty")
+ ?? throw new InvalidOperationException("No CITestsProperty items found in CITestsProperties.props");
+
+ var names = new HashSet();
+ foreach (System.Xml.XmlElement item in items)
+ {
+ names.Add(item.GetAttribute("Include"));
+ }
+ return names;
+ }
}
diff --git a/tests/Infrastructure.Tests/PowerShellScripts/ExpandTestMatrixGitHubTests.cs b/tests/Infrastructure.Tests/PowerShellScripts/ExpandTestMatrixGitHubTests.cs
index 5db73a7984b..10c585f7cd9 100644
--- a/tests/Infrastructure.Tests/PowerShellScripts/ExpandTestMatrixGitHubTests.cs
+++ b/tests/Infrastructure.Tests/PowerShellScripts/ExpandTestMatrixGitHubTests.cs
@@ -599,13 +599,13 @@ public async Task FullPipeline_SplitTestsExpandPerOS()
var e2eEntries = nugetsLinux.Include.Where(e => e.ProjectName == "LinuxE2E").ToArray();
Assert.Single(e2eEntries);
Assert.Equal("ubuntu-latest", e2eEntries[0].RunsOn);
- Assert.True(e2eEntries[0].RequiresNugets);
+ Assert.True(e2eEntries[0].Properties.GetValueOrDefault("requiresNugets"));
// CLI E2E: 1 project × 1 OS = 1, in requires-cli-archive matrix
var cliE2eEntries = cliArchiveMatrix.Include.Where(e => e.ProjectName == "CliE2E").ToArray();
Assert.Single(cliE2eEntries);
Assert.Equal("ubuntu-latest", cliE2eEntries[0].RunsOn);
- Assert.True(cliE2eEntries[0].RequiresCliArchive);
+ Assert.True(cliE2eEntries[0].Properties.GetValueOrDefault("requiresCliArchive"));
// Total no-nugets: 3 + 6 = 9, Total nugets: 1 (linux only), Total cli-archive: 1
Assert.Equal(9, allNoNugets.Length);
diff --git a/tests/Infrastructure.Tests/PowerShellScripts/SplitTestMatrixByDepsTests.cs b/tests/Infrastructure.Tests/PowerShellScripts/SplitTestMatrixByDepsTests.cs
index 86f958177a9..05cb1473711 100644
--- a/tests/Infrastructure.Tests/PowerShellScripts/SplitTestMatrixByDepsTests.cs
+++ b/tests/Infrastructure.Tests/PowerShellScripts/SplitTestMatrixByDepsTests.cs
@@ -51,7 +51,7 @@ public async Task SplitsRequiresNugetsTestsIntoRequiresNugetsBucket()
{
var matrixJson = BuildMatrixJson(
new { name = "Plain", shortname = "p", runs_on = "ubuntu-latest" },
- new { name = "NugetTest", shortname = "nt", runs_on = "ubuntu-latest", requiresNugets = true });
+ new { name = "NugetTest", shortname = "nt", runs_on = "ubuntu-latest", properties = new { requiresNugets = true } });
var result = await RunScript(allTestsMatrix: matrixJson);
@@ -69,9 +69,9 @@ public async Task SplitsRequiresNugetsTestsIntoRequiresNugetsBucket()
public async Task SplitsRequiresNugetsByOs()
{
var matrixJson = BuildMatrixJson(
- new { name = "LinuxNuget", shortname = "ln", runs_on = "ubuntu-latest", requiresNugets = true },
- new { name = "WinNuget", shortname = "wn", runs_on = "windows-latest", requiresNugets = true },
- new { name = "MacNuget", shortname = "mn", runs_on = "macos-latest", requiresNugets = true });
+ new { name = "LinuxNuget", shortname = "ln", runs_on = "ubuntu-latest", properties = new { requiresNugets = true } },
+ new { name = "WinNuget", shortname = "wn", runs_on = "windows-latest", properties = new { requiresNugets = true } },
+ new { name = "MacNuget", shortname = "mn", runs_on = "macos-latest", properties = new { requiresNugets = true } });
var result = await RunScript(allTestsMatrix: matrixJson);
@@ -92,7 +92,7 @@ public async Task SplitsRequiresCliArchiveTestsIntoCliArchiveBucket()
{
var matrixJson = BuildMatrixJson(
new { name = "Plain", shortname = "p", runs_on = "ubuntu-latest" },
- new { name = "CliTest", shortname = "ct", runs_on = "ubuntu-latest", requiresCliArchive = true });
+ new { name = "CliTest", shortname = "ct", runs_on = "ubuntu-latest", properties = new { requiresCliArchive = true } });
var result = await RunScript(allTestsMatrix: matrixJson);
@@ -109,7 +109,7 @@ public async Task SplitsRequiresCliArchiveTestsIntoCliArchiveBucket()
public async Task CliArchiveTakesPriorityOverNugets()
{
var matrixJson = BuildMatrixJson(
- new { name = "Both", shortname = "b", runs_on = "ubuntu-latest", requiresNugets = true, requiresCliArchive = true });
+ new { name = "Both", shortname = "b", runs_on = "ubuntu-latest", properties = new { requiresNugets = true, requiresCliArchive = true } });
var result = await RunScript(allTestsMatrix: matrixJson);
diff --git a/tests/Infrastructure.Tests/Shared/TestDataBuilder.cs b/tests/Infrastructure.Tests/Shared/TestDataBuilder.cs
index fdd356d43d6..7cc9c2e512f 100644
--- a/tests/Infrastructure.Tests/Shared/TestDataBuilder.cs
+++ b/tests/Infrastructure.Tests/Shared/TestDataBuilder.cs
@@ -30,6 +30,7 @@ public static string CreateTestsMetadataJson(
bool requiresNugets = false,
bool requiresTestSdk = false,
bool requiresCliArchive = false,
+ bool enablePlaywrightInstall = false,
string? extraTestArgs = null,
string[]? supportedOSes = null,
Dictionary? runners = null)
@@ -42,9 +43,13 @@ public static string CreateTestsMetadataJson(
SplitTests = "false",
TestSessionTimeout = testSessionTimeout,
TestHangTimeout = testHangTimeout,
- RequiresNugets = requiresNugets ? "true" : null,
- RequiresTestSdk = requiresTestSdk ? "true" : null,
- RequiresCliArchive = requiresCliArchive ? "true" : null,
+ Properties = new Dictionary
+ {
+ ["requiresNugets"] = requiresNugets,
+ ["requiresTestSdk"] = requiresTestSdk,
+ ["requiresCliArchive"] = requiresCliArchive,
+ ["enablePlaywrightInstall"] = enablePlaywrightInstall
+ },
ExtraTestArgs = extraTestArgs,
SupportedOSes = supportedOSes ?? ["windows", "linux", "macos"],
Runners = runners
@@ -87,8 +92,11 @@ public static string CreateSplitTestsMetadataJson(
TestHangTimeout = testHangTimeout,
UncollectedTestsSessionTimeout = uncollectedTestsSessionTimeout,
UncollectedTestsHangTimeout = uncollectedTestsHangTimeout,
- RequiresNugets = requiresNugets ? "true" : null,
- RequiresTestSdk = requiresTestSdk ? "true" : null,
+ Properties = new Dictionary
+ {
+ ["requiresNugets"] = requiresNugets,
+ ["requiresTestSdk"] = requiresTestSdk,
+ },
SupportedOSes = supportedOSes ?? ["windows", "linux", "macos"],
Runners = runners
};
@@ -187,6 +195,7 @@ public static CanonicalMatrixEntry CreateMatrixEntry(
bool requiresNugets = false,
bool requiresTestSdk = false,
bool requiresCliArchive = false,
+ bool enablePlaywrightInstall = false,
string[]? supportedOSes = null,
Dictionary? runners = null)
{
@@ -203,9 +212,13 @@ public static CanonicalMatrixEntry CreateMatrixEntry(
ExtraTestArgs = extraTestArgs ?? "",
TestSessionTimeout = testSessionTimeout,
TestHangTimeout = testHangTimeout,
- RequiresNugets = requiresNugets,
- RequiresTestSdk = requiresTestSdk,
- RequiresCliArchive = requiresCliArchive,
+ Properties = new Dictionary
+ {
+ ["requiresNugets"] = requiresNugets,
+ ["requiresTestSdk"] = requiresTestSdk,
+ ["requiresCliArchive"] = requiresCliArchive,
+ ["enablePlaywrightInstall"] = enablePlaywrightInstall
+ },
SupportedOSes = supportedOSes ?? ["windows", "linux", "macos"],
Runners = runners
};
@@ -237,14 +250,8 @@ private sealed class TestMetadata
[JsonPropertyName("uncollectedTestsHangTimeout")]
public string? UncollectedTestsHangTimeout { get; set; }
- [JsonPropertyName("requiresNugets")]
- public string? RequiresNugets { get; set; }
-
- [JsonPropertyName("requiresTestSdk")]
- public string? RequiresTestSdk { get; set; }
-
- [JsonPropertyName("requiresCliArchive")]
- public string? RequiresCliArchive { get; set; }
+ [JsonPropertyName("properties")]
+ public Dictionary Properties { get; set; } = new();
[JsonPropertyName("extraTestArgs")]
public string? ExtraTestArgs { get; set; }
@@ -310,14 +317,8 @@ public class CanonicalMatrixEntry
[JsonPropertyName("testHangTimeout")]
public string TestHangTimeout { get; set; } = "10m";
- [JsonPropertyName("requiresNugets")]
- public bool RequiresNugets { get; set; }
-
- [JsonPropertyName("requiresTestSdk")]
- public bool RequiresTestSdk { get; set; }
-
- [JsonPropertyName("requiresCliArchive")]
- public bool RequiresCliArchive { get; set; }
+ [JsonPropertyName("properties")]
+ public Dictionary Properties { get; set; } = new();
[JsonPropertyName("splitTests")]
public bool SplitTests { get; set; }