From 23846fe93ee3fd865575f21b794cafa276ef0f69 Mon Sep 17 00:00:00 2001 From: Steven Ma Date: Tue, 18 Aug 2026 11:21:56 -0700 Subject: [PATCH 1/6] Improve quickstart deployment validation Add correlation-backed metadata validation and a maintainer-triggered /validate workflow for Terraform quickstarts. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../schemas/quickstart-metadata.schema.json | 42 ++ .../test_validate_quickstart_metadata.py | 111 ++++++ .../scripts/validate-quickstart-metadata.py | 239 ++++++++++++ .github/workflows/e2e.yaml | 369 +++++++++++++++--- .github/workflows/pr-check.yaml | 19 +- quickstart/CONTRIBUTE.md | 106 +++++ 6 files changed, 832 insertions(+), 54 deletions(-) create mode 100644 .github/schemas/quickstart-metadata.schema.json create mode 100644 .github/scripts/test_validate_quickstart_metadata.py create mode 100644 .github/scripts/validate-quickstart-metadata.py diff --git a/.github/schemas/quickstart-metadata.schema.json b/.github/schemas/quickstart-metadata.schema.json new file mode 100644 index 000000000..699553ac6 --- /dev/null +++ b/.github/schemas/quickstart-metadata.schema.json @@ -0,0 +1,42 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/Azure/terraform/blob/master/.github/schemas/quickstart-metadata.schema.json", + "title": "Azure Terraform quickstart deployment metadata", + "type": "object", + "additionalProperties": false, + "required": [ + "$schema", + "testResult" + ], + "properties": { + "$schema": { + "type": "string", + "const": "../../.github/schemas/quickstart-metadata.schema.json" + }, + "testResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "correlationId", + "timestamp" + ], + "properties": { + "correlationId": { + "type": "string", + "format": "uuid", + "description": "Value supplied to ARM_CORRELATION_REQUEST_ID for the validated Terraform apply." + }, + "timestamp": { + "type": "string", + "format": "date-time", + "description": "UTC time immediately before the validated Terraform apply started." + }, + "terraformVersion": { + "type": "string", + "pattern": "^\\d+\\.\\d+\\.\\d+(?:[-+][0-9A-Za-z.-]+)?$", + "description": "Terraform CLI version used for the apply." + } + } + } + } +} diff --git a/.github/scripts/test_validate_quickstart_metadata.py b/.github/scripts/test_validate_quickstart_metadata.py new file mode 100644 index 000000000..db9d0b17b --- /dev/null +++ b/.github/scripts/test_validate_quickstart_metadata.py @@ -0,0 +1,111 @@ +import importlib.util +import datetime +import json +import pathlib +import tempfile +import unittest + + +SCRIPT = pathlib.Path(__file__).with_name("validate-quickstart-metadata.py") +SPEC = importlib.util.spec_from_file_location("metadata_validator", SCRIPT) +VALIDATOR = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(VALIDATOR) + + +class ValidateMetadataTests(unittest.TestCase): + def current_timestamp(self) -> str: + return ( + datetime.datetime.now(datetime.timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + ) + + def write_metadata(self, data: dict) -> pathlib.Path: + temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(temporary_directory.cleanup) + path = pathlib.Path(temporary_directory.name) / "metadata.json" + path.write_text(json.dumps(data), encoding="utf-8") + return path + + def test_accepts_valid_metadata(self) -> None: + path = self.write_metadata( + { + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "12345678-1234-1234-1234-1234567890ab", + "timestamp": self.current_timestamp(), + "terraformVersion": "1.13.0", + }, + } + ) + + self.assertEqual([], VALIDATOR.validate_metadata(path)) + + def test_rejects_invalid_correlation_id(self) -> None: + path = self.write_metadata( + { + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "not-a-guid", + "timestamp": self.current_timestamp(), + }, + } + ) + + self.assertTrue( + any("dashed UUID" in error for error in VALIDATOR.validate_metadata(path)) + ) + + def test_requires_utc_timestamp(self) -> None: + path = self.write_metadata( + { + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "12345678-1234-1234-1234-1234567890ab", + "timestamp": "2026-08-18T17:00:00-07:00", + }, + } + ) + + self.assertTrue( + any( + "timestamp" in error + for error in VALIDATOR.validate_metadata(path) + ) + ) + + def test_reads_test_result_value(self) -> None: + data = {"testResult": {"correlationId": "example"}} + + self.assertEqual( + "example", + VALIDATOR.test_result_value(data, "correlationId"), + ) + self.assertIsNone(VALIDATOR.test_result_value({}, "correlationId")) + + def test_rejects_stale_timestamp(self) -> None: + path = self.write_metadata( + { + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "12345678-1234-1234-1234-1234567890ab", + "timestamp": "2000-01-01T00:00:00Z", + }, + } + ) + + self.assertTrue( + any("days old" in error for error in VALIDATOR.validate_metadata(path)) + ) + + def test_recognizes_terraform_json_configuration(self) -> None: + self.assertTrue("main.tf.json".endswith(VALIDATOR.CONFIGURATION_SUFFIXES)) + self.assertTrue( + "testing.auto.tfvars.json".endswith(VALIDATOR.CONFIGURATION_SUFFIXES) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/validate-quickstart-metadata.py b/.github/scripts/validate-quickstart-metadata.py new file mode 100644 index 000000000..47605e992 --- /dev/null +++ b/.github/scripts/validate-quickstart-metadata.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 + +import argparse +import datetime +import json +import pathlib +import re +import subprocess +import sys +import uuid + + +CONFIGURATION_SUFFIXES = ( + ".tf", + ".tf.json", + ".tfvars", + ".tfvars.json", + ".tftpl", + ".pkr.hcl", +) +EXPECTED_SCHEMA = "../../.github/schemas/quickstart-metadata.schema.json" +MAX_METADATA_AGE = datetime.timedelta(days=30) +MAX_FUTURE_SKEW = datetime.timedelta(minutes=15) +TERRAFORM_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Validate deployment metadata for changed Terraform quickstarts." + ) + parser.add_argument( + "folders", + nargs="+", + help="Quickstart folders. A comma-delimited argument is also accepted.", + ) + parser.add_argument( + "--base", + help="Git base ref used to require a metadata update when configuration changes.", + ) + parser.add_argument( + "--require-metadata", + action="store_true", + help="Require metadata.json even when no Git base is supplied.", + ) + return parser.parse_args() + + +def expand_folders(values: list[str]) -> list[pathlib.Path]: + folders: list[pathlib.Path] = [] + for value in values: + folders.extend( + pathlib.Path(item.strip()) + for item in value.split(",") + if item.strip() + ) + return sorted(set(folders)) + + +def changed_files(base: str, folder: pathlib.Path) -> list[str]: + result = subprocess.run( + ["git", "diff", "--name-only", f"{base}...HEAD", "--", folder.as_posix()], + check=True, + capture_output=True, + text=True, + ) + return [line for line in result.stdout.splitlines() if line] + + +def metadata_at_ref(base: str, path: pathlib.Path) -> dict | None: + result = subprocess.run( + ["git", "show", f"{base}:{path.as_posix()}"], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None + try: + data = json.loads(result.stdout) + except json.JSONDecodeError: + return None + return data if isinstance(data, dict) else None + + +def test_result_value(data: dict | None, field: str) -> object: + if not data: + return None + test_result = data.get("testResult") + if not isinstance(test_result, dict): + return None + return test_result.get(field) + + +def parse_timestamp(value: object) -> datetime.datetime | None: + if not isinstance(value, str) or not value.endswith("Z"): + return None + try: + parsed = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo != datetime.timezone.utc: + return None + return parsed + + +def validate_metadata(path: pathlib.Path) -> list[str]: + errors: list[str] = [] + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + return [f"{path}: invalid JSON: {error}"] + + if not isinstance(data, dict): + return [f"{path}: root value must be an object"] + + allowed_root = {"$schema", "testResult"} + unexpected_root = sorted(set(data) - allowed_root) + if unexpected_root: + errors.append(f"{path}: unexpected fields: {', '.join(unexpected_root)}") + if data.get("$schema") != EXPECTED_SCHEMA: + errors.append(f"{path}: $schema must be '{EXPECTED_SCHEMA}'") + + test_result = data.get("testResult") + if not isinstance(test_result, dict): + errors.append(f"{path}: testResult must be an object") + return errors + + allowed_result = {"correlationId", "timestamp", "terraformVersion"} + unexpected_result = sorted(set(test_result) - allowed_result) + if unexpected_result: + errors.append( + f"{path}: unexpected testResult fields: {', '.join(unexpected_result)}" + ) + + correlation_id = test_result.get("correlationId") + try: + parsed_id = uuid.UUID(correlation_id) + if str(parsed_id) != correlation_id.lower(): + raise ValueError + except (AttributeError, TypeError, ValueError): + errors.append(f"{path}: testResult.correlationId must be a dashed UUID") + + timestamp = parse_timestamp(test_result.get("timestamp")) + if timestamp is None: + errors.append( + f"{path}: testResult.timestamp must be an RFC 3339 UTC timestamp ending in Z" + ) + else: + now = datetime.datetime.now(datetime.timezone.utc) + if timestamp < now - MAX_METADATA_AGE: + errors.append( + f"{path}: testResult.timestamp must be no more than " + f"{MAX_METADATA_AGE.days} days old" + ) + if timestamp > now + MAX_FUTURE_SKEW: + errors.append( + f"{path}: testResult.timestamp is more than " + f"{int(MAX_FUTURE_SKEW.total_seconds() / 60)} minutes in the future" + ) + + terraform_version = test_result.get("terraformVersion") + if terraform_version is not None and ( + not isinstance(terraform_version, str) + or not TERRAFORM_VERSION_RE.fullmatch(terraform_version) + ): + errors.append( + f"{path}: testResult.terraformVersion must be a semantic version" + ) + + return errors + + +def main() -> int: + args = parse_args() + folders = expand_folders(args.folders) + errors: list[str] = [] + + if not folders: + print("No changed quickstart folders.") + return 0 + + for folder in folders: + metadata_path = folder / "metadata.json" + require_metadata = args.require_metadata + + if args.base: + changed = changed_files(args.base, folder) + configuration_changed = any( + file_name.endswith(CONFIGURATION_SUFFIXES) for file_name in changed + ) + metadata_changed = metadata_path.as_posix() in changed + require_metadata = require_metadata or configuration_changed + if configuration_changed and not metadata_changed: + errors.append( + f"{folder}: Terraform configuration changed but metadata.json " + "was not updated with a fresh correlation ID" + ) + elif configuration_changed and metadata_path.is_file(): + previous = metadata_at_ref(args.base, metadata_path) + try: + current = json.loads(metadata_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + current = None + previous_id = test_result_value(previous, "correlationId") + current_id = test_result_value(current, "correlationId") + if previous_id is not None and current_id == previous_id: + errors.append( + f"{folder}: testResult.correlationId must change when " + "Terraform configuration changes" + ) + previous_timestamp = test_result_value(previous, "timestamp") + current_timestamp = test_result_value(current, "timestamp") + if ( + previous_timestamp is not None + and current_timestamp == previous_timestamp + ): + errors.append( + f"{folder}: testResult.timestamp must change when " + "Terraform configuration changes" + ) + + if require_metadata and not metadata_path.is_file(): + errors.append(f"{folder}: metadata.json is required") + continue + + if metadata_path.is_file(): + errors.extend(validate_metadata(metadata_path)) + + if errors: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + + print(f"Validated deployment metadata for {len(folders)} quickstart folder(s).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index bc41597e9..72ce65a5d 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1,56 +1,331 @@ -name: E2E Test Check +name: Validate Terraform Deployment + on: - pull_request: - types: ['opened', 'synchronize'] - paths: - - '.github/**' - - '.github/workflows/**' - - 'quickstart/**' - - 'test/**' - - 'test/e2e/**' -permissions: - actions: write - contents: read - id-token: write + issue_comment: + types: [created] jobs: - e2e-check: + gate: + name: Authorize validation command + if: >- + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/validate') runs-on: ubuntu-latest - environment: - name: test + timeout-minutes: 12 + permissions: + actions: read + checks: write + contents: read + issues: write + pull-requests: read + + outputs: + check_id: ${{ steps.check.outputs.check_id }} + head_sha: ${{ steps.pr.outputs.head_sha }} + head_repo: ${{ steps.pr.outputs.head_repo }} + sample_path: ${{ steps.pr.outputs.sample_path }} + steps: - - name: Checking for Fork - shell: pwsh + - name: Authorize command and resolve pull request + id: pr + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + script: | + if (context.payload.comment.body.trim() !== '/validate') { + core.setFailed('The supported command is exactly /validate.'); + return; + } + + const allowedCommenters = new Set(['MEMBER', 'OWNER', 'COLLABORATOR']); + const commenterAssociation = context.payload.comment.author_association; + if (!allowedCommenters.has(commenterAssociation)) { + core.setFailed( + `Not authorized to run /validate: ${commenterAssociation}. ` + + 'Only MEMBER, OWNER, or COLLABORATOR may run this command.' + ); + return; + } + + const owner = context.repo.owner; + const repo = context.repo.repo; + const pull_number = context.payload.issue.number; + const [{ data: pr }, files] = await Promise.all([ + github.rest.pulls.get({ owner, repo, pull_number }), + github.paginate( + github.rest.pulls.listFiles, + { owner, repo, pull_number, per_page: 100 } + ) + ]); + + const protectedPrefixes = ['.github/workflows/', '.github/scripts/']; + const protectedFiles = [ + '.github/schemas/quickstart-metadata.schema.json', + 'GNUmakefile', + 'scripts/terraform-validate.sh' + ]; + const touchedProtected = files + .map(file => file.filename) + .filter(name => + protectedPrefixes.some(prefix => name.startsWith(prefix)) || + protectedFiles.includes(name) + ); + const allowedProtectedAuthors = new Set(['MEMBER', 'OWNER']); + if (touchedProtected.length > 0 && !allowedProtectedAuthors.has(pr.author_association)) { + core.setFailed( + `Protected validation files were changed by a PR author with ` + + `association '${pr.author_association}': ${touchedProtected.join(', ')}` + ); + return; + } + + const deployFiles = files + .map(file => file.filename) + .filter(name => + name.startsWith('quickstart/') && + /\.(tf|tf\.json|tfvars|tfvars\.json|tftpl|pkr\.hcl)$/.test(name) + ); + const samplePaths = new Set( + deployFiles.map(name => name.split('/').slice(0, 2).join('/')) + ); + if (samplePaths.size !== 1) { + core.setFailed( + `/validate requires exactly one changed deployable quickstart; found ` + + `${samplePaths.size}: ${[...samplePaths].join(', ') || ''}` + ); + return; + } + + core.setOutput('head_sha', pr.head.sha); + core.setOutput('head_repo', pr.head.repo.full_name); + core.setOutput('sample_path', [...samplePaths][0]); + + - name: React to command + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: 'eyes' + }); + + - name: Start pull request check + id: check + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + env: + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + with: + script: | + const { data } = await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + name: 'terraform-deployment-validation', + head_sha: process.env.HEAD_SHA, + status: 'in_progress', + details_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + output: { + title: 'Validating Terraform deployment', + summary: 'A maintainer ran `/validate`; waiting for static checks, then querying ARM telemetry.' + } + }); + core.setOutput('check_id', data.id); + + - name: Wait for pre-pull-request checks + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + REPOSITORY: ${{ github.repository }} + shell: bash run: | - $isFork = "${{ github.event.pull_request.head.repo.fork }}" - if($isFork -eq "true") { - echo "### WARNING: This workflow is disabled for forked repositories. Please follow the [release branch process](https://azure.github.io/Azure-Verified-Modules/contributing/terraform/terraform-contribution-flow/#5-create-a-pull-request-to-the-upstream-repository) if end to end tests are required." >> $env:GITHUB_STEP_SUMMARY - } - - name: checkout - if: github.event.pull_request.head.repo.fork == false - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 #v3.6.0 - - name: Get changed files - if: github.event.pull_request.head.repo.fork == false - id: changed-files - uses: step-security/changed-files@3dbe17c78367e7d60f00d78ae6781a35be47b4a1 #v45.0.1 + set -euo pipefail + deadline=$(( $(date +%s) + 600 )) + + while (( $(date +%s) < deadline )); do + run_json=$(gh api \ + "/repos/${REPOSITORY}/actions/workflows/pr-check.yaml/runs?head_sha=${HEAD_SHA}&per_page=1" \ + --jq '.workflow_runs[0] // {}') + status=$(jq -r '.status // empty' <<< "${run_json}") + conclusion=$(jq -r '.conclusion // empty' <<< "${run_json}") + + if [[ "${status}" == "completed" ]]; then + if [[ "${conclusion}" == "success" ]]; then + echo "Pre-pull-request checks passed for ${HEAD_SHA}." + exit 0 + fi + echo "Pre-pull-request checks concluded '${conclusion}' for ${HEAD_SHA}." >&2 + exit 1 + fi + + echo "Waiting for pre-pull-request checks (status=${status:-not-started})..." + sleep 30 + done + + echo "Timed out waiting for pre-pull-request checks." >&2 + exit 1 + + validate: + name: Validate contributor deployment + needs: gate + runs-on: ubuntu-latest + timeout-minutes: 5 + environment: + name: adx-readonly + permissions: + contents: read + id-token: write + + steps: + - name: Checkout trusted validator + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: ${{ github.repository }} + ref: ${{ github.event.repository.default_branch }} + path: trusted + sparse-checkout: .github/scripts/validate-quickstart-metadata.py + sparse-checkout-cone-mode: false + + - name: Checkout pull request sample + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: ${{ needs.gate.outputs.head_repo }} + ref: ${{ needs.gate.outputs.head_sha }} + path: pull-request + sparse-checkout: ${{ needs.gate.outputs.sample_path }} + + - name: Validate metadata + env: + SAMPLE_PATH: ${{ needs.gate.outputs.sample_path }} + run: >- + python trusted/.github/scripts/validate-quickstart-metadata.py + --require-metadata + "pull-request/${SAMPLE_PATH}" + + - name: Sign in to Azure with OIDC + uses: azure/login@8216e11d8cd9b42fe925c852af8e76311ff067ac # v2 with: - dir_names: true - separator: "," - dir_names_include_files: "quickstart/*" - files: "quickstart/**" - files_ignore: "**/TestRecord.md" - dir_names_max_depth: 2 - - name: test pr - if: github.event.pull_request.head.repo.fork == false + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + client-id: ${{ secrets.AZURE_UAMI_CLIENT_ID }} + + - name: Validate correlated Terraform operations env: - ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + SAMPLE_PATH: ${{ needs.gate.outputs.sample_path }} + shell: bash run: | - echo "change files" $ALL_CHANGED_FILES - export ARM_OIDC_REQUEST_TOKEN=$ACTIONS_ID_TOKEN_REQUEST_TOKEN - export ARM_OIDC_REQUEST_URL=$ACTIONS_ID_TOKEN_REQUEST_URL - export ARM_SUBSCRIPTION_ID=${{ secrets.ARM_SUBSCRIPTION_ID }} - export ARM_TENANT_ID=${{ secrets.ARM_TENANT_ID }} - export ARM_CLIENT_ID=${{ secrets.ARM_CLIENT_ID }} - export PACKER_VERSION=${{ vars.PACKER_VERSION }} - export CHANGED_FOLDERS="${{ steps.changed-files.outputs.all_changed_files }}" - docker run --rm -v $(pwd):/src -w /src/test --network=host -e ARM_SUBSCRIPTION_ID -e ARM_TENANT_ID -e ARM_CLIENT_ID -e ARM_OIDC_REQUEST_TOKEN -e ARM_OIDC_REQUEST_URL -e ARM_USE_OIDC=true -e CHANGED_FOLDERS mcr.microsoft.com/azterraform:latest sh -c "pkenv install $PACKER_VERSION && go mod tidy && go test -timeout=360m -v ./e2e" \ No newline at end of file + set -euo pipefail + + metadata="pull-request/${SAMPLE_PATH}/metadata.json" + correlation_id=$(jq -r '.testResult.correlationId' "${metadata}") + started_at=$(jq -r '.testResult.timestamp' "${metadata}") + + query='declare query_parameters(cid:string, started:datetime); + ARM_Deployments_Terraform_v2 + | where correlationId == cid + | where PreciseTimeStamp between ((started - 1h) .. (started + 24h)) + | where client == "Terraform" + | summarize + firstSeen=min(PreciseTimeStamp), + lastSeen=max(PreciseTimeStamp), + totalRequests=sum(requestCount), + successfulWrites=sumif(requestCount, httpMethod in ("PUT", "PATCH", "POST", "DELETE") and executionStatus == "Succeeded"), + failedRequests=sumif(requestCount, executionStatus in ("Failed", "Canceled")), + providers=dcount(providerNamespace)' + + properties=$(jq -cn \ + --arg cid "${correlation_id}" \ + --arg started "${started_at}" \ + '{ Parameters: { cid: $cid, started: $started } }') + body=$(jq -n \ + --arg db "APAProd" \ + --arg csl "${query}" \ + --arg properties "${properties}" \ + '{ + db: $db, + csl: $csl, + properties: $properties + }') + + response=$(az rest \ + --method POST \ + --url "https://apadata.westus.kusto.windows.net/v1/rest/query" \ + --resource "https://kusto.kusto.windows.net" \ + --headers "Content-Type=application/json" \ + --body "${body}") + + value() { + local column="$1" + jq -r --arg column "${column}" \ + '.Tables[0] as $table + | ($table.Columns | map(.ColumnName) | index($column)) as $index + | if $index == null then empty else $table.Rows[0][$index] end' \ + <<< "${response}" + } + + total_requests=$(value totalRequests) + successful_writes=$(value successfulWrites) + failed_requests=$(value failedRequests) + first_seen=$(value firstSeen) + last_seen=$(value lastSeen) + providers=$(value providers) + + echo "Correlation ID: ${correlation_id}" + echo "Observed: ${first_seen} through ${last_seen}" + echo "Requests: ${total_requests:-0}; successful writes: ${successful_writes:-0}; failed/canceled: ${failed_requests:-0}; providers: ${providers:-0}" + + if (( ${total_requests:-0} == 0 )); then + echo "No Terraform operations were found for this correlation ID." >&2 + exit 1 + fi + if (( ${successful_writes:-0} == 0 )); then + echo "The correlation ID has no successful Terraform write operations." >&2 + exit 1 + fi + if (( ${failed_requests:-0} != 0 )); then + echo "The correlated Terraform workflow contains failed or canceled operations." >&2 + exit 1 + fi + + { + echo "## Terraform deployment validation" + echo + echo "- Sample: \`${SAMPLE_PATH}\`" + echo "- Correlation ID: \`${correlation_id}\`" + echo "- Successful writes: ${successful_writes}" + echo "- Failed or canceled operations: ${failed_requests}" + echo "- Resource providers: ${providers}" + } >> "${GITHUB_STEP_SUMMARY}" + + report: + name: Report deployment validation + needs: [gate, validate] + if: always() && needs.gate.outputs.check_id != '' + runs-on: ubuntu-latest + permissions: + checks: write + contents: read + steps: + - name: Complete pull request check + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + env: + CHECK_ID: ${{ needs.gate.outputs.check_id }} + RESULT: ${{ needs.validate.result }} + SAMPLE_PATH: ${{ needs.gate.outputs.sample_path }} + with: + script: | + const passed = process.env.RESULT === 'success'; + await github.rest.checks.update({ + owner: context.repo.owner, + repo: context.repo.repo, + check_run_id: Number(process.env.CHECK_ID), + status: 'completed', + conclusion: passed ? 'success' : 'failure', + output: { + title: passed + ? 'Terraform deployment validation passed' + : 'Terraform deployment validation failed', + summary: passed + ? `ARM telemetry confirmed successful Terraform writes for \`${process.env.SAMPLE_PATH}\` with no failed or canceled operations.` + : `Deployment validation failed. Open the workflow run for details, correct the metadata or deployment, then run \`/validate\` again.` + } + }); diff --git a/.github/workflows/pr-check.yaml b/.github/workflows/pr-check.yaml index eb4d6a6a6..5f52c3dc5 100644 --- a/.github/workflows/pr-check.yaml +++ b/.github/workflows/pr-check.yaml @@ -1,10 +1,9 @@ name: Pre Pull Request Check on: pull_request: - types: [ 'opened', 'synchronize' ] + types: [ 'opened', 'reopened', 'synchronize' ] paths: - '.github/**' - - '.github/workflows/**' - 'quickstart/**' jobs: @@ -12,7 +11,9 @@ jobs: runs-on: ubuntu-latest steps: - name: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 #v3.6.0 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 - name: Get changed files id: changed-files uses: step-security/changed-files@3dbe17c78367e7d60f00d78ae6781a35be47b4a1 #v45.0.1 @@ -22,10 +23,14 @@ jobs: dir_names_include_files: "quickstart/*" files: "quickstart/**" dir_names_max_depth: 2 + - name: Validate quickstart deployment metadata + env: + CHANGED_FOLDERS: ${{ steps.changed-files.outputs.all_changed_files }} + run: | + python .github/scripts/validate-quickstart-metadata.py \ + --base "origin/${{ github.base_ref }}" \ + "${CHANGED_FOLDERS}" - name: pr-check run: | export CHANGED_FOLDERS="${{ steps.changed-files.outputs.all_changed_files }}" - if [ -z "${{ github.event.number }}" ]; then - CHANGED_FOLDERS=$(find ./quickstart -maxdepth 1 -mindepth 1 -type d | tr '\n' ',') - fi - docker run --rm -v $(pwd):/src -w /src -e CHANGED_FOLDERS mcr.microsoft.com/azterraform:latest make pr-check + docker run --rm -v "$(pwd):/src" -w /src -e CHANGED_FOLDERS mcr.microsoft.com/azterraform:latest make pr-check diff --git a/quickstart/CONTRIBUTE.md b/quickstart/CONTRIBUTE.md index e69de29bb..d3acfd2db 100644 --- a/quickstart/CONTRIBUTE.md +++ b/quickstart/CONTRIBUTE.md @@ -0,0 +1,106 @@ +# Contributing a Terraform quickstart + +The repository validates two different claims: + +1. **Static quality:** the pull request passes formatting, linting, and repository checks. +2. **Deployment evidence:** a maintainer confirms that the exact correlation ID recorded by the contributor maps to a successful Terraform workflow in Azure Resource Manager telemetry. + +The repository does not deploy pull-request code with privileged credentials. Contributors deploy the quickstart first; maintainers validate the resulting telemetry with `/validate`. + +## Contribution requirements + +A pull request that changes deployable files in `quickstart//` must: + +- Change one deployable quickstart per pull request. +- Pass `terraform fmt`, `terraform init`, `terraform validate`, and `terraform plan`. +- Apply the quickstart to Azure before opening the pull request. +- Set `ARM_CORRELATION_REQUEST_ID` to a new UUID for that apply. +- Add or update `metadata.json` in the quickstart folder with the UUID and apply start time. +- Remove the deployed resources after validation unless the sample documents a reason to retain them. + +The AzureRM and AzAPI providers use `ARM_CORRELATION_REQUEST_ID` as the `x-ms-correlation-request-id` for the provider workflow. Do not reuse an ID from an earlier plan or apply. + +## Deploy and capture validation evidence + +Authenticate to the Azure subscription you use for testing, then run the commands from the quickstart folder. + +### Bash + +```bash +export ARM_CORRELATION_REQUEST_ID="$(python -c 'import uuid; print(uuid.uuid4())')" +export VALIDATION_STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + +terraform init +terraform validate +terraform plan -out=tfplan +terraform apply tfplan + +printf 'correlationId=%s\ntimestamp=%s\n' \ + "$ARM_CORRELATION_REQUEST_ID" \ + "$VALIDATION_STARTED_AT" +``` + +### PowerShell + +```powershell +$env:ARM_CORRELATION_REQUEST_ID = [guid]::NewGuid().ToString() +$validationStartedAt = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + +terraform init +terraform validate +terraform plan -out=tfplan +terraform apply tfplan + +"correlationId=$env:ARM_CORRELATION_REQUEST_ID" +"timestamp=$validationStartedAt" +``` + +Use a saved plan so the reviewed plan and applied configuration are the same local operation. Never commit `tfplan`, Terraform state, credentials, subscription IDs, tenant IDs, or other environment-specific values. + +## Add `metadata.json` + +Create `metadata.json` beside the quickstart's Terraform files: + +```json +{ + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "12345678-1234-1234-1234-1234567890ab", + "timestamp": "2026-08-18T17:00:00Z", + "terraformVersion": "1.13.0" + } +} +``` + +`correlationId` and `timestamp` are required. The timestamp must be no more than 30 days old. `terraformVersion` is optional but recommended. If any `.tf`, `.tf.json`, `.tfvars`, `.tfvars.json`, `.tftpl`, or `.pkr.hcl` file changes, deploy again and replace both required values. + +## Pull request validation + +The **Pre Pull Request Check** workflow runs automatically. It: + +- Detects changed quickstart folders. +- Requires a fresh `metadata.json` update when deployable files change. +- Validates the metadata format. +- Runs the existing repository `pr-check` target. + +After that workflow passes, a repository maintainer comments: + +```text +/validate +``` + +The command is restricted to repository members, owners, and collaborators. It checks out only the changed quickstart, validates its metadata with trusted code from the default branch, signs in to Azure with the `adx-readonly` environment's federated identity, and queries `ARM_Deployments_Terraform_v2` by the supplied correlation ID. + +Validation passes only when telemetry contains at least one successful Terraform write and no failed or canceled operation for that correlated workflow. The workflow publishes the result as the `terraform-deployment-validation` check on the pull request head. + +This is deployment evidence, not source attestation: unlike an ARM template deployment, Terraform does not emit a template hash that ARM can compare with the pull request. Reviewers must still inspect the code and the static checks. + +## Maintainer configuration + +The `adx-readonly` GitHub environment must provide federated OIDC credentials through: + +- `AZURE_TENANT_ID` +- `AZURE_SUBSCRIPTION_ID` +- `AZURE_UAMI_CLIENT_ID` + +The identity needs read-only query access to the `APAProd` database on `https://apadata.westus.kusto.windows.net`. It does not need permissions to deploy or modify Azure resources. \ No newline at end of file From 166ae3713bf0e7cb71084a6583a93158566e17b9 Mon Sep 17 00:00:00 2001 From: Steven Ma Date: Tue, 18 Aug 2026 11:45:49 -0700 Subject: [PATCH 2/6] Normalize changed quickstart folders Filter changed-files output to existing quickstart sample directories before invoking metadata and Terraform validation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/pr-check.yaml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-check.yaml b/.github/workflows/pr-check.yaml index 5f52c3dc5..a57ff8d5c 100644 --- a/.github/workflows/pr-check.yaml +++ b/.github/workflows/pr-check.yaml @@ -23,14 +23,34 @@ jobs: dir_names_include_files: "quickstart/*" files: "quickstart/**" dir_names_max_depth: 2 + - name: Normalize changed quickstart folders + id: changed-folders + env: + CANDIDATES: ${{ steps.changed-files.outputs.all_changed_files }} + shell: bash + run: | + set -euo pipefail + folders=$( + tr ',' '\n' <<< "${CANDIDATES}" | + while IFS= read -r candidate; do + candidate=$(xargs <<< "${candidate}") + if [[ -d "${candidate}" && "${candidate}" =~ ^quickstart/[^/]+$ ]]; then + echo "${candidate}" + fi + done | + sort -u | + paste -sd, - + ) + echo "folders=${folders}" >> "$GITHUB_OUTPUT" + echo "Changed quickstart folders: ${folders:-}" - name: Validate quickstart deployment metadata env: - CHANGED_FOLDERS: ${{ steps.changed-files.outputs.all_changed_files }} + CHANGED_FOLDERS: ${{ steps.changed-folders.outputs.folders }} run: | python .github/scripts/validate-quickstart-metadata.py \ --base "origin/${{ github.base_ref }}" \ "${CHANGED_FOLDERS}" - name: pr-check run: | - export CHANGED_FOLDERS="${{ steps.changed-files.outputs.all_changed_files }}" + export CHANGED_FOLDERS="${{ steps.changed-folders.outputs.folders }}" docker run --rm -v "$(pwd):/src" -w /src -e CHANGED_FOLDERS mcr.microsoft.com/azterraform:latest make pr-check From 2503a6b657b1471ce29786cff97039fde4402ee3 Mon Sep 17 00:00:00 2001 From: Steven Ma Date: Wed, 19 Aug 2026 09:19:46 -0700 Subject: [PATCH 3/6] Close validation rollout gaps Restore PR coverage for E2E test changes and scope metadata freshness checks to deployable or metadata updates. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../test_validate_quickstart_metadata.py | 21 +++++++ .../scripts/validate-quickstart-metadata.py | 24 ++++++-- .github/workflows/e2e-pr.yaml | 57 +++++++++++++++++++ quickstart/CONTRIBUTE.md | 2 + 4 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/e2e-pr.yaml diff --git a/.github/scripts/test_validate_quickstart_metadata.py b/.github/scripts/test_validate_quickstart_metadata.py index db9d0b17b..ab065668c 100644 --- a/.github/scripts/test_validate_quickstart_metadata.py +++ b/.github/scripts/test_validate_quickstart_metadata.py @@ -100,12 +100,33 @@ def test_rejects_stale_timestamp(self) -> None: any("days old" in error for error in VALIDATOR.validate_metadata(path)) ) + def test_allows_stale_timestamp_when_freshness_is_not_required(self) -> None: + path = self.write_metadata( + { + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "12345678-1234-1234-1234-1234567890ab", + "timestamp": "2000-01-01T00:00:00Z", + }, + } + ) + + self.assertEqual( + [], + VALIDATOR.validate_metadata(path, enforce_freshness=False), + ) + def test_recognizes_terraform_json_configuration(self) -> None: self.assertTrue("main.tf.json".endswith(VALIDATOR.CONFIGURATION_SUFFIXES)) self.assertTrue( "testing.auto.tfvars.json".endswith(VALIDATOR.CONFIGURATION_SUFFIXES) ) + def test_metadata_change_requires_metadata(self) -> None: + self.assertTrue( + VALIDATOR.should_require_metadata(False, False, True) + ) + if __name__ == "__main__": unittest.main() diff --git a/.github/scripts/validate-quickstart-metadata.py b/.github/scripts/validate-quickstart-metadata.py index 47605e992..e4fb55b94 100644 --- a/.github/scripts/validate-quickstart-metadata.py +++ b/.github/scripts/validate-quickstart-metadata.py @@ -91,6 +91,12 @@ def test_result_value(data: dict | None, field: str) -> object: return test_result.get(field) +def should_require_metadata( + explicit: bool, configuration_changed: bool, metadata_changed: bool +) -> bool: + return explicit or configuration_changed or metadata_changed + + def parse_timestamp(value: object) -> datetime.datetime | None: if not isinstance(value, str) or not value.endswith("Z"): return None @@ -103,7 +109,9 @@ def parse_timestamp(value: object) -> datetime.datetime | None: return parsed -def validate_metadata(path: pathlib.Path) -> list[str]: +def validate_metadata( + path: pathlib.Path, *, enforce_freshness: bool = True +) -> list[str]: errors: list[str] = [] try: data = json.loads(path.read_text(encoding="utf-8")) @@ -145,7 +153,7 @@ def validate_metadata(path: pathlib.Path) -> list[str]: errors.append( f"{path}: testResult.timestamp must be an RFC 3339 UTC timestamp ending in Z" ) - else: + elif enforce_freshness: now = datetime.datetime.now(datetime.timezone.utc) if timestamp < now - MAX_METADATA_AGE: errors.append( @@ -182,6 +190,7 @@ def main() -> int: for folder in folders: metadata_path = folder / "metadata.json" require_metadata = args.require_metadata + metadata_changed = False if args.base: changed = changed_files(args.base, folder) @@ -189,7 +198,9 @@ def main() -> int: file_name.endswith(CONFIGURATION_SUFFIXES) for file_name in changed ) metadata_changed = metadata_path.as_posix() in changed - require_metadata = require_metadata or configuration_changed + require_metadata = should_require_metadata( + require_metadata, configuration_changed, metadata_changed + ) if configuration_changed and not metadata_changed: errors.append( f"{folder}: Terraform configuration changed but metadata.json " @@ -224,7 +235,12 @@ def main() -> int: continue if metadata_path.is_file(): - errors.extend(validate_metadata(metadata_path)) + errors.extend( + validate_metadata( + metadata_path, + enforce_freshness=require_metadata or metadata_changed, + ) + ) if errors: for error in errors: diff --git a/.github/workflows/e2e-pr.yaml b/.github/workflows/e2e-pr.yaml new file mode 100644 index 000000000..e12ebb066 --- /dev/null +++ b/.github/workflows/e2e-pr.yaml @@ -0,0 +1,57 @@ +name: E2E Test Code Check + +on: + pull_request: + types: [opened, reopened, synchronize] + paths: + - 'test/**' + +jobs: + fork-notice: + if: github.event.pull_request.head.repo.fork + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Report fork limitation + shell: bash + run: | + echo "### E2E tests are disabled for fork pull requests." >> "$GITHUB_STEP_SUMMARY" + echo "A maintainer must reproduce test-code changes on a branch in Azure/terraform." >> "$GITHUB_STEP_SUMMARY" + + e2e-test-code: + if: github.event.pull_request.head.repo.fork == false + runs-on: ubuntu-latest + timeout-minutes: 360 + environment: + name: test + permissions: + contents: read + id-token: write + steps: + - name: Checkout + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Run E2E test suite + env: + ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} + ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} + ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} + PACKER_VERSION: ${{ vars.PACKER_VERSION }} + shell: bash + run: | + set -euo pipefail + export ARM_OIDC_REQUEST_TOKEN="${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" + export ARM_OIDC_REQUEST_URL="${ACTIONS_ID_TOKEN_REQUEST_URL}" + docker run --rm \ + -v "$(pwd):/src" \ + -w /src/test \ + --network=host \ + -e ARM_CLIENT_ID \ + -e ARM_OIDC_REQUEST_TOKEN \ + -e ARM_OIDC_REQUEST_URL \ + -e ARM_SUBSCRIPTION_ID \ + -e ARM_TENANT_ID \ + -e ARM_USE_OIDC=true \ + mcr.microsoft.com/azterraform:latest \ + sh -c "pkenv install '${PACKER_VERSION}' && go mod tidy && go test -timeout=360m -v ./e2e" diff --git a/quickstart/CONTRIBUTE.md b/quickstart/CONTRIBUTE.md index d3acfd2db..74880e324 100644 --- a/quickstart/CONTRIBUTE.md +++ b/quickstart/CONTRIBUTE.md @@ -83,6 +83,8 @@ The **Pre Pull Request Check** workflow runs automatically. It: - Validates the metadata format. - Runs the existing repository `pr-check` target. +Changes to the E2E test harness under `test/**` run separately in the **E2E Test Code Check** workflow. That workflow retains repository-owned Azure execution for internal branches and does not run privileged pull-request code from forks. + After that workflow passes, a repository maintainer comments: ```text From 53c0390b5812cc3053ebc53a2de13563e53ce754 Mon Sep 17 00:00:00 2001 From: Steven Ma Date: Wed, 19 Aug 2026 09:21:54 -0700 Subject: [PATCH 4/6] Use subscriptionless ADX authentication Configure the validation workflow for a dedicated corporate OIDC application with no Azure subscription roles. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/e2e.yaml | 6 +++--- quickstart/CONTRIBUTE.md | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 72ce65a5d..9f5a67e50 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -204,9 +204,9 @@ jobs: - name: Sign in to Azure with OIDC uses: azure/login@8216e11d8cd9b42fe925c852af8e76311ff067ac # v2 with: - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - client-id: ${{ secrets.AZURE_UAMI_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + client-id: ${{ vars.AZURE_UAMI_CLIENT_ID }} + allow-no-subscriptions: true - name: Validate correlated Terraform operations env: diff --git a/quickstart/CONTRIBUTE.md b/quickstart/CONTRIBUTE.md index 74880e324..416560063 100644 --- a/quickstart/CONTRIBUTE.md +++ b/quickstart/CONTRIBUTE.md @@ -99,10 +99,9 @@ This is deployment evidence, not source attestation: unlike an ARM template depl ## Maintainer configuration -The `adx-readonly` GitHub environment must provide federated OIDC credentials through: +The `adx-readonly` GitHub environment must provide these variables for its federated OIDC application: - `AZURE_TENANT_ID` -- `AZURE_SUBSCRIPTION_ID` - `AZURE_UAMI_CLIENT_ID` -The identity needs read-only query access to the `APAProd` database on `https://apadata.westus.kusto.windows.net`. It does not need permissions to deploy or modify Azure resources. \ No newline at end of file +The identity uses subscriptionless OIDC and needs read-only query access to the `APAProd` database on `https://apadata.westus.kusto.windows.net`. It does not need an Azure subscription role or permissions to deploy or modify resources. \ No newline at end of file From d5c78312d10f286e9b84ec47edac2bff63d54692 Mon Sep 17 00:00:00 2001 From: Steven Ma <13874900+stevenjma@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:34:52 -0700 Subject: [PATCH 5/6] Use approved ARMProd correlation telemetry Query the three regional ARM request stores with stable fail-closed snapshots and distinguish synchronous completion from asynchronous acceptance. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../scripts/test_validate_arm_correlation.py | 108 +++++++++ .github/scripts/validate-arm-correlation.py | 211 ++++++++++++++++++ .github/workflows/e2e.yaml | 96 ++------ quickstart/CONTRIBUTE.md | 8 +- 4 files changed, 345 insertions(+), 78 deletions(-) create mode 100644 .github/scripts/test_validate_arm_correlation.py create mode 100644 .github/scripts/validate-arm-correlation.py diff --git a/.github/scripts/test_validate_arm_correlation.py b/.github/scripts/test_validate_arm_correlation.py new file mode 100644 index 000000000..895fdbabb --- /dev/null +++ b/.github/scripts/test_validate_arm_correlation.py @@ -0,0 +1,108 @@ +import importlib.util +import json +import unittest.mock +import pathlib +import unittest + + +SCRIPT = pathlib.Path(__file__).with_name("validate-arm-correlation.py") +SPEC = importlib.util.spec_from_file_location("arm_validator", SCRIPT) +VALIDATOR = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(VALIDATOR) + + +class ValidateArmCorrelationTests(unittest.TestCase): + def test_build_body_uses_string_encoded_properties(self) -> None: + body = json.loads( + VALIDATOR.build_body( + "12345678-1234-1234-1234-1234567890ab", + "2026-10-06T18:00:00Z", + ) + ) + + self.assertEqual("Requests", body["db"]) + self.assertIsInstance(body["properties"], str) + self.assertEqual( + "12345678-1234-1234-1234-1234567890ab", + json.loads(body["properties"])["Parameters"]["cid"], + ) + + def test_primary_row_maps_columns(self) -> None: + response = { + "Tables": [ + { + "Columns": [ + {"ColumnName": "totalWrites"}, + {"ColumnName": "failedWrites"}, + ], + "Rows": [[3, 0]], + } + ] + } + + self.assertEqual( + {"totalWrites": 3, "failedWrites": 0}, + VALIDATOR.primary_row(response), + ) + + @unittest.mock.patch.object(VALIDATOR.shutil, "which") + def test_azure_cli_finds_command(self, which: unittest.mock.Mock) -> None: + which.return_value = "/usr/bin/az" + + with unittest.mock.patch.object(VALIDATOR.os, "name", "posix"): + self.assertEqual(["/usr/bin/az"], VALIDATOR.azure_cli_command()) + + def test_success_requires_writes_and_no_final_failures(self) -> None: + self.assertEqual( + [], + VALIDATOR.validate_summary( + { + "totalWrites": 3, + "completedWrites": 2, + "acceptedAsyncWrites": 1, + "failedWrites": 0, + } + ), + ) + + def test_final_failed_write_is_rejected(self) -> None: + errors = VALIDATOR.validate_summary( + { + "totalWrites": 3, + "completedWrites": 1, + "acceptedAsyncWrites": 1, + "failedWrites": 1, + } + ) + + self.assertTrue(any("final failed ARM write" in error for error in errors)) + + def test_summary_comparison_detects_ingestion_changes(self) -> None: + first = { + "totalWrites": 1, + "completedWrites": 1, + "acceptedAsyncWrites": 0, + "failedWrites": 0, + } + second = { + "totalWrites": 2, + "completedWrites": 1, + "acceptedAsyncWrites": 0, + "failedWrites": 1, + } + + self.assertNotEqual(first, second) + + def test_snapshot_requires_all_cluster_queries(self) -> None: + self.assertTrue(VALIDATOR.snapshot_is_complete([{"totalWrites": 1}], [])) + self.assertFalse( + VALIDATOR.snapshot_is_complete( + [{"totalWrites": 1}], + ["https://armprodweu.westeurope.kusto.windows.net"], + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/validate-arm-correlation.py b/.github/scripts/validate-arm-correlation.py new file mode 100644 index 000000000..8820cd4e6 --- /dev/null +++ b/.github/scripts/validate-arm-correlation.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import pathlib +import shutil +import subprocess +import sys +import time + + +CLUSTERS = ( + "https://armprodeus.eastus.kusto.windows.net", + "https://armprodweu.westeurope.kusto.windows.net", + "https://armprodsea.southeastasia.kusto.windows.net", +) +DATABASE = "Requests" +QUERY = r""" +declare query_parameters(cid:string, started:datetime); +HttpIncomingRequests +| where PreciseTimeStamp between ((started - 1h) .. (started + 24h)) +| where correlationId =~ cid +| where userAgent has "HashiCorp Terraform" + or userAgent has "terraform-provider-azurerm" + or userAgent has "terraform-provider-azapi" +| where httpStatusCode >= 100 +| where httpMethod in ("PUT", "PATCH", "POST", "DELETE") +| extend operationKey = strcat(httpMethod, "|", operationName, "|", targetUri) +| summarize arg_max(PreciseTimeStamp, *) by operationKey +| summarize + firstSeen=min(PreciseTimeStamp), + lastSeen=max(PreciseTimeStamp), + totalWrites=count(), + completedWrites=countif(httpStatusCode between (200 .. 299) and httpStatusCode != 202), + acceptedAsyncWrites=countif(httpStatusCode == 202), + failedWrites=countif(httpStatusCode >= 400), + providers=dcount(targetResourceProvider) +""" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Validate Terraform ARM writes for a workflow correlation ID." + ) + parser.add_argument("--correlation-id", required=True) + parser.add_argument("--started-at", required=True) + parser.add_argument("--attempts", type=int, default=3) + parser.add_argument("--retry-seconds", type=int, default=30) + return parser.parse_args() + + +def build_body(correlation_id: str, started_at: str) -> str: + properties = json.dumps( + {"Parameters": {"cid": correlation_id, "started": started_at}}, + separators=(",", ":"), + ) + return json.dumps( + {"db": DATABASE, "csl": QUERY, "properties": properties}, + separators=(",", ":"), + ) + + +def primary_row(response: dict) -> dict | None: + tables = response.get("Tables", []) + if not tables: + return None + table = tables[0] + rows = table.get("Rows", []) + if not rows: + return None + columns = [column["ColumnName"] for column in table.get("Columns", [])] + return dict(zip(columns, rows[0])) + + +def azure_cli_command() -> list[str]: + command = shutil.which("az") + if command and os.name != "nt": + return [command] + if os.name == "nt" and command: + python = pathlib.Path(command).resolve().parent.parent / "python.exe" + if python.is_file(): + return [str(python), "-IBm", "azure.cli"] + raise FileNotFoundError("Azure CLI was not found on PATH.") + + +def query_cluster(cluster: str, body: str) -> dict | None: + result = subprocess.run( + azure_cli_command() + + [ + "rest", + "--method", + "POST", + "--url", + f"{cluster}/v1/rest/query", + "--resource", + "https://kusto.kusto.windows.net", + "--headers", + "Content-Type=application/json", + "--body", + body, + ], + check=True, + capture_output=True, + text=True, + ) + return primary_row(json.loads(result.stdout)) + + +def validate_summary(summary: dict) -> list[str]: + errors: list[str] = [] + total_writes = int(summary.get("totalWrites") or 0) + completed_writes = int(summary.get("completedWrites") or 0) + accepted_async_writes = int(summary.get("acceptedAsyncWrites") or 0) + failed_writes = int(summary.get("failedWrites") or 0) + if total_writes == 0: + errors.append("No terminal Terraform ARM write operations were found.") + if completed_writes + accepted_async_writes == 0: + errors.append("No successful or accepted Terraform ARM writes were found.") + if failed_writes != 0: + errors.append( + f"The correlated workflow has {failed_writes} final failed ARM write(s)." + ) + return errors + + +def snapshot_is_complete(summaries: list[dict], cluster_errors: list[str]) -> bool: + return bool(summaries) and not cluster_errors + + +def main() -> int: + args = parse_args() + body = build_body(args.correlation_id, args.started_at) + attempts = max(2, args.attempts) + previous_summary = None + last_cluster_errors = [] + + for attempt in range(1, attempts + 1): + summaries = [] + cluster_errors = [] + for cluster in CLUSTERS: + print(f"Querying {cluster} (attempt {attempt})...", file=sys.stderr) + try: + row = query_cluster(cluster, body) + except (json.JSONDecodeError, subprocess.CalledProcessError) as error: + cluster_errors.append(cluster) + print(f"WARNING: {cluster} query failed: {error}", file=sys.stderr) + continue + if row and int(row.get("totalWrites") or 0) > 0: + row["cluster"] = cluster + summaries.append(row) + + last_cluster_errors = cluster_errors + if snapshot_is_complete(summaries, cluster_errors): + summary = { + "correlationId": args.correlation_id, + "clusters": [row["cluster"] for row in summaries], + "firstSeen": min(row["firstSeen"] for row in summaries), + "lastSeen": max(row["lastSeen"] for row in summaries), + "totalWrites": sum(int(row["totalWrites"]) for row in summaries), + "completedWrites": sum( + int(row["completedWrites"]) for row in summaries + ), + "acceptedAsyncWrites": sum( + int(row["acceptedAsyncWrites"]) for row in summaries + ), + "failedWrites": sum(int(row["failedWrites"]) for row in summaries), + "providers": sum(int(row["providers"]) for row in summaries), + } + + if summary == previous_summary: + errors = validate_summary(summary) + print(json.dumps(summary, separators=(",", ":"))) + if errors: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + return 0 + + previous_summary = summary + print( + "Telemetry snapshot changed or was first observed; " + "waiting for a stable snapshot.", + file=sys.stderr, + ) + + if attempt < attempts: + print( + f"Waiting {args.retry_seconds}s for telemetry ingestion.", + file=sys.stderr, + ) + time.sleep(args.retry_seconds) + + if last_cluster_errors: + print( + "ERROR: ARM telemetry queries failed for: " + + ", ".join(last_cluster_errors), + file=sys.stderr, + ) + elif previous_summary is not None: + print( + "ERROR: Correlated ARM telemetry did not stabilize before timeout.", + file=sys.stderr, + ) + else: + print("ERROR: No correlated Terraform ARM writes were found.", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 9f5a67e50..81bcae575 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -182,7 +182,9 @@ jobs: repository: ${{ github.repository }} ref: ${{ github.event.repository.default_branch }} path: trusted - sparse-checkout: .github/scripts/validate-quickstart-metadata.py + sparse-checkout: | + .github/scripts/validate-quickstart-metadata.py + .github/scripts/validate-arm-correlation.py sparse-checkout-cone-mode: false - name: Checkout pull request sample @@ -205,7 +207,7 @@ jobs: uses: azure/login@8216e11d8cd9b42fe925c852af8e76311ff067ac # v2 with: tenant-id: ${{ vars.AZURE_TENANT_ID }} - client-id: ${{ vars.AZURE_UAMI_CLIENT_ID }} + client-id: ${{ vars.AZURE_CLIENT_ID }} allow-no-subscriptions: true - name: Validate correlated Terraform operations @@ -219,81 +221,27 @@ jobs: correlation_id=$(jq -r '.testResult.correlationId' "${metadata}") started_at=$(jq -r '.testResult.timestamp' "${metadata}") - query='declare query_parameters(cid:string, started:datetime); - ARM_Deployments_Terraform_v2 - | where correlationId == cid - | where PreciseTimeStamp between ((started - 1h) .. (started + 24h)) - | where client == "Terraform" - | summarize - firstSeen=min(PreciseTimeStamp), - lastSeen=max(PreciseTimeStamp), - totalRequests=sum(requestCount), - successfulWrites=sumif(requestCount, httpMethod in ("PUT", "PATCH", "POST", "DELETE") and executionStatus == "Succeeded"), - failedRequests=sumif(requestCount, executionStatus in ("Failed", "Canceled")), - providers=dcount(providerNamespace)' - - properties=$(jq -cn \ - --arg cid "${correlation_id}" \ - --arg started "${started_at}" \ - '{ Parameters: { cid: $cid, started: $started } }') - body=$(jq -n \ - --arg db "APAProd" \ - --arg csl "${query}" \ - --arg properties "${properties}" \ - '{ - db: $db, - csl: $csl, - properties: $properties - }') - - response=$(az rest \ - --method POST \ - --url "https://apadata.westus.kusto.windows.net/v1/rest/query" \ - --resource "https://kusto.kusto.windows.net" \ - --headers "Content-Type=application/json" \ - --body "${body}") - - value() { - local column="$1" - jq -r --arg column "${column}" \ - '.Tables[0] as $table - | ($table.Columns | map(.ColumnName) | index($column)) as $index - | if $index == null then empty else $table.Rows[0][$index] end' \ - <<< "${response}" - } - - total_requests=$(value totalRequests) - successful_writes=$(value successfulWrites) - failed_requests=$(value failedRequests) - first_seen=$(value firstSeen) - last_seen=$(value lastSeen) - providers=$(value providers) + summary=$( + python trusted/.github/scripts/validate-arm-correlation.py \ + --correlation-id "${correlation_id}" \ + --started-at "${started_at}" + ) echo "Correlation ID: ${correlation_id}" - echo "Observed: ${first_seen} through ${last_seen}" - echo "Requests: ${total_requests:-0}; successful writes: ${successful_writes:-0}; failed/canceled: ${failed_requests:-0}; providers: ${providers:-0}" - - if (( ${total_requests:-0} == 0 )); then - echo "No Terraform operations were found for this correlation ID." >&2 - exit 1 - fi - if (( ${successful_writes:-0} == 0 )); then - echo "The correlation ID has no successful Terraform write operations." >&2 - exit 1 - fi - if (( ${failed_requests:-0} != 0 )); then - echo "The correlated Terraform workflow contains failed or canceled operations." >&2 - exit 1 - fi + echo "Observed: $(jq -r '.firstSeen' <<< "${summary}") through $(jq -r '.lastSeen' <<< "${summary}")" + echo "Clusters: $(jq -r '.clusters | join(\", \")' <<< "${summary}")" + echo "Writes: $(jq -r '.totalWrites' <<< "${summary}"); completed: $(jq -r '.completedWrites' <<< "${summary}"); async accepted: $(jq -r '.acceptedAsyncWrites' <<< "${summary}"); failed: $(jq -r '.failedWrites' <<< "${summary}")" { - echo "## Terraform deployment validation" + echo "## Terraform ARM activity validation" echo echo "- Sample: \`${SAMPLE_PATH}\`" echo "- Correlation ID: \`${correlation_id}\`" - echo "- Successful writes: ${successful_writes}" - echo "- Failed or canceled operations: ${failed_requests}" - echo "- Resource providers: ${providers}" + echo "- ARM regions: $(jq -r '.clusters | join(\", \")' <<< "${summary}")" + echo "- Synchronous completed writes: $(jq -r '.completedWrites' <<< "${summary}")" + echo "- Asynchronous writes accepted by ARM: $(jq -r '.acceptedAsyncWrites' <<< "${summary}")" + echo "- Final HTTP write failures: $(jq -r '.failedWrites' <<< "${summary}")" + echo "- Resource providers: $(jq -r '.providers' <<< "${summary}")" } >> "${GITHUB_STEP_SUMMARY}" report: @@ -322,10 +270,10 @@ jobs: conclusion: passed ? 'success' : 'failure', output: { title: passed - ? 'Terraform deployment validation passed' - : 'Terraform deployment validation failed', + ? 'Terraform ARM activity validation passed' + : 'Terraform ARM activity validation failed', summary: passed - ? `ARM telemetry confirmed successful Terraform writes for \`${process.env.SAMPLE_PATH}\` with no failed or canceled operations.` - : `Deployment validation failed. Open the workflow run for details, correct the metadata or deployment, then run \`/validate\` again.` + ? `ARM telemetry confirmed completed or accepted Terraform writes for \`${process.env.SAMPLE_PATH}\` with no observed final HTTP write failures.` + : `ARM activity validation failed. Open the workflow run for details, correct the metadata or deployment, then run \`/validate\` again.` } }); diff --git a/quickstart/CONTRIBUTE.md b/quickstart/CONTRIBUTE.md index 416560063..b4c754bd8 100644 --- a/quickstart/CONTRIBUTE.md +++ b/quickstart/CONTRIBUTE.md @@ -91,9 +91,9 @@ After that workflow passes, a repository maintainer comments: /validate ``` -The command is restricted to repository members, owners, and collaborators. It checks out only the changed quickstart, validates its metadata with trusted code from the default branch, signs in to Azure with the `adx-readonly` environment's federated identity, and queries `ARM_Deployments_Terraform_v2` by the supplied correlation ID. +The command is restricted to repository members, owners, and collaborators. It checks out only the changed quickstart, validates its metadata with trusted code from the default branch, signs in to Azure with the `adx-readonly` environment's federated identity, and queries the approved regional ARMProd `Requests.HttpIncomingRequests` datasets by the supplied correlation ID. -Validation passes only when telemetry contains at least one successful Terraform write and no failed or canceled operation for that correlated workflow. The workflow publishes the result as the `terraform-deployment-validation` check on the pull request head. +Validation passes only when telemetry contains at least one synchronously completed or asynchronously accepted Terraform ARM write and no observed final HTTP write failures for that correlated workflow. Preliminary records and failures that were successfully retried are not treated as final failures. A `202 Accepted` result proves that ARM accepted the asynchronous operation; the approved dataset does not expose the response body needed to prove its eventual provisioning state. The workflow publishes the result as the `terraform-deployment-validation` check on the pull request head. This is deployment evidence, not source attestation: unlike an ARM template deployment, Terraform does not emit a template hash that ARM can compare with the pull request. Reviewers must still inspect the code and the static checks. @@ -102,6 +102,6 @@ This is deployment evidence, not source attestation: unlike an ARM template depl The `adx-readonly` GitHub environment must provide these variables for its federated OIDC application: - `AZURE_TENANT_ID` -- `AZURE_UAMI_CLIENT_ID` +- `AZURE_CLIENT_ID` -The identity uses subscriptionless OIDC and needs read-only query access to the `APAProd` database on `https://apadata.westus.kusto.windows.net`. It does not need an Azure subscription role or permissions to deploy or modify resources. \ No newline at end of file +The identity uses subscriptionless OIDC and needs read-only query access to the `Requests` databases on the approved `armprodeus`, `armprodweu`, and `armprodsea` clusters. It does not need an Azure subscription role or permissions to deploy or modify resources. \ No newline at end of file From be84305acbaf1951a92975a393e568317cd825b3 Mon Sep 17 00:00:00 2001 From: Steven Ma <13874900+stevenjma@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:18:43 -0700 Subject: [PATCH 6/6] Close deployment validation review findings Block validation-contract changes from /validate, run validator tests in CI, align timestamp contracts, and state the contributor-supplied evidence boundary precisely. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../schemas/quickstart-metadata.schema.json | 3 ++- .../test_validate_quickstart_metadata.py | 21 +++++++++++++++++++ .../scripts/validate-quickstart-metadata.py | 5 ++++- .github/workflows/e2e.yaml | 20 ++++++++++-------- .github/workflows/pr-check.yaml | 4 ++++ quickstart/CONTRIBUTE.md | 12 ++++++----- 6 files changed, 49 insertions(+), 16 deletions(-) diff --git a/.github/schemas/quickstart-metadata.schema.json b/.github/schemas/quickstart-metadata.schema.json index 699553ac6..23bccd755 100644 --- a/.github/schemas/quickstart-metadata.schema.json +++ b/.github/schemas/quickstart-metadata.schema.json @@ -29,7 +29,8 @@ "timestamp": { "type": "string", "format": "date-time", - "description": "UTC time immediately before the validated Terraform apply started." + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?Z$", + "description": "RFC 3339 UTC time ending in Z, captured immediately before the validated Terraform apply started." }, "terraformVersion": { "type": "string", diff --git a/.github/scripts/test_validate_quickstart_metadata.py b/.github/scripts/test_validate_quickstart_metadata.py index ab065668c..6625c81a7 100644 --- a/.github/scripts/test_validate_quickstart_metadata.py +++ b/.github/scripts/test_validate_quickstart_metadata.py @@ -2,11 +2,13 @@ import datetime import json import pathlib +import re import tempfile import unittest SCRIPT = pathlib.Path(__file__).with_name("validate-quickstart-metadata.py") +SCHEMA = SCRIPT.parent.parent / "schemas" / "quickstart-metadata.schema.json" SPEC = importlib.util.spec_from_file_location("metadata_validator", SCRIPT) VALIDATOR = importlib.util.module_from_spec(SPEC) assert SPEC.loader is not None @@ -76,6 +78,25 @@ def test_requires_utc_timestamp(self) -> None: ) ) + def test_schema_matches_python_utc_timestamp_contract(self) -> None: + schema = json.loads(SCHEMA.read_text(encoding="utf-8")) + pattern = schema["properties"]["testResult"]["properties"]["timestamp"][ + "pattern" + ] + + self.assertEqual(VALIDATOR.TIMESTAMP_RE.pattern, pattern) + self.assertIsNotNone(re.fullmatch(pattern, "2026-08-18T17:00:00Z")) + self.assertIsNotNone(re.fullmatch(pattern, "2026-08-18T17:00:00.123Z")) + for invalid in ( + "2026-08-18T17:00:00-07:00", + "2026-08-18 17:00:00Z", + "20260818T170000Z", + "2026-08-18T17:00Z", + "2026-08-18T17:00:00,123Z", + ): + self.assertIsNone(re.fullmatch(pattern, invalid)) + self.assertIsNone(VALIDATOR.parse_timestamp(invalid)) + def test_reads_test_result_value(self) -> None: data = {"testResult": {"correlationId": "example"}} diff --git a/.github/scripts/validate-quickstart-metadata.py b/.github/scripts/validate-quickstart-metadata.py index e4fb55b94..9c9909752 100644 --- a/.github/scripts/validate-quickstart-metadata.py +++ b/.github/scripts/validate-quickstart-metadata.py @@ -22,6 +22,9 @@ MAX_METADATA_AGE = datetime.timedelta(days=30) MAX_FUTURE_SKEW = datetime.timedelta(minutes=15) TERRAFORM_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$") +TIMESTAMP_RE = re.compile( + r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$" +) def parse_args() -> argparse.Namespace: @@ -98,7 +101,7 @@ def should_require_metadata( def parse_timestamp(value: object) -> datetime.datetime | None: - if not isinstance(value, str) or not value.endswith("Z"): + if not isinstance(value, str) or not TIMESTAMP_RE.fullmatch(value): return None try: parsed = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 81bcae575..8bf292eba 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -64,16 +64,18 @@ jobs: 'scripts/terraform-validate.sh' ]; const touchedProtected = files - .map(file => file.filename) + .flatMap(file => [file.filename, file.previous_filename]) + .filter(Boolean) .filter(name => protectedPrefixes.some(prefix => name.startsWith(prefix)) || protectedFiles.includes(name) - ); - const allowedProtectedAuthors = new Set(['MEMBER', 'OWNER']); - if (touchedProtected.length > 0 && !allowedProtectedAuthors.has(pr.author_association)) { + ) + .filter((name, index, names) => names.indexOf(name) === index); + if (touchedProtected.length > 0) { core.setFailed( - `Protected validation files were changed by a PR author with ` + - `association '${pr.author_association}': ${touchedProtected.join(', ')}` + `/validate cannot run while validation-contract files are changing: ` + + `${touchedProtected.join(', ')}. Merge validation changes separately, ` + + `canary them from the default branch, then validate quickstart changes.` ); return; } @@ -125,8 +127,8 @@ jobs: status: 'in_progress', details_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, output: { - title: 'Validating Terraform deployment', - summary: 'A maintainer ran `/validate`; waiting for static checks, then querying ARM telemetry.' + title: 'Validating contributor-supplied ARM activity evidence', + summary: 'A maintainer ran `/validate`; waiting for static checks, then querying ARM telemetry for the contributor-supplied correlation ID.' } }); core.setOutput('check_id', data.id); @@ -273,7 +275,7 @@ jobs: ? 'Terraform ARM activity validation passed' : 'Terraform ARM activity validation failed', summary: passed - ? `ARM telemetry confirmed completed or accepted Terraform writes for \`${process.env.SAMPLE_PATH}\` with no observed final HTTP write failures.` + ? `The contributor-supplied correlation ID recorded by \`${process.env.SAMPLE_PATH}\` maps to completed or accepted Terraform ARM writes with no observed final HTTP write failures. This does not independently bind the telemetry to the pull request or applied source.` : `ARM activity validation failed. Open the workflow run for details, correct the metadata or deployment, then run \`/validate\` again.` } }); diff --git a/.github/workflows/pr-check.yaml b/.github/workflows/pr-check.yaml index a57ff8d5c..5ae89aa6e 100644 --- a/.github/workflows/pr-check.yaml +++ b/.github/workflows/pr-check.yaml @@ -14,6 +14,10 @@ jobs: uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 0 + - name: Test quickstart validation scripts + run: | + python .github/scripts/test_validate_arm_correlation.py + python .github/scripts/test_validate_quickstart_metadata.py - name: Get changed files id: changed-files uses: step-security/changed-files@3dbe17c78367e7d60f00d78ae6781a35be47b4a1 #v45.0.1 diff --git a/quickstart/CONTRIBUTE.md b/quickstart/CONTRIBUTE.md index b4c754bd8..297282f57 100644 --- a/quickstart/CONTRIBUTE.md +++ b/quickstart/CONTRIBUTE.md @@ -3,9 +3,9 @@ The repository validates two different claims: 1. **Static quality:** the pull request passes formatting, linting, and repository checks. -2. **Deployment evidence:** a maintainer confirms that the exact correlation ID recorded by the contributor maps to a successful Terraform workflow in Azure Resource Manager telemetry. +2. **ARM activity evidence:** a maintainer confirms that the contributor-supplied correlation ID maps to qualifying Terraform ARM write activity with no observed final HTTP write failures. -The repository does not deploy pull-request code with privileged credentials. Contributors deploy the quickstart first; maintainers validate the resulting telemetry with `/validate`. +The quickstart validation workflow does not deploy pull-request quickstart code with repository-owned Azure credentials. Contributors deploy the quickstart first; maintainers validate the resulting telemetry with `/validate`. The separate internal E2E test-harness workflow continues to execute same-repository `test/**` pull-request code using the protected `test` environment. ## Contribution requirements @@ -18,7 +18,7 @@ A pull request that changes deployable files in `quickstart//` must: - Add or update `metadata.json` in the quickstart folder with the UUID and apply start time. - Remove the deployed resources after validation unless the sample documents a reason to retain them. -The AzureRM and AzAPI providers use `ARM_CORRELATION_REQUEST_ID` as the `x-ms-correlation-request-id` for the provider workflow. Do not reuse an ID from an earlier plan or apply. +The AzureRM and AzAPI providers use `ARM_CORRELATION_REQUEST_ID` as the `x-ms-correlation-request-id` for the provider workflow. Do not reuse an ID from an earlier plan or apply. This is a good-faith contributor assertion: validation does not prevent deliberate reuse or independently bind the ID to the pull request, sample, actor, or applied source. ## Deploy and capture validation evidence @@ -93,9 +93,11 @@ After that workflow passes, a repository maintainer comments: The command is restricted to repository members, owners, and collaborators. It checks out only the changed quickstart, validates its metadata with trusted code from the default branch, signs in to Azure with the `adx-readonly` environment's federated identity, and queries the approved regional ARMProd `Requests.HttpIncomingRequests` datasets by the supplied correlation ID. -Validation passes only when telemetry contains at least one synchronously completed or asynchronously accepted Terraform ARM write and no observed final HTTP write failures for that correlated workflow. Preliminary records and failures that were successfully retried are not treated as final failures. A `202 Accepted` result proves that ARM accepted the asynchronous operation; the approved dataset does not expose the response body needed to prove its eventual provisioning state. The workflow publishes the result as the `terraform-deployment-validation` check on the pull request head. +Validation passes only when telemetry for the contributor-supplied correlation ID contains at least one synchronously completed or asynchronously accepted Terraform ARM write and no observed final HTTP write failures. Preliminary records and failures that were successfully retried are not treated as final failures. A `202 Accepted` result proves that ARM accepted the asynchronous operation; the approved dataset does not expose the response body needed to prove its eventual provisioning state. The workflow publishes the result as the `terraform-deployment-validation` check on the pull request head. -This is deployment evidence, not source attestation: unlike an ARM template deployment, Terraform does not emit a template hash that ARM can compare with the pull request. Reviewers must still inspect the code and the static checks. +This is contributor-asserted ARM activity evidence, not proof that Terraform exited successfully and not source attestation. Unlike an ARM template deployment, Terraform does not emit a template hash that ARM can compare with the pull request. Reviewers must still inspect the code and static checks. + +Pull requests that modify the validation workflows, validators, metadata schema, or their transitive static-check implementation cannot use `/validate`. Merge validation-contract changes separately, canary them from the default branch, then validate quickstart changes under the deployed contract. ## Maintainer configuration