diff --git a/.github/schemas/quickstart-metadata.schema.json b/.github/schemas/quickstart-metadata.schema.json new file mode 100644 index 000000000..23bccd755 --- /dev/null +++ b/.github/schemas/quickstart-metadata.schema.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/Azure/terraform/blob/master/.github/schemas/quickstart-metadata.schema.json", + "title": "Azure Terraform quickstart deployment metadata", + "type": "object", + "additionalProperties": false, + "required": [ + "$schema", + "testResult" + ], + "properties": { + "$schema": { + "type": "string", + "const": "../../.github/schemas/quickstart-metadata.schema.json" + }, + "testResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "correlationId", + "timestamp" + ], + "properties": { + "correlationId": { + "type": "string", + "format": "uuid", + "description": "Value supplied to ARM_CORRELATION_REQUEST_ID for the validated Terraform apply." + }, + "timestamp": { + "type": "string", + "format": "date-time", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?Z$", + "description": "RFC 3339 UTC time ending in Z, captured immediately before the validated Terraform apply started." + }, + "terraformVersion": { + "type": "string", + "pattern": "^\\d+\\.\\d+\\.\\d+(?:[-+][0-9A-Za-z.-]+)?$", + "description": "Terraform CLI version used for the apply." + } + } + } + } +} diff --git a/.github/scripts/test_validate_arm_correlation.py b/.github/scripts/test_validate_arm_correlation.py new file mode 100644 index 000000000..895fdbabb --- /dev/null +++ b/.github/scripts/test_validate_arm_correlation.py @@ -0,0 +1,108 @@ +import importlib.util +import json +import unittest.mock +import pathlib +import unittest + + +SCRIPT = pathlib.Path(__file__).with_name("validate-arm-correlation.py") +SPEC = importlib.util.spec_from_file_location("arm_validator", SCRIPT) +VALIDATOR = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(VALIDATOR) + + +class ValidateArmCorrelationTests(unittest.TestCase): + def test_build_body_uses_string_encoded_properties(self) -> None: + body = json.loads( + VALIDATOR.build_body( + "12345678-1234-1234-1234-1234567890ab", + "2026-10-06T18:00:00Z", + ) + ) + + self.assertEqual("Requests", body["db"]) + self.assertIsInstance(body["properties"], str) + self.assertEqual( + "12345678-1234-1234-1234-1234567890ab", + json.loads(body["properties"])["Parameters"]["cid"], + ) + + def test_primary_row_maps_columns(self) -> None: + response = { + "Tables": [ + { + "Columns": [ + {"ColumnName": "totalWrites"}, + {"ColumnName": "failedWrites"}, + ], + "Rows": [[3, 0]], + } + ] + } + + self.assertEqual( + {"totalWrites": 3, "failedWrites": 0}, + VALIDATOR.primary_row(response), + ) + + @unittest.mock.patch.object(VALIDATOR.shutil, "which") + def test_azure_cli_finds_command(self, which: unittest.mock.Mock) -> None: + which.return_value = "/usr/bin/az" + + with unittest.mock.patch.object(VALIDATOR.os, "name", "posix"): + self.assertEqual(["/usr/bin/az"], VALIDATOR.azure_cli_command()) + + def test_success_requires_writes_and_no_final_failures(self) -> None: + self.assertEqual( + [], + VALIDATOR.validate_summary( + { + "totalWrites": 3, + "completedWrites": 2, + "acceptedAsyncWrites": 1, + "failedWrites": 0, + } + ), + ) + + def test_final_failed_write_is_rejected(self) -> None: + errors = VALIDATOR.validate_summary( + { + "totalWrites": 3, + "completedWrites": 1, + "acceptedAsyncWrites": 1, + "failedWrites": 1, + } + ) + + self.assertTrue(any("final failed ARM write" in error for error in errors)) + + def test_summary_comparison_detects_ingestion_changes(self) -> None: + first = { + "totalWrites": 1, + "completedWrites": 1, + "acceptedAsyncWrites": 0, + "failedWrites": 0, + } + second = { + "totalWrites": 2, + "completedWrites": 1, + "acceptedAsyncWrites": 0, + "failedWrites": 1, + } + + self.assertNotEqual(first, second) + + def test_snapshot_requires_all_cluster_queries(self) -> None: + self.assertTrue(VALIDATOR.snapshot_is_complete([{"totalWrites": 1}], [])) + self.assertFalse( + VALIDATOR.snapshot_is_complete( + [{"totalWrites": 1}], + ["https://armprodweu.westeurope.kusto.windows.net"], + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/test_validate_quickstart_metadata.py b/.github/scripts/test_validate_quickstart_metadata.py new file mode 100644 index 000000000..6625c81a7 --- /dev/null +++ b/.github/scripts/test_validate_quickstart_metadata.py @@ -0,0 +1,153 @@ +import importlib.util +import datetime +import json +import pathlib +import re +import tempfile +import unittest + + +SCRIPT = pathlib.Path(__file__).with_name("validate-quickstart-metadata.py") +SCHEMA = SCRIPT.parent.parent / "schemas" / "quickstart-metadata.schema.json" +SPEC = importlib.util.spec_from_file_location("metadata_validator", SCRIPT) +VALIDATOR = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(VALIDATOR) + + +class ValidateMetadataTests(unittest.TestCase): + def current_timestamp(self) -> str: + return ( + datetime.datetime.now(datetime.timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + ) + + def write_metadata(self, data: dict) -> pathlib.Path: + temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(temporary_directory.cleanup) + path = pathlib.Path(temporary_directory.name) / "metadata.json" + path.write_text(json.dumps(data), encoding="utf-8") + return path + + def test_accepts_valid_metadata(self) -> None: + path = self.write_metadata( + { + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "12345678-1234-1234-1234-1234567890ab", + "timestamp": self.current_timestamp(), + "terraformVersion": "1.13.0", + }, + } + ) + + self.assertEqual([], VALIDATOR.validate_metadata(path)) + + def test_rejects_invalid_correlation_id(self) -> None: + path = self.write_metadata( + { + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "not-a-guid", + "timestamp": self.current_timestamp(), + }, + } + ) + + self.assertTrue( + any("dashed UUID" in error for error in VALIDATOR.validate_metadata(path)) + ) + + def test_requires_utc_timestamp(self) -> None: + path = self.write_metadata( + { + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "12345678-1234-1234-1234-1234567890ab", + "timestamp": "2026-08-18T17:00:00-07:00", + }, + } + ) + + self.assertTrue( + any( + "timestamp" in error + for error in VALIDATOR.validate_metadata(path) + ) + ) + + def test_schema_matches_python_utc_timestamp_contract(self) -> None: + schema = json.loads(SCHEMA.read_text(encoding="utf-8")) + pattern = schema["properties"]["testResult"]["properties"]["timestamp"][ + "pattern" + ] + + self.assertEqual(VALIDATOR.TIMESTAMP_RE.pattern, pattern) + self.assertIsNotNone(re.fullmatch(pattern, "2026-08-18T17:00:00Z")) + self.assertIsNotNone(re.fullmatch(pattern, "2026-08-18T17:00:00.123Z")) + for invalid in ( + "2026-08-18T17:00:00-07:00", + "2026-08-18 17:00:00Z", + "20260818T170000Z", + "2026-08-18T17:00Z", + "2026-08-18T17:00:00,123Z", + ): + self.assertIsNone(re.fullmatch(pattern, invalid)) + self.assertIsNone(VALIDATOR.parse_timestamp(invalid)) + + def test_reads_test_result_value(self) -> None: + data = {"testResult": {"correlationId": "example"}} + + self.assertEqual( + "example", + VALIDATOR.test_result_value(data, "correlationId"), + ) + self.assertIsNone(VALIDATOR.test_result_value({}, "correlationId")) + + def test_rejects_stale_timestamp(self) -> None: + path = self.write_metadata( + { + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "12345678-1234-1234-1234-1234567890ab", + "timestamp": "2000-01-01T00:00:00Z", + }, + } + ) + + self.assertTrue( + any("days old" in error for error in VALIDATOR.validate_metadata(path)) + ) + + def test_allows_stale_timestamp_when_freshness_is_not_required(self) -> None: + path = self.write_metadata( + { + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "12345678-1234-1234-1234-1234567890ab", + "timestamp": "2000-01-01T00:00:00Z", + }, + } + ) + + self.assertEqual( + [], + VALIDATOR.validate_metadata(path, enforce_freshness=False), + ) + + def test_recognizes_terraform_json_configuration(self) -> None: + self.assertTrue("main.tf.json".endswith(VALIDATOR.CONFIGURATION_SUFFIXES)) + self.assertTrue( + "testing.auto.tfvars.json".endswith(VALIDATOR.CONFIGURATION_SUFFIXES) + ) + + def test_metadata_change_requires_metadata(self) -> None: + self.assertTrue( + VALIDATOR.should_require_metadata(False, False, True) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/validate-arm-correlation.py b/.github/scripts/validate-arm-correlation.py new file mode 100644 index 000000000..8820cd4e6 --- /dev/null +++ b/.github/scripts/validate-arm-correlation.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import pathlib +import shutil +import subprocess +import sys +import time + + +CLUSTERS = ( + "https://armprodeus.eastus.kusto.windows.net", + "https://armprodweu.westeurope.kusto.windows.net", + "https://armprodsea.southeastasia.kusto.windows.net", +) +DATABASE = "Requests" +QUERY = r""" +declare query_parameters(cid:string, started:datetime); +HttpIncomingRequests +| where PreciseTimeStamp between ((started - 1h) .. (started + 24h)) +| where correlationId =~ cid +| where userAgent has "HashiCorp Terraform" + or userAgent has "terraform-provider-azurerm" + or userAgent has "terraform-provider-azapi" +| where httpStatusCode >= 100 +| where httpMethod in ("PUT", "PATCH", "POST", "DELETE") +| extend operationKey = strcat(httpMethod, "|", operationName, "|", targetUri) +| summarize arg_max(PreciseTimeStamp, *) by operationKey +| summarize + firstSeen=min(PreciseTimeStamp), + lastSeen=max(PreciseTimeStamp), + totalWrites=count(), + completedWrites=countif(httpStatusCode between (200 .. 299) and httpStatusCode != 202), + acceptedAsyncWrites=countif(httpStatusCode == 202), + failedWrites=countif(httpStatusCode >= 400), + providers=dcount(targetResourceProvider) +""" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Validate Terraform ARM writes for a workflow correlation ID." + ) + parser.add_argument("--correlation-id", required=True) + parser.add_argument("--started-at", required=True) + parser.add_argument("--attempts", type=int, default=3) + parser.add_argument("--retry-seconds", type=int, default=30) + return parser.parse_args() + + +def build_body(correlation_id: str, started_at: str) -> str: + properties = json.dumps( + {"Parameters": {"cid": correlation_id, "started": started_at}}, + separators=(",", ":"), + ) + return json.dumps( + {"db": DATABASE, "csl": QUERY, "properties": properties}, + separators=(",", ":"), + ) + + +def primary_row(response: dict) -> dict | None: + tables = response.get("Tables", []) + if not tables: + return None + table = tables[0] + rows = table.get("Rows", []) + if not rows: + return None + columns = [column["ColumnName"] for column in table.get("Columns", [])] + return dict(zip(columns, rows[0])) + + +def azure_cli_command() -> list[str]: + command = shutil.which("az") + if command and os.name != "nt": + return [command] + if os.name == "nt" and command: + python = pathlib.Path(command).resolve().parent.parent / "python.exe" + if python.is_file(): + return [str(python), "-IBm", "azure.cli"] + raise FileNotFoundError("Azure CLI was not found on PATH.") + + +def query_cluster(cluster: str, body: str) -> dict | None: + result = subprocess.run( + azure_cli_command() + + [ + "rest", + "--method", + "POST", + "--url", + f"{cluster}/v1/rest/query", + "--resource", + "https://kusto.kusto.windows.net", + "--headers", + "Content-Type=application/json", + "--body", + body, + ], + check=True, + capture_output=True, + text=True, + ) + return primary_row(json.loads(result.stdout)) + + +def validate_summary(summary: dict) -> list[str]: + errors: list[str] = [] + total_writes = int(summary.get("totalWrites") or 0) + completed_writes = int(summary.get("completedWrites") or 0) + accepted_async_writes = int(summary.get("acceptedAsyncWrites") or 0) + failed_writes = int(summary.get("failedWrites") or 0) + if total_writes == 0: + errors.append("No terminal Terraform ARM write operations were found.") + if completed_writes + accepted_async_writes == 0: + errors.append("No successful or accepted Terraform ARM writes were found.") + if failed_writes != 0: + errors.append( + f"The correlated workflow has {failed_writes} final failed ARM write(s)." + ) + return errors + + +def snapshot_is_complete(summaries: list[dict], cluster_errors: list[str]) -> bool: + return bool(summaries) and not cluster_errors + + +def main() -> int: + args = parse_args() + body = build_body(args.correlation_id, args.started_at) + attempts = max(2, args.attempts) + previous_summary = None + last_cluster_errors = [] + + for attempt in range(1, attempts + 1): + summaries = [] + cluster_errors = [] + for cluster in CLUSTERS: + print(f"Querying {cluster} (attempt {attempt})...", file=sys.stderr) + try: + row = query_cluster(cluster, body) + except (json.JSONDecodeError, subprocess.CalledProcessError) as error: + cluster_errors.append(cluster) + print(f"WARNING: {cluster} query failed: {error}", file=sys.stderr) + continue + if row and int(row.get("totalWrites") or 0) > 0: + row["cluster"] = cluster + summaries.append(row) + + last_cluster_errors = cluster_errors + if snapshot_is_complete(summaries, cluster_errors): + summary = { + "correlationId": args.correlation_id, + "clusters": [row["cluster"] for row in summaries], + "firstSeen": min(row["firstSeen"] for row in summaries), + "lastSeen": max(row["lastSeen"] for row in summaries), + "totalWrites": sum(int(row["totalWrites"]) for row in summaries), + "completedWrites": sum( + int(row["completedWrites"]) for row in summaries + ), + "acceptedAsyncWrites": sum( + int(row["acceptedAsyncWrites"]) for row in summaries + ), + "failedWrites": sum(int(row["failedWrites"]) for row in summaries), + "providers": sum(int(row["providers"]) for row in summaries), + } + + if summary == previous_summary: + errors = validate_summary(summary) + print(json.dumps(summary, separators=(",", ":"))) + if errors: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + return 0 + + previous_summary = summary + print( + "Telemetry snapshot changed or was first observed; " + "waiting for a stable snapshot.", + file=sys.stderr, + ) + + if attempt < attempts: + print( + f"Waiting {args.retry_seconds}s for telemetry ingestion.", + file=sys.stderr, + ) + time.sleep(args.retry_seconds) + + if last_cluster_errors: + print( + "ERROR: ARM telemetry queries failed for: " + + ", ".join(last_cluster_errors), + file=sys.stderr, + ) + elif previous_summary is not None: + print( + "ERROR: Correlated ARM telemetry did not stabilize before timeout.", + file=sys.stderr, + ) + else: + print("ERROR: No correlated Terraform ARM writes were found.", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/validate-quickstart-metadata.py b/.github/scripts/validate-quickstart-metadata.py new file mode 100644 index 000000000..9c9909752 --- /dev/null +++ b/.github/scripts/validate-quickstart-metadata.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 + +import argparse +import datetime +import json +import pathlib +import re +import subprocess +import sys +import uuid + + +CONFIGURATION_SUFFIXES = ( + ".tf", + ".tf.json", + ".tfvars", + ".tfvars.json", + ".tftpl", + ".pkr.hcl", +) +EXPECTED_SCHEMA = "../../.github/schemas/quickstart-metadata.schema.json" +MAX_METADATA_AGE = datetime.timedelta(days=30) +MAX_FUTURE_SKEW = datetime.timedelta(minutes=15) +TERRAFORM_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$") +TIMESTAMP_RE = re.compile( + r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$" +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Validate deployment metadata for changed Terraform quickstarts." + ) + parser.add_argument( + "folders", + nargs="+", + help="Quickstart folders. A comma-delimited argument is also accepted.", + ) + parser.add_argument( + "--base", + help="Git base ref used to require a metadata update when configuration changes.", + ) + parser.add_argument( + "--require-metadata", + action="store_true", + help="Require metadata.json even when no Git base is supplied.", + ) + return parser.parse_args() + + +def expand_folders(values: list[str]) -> list[pathlib.Path]: + folders: list[pathlib.Path] = [] + for value in values: + folders.extend( + pathlib.Path(item.strip()) + for item in value.split(",") + if item.strip() + ) + return sorted(set(folders)) + + +def changed_files(base: str, folder: pathlib.Path) -> list[str]: + result = subprocess.run( + ["git", "diff", "--name-only", f"{base}...HEAD", "--", folder.as_posix()], + check=True, + capture_output=True, + text=True, + ) + return [line for line in result.stdout.splitlines() if line] + + +def metadata_at_ref(base: str, path: pathlib.Path) -> dict | None: + result = subprocess.run( + ["git", "show", f"{base}:{path.as_posix()}"], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None + try: + data = json.loads(result.stdout) + except json.JSONDecodeError: + return None + return data if isinstance(data, dict) else None + + +def test_result_value(data: dict | None, field: str) -> object: + if not data: + return None + test_result = data.get("testResult") + if not isinstance(test_result, dict): + return None + return test_result.get(field) + + +def should_require_metadata( + explicit: bool, configuration_changed: bool, metadata_changed: bool +) -> bool: + return explicit or configuration_changed or metadata_changed + + +def parse_timestamp(value: object) -> datetime.datetime | None: + if not isinstance(value, str) or not TIMESTAMP_RE.fullmatch(value): + return None + try: + parsed = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo != datetime.timezone.utc: + return None + return parsed + + +def validate_metadata( + path: pathlib.Path, *, enforce_freshness: bool = True +) -> list[str]: + errors: list[str] = [] + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + return [f"{path}: invalid JSON: {error}"] + + if not isinstance(data, dict): + return [f"{path}: root value must be an object"] + + allowed_root = {"$schema", "testResult"} + unexpected_root = sorted(set(data) - allowed_root) + if unexpected_root: + errors.append(f"{path}: unexpected fields: {', '.join(unexpected_root)}") + if data.get("$schema") != EXPECTED_SCHEMA: + errors.append(f"{path}: $schema must be '{EXPECTED_SCHEMA}'") + + test_result = data.get("testResult") + if not isinstance(test_result, dict): + errors.append(f"{path}: testResult must be an object") + return errors + + allowed_result = {"correlationId", "timestamp", "terraformVersion"} + unexpected_result = sorted(set(test_result) - allowed_result) + if unexpected_result: + errors.append( + f"{path}: unexpected testResult fields: {', '.join(unexpected_result)}" + ) + + correlation_id = test_result.get("correlationId") + try: + parsed_id = uuid.UUID(correlation_id) + if str(parsed_id) != correlation_id.lower(): + raise ValueError + except (AttributeError, TypeError, ValueError): + errors.append(f"{path}: testResult.correlationId must be a dashed UUID") + + timestamp = parse_timestamp(test_result.get("timestamp")) + if timestamp is None: + errors.append( + f"{path}: testResult.timestamp must be an RFC 3339 UTC timestamp ending in Z" + ) + elif enforce_freshness: + now = datetime.datetime.now(datetime.timezone.utc) + if timestamp < now - MAX_METADATA_AGE: + errors.append( + f"{path}: testResult.timestamp must be no more than " + f"{MAX_METADATA_AGE.days} days old" + ) + if timestamp > now + MAX_FUTURE_SKEW: + errors.append( + f"{path}: testResult.timestamp is more than " + f"{int(MAX_FUTURE_SKEW.total_seconds() / 60)} minutes in the future" + ) + + terraform_version = test_result.get("terraformVersion") + if terraform_version is not None and ( + not isinstance(terraform_version, str) + or not TERRAFORM_VERSION_RE.fullmatch(terraform_version) + ): + errors.append( + f"{path}: testResult.terraformVersion must be a semantic version" + ) + + return errors + + +def main() -> int: + args = parse_args() + folders = expand_folders(args.folders) + errors: list[str] = [] + + if not folders: + print("No changed quickstart folders.") + return 0 + + for folder in folders: + metadata_path = folder / "metadata.json" + require_metadata = args.require_metadata + metadata_changed = False + + if args.base: + changed = changed_files(args.base, folder) + configuration_changed = any( + file_name.endswith(CONFIGURATION_SUFFIXES) for file_name in changed + ) + metadata_changed = metadata_path.as_posix() in changed + require_metadata = should_require_metadata( + require_metadata, configuration_changed, metadata_changed + ) + if configuration_changed and not metadata_changed: + errors.append( + f"{folder}: Terraform configuration changed but metadata.json " + "was not updated with a fresh correlation ID" + ) + elif configuration_changed and metadata_path.is_file(): + previous = metadata_at_ref(args.base, metadata_path) + try: + current = json.loads(metadata_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + current = None + previous_id = test_result_value(previous, "correlationId") + current_id = test_result_value(current, "correlationId") + if previous_id is not None and current_id == previous_id: + errors.append( + f"{folder}: testResult.correlationId must change when " + "Terraform configuration changes" + ) + previous_timestamp = test_result_value(previous, "timestamp") + current_timestamp = test_result_value(current, "timestamp") + if ( + previous_timestamp is not None + and current_timestamp == previous_timestamp + ): + errors.append( + f"{folder}: testResult.timestamp must change when " + "Terraform configuration changes" + ) + + if require_metadata and not metadata_path.is_file(): + errors.append(f"{folder}: metadata.json is required") + continue + + if metadata_path.is_file(): + errors.extend( + validate_metadata( + metadata_path, + enforce_freshness=require_metadata or metadata_changed, + ) + ) + + if errors: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + + print(f"Validated deployment metadata for {len(folders)} quickstart folder(s).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/e2e-pr.yaml b/.github/workflows/e2e-pr.yaml new file mode 100644 index 000000000..e12ebb066 --- /dev/null +++ b/.github/workflows/e2e-pr.yaml @@ -0,0 +1,57 @@ +name: E2E Test Code Check + +on: + pull_request: + types: [opened, reopened, synchronize] + paths: + - 'test/**' + +jobs: + fork-notice: + if: github.event.pull_request.head.repo.fork + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Report fork limitation + shell: bash + run: | + echo "### E2E tests are disabled for fork pull requests." >> "$GITHUB_STEP_SUMMARY" + echo "A maintainer must reproduce test-code changes on a branch in Azure/terraform." >> "$GITHUB_STEP_SUMMARY" + + e2e-test-code: + if: github.event.pull_request.head.repo.fork == false + runs-on: ubuntu-latest + timeout-minutes: 360 + environment: + name: test + permissions: + contents: read + id-token: write + steps: + - name: Checkout + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Run E2E test suite + env: + ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} + ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} + ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} + PACKER_VERSION: ${{ vars.PACKER_VERSION }} + shell: bash + run: | + set -euo pipefail + export ARM_OIDC_REQUEST_TOKEN="${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" + export ARM_OIDC_REQUEST_URL="${ACTIONS_ID_TOKEN_REQUEST_URL}" + docker run --rm \ + -v "$(pwd):/src" \ + -w /src/test \ + --network=host \ + -e ARM_CLIENT_ID \ + -e ARM_OIDC_REQUEST_TOKEN \ + -e ARM_OIDC_REQUEST_URL \ + -e ARM_SUBSCRIPTION_ID \ + -e ARM_TENANT_ID \ + -e ARM_USE_OIDC=true \ + mcr.microsoft.com/azterraform:latest \ + sh -c "pkenv install '${PACKER_VERSION}' && go mod tidy && go test -timeout=360m -v ./e2e" diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index bc41597e9..8bf292eba 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1,56 +1,281 @@ -name: E2E Test Check +name: Validate Terraform Deployment + on: - pull_request: - types: ['opened', 'synchronize'] - paths: - - '.github/**' - - '.github/workflows/**' - - 'quickstart/**' - - 'test/**' - - 'test/e2e/**' -permissions: - actions: write - contents: read - id-token: write + issue_comment: + types: [created] jobs: - e2e-check: + gate: + name: Authorize validation command + if: >- + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/validate') runs-on: ubuntu-latest - environment: - name: test + timeout-minutes: 12 + permissions: + actions: read + checks: write + contents: read + issues: write + pull-requests: read + + outputs: + check_id: ${{ steps.check.outputs.check_id }} + head_sha: ${{ steps.pr.outputs.head_sha }} + head_repo: ${{ steps.pr.outputs.head_repo }} + sample_path: ${{ steps.pr.outputs.sample_path }} + steps: - - name: Checking for Fork - shell: pwsh + - name: Authorize command and resolve pull request + id: pr + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + script: | + if (context.payload.comment.body.trim() !== '/validate') { + core.setFailed('The supported command is exactly /validate.'); + return; + } + + const allowedCommenters = new Set(['MEMBER', 'OWNER', 'COLLABORATOR']); + const commenterAssociation = context.payload.comment.author_association; + if (!allowedCommenters.has(commenterAssociation)) { + core.setFailed( + `Not authorized to run /validate: ${commenterAssociation}. ` + + 'Only MEMBER, OWNER, or COLLABORATOR may run this command.' + ); + return; + } + + const owner = context.repo.owner; + const repo = context.repo.repo; + const pull_number = context.payload.issue.number; + const [{ data: pr }, files] = await Promise.all([ + github.rest.pulls.get({ owner, repo, pull_number }), + github.paginate( + github.rest.pulls.listFiles, + { owner, repo, pull_number, per_page: 100 } + ) + ]); + + const protectedPrefixes = ['.github/workflows/', '.github/scripts/']; + const protectedFiles = [ + '.github/schemas/quickstart-metadata.schema.json', + 'GNUmakefile', + 'scripts/terraform-validate.sh' + ]; + const touchedProtected = files + .flatMap(file => [file.filename, file.previous_filename]) + .filter(Boolean) + .filter(name => + protectedPrefixes.some(prefix => name.startsWith(prefix)) || + protectedFiles.includes(name) + ) + .filter((name, index, names) => names.indexOf(name) === index); + if (touchedProtected.length > 0) { + core.setFailed( + `/validate cannot run while validation-contract files are changing: ` + + `${touchedProtected.join(', ')}. Merge validation changes separately, ` + + `canary them from the default branch, then validate quickstart changes.` + ); + return; + } + + const deployFiles = files + .map(file => file.filename) + .filter(name => + name.startsWith('quickstart/') && + /\.(tf|tf\.json|tfvars|tfvars\.json|tftpl|pkr\.hcl)$/.test(name) + ); + const samplePaths = new Set( + deployFiles.map(name => name.split('/').slice(0, 2).join('/')) + ); + if (samplePaths.size !== 1) { + core.setFailed( + `/validate requires exactly one changed deployable quickstart; found ` + + `${samplePaths.size}: ${[...samplePaths].join(', ') || ''}` + ); + return; + } + + core.setOutput('head_sha', pr.head.sha); + core.setOutput('head_repo', pr.head.repo.full_name); + core.setOutput('sample_path', [...samplePaths][0]); + + - name: React to command + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: 'eyes' + }); + + - name: Start pull request check + id: check + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + env: + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + with: + script: | + const { data } = await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + name: 'terraform-deployment-validation', + head_sha: process.env.HEAD_SHA, + status: 'in_progress', + details_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + output: { + title: 'Validating contributor-supplied ARM activity evidence', + summary: 'A maintainer ran `/validate`; waiting for static checks, then querying ARM telemetry for the contributor-supplied correlation ID.' + } + }); + core.setOutput('check_id', data.id); + + - name: Wait for pre-pull-request checks + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + REPOSITORY: ${{ github.repository }} + shell: bash run: | - $isFork = "${{ github.event.pull_request.head.repo.fork }}" - if($isFork -eq "true") { - echo "### WARNING: This workflow is disabled for forked repositories. Please follow the [release branch process](https://azure.github.io/Azure-Verified-Modules/contributing/terraform/terraform-contribution-flow/#5-create-a-pull-request-to-the-upstream-repository) if end to end tests are required." >> $env:GITHUB_STEP_SUMMARY - } - - name: checkout - if: github.event.pull_request.head.repo.fork == false - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 #v3.6.0 - - name: Get changed files - if: github.event.pull_request.head.repo.fork == false - id: changed-files - uses: step-security/changed-files@3dbe17c78367e7d60f00d78ae6781a35be47b4a1 #v45.0.1 + set -euo pipefail + deadline=$(( $(date +%s) + 600 )) + + while (( $(date +%s) < deadline )); do + run_json=$(gh api \ + "/repos/${REPOSITORY}/actions/workflows/pr-check.yaml/runs?head_sha=${HEAD_SHA}&per_page=1" \ + --jq '.workflow_runs[0] // {}') + status=$(jq -r '.status // empty' <<< "${run_json}") + conclusion=$(jq -r '.conclusion // empty' <<< "${run_json}") + + if [[ "${status}" == "completed" ]]; then + if [[ "${conclusion}" == "success" ]]; then + echo "Pre-pull-request checks passed for ${HEAD_SHA}." + exit 0 + fi + echo "Pre-pull-request checks concluded '${conclusion}' for ${HEAD_SHA}." >&2 + exit 1 + fi + + echo "Waiting for pre-pull-request checks (status=${status:-not-started})..." + sleep 30 + done + + echo "Timed out waiting for pre-pull-request checks." >&2 + exit 1 + + validate: + name: Validate contributor deployment + needs: gate + runs-on: ubuntu-latest + timeout-minutes: 5 + environment: + name: adx-readonly + permissions: + contents: read + id-token: write + + steps: + - name: Checkout trusted validator + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: - dir_names: true - separator: "," - dir_names_include_files: "quickstart/*" - files: "quickstart/**" - files_ignore: "**/TestRecord.md" - dir_names_max_depth: 2 - - name: test pr - if: github.event.pull_request.head.repo.fork == false + repository: ${{ github.repository }} + ref: ${{ github.event.repository.default_branch }} + path: trusted + sparse-checkout: | + .github/scripts/validate-quickstart-metadata.py + .github/scripts/validate-arm-correlation.py + sparse-checkout-cone-mode: false + + - name: Checkout pull request sample + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: ${{ needs.gate.outputs.head_repo }} + ref: ${{ needs.gate.outputs.head_sha }} + path: pull-request + sparse-checkout: ${{ needs.gate.outputs.sample_path }} + + - name: Validate metadata env: - ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + SAMPLE_PATH: ${{ needs.gate.outputs.sample_path }} + run: >- + python trusted/.github/scripts/validate-quickstart-metadata.py + --require-metadata + "pull-request/${SAMPLE_PATH}" + + - name: Sign in to Azure with OIDC + uses: azure/login@8216e11d8cd9b42fe925c852af8e76311ff067ac # v2 + with: + tenant-id: ${{ vars.AZURE_TENANT_ID }} + client-id: ${{ vars.AZURE_CLIENT_ID }} + allow-no-subscriptions: true + + - name: Validate correlated Terraform operations + env: + SAMPLE_PATH: ${{ needs.gate.outputs.sample_path }} + shell: bash run: | - echo "change files" $ALL_CHANGED_FILES - export ARM_OIDC_REQUEST_TOKEN=$ACTIONS_ID_TOKEN_REQUEST_TOKEN - export ARM_OIDC_REQUEST_URL=$ACTIONS_ID_TOKEN_REQUEST_URL - export ARM_SUBSCRIPTION_ID=${{ secrets.ARM_SUBSCRIPTION_ID }} - export ARM_TENANT_ID=${{ secrets.ARM_TENANT_ID }} - export ARM_CLIENT_ID=${{ secrets.ARM_CLIENT_ID }} - export PACKER_VERSION=${{ vars.PACKER_VERSION }} - export CHANGED_FOLDERS="${{ steps.changed-files.outputs.all_changed_files }}" - docker run --rm -v $(pwd):/src -w /src/test --network=host -e ARM_SUBSCRIPTION_ID -e ARM_TENANT_ID -e ARM_CLIENT_ID -e ARM_OIDC_REQUEST_TOKEN -e ARM_OIDC_REQUEST_URL -e ARM_USE_OIDC=true -e CHANGED_FOLDERS mcr.microsoft.com/azterraform:latest sh -c "pkenv install $PACKER_VERSION && go mod tidy && go test -timeout=360m -v ./e2e" \ No newline at end of file + set -euo pipefail + + metadata="pull-request/${SAMPLE_PATH}/metadata.json" + correlation_id=$(jq -r '.testResult.correlationId' "${metadata}") + started_at=$(jq -r '.testResult.timestamp' "${metadata}") + + summary=$( + python trusted/.github/scripts/validate-arm-correlation.py \ + --correlation-id "${correlation_id}" \ + --started-at "${started_at}" + ) + + echo "Correlation ID: ${correlation_id}" + echo "Observed: $(jq -r '.firstSeen' <<< "${summary}") through $(jq -r '.lastSeen' <<< "${summary}")" + echo "Clusters: $(jq -r '.clusters | join(\", \")' <<< "${summary}")" + echo "Writes: $(jq -r '.totalWrites' <<< "${summary}"); completed: $(jq -r '.completedWrites' <<< "${summary}"); async accepted: $(jq -r '.acceptedAsyncWrites' <<< "${summary}"); failed: $(jq -r '.failedWrites' <<< "${summary}")" + + { + echo "## Terraform ARM activity validation" + echo + echo "- Sample: \`${SAMPLE_PATH}\`" + echo "- Correlation ID: \`${correlation_id}\`" + echo "- ARM regions: $(jq -r '.clusters | join(\", \")' <<< "${summary}")" + echo "- Synchronous completed writes: $(jq -r '.completedWrites' <<< "${summary}")" + echo "- Asynchronous writes accepted by ARM: $(jq -r '.acceptedAsyncWrites' <<< "${summary}")" + echo "- Final HTTP write failures: $(jq -r '.failedWrites' <<< "${summary}")" + echo "- Resource providers: $(jq -r '.providers' <<< "${summary}")" + } >> "${GITHUB_STEP_SUMMARY}" + + report: + name: Report deployment validation + needs: [gate, validate] + if: always() && needs.gate.outputs.check_id != '' + runs-on: ubuntu-latest + permissions: + checks: write + contents: read + steps: + - name: Complete pull request check + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + env: + CHECK_ID: ${{ needs.gate.outputs.check_id }} + RESULT: ${{ needs.validate.result }} + SAMPLE_PATH: ${{ needs.gate.outputs.sample_path }} + with: + script: | + const passed = process.env.RESULT === 'success'; + await github.rest.checks.update({ + owner: context.repo.owner, + repo: context.repo.repo, + check_run_id: Number(process.env.CHECK_ID), + status: 'completed', + conclusion: passed ? 'success' : 'failure', + output: { + title: passed + ? 'Terraform ARM activity validation passed' + : 'Terraform ARM activity validation failed', + summary: passed + ? `The contributor-supplied correlation ID recorded by \`${process.env.SAMPLE_PATH}\` maps to completed or accepted Terraform ARM writes with no observed final HTTP write failures. This does not independently bind the telemetry to the pull request or applied source.` + : `ARM activity validation failed. Open the workflow run for details, correct the metadata or deployment, then run \`/validate\` again.` + } + }); diff --git a/.github/workflows/pr-check.yaml b/.github/workflows/pr-check.yaml index eb4d6a6a6..5ae89aa6e 100644 --- a/.github/workflows/pr-check.yaml +++ b/.github/workflows/pr-check.yaml @@ -1,10 +1,9 @@ name: Pre Pull Request Check on: pull_request: - types: [ 'opened', 'synchronize' ] + types: [ 'opened', 'reopened', 'synchronize' ] paths: - '.github/**' - - '.github/workflows/**' - 'quickstart/**' jobs: @@ -12,7 +11,13 @@ jobs: runs-on: ubuntu-latest steps: - name: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 #v3.6.0 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + - name: Test quickstart validation scripts + run: | + python .github/scripts/test_validate_arm_correlation.py + python .github/scripts/test_validate_quickstart_metadata.py - name: Get changed files id: changed-files uses: step-security/changed-files@3dbe17c78367e7d60f00d78ae6781a35be47b4a1 #v45.0.1 @@ -22,10 +27,34 @@ jobs: dir_names_include_files: "quickstart/*" files: "quickstart/**" dir_names_max_depth: 2 + - name: Normalize changed quickstart folders + id: changed-folders + env: + CANDIDATES: ${{ steps.changed-files.outputs.all_changed_files }} + shell: bash + run: | + set -euo pipefail + folders=$( + tr ',' '\n' <<< "${CANDIDATES}" | + while IFS= read -r candidate; do + candidate=$(xargs <<< "${candidate}") + if [[ -d "${candidate}" && "${candidate}" =~ ^quickstart/[^/]+$ ]]; then + echo "${candidate}" + fi + done | + sort -u | + paste -sd, - + ) + echo "folders=${folders}" >> "$GITHUB_OUTPUT" + echo "Changed quickstart folders: ${folders:-}" + - name: Validate quickstart deployment metadata + env: + CHANGED_FOLDERS: ${{ steps.changed-folders.outputs.folders }} + run: | + python .github/scripts/validate-quickstart-metadata.py \ + --base "origin/${{ github.base_ref }}" \ + "${CHANGED_FOLDERS}" - name: pr-check run: | - export CHANGED_FOLDERS="${{ steps.changed-files.outputs.all_changed_files }}" - if [ -z "${{ github.event.number }}" ]; then - CHANGED_FOLDERS=$(find ./quickstart -maxdepth 1 -mindepth 1 -type d | tr '\n' ',') - fi - docker run --rm -v $(pwd):/src -w /src -e CHANGED_FOLDERS mcr.microsoft.com/azterraform:latest make pr-check + export CHANGED_FOLDERS="${{ steps.changed-folders.outputs.folders }}" + docker run --rm -v "$(pwd):/src" -w /src -e CHANGED_FOLDERS mcr.microsoft.com/azterraform:latest make pr-check diff --git a/quickstart/CONTRIBUTE.md b/quickstart/CONTRIBUTE.md index e69de29bb..297282f57 100644 --- a/quickstart/CONTRIBUTE.md +++ b/quickstart/CONTRIBUTE.md @@ -0,0 +1,109 @@ +# Contributing a Terraform quickstart + +The repository validates two different claims: + +1. **Static quality:** the pull request passes formatting, linting, and repository checks. +2. **ARM activity evidence:** a maintainer confirms that the contributor-supplied correlation ID maps to qualifying Terraform ARM write activity with no observed final HTTP write failures. + +The quickstart validation workflow does not deploy pull-request quickstart code with repository-owned Azure credentials. Contributors deploy the quickstart first; maintainers validate the resulting telemetry with `/validate`. The separate internal E2E test-harness workflow continues to execute same-repository `test/**` pull-request code using the protected `test` environment. + +## Contribution requirements + +A pull request that changes deployable files in `quickstart//` must: + +- Change one deployable quickstart per pull request. +- Pass `terraform fmt`, `terraform init`, `terraform validate`, and `terraform plan`. +- Apply the quickstart to Azure before opening the pull request. +- Set `ARM_CORRELATION_REQUEST_ID` to a new UUID for that apply. +- Add or update `metadata.json` in the quickstart folder with the UUID and apply start time. +- Remove the deployed resources after validation unless the sample documents a reason to retain them. + +The AzureRM and AzAPI providers use `ARM_CORRELATION_REQUEST_ID` as the `x-ms-correlation-request-id` for the provider workflow. Do not reuse an ID from an earlier plan or apply. This is a good-faith contributor assertion: validation does not prevent deliberate reuse or independently bind the ID to the pull request, sample, actor, or applied source. + +## Deploy and capture validation evidence + +Authenticate to the Azure subscription you use for testing, then run the commands from the quickstart folder. + +### Bash + +```bash +export ARM_CORRELATION_REQUEST_ID="$(python -c 'import uuid; print(uuid.uuid4())')" +export VALIDATION_STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + +terraform init +terraform validate +terraform plan -out=tfplan +terraform apply tfplan + +printf 'correlationId=%s\ntimestamp=%s\n' \ + "$ARM_CORRELATION_REQUEST_ID" \ + "$VALIDATION_STARTED_AT" +``` + +### PowerShell + +```powershell +$env:ARM_CORRELATION_REQUEST_ID = [guid]::NewGuid().ToString() +$validationStartedAt = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + +terraform init +terraform validate +terraform plan -out=tfplan +terraform apply tfplan + +"correlationId=$env:ARM_CORRELATION_REQUEST_ID" +"timestamp=$validationStartedAt" +``` + +Use a saved plan so the reviewed plan and applied configuration are the same local operation. Never commit `tfplan`, Terraform state, credentials, subscription IDs, tenant IDs, or other environment-specific values. + +## Add `metadata.json` + +Create `metadata.json` beside the quickstart's Terraform files: + +```json +{ + "$schema": "../../.github/schemas/quickstart-metadata.schema.json", + "testResult": { + "correlationId": "12345678-1234-1234-1234-1234567890ab", + "timestamp": "2026-08-18T17:00:00Z", + "terraformVersion": "1.13.0" + } +} +``` + +`correlationId` and `timestamp` are required. The timestamp must be no more than 30 days old. `terraformVersion` is optional but recommended. If any `.tf`, `.tf.json`, `.tfvars`, `.tfvars.json`, `.tftpl`, or `.pkr.hcl` file changes, deploy again and replace both required values. + +## Pull request validation + +The **Pre Pull Request Check** workflow runs automatically. It: + +- Detects changed quickstart folders. +- Requires a fresh `metadata.json` update when deployable files change. +- Validates the metadata format. +- Runs the existing repository `pr-check` target. + +Changes to the E2E test harness under `test/**` run separately in the **E2E Test Code Check** workflow. That workflow retains repository-owned Azure execution for internal branches and does not run privileged pull-request code from forks. + +After that workflow passes, a repository maintainer comments: + +```text +/validate +``` + +The command is restricted to repository members, owners, and collaborators. It checks out only the changed quickstart, validates its metadata with trusted code from the default branch, signs in to Azure with the `adx-readonly` environment's federated identity, and queries the approved regional ARMProd `Requests.HttpIncomingRequests` datasets by the supplied correlation ID. + +Validation passes only when telemetry for the contributor-supplied correlation ID contains at least one synchronously completed or asynchronously accepted Terraform ARM write and no observed final HTTP write failures. Preliminary records and failures that were successfully retried are not treated as final failures. A `202 Accepted` result proves that ARM accepted the asynchronous operation; the approved dataset does not expose the response body needed to prove its eventual provisioning state. The workflow publishes the result as the `terraform-deployment-validation` check on the pull request head. + +This is contributor-asserted ARM activity evidence, not proof that Terraform exited successfully and not source attestation. Unlike an ARM template deployment, Terraform does not emit a template hash that ARM can compare with the pull request. Reviewers must still inspect the code and static checks. + +Pull requests that modify the validation workflows, validators, metadata schema, or their transitive static-check implementation cannot use `/validate`. Merge validation-contract changes separately, canary them from the default branch, then validate quickstart changes under the deployed contract. + +## Maintainer configuration + +The `adx-readonly` GitHub environment must provide these variables for its federated OIDC application: + +- `AZURE_TENANT_ID` +- `AZURE_CLIENT_ID` + +The identity uses subscriptionless OIDC and needs read-only query access to the `Requests` databases on the approved `armprodeus`, `armprodweu`, and `armprodsea` clusters. It does not need an Azure subscription role or permissions to deploy or modify resources. \ No newline at end of file