diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 68cbe7bc..b461939b 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "azure", "description": "Microsoft Azure MCP and Skills integration for cloud resource management, deployments, and Azure services. Manage your Azure infrastructure, monitor applications, and deploy resources directly from Claude Code.", - "version": "1.2.76", + "version": "1.2.77", "author": { "name": "Microsoft", "url": "https://www.microsoft.com" diff --git a/.github/plugins/azure-skills/.claude-plugin/plugin.json b/.github/plugins/azure-skills/.claude-plugin/plugin.json index 68cbe7bc..b461939b 100644 --- a/.github/plugins/azure-skills/.claude-plugin/plugin.json +++ b/.github/plugins/azure-skills/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "azure", "description": "Microsoft Azure MCP and Skills integration for cloud resource management, deployments, and Azure services. Manage your Azure infrastructure, monitor applications, and deploy resources directly from Claude Code.", - "version": "1.2.76", + "version": "1.2.77", "author": { "name": "Microsoft", "url": "https://www.microsoft.com" diff --git a/.github/plugins/azure-skills/.cursor-plugin/plugin.json b/.github/plugins/azure-skills/.cursor-plugin/plugin.json index f240ea7d..ac21908e 100644 --- a/.github/plugins/azure-skills/.cursor-plugin/plugin.json +++ b/.github/plugins/azure-skills/.cursor-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "azure", "description": "Microsoft Azure MCP and Skills integration for cloud resource management, deployments, and Azure services. Manage your Azure infrastructure, monitor applications, and deploy resources directly from Cursor.", - "version": "1.2.76", + "version": "1.2.77", "author": { "name": "Microsoft", "url": "https://www.microsoft.com" diff --git a/.github/plugins/azure-skills/.plugin/plugin.json b/.github/plugins/azure-skills/.plugin/plugin.json index 532bd8ca..5882205b 100644 --- a/.github/plugins/azure-skills/.plugin/plugin.json +++ b/.github/plugins/azure-skills/.plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "azure", "description": "Microsoft Azure MCP and Skills integration for cloud resource management, deployments, and Azure services. Manage your Azure infrastructure, monitor applications, and deploy resources directly from your development environment.", - "version": "1.2.76", + "version": "1.2.77", "author": { "name": "Microsoft", "url": "https://www.microsoft.com" diff --git a/.github/plugins/azure-skills/CHANGELOG.md b/.github/plugins/azure-skills/CHANGELOG.md index a889bb0d..9a9e5fb8 100644 --- a/.github/plugins/azure-skills/CHANGELOG.md +++ b/.github/plugins/azure-skills/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 1.2.77 + +- use consistent line ending ([#3313](https://github.com/microsoft/GitHub-Copilot-for-Azure/pull/3313)) + ## 1.2.76 - docs: add telemetry disclosure to azure-skills README ([#3305](https://github.com/microsoft/GitHub-Copilot-for-Azure/pull/3305)) diff --git a/.github/plugins/azure-skills/skills/azure-app-onboard-prereq/SKILL.md b/.github/plugins/azure-skills/skills/azure-app-onboard-prereq/SKILL.md index 8ff9d058..5897a893 100644 --- a/.github/plugins/azure-skills/skills/azure-app-onboard-prereq/SKILL.md +++ b/.github/plugins/azure-skills/skills/azure-app-onboard-prereq/SKILL.md @@ -4,7 +4,7 @@ description: "Assess whether source code is ready to deploy to Azure — the che license: MIT metadata: author: Microsoft - version: "1.2.2" + version: "1.2.3" --- # Azure App Onboard Prereq — Repository Evaluation diff --git a/.github/plugins/azure-skills/skills/azure-app-onboard-prereq/references/prereq-schemas.ts b/.github/plugins/azure-skills/skills/azure-app-onboard-prereq/references/prereq-schemas.ts index 3ac53a07..faab148c 100644 --- a/.github/plugins/azure-skills/skills/azure-app-onboard-prereq/references/prereq-schemas.ts +++ b/.github/plugins/azure-skills/skills/azure-app-onboard-prereq/references/prereq-schemas.ts @@ -1,80 +1,80 @@ -/** - * Prereq-phase TypeScript interfaces for prereq-output.json. - * - * Standalone schema for azure-app-onboard-prereq — this skill can run independently. - * - * Shared types used across phases (AppOnboardComponent, PostDeployRecommendation, - * DetectedService, AppOnboardContext) are defined in: - * session-schemas.ts (local copy in this skill's references/) - * - * Context schema (AppOnboardContext, AppOnboardIntent) is defined in: - * session-schemas.ts (local copy in this skill's references/) - */ - -// ─── prereq-output.json ────────────────────────────────────────────────────── - -export interface BuildRequirements { - hasNativeModules: boolean; - hasDockerfile: boolean; - /** True when the app can deploy on App Service F1 (Free) SKU */ - f1Viable: boolean; - /** True when Dockerfile uses BuildKit-only syntax (--mount, # syntax=docker/dockerfile:1) */ - hasBuildKitSyntax?: boolean; - /** Port from EXPOSE directive — used for Container Apps targetPort */ - exposedPort?: number; - /** Estimated native module compilation time in seconds (typically 30–300s; - * larger compiled dependencies like scipy may take 600+s). - * Used by deploy phase to set WEBSITES_CONTAINER_START_TIME_LIMIT. */ - estimatedInstallTime?: number; - /** Why F1 is not viable — set when f1Viable is false. - * Examples: "native modules (node-gyp)", "large dependency tree (27 pinned deps)", - * "build-time compilation (TypeScript tsc)", "major migration (Flask 0.12→2.3)" */ - f1BlockReason?: string; -} - -/** Structured warning from prereq evaluation */ -export interface PrereqWarning { - id: string; - component: string; - axis: "build" | "completeness" | "deployability"; - summary: string; - detail: string; - /** What to do about this warning — actionable fix instruction. - * Examples: "Set PGSSLMODE=require env var in IaC", "Add trust proxy setting to Express app" */ - fix: string; - /** When the fix should be applied in the pipeline. - * "prereq" = prereq can fix this NOW with user approval (code/config changes that prevent deploy failures). - * "scaffold" = handled in generated IaC only (env var override, probe config). NEVER modifies user code. - * "deploy-gate" = surface at deploy approval gate for user awareness. No code changes. - * "post-deploy" = informational — add to postDeployRecommendations[], no action during pipeline. */ - fixPhase: "prereq" | "scaffold" | "deploy-gate" | "post-deploy"; -} - -export interface PrereqOutput { - // AppOnboardComponent[] — see session-schemas.ts - components: any[]; - /** Structured warnings — see prereq-artifacts.md for write rules */ - warnings: PrereqWarning[]; - detectedStack: string; - /** Prereq-only: auto-approves readiness gate + simplifies prepare alt analysis. - * Does NOT skip any phase, gate, reference read, or validation. */ - fastTrackEligible: boolean; - overallHealth?: "ready" | "readyWithCaveats" | "blocked"; - /** Build-time requirements detected from manifests and lockfiles */ - buildRequirements?: BuildRequirements; - /** Detected health endpoint path (e.g., "/health", "/api/v1/health/"). - * Used by scaffold for Azure health probe config and deploy for curl health check. - * null if no health endpoint found (triggers W-HEALTH warning). */ - healthEndpoint?: string | null; - /** Application entry point detected from manifest (e.g., "index.js", "main.py", "cmd/main.go"). - * Used by prepare/deploy-strategy.md for startupCommand generation. */ - entryPoint?: string; - /** Structured recommendations derived from WARN findings. - * Merged into prepare-plan.json.postDeployRecommendations[] by the prepare phase. - * PostDeployRecommendation type — see session-schemas.ts */ - postDeployRecommendations?: any[]; - /** First-run init commands (DB migrations, seed data) detected from - * framework signals + migrations/ dir. Prepare prepends required entries - * to deployStrategy.startupCommand. See dependency-compatibility.md § First-Run. */ - initCommands?: { type: string; framework: string; command: string; required: boolean }[]; -} +/** + * Prereq-phase TypeScript interfaces for prereq-output.json. + * + * Standalone schema for azure-app-onboard-prereq — this skill can run independently. + * + * Shared types used across phases (AppOnboardComponent, PostDeployRecommendation, + * DetectedService, AppOnboardContext) are defined in: + * session-schemas.ts (local copy in this skill's references/) + * + * Context schema (AppOnboardContext, AppOnboardIntent) is defined in: + * session-schemas.ts (local copy in this skill's references/) + */ + +// ─── prereq-output.json ────────────────────────────────────────────────────── + +export interface BuildRequirements { + hasNativeModules: boolean; + hasDockerfile: boolean; + /** True when the app can deploy on App Service F1 (Free) SKU */ + f1Viable: boolean; + /** True when Dockerfile uses BuildKit-only syntax (--mount, # syntax=docker/dockerfile:1) */ + hasBuildKitSyntax?: boolean; + /** Port from EXPOSE directive — used for Container Apps targetPort */ + exposedPort?: number; + /** Estimated native module compilation time in seconds (typically 30–300s; + * larger compiled dependencies like scipy may take 600+s). + * Used by deploy phase to set WEBSITES_CONTAINER_START_TIME_LIMIT. */ + estimatedInstallTime?: number; + /** Why F1 is not viable — set when f1Viable is false. + * Examples: "native modules (node-gyp)", "large dependency tree (27 pinned deps)", + * "build-time compilation (TypeScript tsc)", "major migration (Flask 0.12→2.3)" */ + f1BlockReason?: string; +} + +/** Structured warning from prereq evaluation */ +export interface PrereqWarning { + id: string; + component: string; + axis: "build" | "completeness" | "deployability"; + summary: string; + detail: string; + /** What to do about this warning — actionable fix instruction. + * Examples: "Set PGSSLMODE=require env var in IaC", "Add trust proxy setting to Express app" */ + fix: string; + /** When the fix should be applied in the pipeline. + * "prereq" = prereq can fix this NOW with user approval (code/config changes that prevent deploy failures). + * "scaffold" = handled in generated IaC only (env var override, probe config). NEVER modifies user code. + * "deploy-gate" = surface at deploy approval gate for user awareness. No code changes. + * "post-deploy" = informational — add to postDeployRecommendations[], no action during pipeline. */ + fixPhase: "prereq" | "scaffold" | "deploy-gate" | "post-deploy"; +} + +export interface PrereqOutput { + // AppOnboardComponent[] — see session-schemas.ts + components: any[]; + /** Structured warnings — see prereq-artifacts.md for write rules */ + warnings: PrereqWarning[]; + detectedStack: string; + /** Prereq-only: auto-approves readiness gate + simplifies prepare alt analysis. + * Does NOT skip any phase, gate, reference read, or validation. */ + fastTrackEligible: boolean; + overallHealth?: "ready" | "readyWithCaveats" | "blocked"; + /** Build-time requirements detected from manifests and lockfiles */ + buildRequirements?: BuildRequirements; + /** Detected health endpoint path (e.g., "/health", "/api/v1/health/"). + * Used by scaffold for Azure health probe config and deploy for curl health check. + * null if no health endpoint found (triggers W-HEALTH warning). */ + healthEndpoint?: string | null; + /** Application entry point detected from manifest (e.g., "index.js", "main.py", "cmd/main.go"). + * Used by prepare/deploy-strategy.md for startupCommand generation. */ + entryPoint?: string; + /** Structured recommendations derived from WARN findings. + * Merged into prepare-plan.json.postDeployRecommendations[] by the prepare phase. + * PostDeployRecommendation type — see session-schemas.ts */ + postDeployRecommendations?: any[]; + /** First-run init commands (DB migrations, seed data) detected from + * framework signals + migrations/ dir. Prepare prepends required entries + * to deployStrategy.startupCommand. See dependency-compatibility.md § First-Run. */ + initCommands?: { type: string; framework: string; command: string; required: boolean }[]; +} diff --git a/.github/plugins/azure-skills/skills/azure-app-onboard/SKILL.md b/.github/plugins/azure-skills/skills/azure-app-onboard/SKILL.md index 7946e558..5e78ab43 100644 --- a/.github/plugins/azure-skills/skills/azure-app-onboard/SKILL.md +++ b/.github/plugins/azure-skills/skills/azure-app-onboard/SKILL.md @@ -4,7 +4,7 @@ description: "End-to-end orchestrator: from a business idea, app idea, or existi license: MIT metadata: author: Microsoft - version: "1.2.4" + version: "1.2.5" --- # Azure App Onboard diff --git a/.github/plugins/azure-skills/skills/azure-app-onboard/references/session-schemas.ts b/.github/plugins/azure-skills/skills/azure-app-onboard/references/session-schemas.ts index 1630c221..b9da5577 100644 --- a/.github/plugins/azure-skills/skills/azure-app-onboard/references/session-schemas.ts +++ b/.github/plugins/azure-skills/skills/azure-app-onboard/references/session-schemas.ts @@ -1,147 +1,147 @@ -/** - * Context + shared TypeScript interfaces for AppOnboard session artifacts: - * context.json, active-session.json, and shared types used across all phases. - * - * Per-phase schemas (each sub-skill has its own in its references/ folder): - * - azure-app-onboard-prereq/references/prereq-schemas.ts — prereq-output.json - * - prepare/references/prepare-schemas.ts — prepare-plan.json - * - scaffold/references/scaffold-schemas.ts — scaffold-manifest.json - * - deploy/references/deploy-schemas.ts — deploy-result.json - * - * Source of truth for JSON artifacts in `.copilot-azure/sessions/{session-id}/`. - */ - -// ─── shared types (used across all phases) ─────────────────────────────────── - -export interface AppOnboardComponentStack { - language: string; - framework: string; - version: string; -} - -export type ReadinessStatus = "ready" | "fixesApplied" | "needsFixes" | "unknown"; - -export interface AppOnboardComponentReadiness { - status: ReadinessStatus; - fixes: string[]; -} - -export type VerdictLevel = "PASS" | "WARN" | "FAIL" | "SKIPPED"; - -export interface AppOnboardComponentVerdicts { - build: VerdictLevel; - completeness: Exclude; - deployability: Exclude; -} - -export interface AppOnboardComponentFinding { - category: "build" | "completeness" | "deployability"; - verdict: VerdictLevel; - summary: string; - fix: string | null; -} - -export interface AppOnboardComponent { - name: string; - path: string; - stack: AppOnboardComponentStack; - readiness: AppOnboardComponentReadiness; - verdicts?: AppOnboardComponentVerdicts; - findings?: readonly AppOnboardComponentFinding[]; -} - -export interface AppOnboardAzureTarget { - subscriptionId: string; - /** Display name of the subscription (from `az account show --query name`). - * Shown at both approval gates so the user can verify the target. */ - subscriptionName: string; - resourceGroup: string; - region: string; -} - -export interface AppOnboardRepoInfo { - remote: string | null; -} - -export interface AppOnboardOverride { - key: string; - value: string; - reason: string; -} - -export interface AppOnboardAppInfo { - name: string; -} - -export interface PostDeployRecommendation { - title: string; - reason: string; - effort: "low" | "medium" | "high"; - services?: string[]; -} - -export interface DetectedService { - type: string; - version?: string; - source: "compose" | "config" | "code"; -} - -// ─── context.json ───────────────────────────────────────────────────────────── - -export interface AppOnboardIntent { - userPrompt: string; - description: string; - users?: string; - auth?: string; - scale?: string; - budget?: string; - /** Set to true after prereq scan refines intent */ - refinedFromScan?: boolean; - /** Facts discovered by the prereq scan */ - scanDiscoveredFacts?: string[]; -} - -export type AppOnboardPhase = "info" | "prereq" | "prepare" | "scaffold" | "deploy" | "cicd" | "observe"; - -export interface AppOnboardContext { - sessionId: string; - createdUtc: string; - lastModifiedUtc: string; - currentPhase: AppOnboardPhase | null; - completedPhases: readonly AppOnboardPhase[]; - /** Human-readable 1-line summary of where the session stands, updated at each phase exit. - * Displayed in the session picker when the user resumes or switches sessions. */ - statusSummary?: string; - intent: AppOnboardIntent; - components: AppOnboardComponent[]; - azure: AppOnboardAzureTarget; - repo: AppOnboardRepoInfo; - app?: AppOnboardAppInfo; - /** Infrastructure file types detected in repo: dockerfile, terraform, bicep, azure-yaml, github-actions */ - detectedInfra: readonly string[]; - /** Cloud provider targeted by detected IaC. Only populated when `.tf` or `.bicep` files found. - * Used by scaffold to distinguish "existing Azure IaC" (halt) from "non-Azure IaC" (generate Azure TF alongside). */ - detectedInfraProvider?: { - terraform?: "azure" | "gcp" | "aws" | "multi" | "unknown"; - }; - /** Service dependencies parsed from docker-compose, config files, or code imports */ - detectedServices: readonly DetectedService[]; - overrides: AppOnboardOverride[]; - /** Set when a phase routes the pipeline to another skill — e.g. "azure-cloud-migrate" - * (non-Azure cloud SDK deps) or "azure-prepare" (existing azd template). Presence halts - * the greenfield pipeline; the resume path in session-protocol.md clears it and re-runs prereq. */ - routeToSkill?: string; - /** Human-readable reason paired with `routeToSkill` (e.g. "existing-azd-template"). */ - routeReason?: string; -} - -// ─── active-session.json ───────────────────────────────────────────────────── - -/** Pointer file at `.copilot-azure/sessions/active-session.json`. - * Avoids scanning all session folders on startup — read this one file - * to find the active session, then read that session's context.json. */ -export interface ActiveSessionPointer { - activeSessionId: string; -} - +/** + * Context + shared TypeScript interfaces for AppOnboard session artifacts: + * context.json, active-session.json, and shared types used across all phases. + * + * Per-phase schemas (each sub-skill has its own in its references/ folder): + * - azure-app-onboard-prereq/references/prereq-schemas.ts — prereq-output.json + * - prepare/references/prepare-schemas.ts — prepare-plan.json + * - scaffold/references/scaffold-schemas.ts — scaffold-manifest.json + * - deploy/references/deploy-schemas.ts — deploy-result.json + * + * Source of truth for JSON artifacts in `.copilot-azure/sessions/{session-id}/`. + */ + +// ─── shared types (used across all phases) ─────────────────────────────────── + +export interface AppOnboardComponentStack { + language: string; + framework: string; + version: string; +} + +export type ReadinessStatus = "ready" | "fixesApplied" | "needsFixes" | "unknown"; + +export interface AppOnboardComponentReadiness { + status: ReadinessStatus; + fixes: string[]; +} + +export type VerdictLevel = "PASS" | "WARN" | "FAIL" | "SKIPPED"; + +export interface AppOnboardComponentVerdicts { + build: VerdictLevel; + completeness: Exclude; + deployability: Exclude; +} + +export interface AppOnboardComponentFinding { + category: "build" | "completeness" | "deployability"; + verdict: VerdictLevel; + summary: string; + fix: string | null; +} + +export interface AppOnboardComponent { + name: string; + path: string; + stack: AppOnboardComponentStack; + readiness: AppOnboardComponentReadiness; + verdicts?: AppOnboardComponentVerdicts; + findings?: readonly AppOnboardComponentFinding[]; +} + +export interface AppOnboardAzureTarget { + subscriptionId: string; + /** Display name of the subscription (from `az account show --query name`). + * Shown at both approval gates so the user can verify the target. */ + subscriptionName: string; + resourceGroup: string; + region: string; +} + +export interface AppOnboardRepoInfo { + remote: string | null; +} + +export interface AppOnboardOverride { + key: string; + value: string; + reason: string; +} + +export interface AppOnboardAppInfo { + name: string; +} + +export interface PostDeployRecommendation { + title: string; + reason: string; + effort: "low" | "medium" | "high"; + services?: string[]; +} + +export interface DetectedService { + type: string; + version?: string; + source: "compose" | "config" | "code"; +} + +// ─── context.json ───────────────────────────────────────────────────────────── + +export interface AppOnboardIntent { + userPrompt: string; + description: string; + users?: string; + auth?: string; + scale?: string; + budget?: string; + /** Set to true after prereq scan refines intent */ + refinedFromScan?: boolean; + /** Facts discovered by the prereq scan */ + scanDiscoveredFacts?: string[]; +} + +export type AppOnboardPhase = "info" | "prereq" | "prepare" | "scaffold" | "deploy" | "cicd" | "observe"; + +export interface AppOnboardContext { + sessionId: string; + createdUtc: string; + lastModifiedUtc: string; + currentPhase: AppOnboardPhase | null; + completedPhases: readonly AppOnboardPhase[]; + /** Human-readable 1-line summary of where the session stands, updated at each phase exit. + * Displayed in the session picker when the user resumes or switches sessions. */ + statusSummary?: string; + intent: AppOnboardIntent; + components: AppOnboardComponent[]; + azure: AppOnboardAzureTarget; + repo: AppOnboardRepoInfo; + app?: AppOnboardAppInfo; + /** Infrastructure file types detected in repo: dockerfile, terraform, bicep, azure-yaml, github-actions */ + detectedInfra: readonly string[]; + /** Cloud provider targeted by detected IaC. Only populated when `.tf` or `.bicep` files found. + * Used by scaffold to distinguish "existing Azure IaC" (halt) from "non-Azure IaC" (generate Azure TF alongside). */ + detectedInfraProvider?: { + terraform?: "azure" | "gcp" | "aws" | "multi" | "unknown"; + }; + /** Service dependencies parsed from docker-compose, config files, or code imports */ + detectedServices: readonly DetectedService[]; + overrides: AppOnboardOverride[]; + /** Set when a phase routes the pipeline to another skill — e.g. "azure-cloud-migrate" + * (non-Azure cloud SDK deps) or "azure-prepare" (existing azd template). Presence halts + * the greenfield pipeline; the resume path in session-protocol.md clears it and re-runs prereq. */ + routeToSkill?: string; + /** Human-readable reason paired with `routeToSkill` (e.g. "existing-azd-template"). */ + routeReason?: string; +} + +// ─── active-session.json ───────────────────────────────────────────────────── + +/** Pointer file at `.copilot-azure/sessions/active-session.json`. + * Avoids scanning all session folders on startup — read this one file + * to find the active session, then read that session's context.json. */ +export interface ActiveSessionPointer { + activeSessionId: string; +} + // PrereqOutput, BuildRequirements → see azure-app-onboard-prereq/references/prereq-schemas.ts \ No newline at end of file diff --git a/.github/plugins/azure-skills/skills/azure-compliance/SKILL.md b/.github/plugins/azure-skills/skills/azure-compliance/SKILL.md index 28fa6cf4..5f133e41 100644 --- a/.github/plugins/azure-skills/skills/azure-compliance/SKILL.md +++ b/.github/plugins/azure-skills/skills/azure-compliance/SKILL.md @@ -4,7 +4,7 @@ description: "Run Azure compliance and security audits with azqr plus Key Vault license: MIT metadata: author: Microsoft - version: "1.2.2" + version: "1.2.3" --- # Azure Compliance & Security Auditing diff --git a/.github/plugins/azure-skills/skills/azure-compliance/references/azure-resource-graph.md b/.github/plugins/azure-skills/skills/azure-compliance/references/azure-resource-graph.md index fea98c15..762b0a9a 100644 --- a/.github/plugins/azure-skills/skills/azure-compliance/references/azure-resource-graph.md +++ b/.github/plugins/azure-skills/skills/azure-compliance/references/azure-resource-graph.md @@ -1,97 +1,97 @@ -# Azure Resource Graph Queries for Compliance Auditing - -Azure Resource Graph (ARG) enables fast, cross-subscription resource querying using KQL via `az graph query`. Use it for compliance scanning, tag audits, and configuration validation. - -## How to Query - -Use the `extension_cli_generate` MCP tool to generate `az graph query` commands: - -```yaml -mcp_azure_mcp_extension_cli_generate - intent: "query Azure Resource Graph to " - cli-type: "az" -``` - -Or construct directly: - -```bash -az graph query -q "" --query "data[].{name:name, type:type}" -o table -``` - -> ⚠️ **Prerequisite:** `az extension add --name resource-graph` - -## Key Tables - -| Table | Contains | -|-------|----------| -| `Resources` | All ARM resources (name, type, location, properties, tags) | -| `ResourceContainers` | Subscriptions, resource groups, management groups | -| `AuthorizationResources` | Role assignments and role definitions | -| `AdvisorResources` | Azure Advisor recommendations | - -## Compliance Query Patterns - -**Find resources missing a required tag:** - -```kql -Resources -| where isnull(tags['Environment']) or isnull(tags['CostCenter']) -| project name, type, resourceGroup, tags -``` - -**Tag coverage analysis:** - -```kql -Resources -| extend hasEnvTag = isnotnull(tags['Environment']) -| summarize total=count(), tagged=countif(hasEnvTag) by type -| extend coverage=round(100.0 * tagged / total, 1) -| order by coverage asc -``` - -**Find storage accounts without HTTPS enforcement:** - -```kql -Resources -| where type =~ 'microsoft.storage/storageaccounts' -| where properties.supportsHttpsTrafficOnly == false -| project name, resourceGroup, location -``` - -**Find resources with public network access enabled:** - -```kql -Resources -| where properties.publicNetworkAccess =~ 'Enabled' -| project name, type, resourceGroup, location -``` - -**Query role assignments across subscriptions:** - -```kql -AuthorizationResources -| where type == 'microsoft.authorization/roleassignments' -| extend principalType = tostring(properties.principalType) -| summarize count() by principalType -``` - -**Find resource groups without locks:** - -```kql -ResourceContainers -| where type == 'microsoft.resources/subscriptions/resourcegroups' -| project rgName=name, rgId=id -| join kind=leftanti ( - Resources - | where type == 'microsoft.authorization/locks' - | project rgId=tostring(properties.resourceId) -) on rgId -``` - -## Tips - -- Use `=~` for case-insensitive type matching (resource types are lowercase) -- Navigate properties with `properties.fieldName` -- Use `--first N` to limit result count -- Use `--subscriptions` to scope to specific subscriptions -- Combine with `AdvisorResources` for security recommendations +# Azure Resource Graph Queries for Compliance Auditing + +Azure Resource Graph (ARG) enables fast, cross-subscription resource querying using KQL via `az graph query`. Use it for compliance scanning, tag audits, and configuration validation. + +## How to Query + +Use the `extension_cli_generate` MCP tool to generate `az graph query` commands: + +```yaml +mcp_azure_mcp_extension_cli_generate + intent: "query Azure Resource Graph to " + cli-type: "az" +``` + +Or construct directly: + +```bash +az graph query -q "" --query "data[].{name:name, type:type}" -o table +``` + +> ⚠️ **Prerequisite:** `az extension add --name resource-graph` + +## Key Tables + +| Table | Contains | +|-------|----------| +| `Resources` | All ARM resources (name, type, location, properties, tags) | +| `ResourceContainers` | Subscriptions, resource groups, management groups | +| `AuthorizationResources` | Role assignments and role definitions | +| `AdvisorResources` | Azure Advisor recommendations | + +## Compliance Query Patterns + +**Find resources missing a required tag:** + +```kql +Resources +| where isnull(tags['Environment']) or isnull(tags['CostCenter']) +| project name, type, resourceGroup, tags +``` + +**Tag coverage analysis:** + +```kql +Resources +| extend hasEnvTag = isnotnull(tags['Environment']) +| summarize total=count(), tagged=countif(hasEnvTag) by type +| extend coverage=round(100.0 * tagged / total, 1) +| order by coverage asc +``` + +**Find storage accounts without HTTPS enforcement:** + +```kql +Resources +| where type =~ 'microsoft.storage/storageaccounts' +| where properties.supportsHttpsTrafficOnly == false +| project name, resourceGroup, location +``` + +**Find resources with public network access enabled:** + +```kql +Resources +| where properties.publicNetworkAccess =~ 'Enabled' +| project name, type, resourceGroup, location +``` + +**Query role assignments across subscriptions:** + +```kql +AuthorizationResources +| where type == 'microsoft.authorization/roleassignments' +| extend principalType = tostring(properties.principalType) +| summarize count() by principalType +``` + +**Find resource groups without locks:** + +```kql +ResourceContainers +| where type == 'microsoft.resources/subscriptions/resourcegroups' +| project rgName=name, rgId=id +| join kind=leftanti ( + Resources + | where type == 'microsoft.authorization/locks' + | project rgId=tostring(properties.resourceId) +) on rgId +``` + +## Tips + +- Use `=~` for case-insensitive type matching (resource types are lowercase) +- Navigate properties with `properties.fieldName` +- Use `--first N` to limit result count +- Use `--subscriptions` to scope to specific subscriptions +- Combine with `AdvisorResources` for security recommendations diff --git a/.github/plugins/azure-skills/skills/azure-deploy/SKILL.md b/.github/plugins/azure-skills/skills/azure-deploy/SKILL.md index 911caa7e..187e0206 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/SKILL.md +++ b/.github/plugins/azure-skills/skills/azure-deploy/SKILL.md @@ -4,7 +4,7 @@ description: "Execute Azure deployments for ALREADY-PREPARED applications that h license: MIT metadata: author: Microsoft - version: "1.2.1" + version: "1.2.2" --- # Azure Deploy diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/README.md b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/README.md index 68dc0154..59af3a3f 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/README.md +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/README.md @@ -1,11 +1,11 @@ -# Recipes - -Deployment recipes for different infrastructure approaches. - -| Recipe | When to Use | -|--------|-------------| -| [AZD](azd/README.md) | Projects using Azure Developer CLI | -| [AZCLI](azcli/README.md) | Projects using Azure CLI scripts | -| [Bicep](bicep/README.md) | Projects using Bicep templates | -| [Terraform](terraform/README.md) | Projects using Terraform | -| [CI/CD](cicd/README.md) | Pipeline-based deployments | +# Recipes + +Deployment recipes for different infrastructure approaches. + +| Recipe | When to Use | +|--------|-------------| +| [AZD](azd/README.md) | Projects using Azure Developer CLI | +| [AZCLI](azcli/README.md) | Projects using Azure CLI scripts | +| [Bicep](bicep/README.md) | Projects using Bicep templates | +| [Terraform](terraform/README.md) | Projects using Terraform | +| [CI/CD](cicd/README.md) | Pipeline-based deployments | diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azcli/README.md b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azcli/README.md index 05cc4614..a5dd9c3a 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azcli/README.md +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azcli/README.md @@ -1,73 +1,73 @@ -# Azure CLI Deploy Recipe - -Deploy to Azure using Azure CLI. - -## Prerequisites - -- `az` CLI installed → Run `mcp_azure_mcp_extension_cli_install` with `cli-type: az` if needed -- `.azure/deployment-plan.md` exists with status `Validated` -- Bicep/ARM templates exist in `infra/` -- **Subscription and location confirmed** → See [Pre-Deploy Checklist](../../pre-deploy-checklist.md) - -## Workflow - -| Step | Task | Command | -|------|------|---------| -| 1 | **[Pre-deploy checklist](../../pre-deploy-checklist.md)** | Confirm subscription/location with user | -| 2 | Deploy infrastructure | `az deployment sub create` | -| 3 | Deploy application | Service-specific commands | -| 4 | Verify | `az resource list` | -| 5 | **Report** | Present deployed endpoint URLs to the user — see [Verification](verify.md) | - -## Infrastructure Deployment - -### Subscription-Level (Recommended) - -```bash -az deployment sub create \ - --location eastus2 \ - --template-file ./infra/main.bicep \ - --parameters environmentName=dev -``` - -### Resource Group Level - -```bash -az group create --name rg-myapp-dev --location eastus2 - -az deployment group create \ - --resource-group rg-myapp-dev \ - --template-file ./infra/main.bicep \ - --parameters environmentName=dev -``` - -## Application Deployment - -### Container Apps - -```bash -az containerapp update \ - --name \ - --resource-group \ - --image .azurecr.io/myapp:latest -``` - -### App Service - -```bash -az webapp deploy \ - --name \ - --resource-group \ - --src-path ./publish.zip -``` - -### Azure Functions - -```bash -func azure functionapp publish -``` - -## References - -- [Verification steps](./verify.md) -- [Error handling](./errors.md) +# Azure CLI Deploy Recipe + +Deploy to Azure using Azure CLI. + +## Prerequisites + +- `az` CLI installed → Run `mcp_azure_mcp_extension_cli_install` with `cli-type: az` if needed +- `.azure/deployment-plan.md` exists with status `Validated` +- Bicep/ARM templates exist in `infra/` +- **Subscription and location confirmed** → See [Pre-Deploy Checklist](../../pre-deploy-checklist.md) + +## Workflow + +| Step | Task | Command | +|------|------|---------| +| 1 | **[Pre-deploy checklist](../../pre-deploy-checklist.md)** | Confirm subscription/location with user | +| 2 | Deploy infrastructure | `az deployment sub create` | +| 3 | Deploy application | Service-specific commands | +| 4 | Verify | `az resource list` | +| 5 | **Report** | Present deployed endpoint URLs to the user — see [Verification](verify.md) | + +## Infrastructure Deployment + +### Subscription-Level (Recommended) + +```bash +az deployment sub create \ + --location eastus2 \ + --template-file ./infra/main.bicep \ + --parameters environmentName=dev +``` + +### Resource Group Level + +```bash +az group create --name rg-myapp-dev --location eastus2 + +az deployment group create \ + --resource-group rg-myapp-dev \ + --template-file ./infra/main.bicep \ + --parameters environmentName=dev +``` + +## Application Deployment + +### Container Apps + +```bash +az containerapp update \ + --name \ + --resource-group \ + --image .azurecr.io/myapp:latest +``` + +### App Service + +```bash +az webapp deploy \ + --name \ + --resource-group \ + --src-path ./publish.zip +``` + +### Azure Functions + +```bash +func azure functionapp publish +``` + +## References + +- [Verification steps](./verify.md) +- [Error handling](./errors.md) diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azcli/errors.md b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azcli/errors.md index b2094771..f0927b19 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azcli/errors.md +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azcli/errors.md @@ -1,18 +1,18 @@ -# Azure CLI Errors - -| Error | Resolution | -|-------|------------| -| Not authenticated | `az login` | -| Subscription not found | `az account list` | -| Deployment failed | `az deployment sub show --name ` | -| Template error | `az deployment sub validate` | -| Permission denied | Verify RBAC roles | -| Quota exceeded | Request increase or change region | - -## Cleanup (DESTRUCTIVE) - -```bash -az group delete --name --yes -``` - -⚠️ Permanently deletes ALL resources in the group. +# Azure CLI Errors + +| Error | Resolution | +|-------|------------| +| Not authenticated | `az login` | +| Subscription not found | `az account list` | +| Deployment failed | `az deployment sub show --name ` | +| Template error | `az deployment sub validate` | +| Permission denied | Verify RBAC roles | +| Quota exceeded | Request increase or change region | + +## Cleanup (DESTRUCTIVE) + +```bash +az group delete --name --yes +``` + +⚠️ Permanently deletes ALL resources in the group. diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azcli/verify.md b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azcli/verify.md index 9caaf2b0..ba50c855 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azcli/verify.md +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azcli/verify.md @@ -1,69 +1,69 @@ -# Azure CLI Verification - -```bash -az resource list --resource-group --output table -``` - -## Health Check - -```bash -curl -s https:///health | jq . -``` - -## Container Apps - -```bash -az containerapp revision list \ - --name \ - --resource-group \ - --query "[].{name:name, active:properties.active}" \ - --output table -``` - -## App Service - -```bash -az webapp show \ - --name \ - --resource-group \ - --query "{state:state, hostNames:hostNames}" -``` - -## Report Results to User - -> ⛔ **MANDATORY** — You **MUST** present the deployed endpoint URLs to the user in your response. - -Extract endpoints using the appropriate command for the service type: - -```bash -# Container Apps -FQDN=$(az containerapp show --name --resource-group --query "properties.configuration.ingress.fqdn" -o tsv) -echo "https://$FQDN" - -# App Service -HOSTNAME=$(az webapp show --name --resource-group --query "defaultHostName" -o tsv) -echo "https://$HOSTNAME" - -# Static Web Apps -HOSTNAME=$(az staticwebapp show --name --resource-group --query "defaultHostname" -o tsv) -echo "https://$HOSTNAME" -``` - -**PowerShell:** -```powershell -# Container Apps -$Fqdn = az containerapp show --name --resource-group --query "properties.configuration.ingress.fqdn" -o tsv -Write-Output "https://$Fqdn" - -# App Service -$Hostname = az webapp show --name --resource-group --query "defaultHostName" -o tsv -Write-Output "https://$Hostname" - -# Static Web Apps -$Hostname = az staticwebapp show --name --resource-group --query "defaultHostname" -o tsv -Write-Output "https://$Hostname" -``` - -> ⚠️ **These commands return bare hostnames without a scheme.** Always prepend `https://` when presenting URLs to the user. For example, report `https://myapp.azurewebsites.net`, never `myapp.azurewebsites.net`. - -Present a summary including all service URLs as fully-qualified `https://` links. Do NOT end your response without including them. +# Azure CLI Verification + +```bash +az resource list --resource-group --output table +``` + +## Health Check + +```bash +curl -s https:///health | jq . +``` + +## Container Apps + +```bash +az containerapp revision list \ + --name \ + --resource-group \ + --query "[].{name:name, active:properties.active}" \ + --output table +``` + +## App Service + +```bash +az webapp show \ + --name \ + --resource-group \ + --query "{state:state, hostNames:hostNames}" +``` + +## Report Results to User + +> ⛔ **MANDATORY** — You **MUST** present the deployed endpoint URLs to the user in your response. + +Extract endpoints using the appropriate command for the service type: + +```bash +# Container Apps +FQDN=$(az containerapp show --name --resource-group --query "properties.configuration.ingress.fqdn" -o tsv) +echo "https://$FQDN" + +# App Service +HOSTNAME=$(az webapp show --name --resource-group --query "defaultHostName" -o tsv) +echo "https://$HOSTNAME" + +# Static Web Apps +HOSTNAME=$(az staticwebapp show --name --resource-group --query "defaultHostname" -o tsv) +echo "https://$HOSTNAME" +``` + +**PowerShell:** +```powershell +# Container Apps +$Fqdn = az containerapp show --name --resource-group --query "properties.configuration.ingress.fqdn" -o tsv +Write-Output "https://$Fqdn" + +# App Service +$Hostname = az webapp show --name --resource-group --query "defaultHostName" -o tsv +Write-Output "https://$Hostname" + +# Static Web Apps +$Hostname = az staticwebapp show --name --resource-group --query "defaultHostname" -o tsv +Write-Output "https://$Hostname" +``` + +> ⚠️ **These commands return bare hostnames without a scheme.** Always prepend `https://` when presenting URLs to the user. For example, report `https://myapp.azurewebsites.net`, never `myapp.azurewebsites.net`. + +Present a summary including all service URLs as fully-qualified `https://` links. Do NOT end your response without including them. diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azd/README.md b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azd/README.md index ef471b07..cddb02f9 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azd/README.md +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azd/README.md @@ -1,81 +1,81 @@ -# AZD Deploy Recipe - -Deploy to Azure using Azure Developer CLI (azd). - -> 💡 **Note:** azd supports both Bicep and Terraform as IaC providers. The deployment workflow is identical regardless of which you use. - -## Prerequisites - -- `azd` CLI installed → Run `mcp_azure_mcp_extension_cli_install` with `cli-type: azd` if needed -- `.azure/deployment-plan.md` exists with status `Validated` -- `azure.yaml` exists and validated -- Infrastructure files exist (Bicep: `infra/main.bicep`, Terraform: `infra/*.tf`) -- **AZD environment configured** → Done in azure-validate -- **Subscription and location confirmed** → See [Pre-deploy Checklist](../../pre-deploy-checklist.md) - -## Workflow - -| Step | Task | Command | -|------|------|---------| -| 1 | **Verify environment** | `azd env get-values` — Confirm AZURE_SUBSCRIPTION_ID and AZURE_LOCATION set | -| 2 | **Provision infrastructure** | `azd provision --no-prompt` | -| 3 | **RBAC health check** *(Container Apps + ACR only)* | After provisioning, verify `AcrPull` role has propagated before deploying — see [Pre-Deploy Checklist](../../pre-deploy-checklist.md#container-apps--acr--pre-deploy-rbac-health-check) | -| 4 | **Deploy application** | `azd deploy --no-prompt` | -| 5 | **Post-Deploy** | [Post-Deployment Steps](post-deployment.md) — If using SQL + managed identity | -| 6 | **Verify** | See [Verification](verify.md) | -| 7 | **Report** | Present deployed endpoint URLs to the user — see [Verification](verify.md) Step 3 | - -> ⚠️ **Important:** For Container Apps that use a managed identity to pull from ACR, always run `azd provision` and `azd deploy` as **separate steps** (not `azd up`) and complete the RBAC health check between them. This ensures the managed identity `AcrPull` role assignment has propagated before the Container App revision attempts to pull the image. - -> ⚠️ **Important:** For .NET Aspire projects or projects using azd "limited mode" (no explicit `infra/` folder), verify that `azd provision` populated all required environment variables. If `azd deploy` fails with errors about missing `AZURE_CONTAINER_REGISTRY_ENDPOINT`, `AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID`, or `MANAGED_IDENTITY_CLIENT_ID`, see [Error Handling](errors.md#missing-container-registry-variables) for the resolution. - -## Common Mistakes - -| ❌ Wrong | Why It Fails | -|----------|-------------| -| `azd up --location eastus2` | `--location` is not a valid flag for `azd up` | -| `azd up` without `azd env new` | Prompts for input, fails with `--no-prompt` | -| `mkdir .azure` then `azd env new --no-prompt` | Creates env folder structure incorrectly | -| Setting AZURE_LOCATION without checking RG | "Invalid resource group location" if RG exists elsewhere | -| Ignoring `azd-service-name` tag conflicts in same RG | "found '2' resources tagged with..." error | -| `language: html` or `language: static` | Not valid - use `language: js` with `dist: .` for static sites | - -## Deployment Commands - -> ⚠️ `azd up` takes 5-15 min. Run with output **streamed visibly to the user** — do NOT run silently or suppress output. The user must see provisioning progress in real time. - -### Full Deployment - -Provisions infrastructure AND deploys application: - -```bash -azd up --no-prompt -``` - -### Infrastructure Only - -```bash -azd provision --no-prompt -``` - -### Application Only - -Deploy code to existing infrastructure: - -```bash -azd deploy --no-prompt -``` - -### Single Service - -```bash -azd deploy api --no-prompt -``` - -## References - -- [Pre-deploy Checklist](../../pre-deploy-checklist.md) — **REQUIRED** -- [Post-Deployment Steps](post-deployment.md) — SQL + managed identity setup -- [Azure Functions Deployment](functions-deploy.md) -- [Verification](verify.md) -- [Error Handling](errors.md) +# AZD Deploy Recipe + +Deploy to Azure using Azure Developer CLI (azd). + +> 💡 **Note:** azd supports both Bicep and Terraform as IaC providers. The deployment workflow is identical regardless of which you use. + +## Prerequisites + +- `azd` CLI installed → Run `mcp_azure_mcp_extension_cli_install` with `cli-type: azd` if needed +- `.azure/deployment-plan.md` exists with status `Validated` +- `azure.yaml` exists and validated +- Infrastructure files exist (Bicep: `infra/main.bicep`, Terraform: `infra/*.tf`) +- **AZD environment configured** → Done in azure-validate +- **Subscription and location confirmed** → See [Pre-deploy Checklist](../../pre-deploy-checklist.md) + +## Workflow + +| Step | Task | Command | +|------|------|---------| +| 1 | **Verify environment** | `azd env get-values` — Confirm AZURE_SUBSCRIPTION_ID and AZURE_LOCATION set | +| 2 | **Provision infrastructure** | `azd provision --no-prompt` | +| 3 | **RBAC health check** *(Container Apps + ACR only)* | After provisioning, verify `AcrPull` role has propagated before deploying — see [Pre-Deploy Checklist](../../pre-deploy-checklist.md#container-apps--acr--pre-deploy-rbac-health-check) | +| 4 | **Deploy application** | `azd deploy --no-prompt` | +| 5 | **Post-Deploy** | [Post-Deployment Steps](post-deployment.md) — If using SQL + managed identity | +| 6 | **Verify** | See [Verification](verify.md) | +| 7 | **Report** | Present deployed endpoint URLs to the user — see [Verification](verify.md) Step 3 | + +> ⚠️ **Important:** For Container Apps that use a managed identity to pull from ACR, always run `azd provision` and `azd deploy` as **separate steps** (not `azd up`) and complete the RBAC health check between them. This ensures the managed identity `AcrPull` role assignment has propagated before the Container App revision attempts to pull the image. + +> ⚠️ **Important:** For .NET Aspire projects or projects using azd "limited mode" (no explicit `infra/` folder), verify that `azd provision` populated all required environment variables. If `azd deploy` fails with errors about missing `AZURE_CONTAINER_REGISTRY_ENDPOINT`, `AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID`, or `MANAGED_IDENTITY_CLIENT_ID`, see [Error Handling](errors.md#missing-container-registry-variables) for the resolution. + +## Common Mistakes + +| ❌ Wrong | Why It Fails | +|----------|-------------| +| `azd up --location eastus2` | `--location` is not a valid flag for `azd up` | +| `azd up` without `azd env new` | Prompts for input, fails with `--no-prompt` | +| `mkdir .azure` then `azd env new --no-prompt` | Creates env folder structure incorrectly | +| Setting AZURE_LOCATION without checking RG | "Invalid resource group location" if RG exists elsewhere | +| Ignoring `azd-service-name` tag conflicts in same RG | "found '2' resources tagged with..." error | +| `language: html` or `language: static` | Not valid - use `language: js` with `dist: .` for static sites | + +## Deployment Commands + +> ⚠️ `azd up` takes 5-15 min. Run with output **streamed visibly to the user** — do NOT run silently or suppress output. The user must see provisioning progress in real time. + +### Full Deployment + +Provisions infrastructure AND deploys application: + +```bash +azd up --no-prompt +``` + +### Infrastructure Only + +```bash +azd provision --no-prompt +``` + +### Application Only + +Deploy code to existing infrastructure: + +```bash +azd deploy --no-prompt +``` + +### Single Service + +```bash +azd deploy api --no-prompt +``` + +## References + +- [Pre-deploy Checklist](../../pre-deploy-checklist.md) — **REQUIRED** +- [Post-Deployment Steps](post-deployment.md) — SQL + managed identity setup +- [Azure Functions Deployment](functions-deploy.md) +- [Verification](verify.md) +- [Error Handling](errors.md) diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azd/errors.md b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azd/errors.md index 71bb207a..d933d04b 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azd/errors.md +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azd/errors.md @@ -1,313 +1,313 @@ -# AZD Errors - -## Deployment Runtime Errors - -These errors occur **during** `azd up` execution: - -| Error | Cause | Resolution | -|-------|-------|------------| -| `unknown flag: --location` | `azd up` doesn't accept `--location` | Use `azd env set AZURE_LOCATION ` before `azd up` | -| Provision failed | Bicep template errors | Check detailed error in output | -| Deploy failed | Build or Docker errors | Check build logs | -| Package failed | Missing Dockerfile or deps | Verify Dockerfile exists and dependencies | -| Quota exceeded | Subscription limits | Request increase or change region | -| `PrincipalId '...' has type 'ServicePrincipal', which is different from specified PrincipalType 'User'` | Base template RBAC assigns roles with `principalType: 'User'` but deploying identity is a service principal (CI/CD) | Set `allowUserIdentityPrincipal: false` in the `storageEndpointConfig` variable in `infra/main.bicep`. Do NOT try clearing `AZURE_PRINCIPAL_ID` — azd repopulates it. See [Principal Type Mismatch](#principal-type-mismatch). | -| `ImagePullBackOff` or `azd up` hangs during provision for Container Apps | Container App references an image that doesn't exist in ACR yet | See [Container Apps Bootstrap Problem](#container-apps-bootstrap-problem) | -| `unauthorized: authentication required` on `docker push` to ACR | ACR auth token expired or scoped incorrectly | See [ACR Authentication Failures](#acr-authentication-failures) | -| `could not determine container registry endpoint` | Missing `AZURE_CONTAINER_REGISTRY_ENDPOINT` | See [Missing Container Registry Variables](#missing-container-registry-variables) | -| `map has no entry for key "AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID"` | Missing managed identity env vars | See [Missing Container Registry Variables](#missing-container-registry-variables) | -| `map has no entry for key "MANAGED_IDENTITY_CLIENT_ID"` | Missing managed identity client ID | See [Missing Container Registry Variables](#missing-container-registry-variables) | -| `Operation expired` / revision creation timeout (900s) | RBAC propagation delay — Container App's managed identity doesn't have `AcrPull` on ACR yet | See [Container App Revision Timeout](#container-app-revision-timeout) | -| `found '2' resources tagged with 'azd-service-name: '` | Previous deployment left duplicate-tagged resources in same RG | **Preferred**: Create fresh env with `azd env new --no-prompt`, set subscription/location, redeploy. **Alternative**: Delete conflicting resources (requires `ask_user`). | -| Literal `{{ .Env.* }}` in Terraform errors | azd does not interpolate template variables in `.tfvars.json` | See [Unresolved Terraform Template Variables](#unresolved-terraform-template-variables) | - -> ℹ️ **Pre-flight validation**: Run `azure-validate` before deployment to catch configuration errors early. See [Pre-Deploy Checklist](../../pre-deploy-checklist.md). - -## Container App Revision Timeout - -**Symptom:** `azd up` provisions infrastructure successfully but the Container App revision creation times out after ~900 seconds. The Container App enters a `Failed` provisioning state with no active revision. The `azd` output shows `Operation expired` or `The operation did not complete within the permitted time`. - -**Cause:** Azure RBAC propagation delay. When `azd up` runs both `azd provision` and `azd deploy` in a single step: -1. Bicep creates the Container App with a system-assigned managed identity and a public placeholder image -2. Bicep creates an `AcrPull` role assignment for that identity on ACR in a separate module using the two-phase deployment pattern -3. `azd deploy` immediately pushes the real image and creates a new Container App revision -4. The revision tries to pull the image from ACR, but the `AcrPull` role assignment hasn't propagated yet (can take 1–5 minutes) -5. The image pull fails repeatedly until the 900-second timeout is reached - -**Solution:** - -1. **Verify the Container App state:** -```bash -az containerapp show --name --resource-group \ - --query "{provisioningState:properties.provisioningState, latestRevision:properties.latestRevisionName}" -o json -``` - -2. **Confirm the AcrPull role exists (it may have propagated by now):** -```bash -PRINCIPAL_ID=$(az containerapp identity show --name --resource-group --query principalId -o tsv) -az role assignment list --scope $(az acr show --name --resource-group --query id -o tsv) \ - --assignee-object-id "$PRINCIPAL_ID" --query "[].roleDefinitionName" -o tsv -``` - -**PowerShell:** -```powershell -$PrincipalId = az containerapp identity show --name --resource-group --query principalId -o tsv -$AcrScope = az acr show --name --resource-group --query id -o tsv -az role assignment list --scope $AcrScope --assignee-object-id $PrincipalId --query "[].roleDefinitionName" -o tsv -``` - -3. **If AcrPull is missing, assign it:** -```bash -az role assignment create \ - --assignee-object-id "$PRINCIPAL_ID" \ - --assignee-principal-type ServicePrincipal \ - --role AcrPull \ - --scope $(az acr show --name --resource-group --query id -o tsv) -``` - -**PowerShell:** -```powershell -$AcrScope = az acr show --name --resource-group --query id -o tsv -az role assignment create ` - --assignee-object-id $PrincipalId ` - --assignee-principal-type ServicePrincipal ` - --role AcrPull ` - --scope $AcrScope -``` - -4. **Wait for propagation, then redeploy:** -```bash -azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT $(az acr show --name --resource-group --query loginServer -o tsv) - -for attempt in 1 2 3 4 5; do - echo "Waiting for RBAC propagation (attempt $attempt/5)..." - sleep 60 - - if azd deploy --no-prompt; then - echo "Deployment succeeded after RBAC propagation." - break - fi - - if [ "$attempt" -eq 5 ]; then - echo "Deployment still failing after 5 minutes. Re-check AcrPull assignment and Container App revision status." - exit 1 - fi -done -``` - -**PowerShell:** -```powershell -azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT (az acr show --name --resource-group --query loginServer -o tsv) - -for ($attempt = 1; $attempt -le 5; $attempt++) { - Write-Output "Waiting for RBAC propagation (attempt $attempt/5)..." - Start-Sleep -Seconds 60 - - azd deploy --no-prompt - if ($LASTEXITCODE -eq 0) { - Write-Output "Deployment succeeded after RBAC propagation." - break - } - - if ($attempt -eq 5) { - Write-Output "Deployment still failing after 5 minutes. Re-check AcrPull assignment and Container App revision status." - exit 1 - } -} -``` - -> 💡 **Prevention:** To avoid this in future deployments, ensure the Bicep template includes the `AcrPull` role assignment with `principalType: 'ServicePrincipal'`, and consider using `azd provision` + `azd deploy` as separate steps instead of `azd up` to allow RBAC propagation time between infrastructure creation and app deployment. - -## Container Apps Bootstrap Problem - -**Symptom:** `azd up` hangs or fails during provisioning with `ImagePullBackOff`, or the Container App cannot start because the referenced image doesn't exist in ACR yet. - -**Cause:** The Bicep template creates the Container App referencing an ACR image, but that image doesn't exist until `azd deploy` builds and pushes it. This chicken-and-egg problem blocks provisioning. - -**Solution — use two-phase deployment:** - -```bash -# Phase 1: Provision infrastructure (Container App uses placeholder image) -azd provision --no-prompt - -# Phase 2: Build, push, and update Container App with real image -azd deploy --no-prompt -``` - -> ⚠️ This requires the Bicep template to use a placeholder image parameter (e.g., `mcr.microsoft.com/azuredocs/containerapps-helloworld:latest`) so provisioning succeeds without the app image. If the Bicep hardcodes the ACR image reference, update it to accept a `containerImageName` parameter with a placeholder default before provisioning. - -> ⚠️ Do **NOT** repeatedly poll a hanging `azd up` — if there is no provisioning progress or you continue to see `ImagePullBackOff` events for several minutes during a Container Apps deployment, stop it and switch to the two-phase approach above. - -## ACR Authentication Failures - -**Symptom:** `docker push` fails with `unauthorized: authentication required` even after `az acr login` succeeds. - -**Solution — try these methods in order:** - -```bash -# Method 1: AAD-based login (preferred) -az acr login --name -docker push .azurecr.io/: - -# Method 2: Admin credentials (fallback) -ACR_USER=$(az acr credential show --name --query username -o tsv) -ACR_PASS=$(az acr credential show --name --query "passwords[0].value" -o tsv) -docker login .azurecr.io -u "$ACR_USER" -p "$ACR_PASS" -docker push .azurecr.io/: -``` - -**PowerShell (Method 2):** -```powershell -$AcrUser = az acr credential show --name --query username -o tsv -$AcrPass = az acr credential show --name --query "passwords[0].value" -o tsv -docker login .azurecr.io -u $AcrUser -p $AcrPass -docker push .azurecr.io/: -``` - -> 💡 **Tip:** Prefer `azd deploy` over manual `docker push` — azd handles ACR authentication automatically. - -## Missing Container Registry Variables - -**Symptom:** Errors during `azd deploy` about missing container registry or managed identity environment variables: - -``` -ERROR: could not determine container registry endpoint, ensure 'registry' has been set in the docker options or 'AZURE_CONTAINER_REGISTRY_ENDPOINT' environment variable has been set -``` - -Or: - -``` -ERROR: failed executing template file: template: manifest template:6:14: executing "manifest template" at <.Env.AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID>: map has no entry for key "AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID" -``` - -Or: - -``` -ERROR: failed executing template file: template: manifest template:39:26: executing "manifest template" at <.Env.MANAGED_IDENTITY_CLIENT_ID>: map has no entry for key "MANAGED_IDENTITY_CLIENT_ID" -``` - -**Cause:** This typically occurs with .NET Aspire projects using azd "limited mode" (in-memory infrastructure generation without explicit `infra/` folder). The `azd provision` command creates the Azure Container Registry and Managed Identity resources but doesn't automatically populate the environment variables that `azd deploy` needs to reference them. - -> ⚠️ **Prevention is Better:** For .NET Aspire projects, this issue should be addressed PROACTIVELY before deployment by setting up environment variables after `azd init` but before `azd up`. This avoids deployment failures entirely. - -**Solution:** - -After `azd provision` succeeds, manually set the missing environment variables by querying the provisioned resources: - -```bash -# Get the resource group name (typically rg-{environment-name}) -azd env get-values - -# Set container registry endpoint -azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT $(az acr list --resource-group --query "[0].loginServer" -o tsv) - -# Set managed identity resource ID -azd env set AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID $(az identity list --resource-group --query "[0].id" -o tsv) - -# Set managed identity client ID -azd env set MANAGED_IDENTITY_CLIENT_ID $(az identity list --resource-group --query "[0].clientId" -o tsv) -``` - -**PowerShell:** -```powershell -# Set container registry endpoint -azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT (az acr list --resource-group --query "[0].loginServer" -o tsv) - -# Set managed identity resource ID -azd env set AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID (az identity list --resource-group --query "[0].id" -o tsv) - -# Set managed identity client ID -azd env set MANAGED_IDENTITY_CLIENT_ID (az identity list --resource-group --query "[0].clientId" -o tsv) -``` - -After setting these variables, retry the deployment: -```bash -azd deploy --no-prompt -``` - -> 💡 **Tip:** This issue is specific to Aspire limited mode. Manually setting these environment variables after `azd provision` is the recommended workaround. - -## Unresolved Terraform Template Variables - -**Symptom:** Terraform receives literal Go-style template strings instead of resolved values during `azd provision`: - -``` -Error: Invalid value for variable "environment_name" - The value "{{ .Env.AZURE_ENV_NAME }}" is not valid. -``` - -Or Terraform silently uses the literal string, causing resource naming failures, state conflicts, and cascading errors that lead to deployment timeouts. - -**Cause:** azd reads `infra/main.tfvars.json`, substitutes `${VAR}` references using its built-in envsubst, and passes the resolved file to Terraform via `-var-file=`. Go-style `{{ .Env.* }}` variables are only processed in `azure.yaml` and service manifests — they are **NOT** interpolated in `.tfvars.json` or any Terraform variable files. If `azure-prepare` generated a `main.tfvars.json` with Go-style template expressions, those literal strings are passed to Terraform. - -**Solution:** - -1. **Fix the syntax** in `infra/main.tfvars.json` — replace Go-style `{{ .Env.* }}` with `${VAR}`: - ```json - { - "environment_name": "${AZURE_ENV_NAME}", - "location": "${AZURE_LOCATION}", - "subscription_id": "${AZURE_SUBSCRIPTION_ID}" - } - ``` - -2. **Or use `TF_VAR_*` environment variables** if you don't have `main.tfvars.json`: - ```bash - azd env set TF_VAR_environment_name "$(azd env get-value AZURE_ENV_NAME)" - azd env set TF_VAR_location "$(azd env get-value AZURE_LOCATION)" - azd env set TF_VAR_subscription_id "$(azd env get-value AZURE_SUBSCRIPTION_ID)" - ``` - -3. **Ensure `variables.tf`** declares all required variables: - ```hcl - variable "environment_name" { type = string } - variable "location" { type = string } - ``` - -4. **Re-run deployment:** - ```bash - azd up --no-prompt - ``` - -> ⚠️ **Prevention:** This issue should be caught by `azure-validate` Step 10 (Template Variable Resolution Check) before deployment. If you encounter it, re-run validation after fixing. - -## Retry - -After fixing the issue: -```bash -azd up --no-prompt -``` - -## Principal Type Mismatch - -**Symptom:** `azd up` fails during provisioning with: - -``` -PrincipalId '...' has type 'ServicePrincipal', which is different from specified PrincipalType 'User' -``` - -**Cause:** Many AZD templates (e.g., `functions-quickstart-python-http-azd`) include RBAC role assignments for the deploying user with hardcoded `principalType: 'User'`. This is controlled by an `allowUserIdentityPrincipal` flag in `main.bicep`'s `storageEndpointConfig` variable. When deploying from CI/CD with a service principal, `azd` sets `AZURE_PRINCIPAL_ID` to that service principal's object ID, but the Bicep still tries to create a role assignment with `principalType: 'User'`, causing ARM to reject it. - -**Solution:** - -In `infra/main.bicep`, find the `storageEndpointConfig` variable and set `allowUserIdentityPrincipal` to `false`: - -```bicep -var storageEndpointConfig = { - enableBlob: true - enableQueue: false - enableTable: false - enableFiles: false - allowUserIdentityPrincipal: false // Set to false for service principal deployments -} -``` - -> ⚠️ **Do NOT** try to fix this by running `azd env set AZURE_PRINCIPAL_ID ""`. The `azd` CLI repopulates this value from the current auth context, so clearing it has no effect. - -## Cleanup (DESTRUCTIVE) - -```bash -azd down --force --purge -``` - -⚠️ Permanently deletes ALL resources including databases and Key Vaults. +# AZD Errors + +## Deployment Runtime Errors + +These errors occur **during** `azd up` execution: + +| Error | Cause | Resolution | +|-------|-------|------------| +| `unknown flag: --location` | `azd up` doesn't accept `--location` | Use `azd env set AZURE_LOCATION ` before `azd up` | +| Provision failed | Bicep template errors | Check detailed error in output | +| Deploy failed | Build or Docker errors | Check build logs | +| Package failed | Missing Dockerfile or deps | Verify Dockerfile exists and dependencies | +| Quota exceeded | Subscription limits | Request increase or change region | +| `PrincipalId '...' has type 'ServicePrincipal', which is different from specified PrincipalType 'User'` | Base template RBAC assigns roles with `principalType: 'User'` but deploying identity is a service principal (CI/CD) | Set `allowUserIdentityPrincipal: false` in the `storageEndpointConfig` variable in `infra/main.bicep`. Do NOT try clearing `AZURE_PRINCIPAL_ID` — azd repopulates it. See [Principal Type Mismatch](#principal-type-mismatch). | +| `ImagePullBackOff` or `azd up` hangs during provision for Container Apps | Container App references an image that doesn't exist in ACR yet | See [Container Apps Bootstrap Problem](#container-apps-bootstrap-problem) | +| `unauthorized: authentication required` on `docker push` to ACR | ACR auth token expired or scoped incorrectly | See [ACR Authentication Failures](#acr-authentication-failures) | +| `could not determine container registry endpoint` | Missing `AZURE_CONTAINER_REGISTRY_ENDPOINT` | See [Missing Container Registry Variables](#missing-container-registry-variables) | +| `map has no entry for key "AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID"` | Missing managed identity env vars | See [Missing Container Registry Variables](#missing-container-registry-variables) | +| `map has no entry for key "MANAGED_IDENTITY_CLIENT_ID"` | Missing managed identity client ID | See [Missing Container Registry Variables](#missing-container-registry-variables) | +| `Operation expired` / revision creation timeout (900s) | RBAC propagation delay — Container App's managed identity doesn't have `AcrPull` on ACR yet | See [Container App Revision Timeout](#container-app-revision-timeout) | +| `found '2' resources tagged with 'azd-service-name: '` | Previous deployment left duplicate-tagged resources in same RG | **Preferred**: Create fresh env with `azd env new --no-prompt`, set subscription/location, redeploy. **Alternative**: Delete conflicting resources (requires `ask_user`). | +| Literal `{{ .Env.* }}` in Terraform errors | azd does not interpolate template variables in `.tfvars.json` | See [Unresolved Terraform Template Variables](#unresolved-terraform-template-variables) | + +> ℹ️ **Pre-flight validation**: Run `azure-validate` before deployment to catch configuration errors early. See [Pre-Deploy Checklist](../../pre-deploy-checklist.md). + +## Container App Revision Timeout + +**Symptom:** `azd up` provisions infrastructure successfully but the Container App revision creation times out after ~900 seconds. The Container App enters a `Failed` provisioning state with no active revision. The `azd` output shows `Operation expired` or `The operation did not complete within the permitted time`. + +**Cause:** Azure RBAC propagation delay. When `azd up` runs both `azd provision` and `azd deploy` in a single step: +1. Bicep creates the Container App with a system-assigned managed identity and a public placeholder image +2. Bicep creates an `AcrPull` role assignment for that identity on ACR in a separate module using the two-phase deployment pattern +3. `azd deploy` immediately pushes the real image and creates a new Container App revision +4. The revision tries to pull the image from ACR, but the `AcrPull` role assignment hasn't propagated yet (can take 1–5 minutes) +5. The image pull fails repeatedly until the 900-second timeout is reached + +**Solution:** + +1. **Verify the Container App state:** +```bash +az containerapp show --name --resource-group \ + --query "{provisioningState:properties.provisioningState, latestRevision:properties.latestRevisionName}" -o json +``` + +2. **Confirm the AcrPull role exists (it may have propagated by now):** +```bash +PRINCIPAL_ID=$(az containerapp identity show --name --resource-group --query principalId -o tsv) +az role assignment list --scope $(az acr show --name --resource-group --query id -o tsv) \ + --assignee-object-id "$PRINCIPAL_ID" --query "[].roleDefinitionName" -o tsv +``` + +**PowerShell:** +```powershell +$PrincipalId = az containerapp identity show --name --resource-group --query principalId -o tsv +$AcrScope = az acr show --name --resource-group --query id -o tsv +az role assignment list --scope $AcrScope --assignee-object-id $PrincipalId --query "[].roleDefinitionName" -o tsv +``` + +3. **If AcrPull is missing, assign it:** +```bash +az role assignment create \ + --assignee-object-id "$PRINCIPAL_ID" \ + --assignee-principal-type ServicePrincipal \ + --role AcrPull \ + --scope $(az acr show --name --resource-group --query id -o tsv) +``` + +**PowerShell:** +```powershell +$AcrScope = az acr show --name --resource-group --query id -o tsv +az role assignment create ` + --assignee-object-id $PrincipalId ` + --assignee-principal-type ServicePrincipal ` + --role AcrPull ` + --scope $AcrScope +``` + +4. **Wait for propagation, then redeploy:** +```bash +azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT $(az acr show --name --resource-group --query loginServer -o tsv) + +for attempt in 1 2 3 4 5; do + echo "Waiting for RBAC propagation (attempt $attempt/5)..." + sleep 60 + + if azd deploy --no-prompt; then + echo "Deployment succeeded after RBAC propagation." + break + fi + + if [ "$attempt" -eq 5 ]; then + echo "Deployment still failing after 5 minutes. Re-check AcrPull assignment and Container App revision status." + exit 1 + fi +done +``` + +**PowerShell:** +```powershell +azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT (az acr show --name --resource-group --query loginServer -o tsv) + +for ($attempt = 1; $attempt -le 5; $attempt++) { + Write-Output "Waiting for RBAC propagation (attempt $attempt/5)..." + Start-Sleep -Seconds 60 + + azd deploy --no-prompt + if ($LASTEXITCODE -eq 0) { + Write-Output "Deployment succeeded after RBAC propagation." + break + } + + if ($attempt -eq 5) { + Write-Output "Deployment still failing after 5 minutes. Re-check AcrPull assignment and Container App revision status." + exit 1 + } +} +``` + +> 💡 **Prevention:** To avoid this in future deployments, ensure the Bicep template includes the `AcrPull` role assignment with `principalType: 'ServicePrincipal'`, and consider using `azd provision` + `azd deploy` as separate steps instead of `azd up` to allow RBAC propagation time between infrastructure creation and app deployment. + +## Container Apps Bootstrap Problem + +**Symptom:** `azd up` hangs or fails during provisioning with `ImagePullBackOff`, or the Container App cannot start because the referenced image doesn't exist in ACR yet. + +**Cause:** The Bicep template creates the Container App referencing an ACR image, but that image doesn't exist until `azd deploy` builds and pushes it. This chicken-and-egg problem blocks provisioning. + +**Solution — use two-phase deployment:** + +```bash +# Phase 1: Provision infrastructure (Container App uses placeholder image) +azd provision --no-prompt + +# Phase 2: Build, push, and update Container App with real image +azd deploy --no-prompt +``` + +> ⚠️ This requires the Bicep template to use a placeholder image parameter (e.g., `mcr.microsoft.com/azuredocs/containerapps-helloworld:latest`) so provisioning succeeds without the app image. If the Bicep hardcodes the ACR image reference, update it to accept a `containerImageName` parameter with a placeholder default before provisioning. + +> ⚠️ Do **NOT** repeatedly poll a hanging `azd up` — if there is no provisioning progress or you continue to see `ImagePullBackOff` events for several minutes during a Container Apps deployment, stop it and switch to the two-phase approach above. + +## ACR Authentication Failures + +**Symptom:** `docker push` fails with `unauthorized: authentication required` even after `az acr login` succeeds. + +**Solution — try these methods in order:** + +```bash +# Method 1: AAD-based login (preferred) +az acr login --name +docker push .azurecr.io/: + +# Method 2: Admin credentials (fallback) +ACR_USER=$(az acr credential show --name --query username -o tsv) +ACR_PASS=$(az acr credential show --name --query "passwords[0].value" -o tsv) +docker login .azurecr.io -u "$ACR_USER" -p "$ACR_PASS" +docker push .azurecr.io/: +``` + +**PowerShell (Method 2):** +```powershell +$AcrUser = az acr credential show --name --query username -o tsv +$AcrPass = az acr credential show --name --query "passwords[0].value" -o tsv +docker login .azurecr.io -u $AcrUser -p $AcrPass +docker push .azurecr.io/: +``` + +> 💡 **Tip:** Prefer `azd deploy` over manual `docker push` — azd handles ACR authentication automatically. + +## Missing Container Registry Variables + +**Symptom:** Errors during `azd deploy` about missing container registry or managed identity environment variables: + +``` +ERROR: could not determine container registry endpoint, ensure 'registry' has been set in the docker options or 'AZURE_CONTAINER_REGISTRY_ENDPOINT' environment variable has been set +``` + +Or: + +``` +ERROR: failed executing template file: template: manifest template:6:14: executing "manifest template" at <.Env.AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID>: map has no entry for key "AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID" +``` + +Or: + +``` +ERROR: failed executing template file: template: manifest template:39:26: executing "manifest template" at <.Env.MANAGED_IDENTITY_CLIENT_ID>: map has no entry for key "MANAGED_IDENTITY_CLIENT_ID" +``` + +**Cause:** This typically occurs with .NET Aspire projects using azd "limited mode" (in-memory infrastructure generation without explicit `infra/` folder). The `azd provision` command creates the Azure Container Registry and Managed Identity resources but doesn't automatically populate the environment variables that `azd deploy` needs to reference them. + +> ⚠️ **Prevention is Better:** For .NET Aspire projects, this issue should be addressed PROACTIVELY before deployment by setting up environment variables after `azd init` but before `azd up`. This avoids deployment failures entirely. + +**Solution:** + +After `azd provision` succeeds, manually set the missing environment variables by querying the provisioned resources: + +```bash +# Get the resource group name (typically rg-{environment-name}) +azd env get-values + +# Set container registry endpoint +azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT $(az acr list --resource-group --query "[0].loginServer" -o tsv) + +# Set managed identity resource ID +azd env set AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID $(az identity list --resource-group --query "[0].id" -o tsv) + +# Set managed identity client ID +azd env set MANAGED_IDENTITY_CLIENT_ID $(az identity list --resource-group --query "[0].clientId" -o tsv) +``` + +**PowerShell:** +```powershell +# Set container registry endpoint +azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT (az acr list --resource-group --query "[0].loginServer" -o tsv) + +# Set managed identity resource ID +azd env set AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID (az identity list --resource-group --query "[0].id" -o tsv) + +# Set managed identity client ID +azd env set MANAGED_IDENTITY_CLIENT_ID (az identity list --resource-group --query "[0].clientId" -o tsv) +``` + +After setting these variables, retry the deployment: +```bash +azd deploy --no-prompt +``` + +> 💡 **Tip:** This issue is specific to Aspire limited mode. Manually setting these environment variables after `azd provision` is the recommended workaround. + +## Unresolved Terraform Template Variables + +**Symptom:** Terraform receives literal Go-style template strings instead of resolved values during `azd provision`: + +``` +Error: Invalid value for variable "environment_name" + The value "{{ .Env.AZURE_ENV_NAME }}" is not valid. +``` + +Or Terraform silently uses the literal string, causing resource naming failures, state conflicts, and cascading errors that lead to deployment timeouts. + +**Cause:** azd reads `infra/main.tfvars.json`, substitutes `${VAR}` references using its built-in envsubst, and passes the resolved file to Terraform via `-var-file=`. Go-style `{{ .Env.* }}` variables are only processed in `azure.yaml` and service manifests — they are **NOT** interpolated in `.tfvars.json` or any Terraform variable files. If `azure-prepare` generated a `main.tfvars.json` with Go-style template expressions, those literal strings are passed to Terraform. + +**Solution:** + +1. **Fix the syntax** in `infra/main.tfvars.json` — replace Go-style `{{ .Env.* }}` with `${VAR}`: + ```json + { + "environment_name": "${AZURE_ENV_NAME}", + "location": "${AZURE_LOCATION}", + "subscription_id": "${AZURE_SUBSCRIPTION_ID}" + } + ``` + +2. **Or use `TF_VAR_*` environment variables** if you don't have `main.tfvars.json`: + ```bash + azd env set TF_VAR_environment_name "$(azd env get-value AZURE_ENV_NAME)" + azd env set TF_VAR_location "$(azd env get-value AZURE_LOCATION)" + azd env set TF_VAR_subscription_id "$(azd env get-value AZURE_SUBSCRIPTION_ID)" + ``` + +3. **Ensure `variables.tf`** declares all required variables: + ```hcl + variable "environment_name" { type = string } + variable "location" { type = string } + ``` + +4. **Re-run deployment:** + ```bash + azd up --no-prompt + ``` + +> ⚠️ **Prevention:** This issue should be caught by `azure-validate` Step 10 (Template Variable Resolution Check) before deployment. If you encounter it, re-run validation after fixing. + +## Retry + +After fixing the issue: +```bash +azd up --no-prompt +``` + +## Principal Type Mismatch + +**Symptom:** `azd up` fails during provisioning with: + +``` +PrincipalId '...' has type 'ServicePrincipal', which is different from specified PrincipalType 'User' +``` + +**Cause:** Many AZD templates (e.g., `functions-quickstart-python-http-azd`) include RBAC role assignments for the deploying user with hardcoded `principalType: 'User'`. This is controlled by an `allowUserIdentityPrincipal` flag in `main.bicep`'s `storageEndpointConfig` variable. When deploying from CI/CD with a service principal, `azd` sets `AZURE_PRINCIPAL_ID` to that service principal's object ID, but the Bicep still tries to create a role assignment with `principalType: 'User'`, causing ARM to reject it. + +**Solution:** + +In `infra/main.bicep`, find the `storageEndpointConfig` variable and set `allowUserIdentityPrincipal` to `false`: + +```bicep +var storageEndpointConfig = { + enableBlob: true + enableQueue: false + enableTable: false + enableFiles: false + allowUserIdentityPrincipal: false // Set to false for service principal deployments +} +``` + +> ⚠️ **Do NOT** try to fix this by running `azd env set AZURE_PRINCIPAL_ID ""`. The `azd` CLI repopulates this value from the current auth context, so clearing it has no effect. + +## Cleanup (DESTRUCTIVE) + +```bash +azd down --force --purge +``` + +⚠️ Permanently deletes ALL resources including databases and Key Vaults. diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azd/verify.md b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azd/verify.md index 85b356d0..a20a6b2b 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azd/verify.md +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/azd/verify.md @@ -1,169 +1,169 @@ -# AZD Verification - -Verify deployment success and application health. - -## Step 1: Verify Resources - -```bash -azd show -``` - -Expected output: -```text -Showing deployed resources: - Resource Group: rg-myapp-dev - Services: - api - Endpoint: https://api-xxxx.azurecontainerapps.io -``` - -## Step 2: Health Check - -```bash -# Get endpoint -ENDPOINT=$(azd env get-values | grep -E "SERVICE_.*_URI|.*_ENDPOINT" | head -1 | cut -d'=' -f2) - -# Test endpoint -curl -f "$ENDPOINT/health" || curl -f "$ENDPOINT" -``` - -**PowerShell:** -```powershell -# Get endpoint -$Endpoint = azd env get-values | Select-String -Pattern 'SERVICE_.*_URI|.*_ENDPOINT' | - Select-Object -First 1 | ForEach-Object { ($_ -split '=', 2)[1] } - -# Test endpoint -try { Invoke-WebRequest "$Endpoint/health" } catch { Invoke-WebRequest $Endpoint } -``` - -Expected: HTTP 200 response. - -## Step 3: Report Results to User - -> ⛔ **MANDATORY** — You **MUST** present the deployed endpoint URLs to the user in your response. A deployment is not considered complete until the user has received the URLs. - -> ⛔ **ALWAYS run `azd show`** after deployment to extract endpoint URLs. Do NOT rely on the `azd deploy` output alone — it is frequently truncated before endpoint URLs appear. - -```bash -azd show -``` - -Parse the `Endpoint:` lines from the `azd show` output to collect all deployed service URLs. - -**Present a summary to the user that includes:** - -| Item | Source | -|------|--------| -| Deployed service endpoint(s) | `Endpoint:` lines from `azd show` | -| Aspire Dashboard URL (if applicable) | `Aspire Dashboard:` line from `azd show` | -| Azure Portal deployment link (if available) | Portal URL from provisioning output | - -Example response format: - -```text -✅ Deployment succeeded! - -| Service | Endpoint | -|---------|----------| -| apiservice | https://apiservice.xxx.azurecontainerapps.io | - -Aspire Dashboard: https://aspire-dashboard.xxx.azurecontainerapps.io -``` - -> ⚠️ **Always use fully-qualified URLs with the `https://` scheme.** If a command returns a bare hostname (e.g. `myapp.azurestaticapps.net`), prepend `https://` before presenting it to the user. - -## Step 4: Post-Deployment Verification (if applicable) - -For deployments with Azure SQL Database and managed identity: - -### Verify SQL Access - -> ⚠️ **Warning:** `az sql db query` requires the `rdbms-connect` extension: `az extension add --name rdbms-connect --yes` - -```bash -# Load environment variables -eval $(azd env get-values) - -# Check managed identity user exists in database -az sql db query \ - --server "$SQL_SERVER" \ - --database "$SQL_DATABASE" \ - --resource-group "$AZURE_RESOURCE_GROUP" \ - --auth-mode ActiveDirectoryDefault \ - --queries "SELECT name, type_desc FROM sys.database_principals WHERE type = 'E'" -``` - -**PowerShell:** -```powershell -# Load environment variables -azd env get-values | ForEach-Object { - $name, $value = $_.Split('=', 2) - Set-Item "env:$name" $value -} - -# Check managed identity user exists in database -az sql db query ` - --server $env:SQL_SERVER ` - --database $env:SQL_DATABASE ` - --resource-group $env:AZURE_RESOURCE_GROUP ` - --auth-mode ActiveDirectoryDefault ` - --queries "SELECT name, type_desc FROM sys.database_principals WHERE type = 'E'" -``` - -**Expected:** Should list the App Service or Container App managed identity. - -### Verify Database Schema - -For EF Core applications: - -```bash -# Check tables exist -az sql db query \ - --server "$SQL_SERVER" \ - --database "$SQL_DATABASE" \ - --resource-group "$AZURE_RESOURCE_GROUP" \ - --auth-mode ActiveDirectoryDefault \ - --queries "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE'" -``` - -**PowerShell:** -```powershell -az sql db query ` - --server $env:SQL_SERVER ` - --database $env:SQL_DATABASE ` - --resource-group $env:AZURE_RESOURCE_GROUP ` - --auth-mode ActiveDirectoryDefault ` - --queries "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE'" -``` - -**Expected:** Should list application tables (not just `__EFMigrationsHistory`). - -### Check Application Logs - -```bash -# For App Service -az webapp log tail --name --resource-group - -# For Container Apps -az containerapp logs show --name --resource-group --follow -``` - -**Look for:** -- ✅ No SQL authentication errors -- ✅ Successful database connection -- ✅ Application started successfully - -## Common Issues - -| Symptom | Cause | Fix | -|---------|-------|-----| -| HTTP 500 on startup | SQL authentication failure | See [sql-managed-identity.md](sql-managed-identity.md) | -| "Invalid object name" errors | Migrations not applied | See [ef-migrations.md](ef-migrations.md) | -| Endpoint not accessible | Service still starting | Wait 1-2 minutes, retry | -| Health check fails | Application error | Check logs with `az webapp log tail` | - -## References - -- [Post-Deployment Steps](post-deployment.md) -- [SQL Managed Identity Access](sql-managed-identity.md) -- [EF Core Migrations](ef-migrations.md) +# AZD Verification + +Verify deployment success and application health. + +## Step 1: Verify Resources + +```bash +azd show +``` + +Expected output: +```text +Showing deployed resources: + Resource Group: rg-myapp-dev + Services: + api - Endpoint: https://api-xxxx.azurecontainerapps.io +``` + +## Step 2: Health Check + +```bash +# Get endpoint +ENDPOINT=$(azd env get-values | grep -E "SERVICE_.*_URI|.*_ENDPOINT" | head -1 | cut -d'=' -f2) + +# Test endpoint +curl -f "$ENDPOINT/health" || curl -f "$ENDPOINT" +``` + +**PowerShell:** +```powershell +# Get endpoint +$Endpoint = azd env get-values | Select-String -Pattern 'SERVICE_.*_URI|.*_ENDPOINT' | + Select-Object -First 1 | ForEach-Object { ($_ -split '=', 2)[1] } + +# Test endpoint +try { Invoke-WebRequest "$Endpoint/health" } catch { Invoke-WebRequest $Endpoint } +``` + +Expected: HTTP 200 response. + +## Step 3: Report Results to User + +> ⛔ **MANDATORY** — You **MUST** present the deployed endpoint URLs to the user in your response. A deployment is not considered complete until the user has received the URLs. + +> ⛔ **ALWAYS run `azd show`** after deployment to extract endpoint URLs. Do NOT rely on the `azd deploy` output alone — it is frequently truncated before endpoint URLs appear. + +```bash +azd show +``` + +Parse the `Endpoint:` lines from the `azd show` output to collect all deployed service URLs. + +**Present a summary to the user that includes:** + +| Item | Source | +|------|--------| +| Deployed service endpoint(s) | `Endpoint:` lines from `azd show` | +| Aspire Dashboard URL (if applicable) | `Aspire Dashboard:` line from `azd show` | +| Azure Portal deployment link (if available) | Portal URL from provisioning output | + +Example response format: + +```text +✅ Deployment succeeded! + +| Service | Endpoint | +|---------|----------| +| apiservice | https://apiservice.xxx.azurecontainerapps.io | + +Aspire Dashboard: https://aspire-dashboard.xxx.azurecontainerapps.io +``` + +> ⚠️ **Always use fully-qualified URLs with the `https://` scheme.** If a command returns a bare hostname (e.g. `myapp.azurestaticapps.net`), prepend `https://` before presenting it to the user. + +## Step 4: Post-Deployment Verification (if applicable) + +For deployments with Azure SQL Database and managed identity: + +### Verify SQL Access + +> ⚠️ **Warning:** `az sql db query` requires the `rdbms-connect` extension: `az extension add --name rdbms-connect --yes` + +```bash +# Load environment variables +eval $(azd env get-values) + +# Check managed identity user exists in database +az sql db query \ + --server "$SQL_SERVER" \ + --database "$SQL_DATABASE" \ + --resource-group "$AZURE_RESOURCE_GROUP" \ + --auth-mode ActiveDirectoryDefault \ + --queries "SELECT name, type_desc FROM sys.database_principals WHERE type = 'E'" +``` + +**PowerShell:** +```powershell +# Load environment variables +azd env get-values | ForEach-Object { + $name, $value = $_.Split('=', 2) + Set-Item "env:$name" $value +} + +# Check managed identity user exists in database +az sql db query ` + --server $env:SQL_SERVER ` + --database $env:SQL_DATABASE ` + --resource-group $env:AZURE_RESOURCE_GROUP ` + --auth-mode ActiveDirectoryDefault ` + --queries "SELECT name, type_desc FROM sys.database_principals WHERE type = 'E'" +``` + +**Expected:** Should list the App Service or Container App managed identity. + +### Verify Database Schema + +For EF Core applications: + +```bash +# Check tables exist +az sql db query \ + --server "$SQL_SERVER" \ + --database "$SQL_DATABASE" \ + --resource-group "$AZURE_RESOURCE_GROUP" \ + --auth-mode ActiveDirectoryDefault \ + --queries "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE'" +``` + +**PowerShell:** +```powershell +az sql db query ` + --server $env:SQL_SERVER ` + --database $env:SQL_DATABASE ` + --resource-group $env:AZURE_RESOURCE_GROUP ` + --auth-mode ActiveDirectoryDefault ` + --queries "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE'" +``` + +**Expected:** Should list application tables (not just `__EFMigrationsHistory`). + +### Check Application Logs + +```bash +# For App Service +az webapp log tail --name --resource-group + +# For Container Apps +az containerapp logs show --name --resource-group --follow +``` + +**Look for:** +- ✅ No SQL authentication errors +- ✅ Successful database connection +- ✅ Application started successfully + +## Common Issues + +| Symptom | Cause | Fix | +|---------|-------|-----| +| HTTP 500 on startup | SQL authentication failure | See [sql-managed-identity.md](sql-managed-identity.md) | +| "Invalid object name" errors | Migrations not applied | See [ef-migrations.md](ef-migrations.md) | +| Endpoint not accessible | Service still starting | Wait 1-2 minutes, retry | +| Health check fails | Application error | Check logs with `az webapp log tail` | + +## References + +- [Post-Deployment Steps](post-deployment.md) +- [SQL Managed Identity Access](sql-managed-identity.md) +- [EF Core Migrations](ef-migrations.md) diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/bicep/README.md b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/bicep/README.md index 535581fd..dfcd3324 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/bicep/README.md +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/bicep/README.md @@ -1,97 +1,97 @@ -# Bicep Deploy Recipe - -Deploy to Azure using Bicep templates directly. - -## Prerequisites - -- `az` CLI installed with Bicep extension -- `.azure/deployment-plan.md` exists with status `Validated` -- Bicep templates exist in `infra/` -- **Subscription and location confirmed** → See [Pre-Deploy Checklist](../../pre-deploy-checklist.md) - -## Workflow - -| Step | Task | Command | -|------|------|---------| -| 1 | **[Pre-deploy checklist](../../pre-deploy-checklist.md)** | Confirm subscription/location with user | -| 2 | Build (optional) | `az bicep build --file main.bicep` | -| 3 | Deploy | `az deployment sub create` | -| 4 | Verify | `az resource list` | -| 5 | **Report** | Present deployed endpoint URLs to the user — see [Verification](verify.md) | - -## Deployment Commands - -### Subscription-Level Deployment - -```bash -az deployment sub create \ - --location eastus2 \ - --template-file ./infra/main.bicep \ - --parameters ./infra/main.parameters.json -``` - -### Resource Group Deployment - -```bash -az deployment group create \ - --resource-group rg-myapp-dev \ - --template-file ./infra/main.bicep \ - --parameters ./infra/main.parameters.json -``` - -### With Inline Parameters - -```bash -az deployment sub create \ - --location eastus2 \ - --template-file ./infra/main.bicep \ - --parameters environmentName=dev location=eastus2 -``` - -### What-If (Preview Changes) - -```bash -az deployment sub what-if \ - --location eastus2 \ - --template-file ./infra/main.bicep \ - --parameters environmentName=dev -``` - -## Get Deployment Outputs - -```bash -az deployment sub show \ - --name main \ - --query properties.outputs -``` - -## References - -- [Verification steps](./verify.md) -- [Error handling](./errors.md) - -## MCP Tools - -| Tool | Purpose | -|------|---------| -| `mcp_bicep_get_bicep_best_practices` | Best practices | -| `mcp_bicep_get_az_resource_type_schema` | Resource schemas | -| `mcp_bicep_list_avm_metadata` | Azure Verified Modules | - -## AVM Verification Before Deploy - -Before running deployment commands, verify generated templates followed AVM-first module selection: - -1. AVM Bicep Pattern Modules (prefer AVM+AZD patterns) -2. AVM Bicep Resource Modules -3. AVM Bicep Utility Modules - -If no AVM+AZD pattern module is available, fallback must remain within AVM modules (resource -> utility). - -## Cleanup (DESTRUCTIVE) - -```bash -az group delete --name --yes -``` - -⚠️ Permanently deletes ALL resources in the group. +# Bicep Deploy Recipe + +Deploy to Azure using Bicep templates directly. + +## Prerequisites + +- `az` CLI installed with Bicep extension +- `.azure/deployment-plan.md` exists with status `Validated` +- Bicep templates exist in `infra/` +- **Subscription and location confirmed** → See [Pre-Deploy Checklist](../../pre-deploy-checklist.md) + +## Workflow + +| Step | Task | Command | +|------|------|---------| +| 1 | **[Pre-deploy checklist](../../pre-deploy-checklist.md)** | Confirm subscription/location with user | +| 2 | Build (optional) | `az bicep build --file main.bicep` | +| 3 | Deploy | `az deployment sub create` | +| 4 | Verify | `az resource list` | +| 5 | **Report** | Present deployed endpoint URLs to the user — see [Verification](verify.md) | + +## Deployment Commands + +### Subscription-Level Deployment + +```bash +az deployment sub create \ + --location eastus2 \ + --template-file ./infra/main.bicep \ + --parameters ./infra/main.parameters.json +``` + +### Resource Group Deployment + +```bash +az deployment group create \ + --resource-group rg-myapp-dev \ + --template-file ./infra/main.bicep \ + --parameters ./infra/main.parameters.json +``` + +### With Inline Parameters + +```bash +az deployment sub create \ + --location eastus2 \ + --template-file ./infra/main.bicep \ + --parameters environmentName=dev location=eastus2 +``` + +### What-If (Preview Changes) + +```bash +az deployment sub what-if \ + --location eastus2 \ + --template-file ./infra/main.bicep \ + --parameters environmentName=dev +``` + +## Get Deployment Outputs + +```bash +az deployment sub show \ + --name main \ + --query properties.outputs +``` + +## References + +- [Verification steps](./verify.md) +- [Error handling](./errors.md) + +## MCP Tools + +| Tool | Purpose | +|------|---------| +| `mcp_bicep_get_bicep_best_practices` | Best practices | +| `mcp_bicep_get_az_resource_type_schema` | Resource schemas | +| `mcp_bicep_list_avm_metadata` | Azure Verified Modules | + +## AVM Verification Before Deploy + +Before running deployment commands, verify generated templates followed AVM-first module selection: + +1. AVM Bicep Pattern Modules (prefer AVM+AZD patterns) +2. AVM Bicep Resource Modules +3. AVM Bicep Utility Modules + +If no AVM+AZD pattern module is available, fallback must remain within AVM modules (resource -> utility). + +## Cleanup (DESTRUCTIVE) + +```bash +az group delete --name --yes +``` + +⚠️ Permanently deletes ALL resources in the group. diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/bicep/errors.md b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/bicep/errors.md index 0309f64c..88a82773 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/bicep/errors.md +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/bicep/errors.md @@ -1,18 +1,18 @@ -# Bicep Errors - -| Error | Resolution | -|-------|------------| -| Syntax error | `az bicep build` to check | -| Missing parameter | Add to parameters file | -| Invalid property | Check `mcp_bicep_get_az_resource_type_schema` | -| Resource conflict | Check existing resources | -| Deployment failed | `az deployment sub show --name ` | -| Permission denied | Verify RBAC roles | - -## Cleanup (DESTRUCTIVE) - -```bash -az group delete --name --yes -``` - -⚠️ Permanently deletes ALL resources in the group. +# Bicep Errors + +| Error | Resolution | +|-------|------------| +| Syntax error | `az bicep build` to check | +| Missing parameter | Add to parameters file | +| Invalid property | Check `mcp_bicep_get_az_resource_type_schema` | +| Resource conflict | Check existing resources | +| Deployment failed | `az deployment sub show --name ` | +| Permission denied | Verify RBAC roles | + +## Cleanup (DESTRUCTIVE) + +```bash +az group delete --name --yes +``` + +⚠️ Permanently deletes ALL resources in the group. diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/bicep/verify.md b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/bicep/verify.md index aab25ad4..45d7ff84 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/bicep/verify.md +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/bicep/verify.md @@ -1,31 +1,31 @@ -# Bicep Verification - -```bash -az resource list --resource-group --output table -``` - -## Get Deployment Outputs - -```bash -az deployment sub show \ - --name main \ - --query properties.outputs -``` - -## Health Check - -```bash -curl -s https:///health | jq . -``` - -## Report Results to User - -> ⛔ **MANDATORY** — You **MUST** present the deployed endpoint URLs to the user in your response. - -Extract endpoints from deployment outputs: - -```bash -az deployment sub show --name main --query properties.outputs -``` - -Present a summary including all service URLs as fully-qualified `https://` links. If a deployment output returns a bare hostname (e.g. `myapp.azurewebsites.net`), always prepend `https://`. Do NOT end your response without including them. +# Bicep Verification + +```bash +az resource list --resource-group --output table +``` + +## Get Deployment Outputs + +```bash +az deployment sub show \ + --name main \ + --query properties.outputs +``` + +## Health Check + +```bash +curl -s https:///health | jq . +``` + +## Report Results to User + +> ⛔ **MANDATORY** — You **MUST** present the deployed endpoint URLs to the user in your response. + +Extract endpoints from deployment outputs: + +```bash +az deployment sub show --name main --query properties.outputs +``` + +Present a summary including all service URLs as fully-qualified `https://` links. If a deployment output returns a bare hostname (e.g. `myapp.azurewebsites.net`), always prepend `https://`. Do NOT end your response without including them. diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/README.md b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/README.md index 73d9ee09..3035cf8e 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/README.md +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/README.md @@ -1,43 +1,43 @@ -# CI/CD Deploy Recipe - -Deploy to Azure using automated pipelines. - -## Prerequisites - -- `.azure/deployment-plan.md` exists with status `Validated` -- Azure Service Principal or federated credentials configured -- Pipeline file exists (`.github/workflows/` or `azure-pipelines.yml`) - -## GitHub Actions - -| Example | Description | -|---------|-------------| -| [github-azd.yml](examples/github-azd.yml) | AZD deployment workflow | -| [github-bicep.yml](examples/github-bicep.yml) | Bicep infrastructure deployment | - -## Azure DevOps - -| Example | Description | -|---------|-------------| -| [azdo-azd.yml](examples/azdo-azd.yml) | Basic AZD pipeline | -| [azdo-multistage.yml](examples/azdo-multistage.yml) | Multi-stage with approvals | - -## Setup Requirements - -### GitHub Actions - -1. Create Azure Service Principal with federated credentials -2. Add secrets: `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID` -3. Add variables: `AZURE_ENV_NAME`, `AZURE_LOCATION` -4. Create environments with protection rules - -### Azure DevOps - -1. Create Service Connection to Azure -2. Create Variable Groups per environment -3. Create Environments with approval gates - -## References - -- [Verification steps](./verify.md) -- [Error handling](./errors.md) +# CI/CD Deploy Recipe + +Deploy to Azure using automated pipelines. + +## Prerequisites + +- `.azure/deployment-plan.md` exists with status `Validated` +- Azure Service Principal or federated credentials configured +- Pipeline file exists (`.github/workflows/` or `azure-pipelines.yml`) + +## GitHub Actions + +| Example | Description | +|---------|-------------| +| [github-azd.yml](examples/github-azd.yml) | AZD deployment workflow | +| [github-bicep.yml](examples/github-bicep.yml) | Bicep infrastructure deployment | + +## Azure DevOps + +| Example | Description | +|---------|-------------| +| [azdo-azd.yml](examples/azdo-azd.yml) | Basic AZD pipeline | +| [azdo-multistage.yml](examples/azdo-multistage.yml) | Multi-stage with approvals | + +## Setup Requirements + +### GitHub Actions + +1. Create Azure Service Principal with federated credentials +2. Add secrets: `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID` +3. Add variables: `AZURE_ENV_NAME`, `AZURE_LOCATION` +4. Create environments with protection rules + +### Azure DevOps + +1. Create Service Connection to Azure +2. Create Variable Groups per environment +3. Create Environments with approval gates + +## References + +- [Verification steps](./verify.md) +- [Error handling](./errors.md) diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/errors.md b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/errors.md index 63c3b639..f75b8b2e 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/errors.md +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/errors.md @@ -1,17 +1,17 @@ -# CI/CD Errors - -| Error | Resolution | -|-------|------------| -| Authentication failed | Check service principal/federated credentials | -| Missing secrets | Add required secrets to repository | -| Missing variables | Add required variables | -| Pipeline timeout | Increase timeout or optimize deployment | -| Approval pending | Request approval in environment settings | - -## GitHub Actions Debugging - -Check workflow logs in Actions tab for detailed error messages. - -## Azure DevOps Debugging - -Check pipeline run logs for detailed error messages. +# CI/CD Errors + +| Error | Resolution | +|-------|------------| +| Authentication failed | Check service principal/federated credentials | +| Missing secrets | Add required secrets to repository | +| Missing variables | Add required variables | +| Pipeline timeout | Increase timeout or optimize deployment | +| Approval pending | Request approval in environment settings | + +## GitHub Actions Debugging + +Check workflow logs in Actions tab for detailed error messages. + +## Azure DevOps Debugging + +Check pipeline run logs for detailed error messages. diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/examples/azdo-azd.yml b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/examples/azdo-azd.yml index da115a50..9286d39a 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/examples/azdo-azd.yml +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/examples/azdo-azd.yml @@ -1,22 +1,22 @@ -trigger: - branches: - include: [main] - -pool: - vmImage: 'ubuntu-latest' - -stages: - - stage: Deploy - jobs: - - job: DeployToAzure - steps: - - task: setup-azd@0 - - - task: AzureCLI@2 - inputs: - azureSubscription: 'azure-service-connection' - scriptType: 'bash' - inlineScript: azd up --no-prompt - env: - AZURE_ENV_NAME: $(AZURE_ENV_NAME) - AZURE_LOCATION: $(AZURE_LOCATION) +trigger: + branches: + include: [main] + +pool: + vmImage: 'ubuntu-latest' + +stages: + - stage: Deploy + jobs: + - job: DeployToAzure + steps: + - task: setup-azd@0 + + - task: AzureCLI@2 + inputs: + azureSubscription: 'azure-service-connection' + scriptType: 'bash' + inlineScript: azd up --no-prompt + env: + AZURE_ENV_NAME: $(AZURE_ENV_NAME) + AZURE_LOCATION: $(AZURE_LOCATION) diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/examples/azdo-multistage.yml b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/examples/azdo-multistage.yml index e81599a4..9fb5a968 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/examples/azdo-multistage.yml +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/examples/azdo-multistage.yml @@ -1,29 +1,29 @@ -stages: - - stage: Dev - jobs: - - deployment: DeployDev - environment: dev - strategy: - runOnce: - deploy: - steps: - - task: AzureCLI@2 - inputs: - azureSubscription: 'azure-dev' - scriptType: 'bash' - inlineScript: azd up --no-prompt --environment dev - - - stage: Prod - dependsOn: Dev - jobs: - - deployment: DeployProd - environment: prod # Configure approval in Azure DevOps - strategy: - runOnce: - deploy: - steps: - - task: AzureCLI@2 - inputs: - azureSubscription: 'azure-prod' - scriptType: 'bash' - inlineScript: azd up --no-prompt --environment prod +stages: + - stage: Dev + jobs: + - deployment: DeployDev + environment: dev + strategy: + runOnce: + deploy: + steps: + - task: AzureCLI@2 + inputs: + azureSubscription: 'azure-dev' + scriptType: 'bash' + inlineScript: azd up --no-prompt --environment dev + + - stage: Prod + dependsOn: Dev + jobs: + - deployment: DeployProd + environment: prod # Configure approval in Azure DevOps + strategy: + runOnce: + deploy: + steps: + - task: AzureCLI@2 + inputs: + azureSubscription: 'azure-prod' + scriptType: 'bash' + inlineScript: azd up --no-prompt --environment prod diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/examples/github-azd.yml b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/examples/github-azd.yml index b62213ce..80eba505 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/examples/github-azd.yml +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/examples/github-azd.yml @@ -1,35 +1,35 @@ -name: Deploy to Azure - -on: - push: - branches: [main] - workflow_dispatch: - -permissions: - id-token: write - contents: read - -jobs: - deploy: - runs-on: ubuntu-latest - environment: dev - - steps: - - uses: actions/checkout@v4 - - - name: Install azd - uses: Azure/setup-azd@v1 - - - name: Azure Login - uses: azure/login@v2 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: Deploy - run: azd up --no-prompt - env: - AZURE_ENV_NAME: ${{ vars.AZURE_ENV_NAME }} - AZURE_LOCATION: ${{ vars.AZURE_LOCATION }} - AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} +name: Deploy to Azure + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + deploy: + runs-on: ubuntu-latest + environment: dev + + steps: + - uses: actions/checkout@v4 + + - name: Install azd + uses: Azure/setup-azd@v1 + + - name: Azure Login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Deploy + run: azd up --no-prompt + env: + AZURE_ENV_NAME: ${{ vars.AZURE_ENV_NAME }} + AZURE_LOCATION: ${{ vars.AZURE_LOCATION }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/examples/github-bicep.yml b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/examples/github-bicep.yml index 5cbc06fc..14d3cdd3 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/examples/github-bicep.yml +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/examples/github-bicep.yml @@ -1,27 +1,27 @@ -name: Deploy Infrastructure - -on: - push: - branches: [main] - paths: ['infra/**'] - -jobs: - deploy: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - uses: azure/login@v2 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - uses: azure/arm-deploy@v2 - with: - scope: subscription - subscriptionId: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - region: eastus - template: ./infra/main.bicep - parameters: environmentName=${{ vars.ENVIRONMENT }} +name: Deploy Infrastructure + +on: + push: + branches: [main] + paths: ['infra/**'] + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - uses: azure/arm-deploy@v2 + with: + scope: subscription + subscriptionId: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + region: eastus + template: ./infra/main.bicep + parameters: environmentName=${{ vars.ENVIRONMENT }} diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/verify.md b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/verify.md index 02c5c686..aaaab4be 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/verify.md +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/cicd/verify.md @@ -1,23 +1,23 @@ -# CI/CD Verification - -Check pipeline run status: -- **GitHub**: Actions tab → workflow run -- **Azure DevOps**: Pipelines → pipeline run - -## Verify Deployed Resources - -```bash -az resource list --resource-group --output table -``` - -## Health Check - -```bash -curl -s https:///health | jq . -``` - -## Report Results to User - -> ⛔ **MANDATORY** — You **MUST** present the deployed endpoint URLs to the user in your response. - -Extract endpoints from the pipeline output or query them directly via `az` CLI. Present a summary including all service URLs as fully-qualified `https://` links. If any command returns a bare hostname (e.g. `myapp.azurewebsites.net`), always prepend `https://`. Do NOT end your response without including them. +# CI/CD Verification + +Check pipeline run status: +- **GitHub**: Actions tab → workflow run +- **Azure DevOps**: Pipelines → pipeline run + +## Verify Deployed Resources + +```bash +az resource list --resource-group --output table +``` + +## Health Check + +```bash +curl -s https:///health | jq . +``` + +## Report Results to User + +> ⛔ **MANDATORY** — You **MUST** present the deployed endpoint URLs to the user in your response. + +Extract endpoints from the pipeline output or query them directly via `az` CLI. Present a summary including all service URLs as fully-qualified `https://` links. If any command returns a bare hostname (e.g. `myapp.azurewebsites.net`), always prepend `https://`. Do NOT end your response without including them. diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/terraform/README.md b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/terraform/README.md index f6fcfd16..188af0c6 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/terraform/README.md +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/terraform/README.md @@ -1,119 +1,119 @@ -# Terraform Deploy Recipe - -Deploy to Azure using Terraform. - -## Prerequisites - -- Terraform CLI installed -- `.azure/deployment-plan.md` exists with status `Validated` -- Terraform initialized (`terraform init`) -- Plan validated (`terraform plan`) -- **Subscription and location confirmed** → See [Pre-Deploy Checklist](../../pre-deploy-checklist.md) - -## Workflow - -| Step | Task | Command | -|------|------|---------| -| 1 | **[Pre-deploy checklist](../../pre-deploy-checklist.md)** | Confirm subscription/location with user | -| 2 | Select workspace | `terraform workspace select ` | -| 3 | Apply | `terraform apply tfplan` | -| 4 | Get outputs | `terraform output` | -| 5 | Deploy app | Service-specific commands | -| 6 | **Report** | Present deployed endpoint URLs to the user — see [Verification](verify.md) | - -## Deployment Commands - -### Apply with Plan File (Recommended) - -```bash -terraform plan -out=tfplan -terraform apply tfplan -``` - -### Auto-Approve (CI/CD) - -```bash -terraform apply -auto-approve -``` - -### With Variables - -```bash -terraform apply \ - -var="environment=prod" \ - -var="location=westus2" \ - -auto-approve -``` - -### Target Specific Resource - -```bash -terraform apply -target=azurerm_container_app.api -``` - -## Get Outputs - -```bash -terraform output -terraform output -json -terraform output api_url -``` - -## Application Deployment - -After infrastructure is deployed: - -### Container Apps (Two-Phase Deployment) - -Container Apps with ACR require a two-phase flow because the app image doesn't exist in ACR during initial `terraform apply`. See the **azure-prepare** skill's `references/services/container-apps/terraform.md` for the full pattern. - -```bash -ACR_NAME=$(terraform output -raw acr_name) -ACR_SERVER=$(terraform output -raw acr_login_server) -APP_NAME=$(terraform output -raw container_app_name) -RG_NAME=$(terraform output -raw resource_group_name) - -# 1. Build and push the application image to ACR -az acr build --registry $ACR_NAME --image myapp:latest ./src/api - -# 2. Configure the registry/identity link (managed identity, no passwords) -az containerapp registry set \ - --name $APP_NAME \ - --resource-group $RG_NAME \ - --server $ACR_SERVER \ - --identity system - -# 3. Update the Container App to use the real image -az containerapp update \ - --name $APP_NAME \ - --resource-group $RG_NAME \ - --image $ACR_SERVER/myapp:latest -``` - -**PowerShell:** -```powershell -$AcrName = terraform output -raw acr_name -$AcrServer = terraform output -raw acr_login_server -$AppName = terraform output -raw container_app_name -$RgName = terraform output -raw resource_group_name - -az acr build --registry $AcrName --image myapp:latest ./src/api - -az containerapp registry set ` - --name $AppName ` - --resource-group $RgName ` - --server $AcrServer ` - --identity system - -az containerapp update ` - --name $AppName ` - --resource-group $RgName ` - --image "$AcrServer/myapp:latest" -``` - -> ⚠️ **Warning:** Step 2 requires the `AcrPull` role assignment to have propagated (1–5 minutes). If the update fails, wait and retry. See the [RBAC propagation health check](../../pre-deploy-checklist.md#container-apps--acr--pre-deploy-rbac-health-check). - -## References - -- [Verification steps](./verify.md) -- [Error handling](./errors.md) +# Terraform Deploy Recipe + +Deploy to Azure using Terraform. + +## Prerequisites + +- Terraform CLI installed +- `.azure/deployment-plan.md` exists with status `Validated` +- Terraform initialized (`terraform init`) +- Plan validated (`terraform plan`) +- **Subscription and location confirmed** → See [Pre-Deploy Checklist](../../pre-deploy-checklist.md) + +## Workflow + +| Step | Task | Command | +|------|------|---------| +| 1 | **[Pre-deploy checklist](../../pre-deploy-checklist.md)** | Confirm subscription/location with user | +| 2 | Select workspace | `terraform workspace select ` | +| 3 | Apply | `terraform apply tfplan` | +| 4 | Get outputs | `terraform output` | +| 5 | Deploy app | Service-specific commands | +| 6 | **Report** | Present deployed endpoint URLs to the user — see [Verification](verify.md) | + +## Deployment Commands + +### Apply with Plan File (Recommended) + +```bash +terraform plan -out=tfplan +terraform apply tfplan +``` + +### Auto-Approve (CI/CD) + +```bash +terraform apply -auto-approve +``` + +### With Variables + +```bash +terraform apply \ + -var="environment=prod" \ + -var="location=westus2" \ + -auto-approve +``` + +### Target Specific Resource + +```bash +terraform apply -target=azurerm_container_app.api +``` + +## Get Outputs + +```bash +terraform output +terraform output -json +terraform output api_url +``` + +## Application Deployment + +After infrastructure is deployed: + +### Container Apps (Two-Phase Deployment) + +Container Apps with ACR require a two-phase flow because the app image doesn't exist in ACR during initial `terraform apply`. See the **azure-prepare** skill's `references/services/container-apps/terraform.md` for the full pattern. + +```bash +ACR_NAME=$(terraform output -raw acr_name) +ACR_SERVER=$(terraform output -raw acr_login_server) +APP_NAME=$(terraform output -raw container_app_name) +RG_NAME=$(terraform output -raw resource_group_name) + +# 1. Build and push the application image to ACR +az acr build --registry $ACR_NAME --image myapp:latest ./src/api + +# 2. Configure the registry/identity link (managed identity, no passwords) +az containerapp registry set \ + --name $APP_NAME \ + --resource-group $RG_NAME \ + --server $ACR_SERVER \ + --identity system + +# 3. Update the Container App to use the real image +az containerapp update \ + --name $APP_NAME \ + --resource-group $RG_NAME \ + --image $ACR_SERVER/myapp:latest +``` + +**PowerShell:** +```powershell +$AcrName = terraform output -raw acr_name +$AcrServer = terraform output -raw acr_login_server +$AppName = terraform output -raw container_app_name +$RgName = terraform output -raw resource_group_name + +az acr build --registry $AcrName --image myapp:latest ./src/api + +az containerapp registry set ` + --name $AppName ` + --resource-group $RgName ` + --server $AcrServer ` + --identity system + +az containerapp update ` + --name $AppName ` + --resource-group $RgName ` + --image "$AcrServer/myapp:latest" +``` + +> ⚠️ **Warning:** Step 2 requires the `AcrPull` role assignment to have propagated (1–5 minutes). If the update fails, wait and retry. See the [RBAC propagation health check](../../pre-deploy-checklist.md#container-apps--acr--pre-deploy-rbac-health-check). + +## References + +- [Verification steps](./verify.md) +- [Error handling](./errors.md) diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/terraform/errors.md b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/terraform/errors.md index 22503096..421832d7 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/terraform/errors.md +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/terraform/errors.md @@ -1,23 +1,23 @@ -# Terraform Errors - -| Error | Resolution | -|-------|------------| -| State lock error | Wait or `terraform force-unlock ` | -| Resource exists | `terraform import ` | -| Backend denied | Check storage permissions | -| Provider error | `terraform init -upgrade` | -| Literal `{{ .Env.* }}` in variable values | Fix syntax in `main.tfvars.json`: use `${VAR}` (e.g., `${AZURE_ENV_NAME}`), not Go-style `{{ .Env.* }}`. See [AZD Errors](../azd/errors.md#unresolved-terraform-template-variables) | -| State cleared on each `azd provision` | azd copies Terraform config to `.azure//infra/` on each run. Use a remote state backend to persist state across runs | - -## Cleanup (DESTRUCTIVE) - -```bash -terraform destroy -auto-approve -``` - -Selective: -```bash -terraform destroy -target=azurerm_container_app.api -``` - -⚠️ Permanently deletes resources. +# Terraform Errors + +| Error | Resolution | +|-------|------------| +| State lock error | Wait or `terraform force-unlock ` | +| Resource exists | `terraform import ` | +| Backend denied | Check storage permissions | +| Provider error | `terraform init -upgrade` | +| Literal `{{ .Env.* }}` in variable values | Fix syntax in `main.tfvars.json`: use `${VAR}` (e.g., `${AZURE_ENV_NAME}`), not Go-style `{{ .Env.* }}`. See [AZD Errors](../azd/errors.md#unresolved-terraform-template-variables) | +| State cleared on each `azd provision` | azd copies Terraform config to `.azure//infra/` on each run. Use a remote state backend to persist state across runs | + +## Cleanup (DESTRUCTIVE) + +```bash +terraform destroy -auto-approve +``` + +Selective: +```bash +terraform destroy -target=azurerm_container_app.api +``` + +⚠️ Permanently deletes resources. diff --git a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/terraform/verify.md b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/terraform/verify.md index 9f47a505..cef3e39b 100644 --- a/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/terraform/verify.md +++ b/.github/plugins/azure-skills/skills/azure-deploy/references/recipes/terraform/verify.md @@ -1,30 +1,30 @@ -# Terraform Verification - -```bash -terraform output -terraform output -json -``` - -## Health Check - -```bash -curl -s https://$(terraform output -raw api_url)/health | jq . -``` - -## Resource Check - -```bash -az resource list --resource-group $(terraform output -raw resource_group_name) --output table -``` - -## Report Results to User - -> ⛔ **MANDATORY** — You **MUST** present the deployed endpoint URLs to the user in your response. - -Extract endpoints from Terraform outputs: - -```bash -terraform output -raw api_url -``` - -Present a summary including all service URLs as fully-qualified `https://` links. If a Terraform output returns a bare hostname (e.g. `myapp.azurewebsites.net`), always prepend `https://`. Do NOT end your response without including them. +# Terraform Verification + +```bash +terraform output +terraform output -json +``` + +## Health Check + +```bash +curl -s https://$(terraform output -raw api_url)/health | jq . +``` + +## Resource Check + +```bash +az resource list --resource-group $(terraform output -raw resource_group_name) --output table +``` + +## Report Results to User + +> ⛔ **MANDATORY** — You **MUST** present the deployed endpoint URLs to the user in your response. + +Extract endpoints from Terraform outputs: + +```bash +terraform output -raw api_url +``` + +Present a summary including all service URLs as fully-qualified `https://` links. If a Terraform output returns a bare hostname (e.g. `myapp.azurewebsites.net`), always prepend `https://`. Do NOT end your response without including them. diff --git a/.github/plugins/azure-skills/skills/azure-diagnostics/SKILL.md b/.github/plugins/azure-skills/skills/azure-diagnostics/SKILL.md index b313ce4e..f0f6ae54 100644 --- a/.github/plugins/azure-skills/skills/azure-diagnostics/SKILL.md +++ b/.github/plugins/azure-skills/skills/azure-diagnostics/SKILL.md @@ -4,7 +4,7 @@ description: "Debug Azure production issues on Azure using AppLens, Azure Monito license: MIT metadata: author: Microsoft - version: "1.2.6" + version: "1.2.7" --- # Azure Diagnostics diff --git a/.github/plugins/azure-skills/skills/azure-diagnostics/references/azure-resource-graph.md b/.github/plugins/azure-skills/skills/azure-diagnostics/references/azure-resource-graph.md index 621bc24e..0c1d78dd 100644 --- a/.github/plugins/azure-skills/skills/azure-diagnostics/references/azure-resource-graph.md +++ b/.github/plugins/azure-skills/skills/azure-diagnostics/references/azure-resource-graph.md @@ -1,93 +1,93 @@ -# Azure Resource Graph Queries for Diagnostics - -Azure Resource Graph (ARG) enables fast, cross-subscription resource querying using KQL via `az graph query`. Use it to check resource health, find degraded resources, and correlate incidents. - -## How to Query - -Use the `extension_cli_generate` MCP tool to generate `az graph query` commands: - -```yaml -mcp_azure_mcp_extension_cli_generate - intent: "query Azure Resource Graph to " - cli-type: "az" -``` - -Or construct directly: - -```bash -az graph query -q "" --query "data[].{name:name, type:type}" -o table -``` - -> ⚠️ **Prerequisite:** `az extension add --name resource-graph` - -## Key Tables - -| Table | Contains | -|-------|----------| -| `Resources` | All ARM resources (name, type, location, properties, tags) | -| `HealthResources` | Resource health availability status | -| `ServiceHealthResources` | Azure service health events and incidents | -| `ResourceContainers` | Subscriptions, resource groups, management groups | - -## Diagnostics Query Patterns - -**Check resource health status across resources:** - -```kql -HealthResources -| where type =~ 'microsoft.resourcehealth/availabilitystatuses' -| project name, availabilityState=properties.availabilityState, reasonType=properties.reasonType -``` - -**Find resources in unhealthy or degraded state:** - -```kql -HealthResources -| where type =~ 'microsoft.resourcehealth/availabilitystatuses' -| where properties.availabilityState != 'Available' -| project name, state=properties.availabilityState, reason=properties.reasonType, summary=properties.summary -``` - -**Query active service health incidents:** - -```kql -ServiceHealthResources -| where type =~ 'microsoft.resourcehealth/events' -| where properties.Status == 'Active' -| project name, title=properties.Title, impact=properties.Impact, status=properties.Status -``` - -**Find resources by provisioning state (failed/stuck deployments):** - -```kql -Resources -| where properties.provisioningState != 'Succeeded' -| project name, type, resourceGroup, provisioningState=properties.provisioningState -``` - -**Find App Services in stopped or error state:** - -```kql -Resources -| where type =~ 'microsoft.web/sites' -| where properties.state != 'Running' -| project name, state=properties.state, resourceGroup, location -``` - -**Find Container Apps with provisioning issues:** - -```kql -Resources -| where type =~ 'microsoft.app/containerapps' -| where properties.provisioningState != 'Succeeded' -| project name, provisioningState=properties.provisioningState, resourceGroup -``` - -## Tips - -- Use `=~` for case-insensitive type matching (resource types are lowercase) -- Navigate properties with `properties.fieldName` -- Use `--first N` to limit result count -- Use `--subscriptions` to scope to specific subscriptions -- Combine ARG health data with Azure Monitor metrics for full picture -- Check `HealthResources` before deep-diving into application logs +# Azure Resource Graph Queries for Diagnostics + +Azure Resource Graph (ARG) enables fast, cross-subscription resource querying using KQL via `az graph query`. Use it to check resource health, find degraded resources, and correlate incidents. + +## How to Query + +Use the `extension_cli_generate` MCP tool to generate `az graph query` commands: + +```yaml +mcp_azure_mcp_extension_cli_generate + intent: "query Azure Resource Graph to " + cli-type: "az" +``` + +Or construct directly: + +```bash +az graph query -q "" --query "data[].{name:name, type:type}" -o table +``` + +> ⚠️ **Prerequisite:** `az extension add --name resource-graph` + +## Key Tables + +| Table | Contains | +|-------|----------| +| `Resources` | All ARM resources (name, type, location, properties, tags) | +| `HealthResources` | Resource health availability status | +| `ServiceHealthResources` | Azure service health events and incidents | +| `ResourceContainers` | Subscriptions, resource groups, management groups | + +## Diagnostics Query Patterns + +**Check resource health status across resources:** + +```kql +HealthResources +| where type =~ 'microsoft.resourcehealth/availabilitystatuses' +| project name, availabilityState=properties.availabilityState, reasonType=properties.reasonType +``` + +**Find resources in unhealthy or degraded state:** + +```kql +HealthResources +| where type =~ 'microsoft.resourcehealth/availabilitystatuses' +| where properties.availabilityState != 'Available' +| project name, state=properties.availabilityState, reason=properties.reasonType, summary=properties.summary +``` + +**Query active service health incidents:** + +```kql +ServiceHealthResources +| where type =~ 'microsoft.resourcehealth/events' +| where properties.Status == 'Active' +| project name, title=properties.Title, impact=properties.Impact, status=properties.Status +``` + +**Find resources by provisioning state (failed/stuck deployments):** + +```kql +Resources +| where properties.provisioningState != 'Succeeded' +| project name, type, resourceGroup, provisioningState=properties.provisioningState +``` + +**Find App Services in stopped or error state:** + +```kql +Resources +| where type =~ 'microsoft.web/sites' +| where properties.state != 'Running' +| project name, state=properties.state, resourceGroup, location +``` + +**Find Container Apps with provisioning issues:** + +```kql +Resources +| where type =~ 'microsoft.app/containerapps' +| where properties.provisioningState != 'Succeeded' +| project name, provisioningState=properties.provisioningState, resourceGroup +``` + +## Tips + +- Use `=~` for case-insensitive type matching (resource types are lowercase) +- Navigate properties with `properties.fieldName` +- Use `--first N` to limit result count +- Use `--subscriptions` to scope to specific subscriptions +- Combine ARG health data with Azure Monitor metrics for full picture +- Check `HealthResources` before deep-diving into application logs diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/SKILL.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/SKILL.md index eccc653c..5bc6f5c5 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/SKILL.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/SKILL.md @@ -4,7 +4,7 @@ description: "Architect and provision enterprise Azure infrastructure from workl license: MIT metadata: author: Microsoft - version: "1.4.1" + version: "1.4.2" --- # Azure Enterprise Infra Planner diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/bicep-generation.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/bicep-generation.md index 1db8b025..c84abe0e 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/bicep-generation.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/bicep-generation.md @@ -1,93 +1,93 @@ -# Bicep Generation - -Generate Bicep IaC files from the approved infrastructure plan. - -> Important: All Bicep files must be created under `/infra/`. Never place `.bicep` files in the project root or in `.azure/`. - -## File Structure - -Generate files under `/infra/`: - -``` -infra/ -├── main.bicep # Orchestrator — deploys all modules -├── main.bicepparam # Parameter values -└── modules/ - ├── storage.bicep # One module per resource or logical group - ├── compute.bicep - ├── networking.bicep - └── monitoring.bicep -``` - -## Generation Steps - -1. Create `infra/` directory — create `/infra/` and `/infra/modules/` directories. All files in subsequent steps go here. -2. Read plan — load `/.azure/infrastructure-plan.json`, verify `meta.status === "approved"` -3. Fetch Bicep schemas — for each resource in the plan, use a sub-agent to call `bicepschema_get` with `resource-type` set to the ARM type from the relevant [resources/](resources/README.md) category file (e.g., `Microsoft.ContainerService/managedClusters`). Instruct the sub-agent: "Return the full property structure for {ARM type}: required properties, allowed values, child resources. ≤500 tokens." Use this output — not training data — to generate correct resource definitions. - -> The schema tool returns only the schema for the exact type requested. Sub-resource types (e.g., `Microsoft.Network/virtualNetworks/subnets`) return a smaller, focused schema but miss parent-level properties (e.g., VNet `encryption` lives on the parent, not the subnet sub-resource). Strategy: -> - Start with sub-resource types when validating child resources — smaller responses (~25KB vs ~95KB), easier to summarize -> - Fetch the parent type separately when you need parent-level properties (encryption, tags, SKU) — delegate to a sub-agent with specific property extraction instructions to manage the large response - -4. Generate modules — group resources by category; one `.bicep` file per group under `infra/modules/`. Use the schema from step 3 for property names, allowed values, and required fields. -5. Generate main.bicep — write `infra/main.bicep` that imports all modules and passes parameters -6. Generate parameters — create `infra/main.bicepparam` with environment-specific values - -## Bicep Conventions - -- Use `@description()` decorators on all parameters -- Use `@secure()` for secrets and connection strings -- Choose `targetScope` in `main.bicep` based on the deployment plan: - - For single resource group deployments, set `targetScope = 'resourceGroup'` and deploy with `az deployment group create`. - - For subscription-scope deployments (for example, resources across multiple resource groups or subscription-level resources), set `targetScope = 'subscription'` and deploy with `az deployment sub create`. -- Use `existing` keyword for referencing pre-existing resources -- Output resource IDs and endpoints needed by other resources -- Use `dependsOn` only when implicit dependencies are insufficient - -## Parameter File Format - -```bicep -using './main.bicep' - -param location = 'eastus' -param environmentName = 'prod' -param workloadName = 'datapipeline' -``` - -## Multi-Environment - -For multi-environment plans, generate one parameter file per environment: - -```txt -infra/ -├── main.bicep -├── main.dev.bicepparam -├── main.staging.bicepparam -└── main.prod.bicepparam -``` - -## Validation Before Deployment - -Run `az bicep build --file infra/main.bicep` to validate syntax before deploying. - -## Correctness Checklist (must pass `az bicep build` with zero errors) - -Generate against these rules, then run `az bicep build` and fix in-place until clean. These are the -failures that most often break validation: - -1. **No undeclared symbols.** Every `param`, `var`, `resource`, and `module` symbol you reference is - declared in the same file. Cross-file values flow only through `module` params and `output`s. -2. **Cross-module outputs (BCP053).** When one module consumes `moduleX.outputs.Y`, that module MUST - declare `output Y ...`. Verify every consumed output exists on the producing module. -3. **`existing` references are complete.** Referenced resources use the `existing` keyword with the - correct type, `name` (and `scope`/`parent` where required); never emit a new `resource` for them. -4. **Required properties present.** Use the schema fetched in step 3 — include every required property - and use only allowed enum values and a valid, real `@apiVersion` for each type. -5. **Types match.** Parameter/variable types match their usage; no string passed where an object/int is - expected; array vs. single-object usage is consistent. -6. **`main.bicepparam` matches `main.bicep`.** Every `param` assigned in `.bicepparam` exists in - `main.bicep`; every required (non-defaulted) param is assigned; `using` points at `./main.bicep`. -7. **`targetScope` matches the deploy command** and any `resourceGroup()`/`subscription()` usage. -8. **No secrets in output.** Never `output` a secret; mark secret params `@secure()`. - -If `az bicep build` is unavailable, self-review every item above before presenting. +# Bicep Generation + +Generate Bicep IaC files from the approved infrastructure plan. + +> Important: All Bicep files must be created under `/infra/`. Never place `.bicep` files in the project root or in `.azure/`. + +## File Structure + +Generate files under `/infra/`: + +``` +infra/ +├── main.bicep # Orchestrator — deploys all modules +├── main.bicepparam # Parameter values +└── modules/ + ├── storage.bicep # One module per resource or logical group + ├── compute.bicep + ├── networking.bicep + └── monitoring.bicep +``` + +## Generation Steps + +1. Create `infra/` directory — create `/infra/` and `/infra/modules/` directories. All files in subsequent steps go here. +2. Read plan — load `/.azure/infrastructure-plan.json`, verify `meta.status === "approved"` +3. Fetch Bicep schemas — for each resource in the plan, use a sub-agent to call `bicepschema_get` with `resource-type` set to the ARM type from the relevant [resources/](resources/README.md) category file (e.g., `Microsoft.ContainerService/managedClusters`). Instruct the sub-agent: "Return the full property structure for {ARM type}: required properties, allowed values, child resources. ≤500 tokens." Use this output — not training data — to generate correct resource definitions. + +> The schema tool returns only the schema for the exact type requested. Sub-resource types (e.g., `Microsoft.Network/virtualNetworks/subnets`) return a smaller, focused schema but miss parent-level properties (e.g., VNet `encryption` lives on the parent, not the subnet sub-resource). Strategy: +> - Start with sub-resource types when validating child resources — smaller responses (~25KB vs ~95KB), easier to summarize +> - Fetch the parent type separately when you need parent-level properties (encryption, tags, SKU) — delegate to a sub-agent with specific property extraction instructions to manage the large response + +4. Generate modules — group resources by category; one `.bicep` file per group under `infra/modules/`. Use the schema from step 3 for property names, allowed values, and required fields. +5. Generate main.bicep — write `infra/main.bicep` that imports all modules and passes parameters +6. Generate parameters — create `infra/main.bicepparam` with environment-specific values + +## Bicep Conventions + +- Use `@description()` decorators on all parameters +- Use `@secure()` for secrets and connection strings +- Choose `targetScope` in `main.bicep` based on the deployment plan: + - For single resource group deployments, set `targetScope = 'resourceGroup'` and deploy with `az deployment group create`. + - For subscription-scope deployments (for example, resources across multiple resource groups or subscription-level resources), set `targetScope = 'subscription'` and deploy with `az deployment sub create`. +- Use `existing` keyword for referencing pre-existing resources +- Output resource IDs and endpoints needed by other resources +- Use `dependsOn` only when implicit dependencies are insufficient + +## Parameter File Format + +```bicep +using './main.bicep' + +param location = 'eastus' +param environmentName = 'prod' +param workloadName = 'datapipeline' +``` + +## Multi-Environment + +For multi-environment plans, generate one parameter file per environment: + +```txt +infra/ +├── main.bicep +├── main.dev.bicepparam +├── main.staging.bicepparam +└── main.prod.bicepparam +``` + +## Validation Before Deployment + +Run `az bicep build --file infra/main.bicep` to validate syntax before deploying. + +## Correctness Checklist (must pass `az bicep build` with zero errors) + +Generate against these rules, then run `az bicep build` and fix in-place until clean. These are the +failures that most often break validation: + +1. **No undeclared symbols.** Every `param`, `var`, `resource`, and `module` symbol you reference is + declared in the same file. Cross-file values flow only through `module` params and `output`s. +2. **Cross-module outputs (BCP053).** When one module consumes `moduleX.outputs.Y`, that module MUST + declare `output Y ...`. Verify every consumed output exists on the producing module. +3. **`existing` references are complete.** Referenced resources use the `existing` keyword with the + correct type, `name` (and `scope`/`parent` where required); never emit a new `resource` for them. +4. **Required properties present.** Use the schema fetched in step 3 — include every required property + and use only allowed enum values and a valid, real `@apiVersion` for each type. +5. **Types match.** Parameter/variable types match their usage; no string passed where an object/int is + expected; array vs. single-object usage is consistent. +6. **`main.bicepparam` matches `main.bicep`.** Every `param` assigned in `.bicepparam` exists in + `main.bicep`; every required (non-defaulted) param is assigned; `using` points at `./main.bicep`. +7. **`targetScope` matches the deploy command** and any `resourceGroup()`/`subscription()` usage. +8. **No secrets in output.** Never `output` a secret; mark secret params `@secure()`. + +If `az bicep build` is unavailable, self-review every item above before presenting. diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/README.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/README.md index c346aa22..31b85471 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/README.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/README.md @@ -1,15 +1,15 @@ -# Pairing Constraints Index - -| Sub-Category | File | Resources | -|-------------|------|-----------| -| AI & ML | [ai-ml.md](ai-ml.md) | AI Search, Cognitive Services, ML Workspace | -| Compute (PaaS) | [compute-apps.md](compute-apps.md) | App Service, ASP, Function App, Container App, CAE, ACR, Static Web App | -| Compute (IaaS) | [compute-infra.md](compute-infra.md) | AKS, VM, VMSS, Availability Set, Managed Disk | -| Data (Relational) | [data-relational.md](data-relational.md) | SQL Server, SQL Database, MySQL, PostgreSQL | -| Data (Analytics) | [data-analytics.md](data-analytics.md) | Cosmos DB, Redis, Storage, Data Factory, Synapse | -| Messaging | [messaging.md](messaging.md) | Event Grid, Event Hub, Service Bus | -| Monitoring | [monitoring.md](monitoring.md) | App Insights, Log Analytics | -| Networking (Core) | [networking-core.md](networking-core.md) | VNet, Subnet, NSG, Route Table, NIC, Public IP, NAT Gateway | -| Networking (Traffic) | [networking-traffic.md](networking-traffic.md) | App Gateway, Front Door, LB, APIM | -| Networking (Connectivity) | [networking-connectivity.md](networking-connectivity.md) | Bastion, Firewall, VPN GW, DNS, Private DNS, PE | -| Security | [security.md](security.md) | Key Vault, Managed Identity | +# Pairing Constraints Index + +| Sub-Category | File | Resources | +|-------------|------|-----------| +| AI & ML | [ai-ml.md](ai-ml.md) | AI Search, Cognitive Services, ML Workspace | +| Compute (PaaS) | [compute-apps.md](compute-apps.md) | App Service, ASP, Function App, Container App, CAE, ACR, Static Web App | +| Compute (IaaS) | [compute-infra.md](compute-infra.md) | AKS, VM, VMSS, Availability Set, Managed Disk | +| Data (Relational) | [data-relational.md](data-relational.md) | SQL Server, SQL Database, MySQL, PostgreSQL | +| Data (Analytics) | [data-analytics.md](data-analytics.md) | Cosmos DB, Redis, Storage, Data Factory, Synapse | +| Messaging | [messaging.md](messaging.md) | Event Grid, Event Hub, Service Bus | +| Monitoring | [monitoring.md](monitoring.md) | App Insights, Log Analytics | +| Networking (Core) | [networking-core.md](networking-core.md) | VNet, Subnet, NSG, Route Table, NIC, Public IP, NAT Gateway | +| Networking (Traffic) | [networking-traffic.md](networking-traffic.md) | App Gateway, Front Door, LB, APIM | +| Networking (Connectivity) | [networking-connectivity.md](networking-connectivity.md) | Bastion, Firewall, VPN GW, DNS, Private DNS, PE | +| Security | [security.md](security.md) | Key Vault, Managed Identity | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/ai-ml.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/ai-ml.md index 3fb42780..1fe3b2c6 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/ai-ml.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/ai-ml.md @@ -1,36 +1,36 @@ -# AI & ML Pairing Constraints - -### AI Search - -| Paired With | Constraint | -|-------------|------------| -| **Cognitive Services (AI Enrichment)** | For AI enrichment pipelines (skillsets), attach a Cognitive Services account. Must be `kind: 'CognitiveServices'` or `kind: 'AIServices'` and in the same region as the search service. | -| **Storage Account (Indexers)** | Indexer data sources support Blob, Table, and File storage. Storage must be accessible (same VNet or public). For managed identity access, assign `Storage Blob Data Reader` role. | -| **Cosmos DB (Indexers)** | Indexer data source for Cosmos DB. Requires connection string or managed identity with `Cosmos DB Account Reader Role`. | -| **SQL Database (Indexers)** | Indexer data source. Requires change tracking enabled on the source table. | -| **Private Endpoints** | When `publicNetworkAccess: 'Disabled'`, create shared private link resources for outbound connections to data sources. | -| **Managed Identity** | Assign system or user-assigned identity for secure connections to data sources. Use RBAC instead of connection strings. | -| **Semantic Search** | Requires `semanticSearch` property set to `free` or `standard`. Available on `basic` and above SKUs. | - -### Cognitive Services - -| Paired With | Constraint | -|-------------|------------| -| **Azure OpenAI Deployments** | When `kind: 'OpenAI'` or `kind: 'AIServices'`, create model deployments as child resource `accounts/deployments`. | -| **Microsoft Entra ID Auth** | Requires `customSubDomainName` to be set. Without it, only API key auth works. | -| **Private Endpoint** | Requires `customSubDomainName`. Set `publicNetworkAccess: 'Disabled'` and configure private DNS zone. | -| **Key Vault (CMK)** | When using customer-managed keys, Key Vault must have soft-delete and purge protection enabled. Set `encryption.keySource: 'Microsoft.KeyVault'`. | -| **Storage Account** | When using `userOwnedStorage`, the storage account must be in the same region. Required for certain features (e.g., batch translation). | -| **AI Foundry Hub** | When `kind: 'AIServices'` with `allowProjectManagement: true`, can manage Foundry projects as child resources (`accounts/projects`). | -| **VNet Integration** | Configure `networkAcls` with `defaultAction: 'Deny'` and add virtual network rules. Set `bypass: 'AzureServices'` to allow trusted Azure services. | - -### ML Workspace - -| Paired With | Constraint | -|-------------|------------| -| **Storage Account** | Must be linked via `properties.storageAccount`. Cannot change after creation. Use `StorageV2` kind with standard SKU. | -| **Key Vault** | Must be linked via `properties.keyVault`. Cannot change after creation. Requires soft-delete enabled. | -| **Application Insights** | Linked via `properties.applicationInsights`. Should use workspace-based App Insights (backed by Log Analytics). | -| **Container Registry** | Optional but recommended for custom environments. Linked via `properties.containerRegistry`. | -| **Hub workspace (kind=Project)** | Must set `properties.hubResourceId` to the parent Hub's ARM resource ID. The Project inherits the Hub's linked resources. | -| **VNet Integration** | When `managedNetwork.isolationMode` is `AllowOnlyApprovedOutbound`, must configure outbound rules for all dependent services. | +# AI & ML Pairing Constraints + +### AI Search + +| Paired With | Constraint | +|-------------|------------| +| **Cognitive Services (AI Enrichment)** | For AI enrichment pipelines (skillsets), attach a Cognitive Services account. Must be `kind: 'CognitiveServices'` or `kind: 'AIServices'` and in the same region as the search service. | +| **Storage Account (Indexers)** | Indexer data sources support Blob, Table, and File storage. Storage must be accessible (same VNet or public). For managed identity access, assign `Storage Blob Data Reader` role. | +| **Cosmos DB (Indexers)** | Indexer data source for Cosmos DB. Requires connection string or managed identity with `Cosmos DB Account Reader Role`. | +| **SQL Database (Indexers)** | Indexer data source. Requires change tracking enabled on the source table. | +| **Private Endpoints** | When `publicNetworkAccess: 'Disabled'`, create shared private link resources for outbound connections to data sources. | +| **Managed Identity** | Assign system or user-assigned identity for secure connections to data sources. Use RBAC instead of connection strings. | +| **Semantic Search** | Requires `semanticSearch` property set to `free` or `standard`. Available on `basic` and above SKUs. | + +### Cognitive Services + +| Paired With | Constraint | +|-------------|------------| +| **Azure OpenAI Deployments** | When `kind: 'OpenAI'` or `kind: 'AIServices'`, create model deployments as child resource `accounts/deployments`. | +| **Microsoft Entra ID Auth** | Requires `customSubDomainName` to be set. Without it, only API key auth works. | +| **Private Endpoint** | Requires `customSubDomainName`. Set `publicNetworkAccess: 'Disabled'` and configure private DNS zone. | +| **Key Vault (CMK)** | When using customer-managed keys, Key Vault must have soft-delete and purge protection enabled. Set `encryption.keySource: 'Microsoft.KeyVault'`. | +| **Storage Account** | When using `userOwnedStorage`, the storage account must be in the same region. Required for certain features (e.g., batch translation). | +| **AI Foundry Hub** | When `kind: 'AIServices'` with `allowProjectManagement: true`, can manage Foundry projects as child resources (`accounts/projects`). | +| **VNet Integration** | Configure `networkAcls` with `defaultAction: 'Deny'` and add virtual network rules. Set `bypass: 'AzureServices'` to allow trusted Azure services. | + +### ML Workspace + +| Paired With | Constraint | +|-------------|------------| +| **Storage Account** | Must be linked via `properties.storageAccount`. Cannot change after creation. Use `StorageV2` kind with standard SKU. | +| **Key Vault** | Must be linked via `properties.keyVault`. Cannot change after creation. Requires soft-delete enabled. | +| **Application Insights** | Linked via `properties.applicationInsights`. Should use workspace-based App Insights (backed by Log Analytics). | +| **Container Registry** | Optional but recommended for custom environments. Linked via `properties.containerRegistry`. | +| **Hub workspace (kind=Project)** | Must set `properties.hubResourceId` to the parent Hub's ARM resource ID. The Project inherits the Hub's linked resources. | +| **VNet Integration** | When `managedNetwork.isolationMode` is `AllowOnlyApprovedOutbound`, must configure outbound rules for all dependent services. | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/compute-apps.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/compute-apps.md index d2c0f13c..66e53e1b 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/compute-apps.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/compute-apps.md @@ -1,89 +1,89 @@ -# Compute (PaaS) Pairing Constraints - -### App Service - -| Paired With | Constraint | -|-------------|------------| -| **App Service Plan** | Must be in the same region. Linux apps need Linux plan (`reserved: true`). Windows apps need Windows plan. | -| **Deployment Slots** | Only available on Standard or higher plan tiers. Free and Basic do not support slots. | -| **VNet Integration** | Requires Basic or higher plan tier. Subnet must be delegated to `Microsoft.Web/serverFarms`. VNet integration subnet must be a different subnet than any Private Endpoint subnet. | -| **Private Endpoints** | Requires Basic or higher plan tier. Not available on Free or Shared tiers. | -| **Custom Domain** | Requires Shared (D1) or higher tier for custom domains. Free tier only supports `*.azurewebsites.net`. Managed certificates require Basic or higher. | -| **Application Insights** | Set `APPLICATIONINSIGHTS_CONNECTION_STRING` in app settings. | -| **Key Vault References** | Use `@Microsoft.KeyVault(SecretUri=...)` in app settings. Requires managed identity with Key Vault access. | -| **Managed Identity** | Enable `identity.type: 'SystemAssigned'` or `'UserAssigned'` for passwordless auth to other Azure resources. | - -### App Service Plan - -| Paired With | Constraint | -|-------------|------------| -| **Function App** | Consumption (`Y1`) and Flex (`FC1`) plans cannot be shared with web apps. EP plans can host both functions and web apps. | -| **Linux Apps** | Linux plan (`reserved: true`) cannot host Windows apps and vice versa. | -| **Zone Redundancy** | Requires Premium v3 (`P1v3`+) or Isolated v2. Minimum 3 instances. | -| **Deployment Slots** | Slots share plan capacity. Standard+ tier required. Slots are not available on Free/Basic. | -| **Auto-scale** | Not available on Free/Shared/Basic. Standard+ required for manual scale, auto-scale. | -| **VNet Integration** | Requires Basic or higher. Subnet must be delegated to `Microsoft.Web/serverFarms`. Minimum subnet size /28 (or /26 for multi-plan subnet join). VNet integration subnet must be a different subnet than any Private Endpoint subnet. | -| **Private Endpoints** | Requires Basic tier or higher. Not available on Free or Shared tiers. | -| **Isolated Compute** | Dedicated single-tenant compute requires IsolatedV2 (`I1v2`+) tier. | -| **Free/Shared Tiers** | Free (`F1`) and Shared (`D1`) use shared compute with no VNet integration, no private endpoints, no deployment slots, no Always On, and no auto-scale. Managed Identity is available but limited. | - -### Container App - -| Paired With | Constraint | -|-------------|------------| -| **Container Apps Environment** | Must reference `environmentId`. Environment must exist in the same region. | -| **VNet** | VNet integration is configured on the **Environment**, not the individual app. Environment needs a dedicated subnet with minimum /23 prefix for Consumption-only environments or /27 for workload profiles environments. | -| **Container Registry** | Requires registry credentials in `configuration.registries[]` or managed identity-based pull. | -| **Dapr** | Enable via `configuration.dapr.enabled: true`. Dapr components are configured on the Environment. | -| **CPU/Memory** | CPU and memory must follow valid combinations: 0.25 cores/0.5Gi, 0.5/1Gi, 1/2Gi, 2/4Gi, 4/8Gi (consumption). | -| **Scale Rules** | KEDA-based scale rules reference secrets by name — secrets must be defined in `configuration.secrets[]`. | - -### Container Apps Environment - -| Paired With | Constraint | -|-------------|------------| -| **Container App** | Container Apps reference the environment via `properties.environmentId`. Apps and environment must be in the same region. | -| **Log Analytics Workspace** | Provide `customerId` and `sharedKey` in `appLogsConfiguration`. Workspace must exist before the environment. | -| **VNet / Subnet** | Subnet must have a minimum /23 prefix for Consumption-only environments or /27 for workload profiles environments. Subnet must be dedicated to the Container Apps Environment (no other resources). Workload Profiles: subnet must be delegated to `Microsoft.App/environments`. Consumption-only: subnet MUST NOT be delegated to any service. | -| **Zone Redundancy** | Requires VNet integration. Zone-redundant environments need a /23 subnet in a region with availability zones. | -| **Internal Environment** | When `internal: true`, no public endpoint is created. Requires custom DNS or Private DNS Zone and a VNet with connectivity to clients. | -| **Workload Profiles** | At least one `Consumption` profile must be defined when using workload profiles. Dedicated profiles require `minimumCount` and `maximumCount`. | -| **Workload Profiles vs Consumption-only** | UDR support, NAT Gateway egress, private endpoints, and remote gateway peering are only available with Workload Profiles environments — not Consumption-only. | -| **Network Immutability** | Network type (Workload Profiles vs Consumption-only) is immutable after creation. Cannot change between environment types. | -| **IPv6** | IPv6 is not supported for either Workload Profiles or Consumption-only environments. | -| **VNet Move** | VNet-integrated environments cannot be moved to a different resource group or subscription while in use. | - -### Container Registry - -| Paired With | Constraint | -|-------------|------------| -| **AKS** | AKS needs `acrPull` role assignment on the registry. Use managed identity (attach via `az aks update --attach-acr`). | -| **Container App** | Reference in `configuration.registries[]`. Use managed identity or admin credentials. | -| **ML Workspace** | Referenced as `containerRegistry` property. Used for custom training/inference images. | -| **Private Endpoint** | Premium SKU required. Set `publicNetworkAccess: 'Disabled'`. | -| **Geo-Replication** | Premium SKU required. Configure via child `replications` resource. | -| **CMK** | Premium SKU required. Needs user-assigned identity with Key Vault access. | - -### Function App - -| Paired With | Constraint | -|-------------|------------| -| **Storage Account** | Must use `StorageV2` or `Storage` kind. `BlobStorage`, `BlockBlobStorage`, `FileStorage` not supported (need Queue + Table). | -| **Storage (Consumption)** | Consumption plan cannot use VNet-secured storage. Only Premium/Dedicated support VNet-restricted storage. | -| **Storage (ZRS)** | Zone-redundant functions require `Standard_ZRS` storage SKU. | -| **App Service Plan** | Plan must be in the same region. Linux functions need Linux plan (`reserved: true`). | -| **VNet Integration** | Requires Premium (EP) or Dedicated plan. Consumption does not support VNet integration (use Flex Consumption). | -| **Application Insights** | Set `APPINSIGHTS_INSTRUMENTATIONKEY` or `APPLICATIONINSIGHTS_CONNECTION_STRING` in app settings. | -| **Key Vault References** | App settings can use `@Microsoft.KeyVault(SecretUri=...)` syntax. Requires managed identity with Key Vault access. | - -### Static Web App - -| Paired With | Constraint | -|-------------|------------| -| **GitHub Repository** | Provide `repositoryUrl`, `branch`, and `repositoryToken`. A GitHub Actions workflow is auto-created in the repo. | -| **Azure DevOps** | Set `provider: 'DevOps'`. Provide `repositoryUrl` and `branch`. Pipeline is configured separately. | -| **Azure Functions (managed)** | API location in `buildProperties.apiLocation` deploys a managed Functions backend. Limited to HTTP triggers, C#, JavaScript, Python, Java. | -| **Linked Backend** | Use `linkedBackends` child resource to connect an existing Function App, Container App, or App Service as the API backend. Standard SKU required. | -| **Private Endpoint** | Only available with `Standard` SKU. Set up a Private Endpoint to restrict access to the static web app. | -| **Custom Domain** | Custom domains are child resources. Require DNS CNAME or TXT validation. Free SSL certificates are auto-provisioned. | -| **Enterprise-Grade CDN** | `Standard` SKU only. Enables Azure Front Door integration for advanced caching and edge capabilities. | +# Compute (PaaS) Pairing Constraints + +### App Service + +| Paired With | Constraint | +|-------------|------------| +| **App Service Plan** | Must be in the same region. Linux apps need Linux plan (`reserved: true`). Windows apps need Windows plan. | +| **Deployment Slots** | Only available on Standard or higher plan tiers. Free and Basic do not support slots. | +| **VNet Integration** | Requires Basic or higher plan tier. Subnet must be delegated to `Microsoft.Web/serverFarms`. VNet integration subnet must be a different subnet than any Private Endpoint subnet. | +| **Private Endpoints** | Requires Basic or higher plan tier. Not available on Free or Shared tiers. | +| **Custom Domain** | Requires Shared (D1) or higher tier for custom domains. Free tier only supports `*.azurewebsites.net`. Managed certificates require Basic or higher. | +| **Application Insights** | Set `APPLICATIONINSIGHTS_CONNECTION_STRING` in app settings. | +| **Key Vault References** | Use `@Microsoft.KeyVault(SecretUri=...)` in app settings. Requires managed identity with Key Vault access. | +| **Managed Identity** | Enable `identity.type: 'SystemAssigned'` or `'UserAssigned'` for passwordless auth to other Azure resources. | + +### App Service Plan + +| Paired With | Constraint | +|-------------|------------| +| **Function App** | Consumption (`Y1`) and Flex (`FC1`) plans cannot be shared with web apps. EP plans can host both functions and web apps. | +| **Linux Apps** | Linux plan (`reserved: true`) cannot host Windows apps and vice versa. | +| **Zone Redundancy** | Requires Premium v3 (`P1v3`+) or Isolated v2. Minimum 3 instances. | +| **Deployment Slots** | Slots share plan capacity. Standard+ tier required. Slots are not available on Free/Basic. | +| **Auto-scale** | Not available on Free/Shared/Basic. Standard+ required for manual scale, auto-scale. | +| **VNet Integration** | Requires Basic or higher. Subnet must be delegated to `Microsoft.Web/serverFarms`. Minimum subnet size /28 (or /26 for multi-plan subnet join). VNet integration subnet must be a different subnet than any Private Endpoint subnet. | +| **Private Endpoints** | Requires Basic tier or higher. Not available on Free or Shared tiers. | +| **Isolated Compute** | Dedicated single-tenant compute requires IsolatedV2 (`I1v2`+) tier. | +| **Free/Shared Tiers** | Free (`F1`) and Shared (`D1`) use shared compute with no VNet integration, no private endpoints, no deployment slots, no Always On, and no auto-scale. Managed Identity is available but limited. | + +### Container App + +| Paired With | Constraint | +|-------------|------------| +| **Container Apps Environment** | Must reference `environmentId`. Environment must exist in the same region. | +| **VNet** | VNet integration is configured on the **Environment**, not the individual app. Environment needs a dedicated subnet with minimum /23 prefix for Consumption-only environments or /27 for workload profiles environments. | +| **Container Registry** | Requires registry credentials in `configuration.registries[]` or managed identity-based pull. | +| **Dapr** | Enable via `configuration.dapr.enabled: true`. Dapr components are configured on the Environment. | +| **CPU/Memory** | CPU and memory must follow valid combinations: 0.25 cores/0.5Gi, 0.5/1Gi, 1/2Gi, 2/4Gi, 4/8Gi (consumption). | +| **Scale Rules** | KEDA-based scale rules reference secrets by name — secrets must be defined in `configuration.secrets[]`. | + +### Container Apps Environment + +| Paired With | Constraint | +|-------------|------------| +| **Container App** | Container Apps reference the environment via `properties.environmentId`. Apps and environment must be in the same region. | +| **Log Analytics Workspace** | Provide `customerId` and `sharedKey` in `appLogsConfiguration`. Workspace must exist before the environment. | +| **VNet / Subnet** | Subnet must have a minimum /23 prefix for Consumption-only environments or /27 for workload profiles environments. Subnet must be dedicated to the Container Apps Environment (no other resources). Workload Profiles: subnet must be delegated to `Microsoft.App/environments`. Consumption-only: subnet MUST NOT be delegated to any service. | +| **Zone Redundancy** | Requires VNet integration. Zone-redundant environments need a /23 subnet in a region with availability zones. | +| **Internal Environment** | When `internal: true`, no public endpoint is created. Requires custom DNS or Private DNS Zone and a VNet with connectivity to clients. | +| **Workload Profiles** | At least one `Consumption` profile must be defined when using workload profiles. Dedicated profiles require `minimumCount` and `maximumCount`. | +| **Workload Profiles vs Consumption-only** | UDR support, NAT Gateway egress, private endpoints, and remote gateway peering are only available with Workload Profiles environments — not Consumption-only. | +| **Network Immutability** | Network type (Workload Profiles vs Consumption-only) is immutable after creation. Cannot change between environment types. | +| **IPv6** | IPv6 is not supported for either Workload Profiles or Consumption-only environments. | +| **VNet Move** | VNet-integrated environments cannot be moved to a different resource group or subscription while in use. | + +### Container Registry + +| Paired With | Constraint | +|-------------|------------| +| **AKS** | AKS needs `acrPull` role assignment on the registry. Use managed identity (attach via `az aks update --attach-acr`). | +| **Container App** | Reference in `configuration.registries[]`. Use managed identity or admin credentials. | +| **ML Workspace** | Referenced as `containerRegistry` property. Used for custom training/inference images. | +| **Private Endpoint** | Premium SKU required. Set `publicNetworkAccess: 'Disabled'`. | +| **Geo-Replication** | Premium SKU required. Configure via child `replications` resource. | +| **CMK** | Premium SKU required. Needs user-assigned identity with Key Vault access. | + +### Function App + +| Paired With | Constraint | +|-------------|------------| +| **Storage Account** | Must use `StorageV2` or `Storage` kind. `BlobStorage`, `BlockBlobStorage`, `FileStorage` not supported (need Queue + Table). | +| **Storage (Consumption)** | Consumption plan cannot use VNet-secured storage. Only Premium/Dedicated support VNet-restricted storage. | +| **Storage (ZRS)** | Zone-redundant functions require `Standard_ZRS` storage SKU. | +| **App Service Plan** | Plan must be in the same region. Linux functions need Linux plan (`reserved: true`). | +| **VNet Integration** | Requires Premium (EP) or Dedicated plan. Consumption does not support VNet integration (use Flex Consumption). | +| **Application Insights** | Set `APPINSIGHTS_INSTRUMENTATIONKEY` or `APPLICATIONINSIGHTS_CONNECTION_STRING` in app settings. | +| **Key Vault References** | App settings can use `@Microsoft.KeyVault(SecretUri=...)` syntax. Requires managed identity with Key Vault access. | + +### Static Web App + +| Paired With | Constraint | +|-------------|------------| +| **GitHub Repository** | Provide `repositoryUrl`, `branch`, and `repositoryToken`. A GitHub Actions workflow is auto-created in the repo. | +| **Azure DevOps** | Set `provider: 'DevOps'`. Provide `repositoryUrl` and `branch`. Pipeline is configured separately. | +| **Azure Functions (managed)** | API location in `buildProperties.apiLocation` deploys a managed Functions backend. Limited to HTTP triggers, C#, JavaScript, Python, Java. | +| **Linked Backend** | Use `linkedBackends` child resource to connect an existing Function App, Container App, or App Service as the API backend. Standard SKU required. | +| **Private Endpoint** | Only available with `Standard` SKU. Set up a Private Endpoint to restrict access to the static web app. | +| **Custom Domain** | Custom domains are child resources. Require DNS CNAME or TXT validation. Free SSL certificates are auto-provisioned. | +| **Enterprise-Grade CDN** | `Standard` SKU only. Enables Azure Front Door integration for advanced caching and edge capabilities. | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/compute-infra.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/compute-infra.md index 0bbe5f24..b5640bfa 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/compute-infra.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/compute-infra.md @@ -1,57 +1,57 @@ -# Compute (IaaS) Pairing Constraints - -### AKS Cluster - -| Paired With | Constraint | -|-------------|------------| -| **VNet / Subnet** | With Azure CNI, subnet must have enough IPs for nodes + pods (30 pods/node default × node count). Subnet cannot have other delegations. Reserved CIDR ranges cannot be used: `169.254.0.0/16`, `172.30.0.0/16`, `172.31.0.0/16`, `192.0.2.0/24`. | -| **Pod CIDR** | Pod CIDR must not overlap with cluster subnet, peered VNets, ExpressRoute, or VPN address spaces. Overlapping causes SNAT/routing issues. | -| **kubenet** | Kubenet uses NAT — subnet only needs IPs for nodes. Less IP pressure but no direct pod-to-VNet connectivity. Kubenet is retiring March 2028 — migrate to CNI Overlay. Not supported by Application Gateway for Containers. | -| **CNI Overlay** | CNI Overlay does not support VM availability sets (must use VMSS-based node pools), virtual nodes, or DCsv2-series VMs (use DCasv5/DCadsv5 instead). | -| **Dual-stack CNI Overlay** | IPv4+IPv6 dual-stack disables Azure/Calico network policies, NAT gateway, and virtual nodes. | -| **Key Vault** | Enable `azureKeyvaultSecretsProvider` addon. Use `enableRbacAuthorization: true` on Key Vault with managed identity. | -| **Container Registry** | Attach ACR via `acrPull` role assignment on cluster identity, or use `imagePullSecrets`. | -| **Log Analytics** | Enable `omsagent` addon with `config.logAnalyticsWorkspaceResourceID` pointing to workspace. | -| **Load Balancer** | AKS creates a managed Standard LB by default (`loadBalancerSku: 'standard'`). | -| **System Pool** | At least one agent pool must have `mode: 'System'`. System pools run critical pods (CoreDNS, tunnelfront). | - -### Availability Set - -| Paired With | Constraint | -|-------------|------------| -| **Virtual Machine** | VMs must be in the same resource group. Set `vm.properties.availabilitySet.id`. | -| **Availability Zones** | Cannot combine with zones — availability zones supersede availability sets for zone-redundant architectures. | -| **Managed Disks** | `sku.name` must be `Aligned` when VMs use managed disks. | -| **VM Scale Set** | A VM cannot be in both an availability set and a VMSS. | - -### Managed Disk - -| Paired With | Constraint | -|-------------|------------| -| **Virtual Machine** | Attach via `storageProfile.osDisk` or `storageProfile.dataDisks`. Disk must be in same region. | -| **Availability Zone** | `PremiumV2_LRS` and `UltraSSD_LRS` require zone specification. | -| **Premium SSD v2** | Cannot be used as OS disk (data disks only). Does not support host caching (`ReadOnly`/`ReadWrite` unavailable). Requires zonal VM deployment. Cannot mix with other storage types on SQL Server VMs. | -| **Key Vault (CMK)** | Requires a Disk Encryption Set pointing to Key Vault key. Key Vault must have purge protection enabled. | - -### Virtual Machine - -| Paired With | Constraint | -|-------------|------------| -| **NIC** | At least one NIC required via `networkProfile.networkInterfaces`. NIC must be in the same region. | -| **Availability Set** | Cannot combine with `virtualMachineScaleSet` or availability zones. Set `availabilitySet.id`. | -| **Availability Zone** | Cannot combine with availability sets. Set `zones: ['1']` (string array). | -| **Managed Disk (Premium SSD)** | Not all VM sizes support Premium storage — check size docs for compatibility. | -| **Managed Disk (UltraSSD)** | Requires `additionalCapabilities.ultraSSDEnabled: true`. Cannot enable on a running VM — requires stop/deallocate first. | -| **Managed Disk (Premium SSD v2)** | Premium SSD v2 cannot be used as OS disk (data disks only). Does not support host caching (ReadOnly/ReadWrite unavailable). Requires zonal VM deployment. Cannot mix Premium SSD v2 with other storage types on SQL Server VMs. | -| **Dedicated Host** | Cannot specify both `host` and `hostGroup`. | -| **Boot Diagnostics Storage** | Cannot use Premium or ZRS storage. Use `Standard_LRS` or `Standard_GRS`. | -| **CNI Overlay (AKS)** | DCsv2-series VMs are not supported with Azure CNI Overlay. Use DCasv5/DCadsv5 for confidential computing. | - -### VM Scale Set - -| Paired With | Constraint | -|-------------|------------| -| **Subnet** | Network interfaces defined inline in `virtualMachineProfile.networkProfile`. Subnet must be in same region. | -| **Load Balancer** | Reference backend pool ID in NIC IP configuration. | -| **Orchestration Mode** | `Flexible` is the modern default. `Uniform` requires `upgradePolicy`. | -| **Availability Zone** | Set `zones: ['1', '2', '3']` for zone distribution. Cannot combine with availability sets. | +# Compute (IaaS) Pairing Constraints + +### AKS Cluster + +| Paired With | Constraint | +|-------------|------------| +| **VNet / Subnet** | With Azure CNI, subnet must have enough IPs for nodes + pods (30 pods/node default × node count). Subnet cannot have other delegations. Reserved CIDR ranges cannot be used: `169.254.0.0/16`, `172.30.0.0/16`, `172.31.0.0/16`, `192.0.2.0/24`. | +| **Pod CIDR** | Pod CIDR must not overlap with cluster subnet, peered VNets, ExpressRoute, or VPN address spaces. Overlapping causes SNAT/routing issues. | +| **kubenet** | Kubenet uses NAT — subnet only needs IPs for nodes. Less IP pressure but no direct pod-to-VNet connectivity. Kubenet is retiring March 2028 — migrate to CNI Overlay. Not supported by Application Gateway for Containers. | +| **CNI Overlay** | CNI Overlay does not support VM availability sets (must use VMSS-based node pools), virtual nodes, or DCsv2-series VMs (use DCasv5/DCadsv5 instead). | +| **Dual-stack CNI Overlay** | IPv4+IPv6 dual-stack disables Azure/Calico network policies, NAT gateway, and virtual nodes. | +| **Key Vault** | Enable `azureKeyvaultSecretsProvider` addon. Use `enableRbacAuthorization: true` on Key Vault with managed identity. | +| **Container Registry** | Attach ACR via `acrPull` role assignment on cluster identity, or use `imagePullSecrets`. | +| **Log Analytics** | Enable `omsagent` addon with `config.logAnalyticsWorkspaceResourceID` pointing to workspace. | +| **Load Balancer** | AKS creates a managed Standard LB by default (`loadBalancerSku: 'standard'`). | +| **System Pool** | At least one agent pool must have `mode: 'System'`. System pools run critical pods (CoreDNS, tunnelfront). | + +### Availability Set + +| Paired With | Constraint | +|-------------|------------| +| **Virtual Machine** | VMs must be in the same resource group. Set `vm.properties.availabilitySet.id`. | +| **Availability Zones** | Cannot combine with zones — availability zones supersede availability sets for zone-redundant architectures. | +| **Managed Disks** | `sku.name` must be `Aligned` when VMs use managed disks. | +| **VM Scale Set** | A VM cannot be in both an availability set and a VMSS. | + +### Managed Disk + +| Paired With | Constraint | +|-------------|------------| +| **Virtual Machine** | Attach via `storageProfile.osDisk` or `storageProfile.dataDisks`. Disk must be in same region. | +| **Availability Zone** | `PremiumV2_LRS` and `UltraSSD_LRS` require zone specification. | +| **Premium SSD v2** | Cannot be used as OS disk (data disks only). Does not support host caching (`ReadOnly`/`ReadWrite` unavailable). Requires zonal VM deployment. Cannot mix with other storage types on SQL Server VMs. | +| **Key Vault (CMK)** | Requires a Disk Encryption Set pointing to Key Vault key. Key Vault must have purge protection enabled. | + +### Virtual Machine + +| Paired With | Constraint | +|-------------|------------| +| **NIC** | At least one NIC required via `networkProfile.networkInterfaces`. NIC must be in the same region. | +| **Availability Set** | Cannot combine with `virtualMachineScaleSet` or availability zones. Set `availabilitySet.id`. | +| **Availability Zone** | Cannot combine with availability sets. Set `zones: ['1']` (string array). | +| **Managed Disk (Premium SSD)** | Not all VM sizes support Premium storage — check size docs for compatibility. | +| **Managed Disk (UltraSSD)** | Requires `additionalCapabilities.ultraSSDEnabled: true`. Cannot enable on a running VM — requires stop/deallocate first. | +| **Managed Disk (Premium SSD v2)** | Premium SSD v2 cannot be used as OS disk (data disks only). Does not support host caching (ReadOnly/ReadWrite unavailable). Requires zonal VM deployment. Cannot mix Premium SSD v2 with other storage types on SQL Server VMs. | +| **Dedicated Host** | Cannot specify both `host` and `hostGroup`. | +| **Boot Diagnostics Storage** | Cannot use Premium or ZRS storage. Use `Standard_LRS` or `Standard_GRS`. | +| **CNI Overlay (AKS)** | DCsv2-series VMs are not supported with Azure CNI Overlay. Use DCasv5/DCadsv5 for confidential computing. | + +### VM Scale Set + +| Paired With | Constraint | +|-------------|------------| +| **Subnet** | Network interfaces defined inline in `virtualMachineProfile.networkProfile`. Subnet must be in same region. | +| **Load Balancer** | Reference backend pool ID in NIC IP configuration. | +| **Orchestration Mode** | `Flexible` is the modern default. `Uniform` requires `upgradePolicy`. | +| **Availability Zone** | Set `zones: ['1', '2', '3']` for zone distribution. Cannot combine with availability sets. | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/data-analytics.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/data-analytics.md index 693a48e6..81efb574 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/data-analytics.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/data-analytics.md @@ -1,63 +1,63 @@ -# Data (Analytics) Pairing Constraints - -### Cosmos DB - -| Paired With | Constraint | -|-------------|------------| -| **Multi-region writes** | `consistencyPolicy.defaultConsistencyLevel` cannot be `Strong` when `enableMultipleWriteLocations: true`. | -| **Strong consistency** | Strong consistency with regions >5000 miles apart is blocked by default (requires support ticket to enable). Strong and Bounded Staleness reads cost 2× RU/s compared to Session/Consistent Prefix/Eventual. | -| **Serverless** | Cannot combine `EnableServerless` capability with multi-region writes or analytical store. Serverless is single-region only — cannot add regions. No shared throughput databases. Cannot provision throughput (auto-managed; settings return error). Merge partitions not available for serverless accounts. | -| **Free tier** | Only one free-tier account per subscription. Cannot combine with multi-region writes. | -| **VNet** | Set `isVirtualNetworkFilterEnabled: true` and configure `virtualNetworkRules[]` with subnet IDs. Subnets need `Microsoft.AzureCosmosDB` service endpoint. | -| **Private Endpoint** | Set `publicNetworkAccess: 'Disabled'` when using private endpoints exclusively. One Private DNS Zone record per DNS name — multiple private endpoints in different regions need separate Private DNS Zones. | -| **Key Vault (CMK)** | Requires `keyVaultKeyUri` in encryption config. Key Vault must be in same region. | -| **Merge Partitions** | Not available for serverless or multi-region write accounts. Single-region provisioned throughput only. | - -### Redis Cache - -| Paired With | Constraint | -|-------------|------------| -| **VNet** | Only Premium SKU supports VNet injection via `subnetId`. Basic/Standard use firewall rules only. | -| **VNet + Private Endpoint** | VNet injection and private endpoint are mutually exclusive — cannot use both on the same cache. | -| **Private Endpoint** | Available for Basic, Standard, Premium, and Enterprise tiers. Set `publicNetworkAccess: 'Disabled'` when using private endpoints. Premium with clustering supports max 1 private link; non-clustered supports up to 100. | -| **Clustering** | Only Premium SKU supports `shardCount`. Basic and Standard are single-node/two-node only. | -| **Persistence** | Only Premium SKU supports RDB/AOF persistence. Requires a storage account for RDB exports. | -| **Geo-replication** | Only Premium SKU. Primary and secondary must be Premium with same shard count. Passive geo-replication with private endpoints requires unlinking geo-replication first, adding private link, then re-linking. | -| **Zones** | Zone redundancy requires Premium SKU with multiple replicas. | -| **Tier Scaling** | Cannot scale down tiers (Enterprise → lower, Premium → Standard/Basic, Standard → Basic). Cannot scale between Enterprise and Enterprise Flash, or from Basic/Standard/Premium to Enterprise/Flash — must create a new cache. | -| **Enterprise/Flash** | Firewall rules and `publicNetworkAccess` flag are not available on Enterprise/Enterprise Flash tiers. | -| **Azure Lighthouse** | Azure Lighthouse + VNet injection is not supported. Use private links instead. | - -### Storage Account - -| Paired With | Constraint | -|-------------|------------| -| **Azure Functions** | Must use `StorageV2` or `Storage` kind. `BlobStorage`, `BlockBlobStorage`, `FileStorage` not supported (missing Queue/Table). | -| **Functions (Consumption plan)** | Cannot use network-secured storage (VNet rules). Only Premium/Dedicated plans support VNet-restricted storage. | -| **Functions (zone-redundant)** | Must use ZRS SKU (`Standard_ZRS`). LRS/GRS not sufficient. | -| **VM Boot Diagnostics** | Cannot use Premium storage or ZRS. Use `Standard_LRS` or `Standard_GRS`. Managed boot diagnostics (no storage account required) is also available. | -| **CMK Encryption** | Key Vault must have `enableSoftDelete: true` AND `enablePurgeProtection: true`. | -| **CMK at creation** | Requires user-assigned managed identity (system-assigned only works for existing accounts). | -| **Geo-redundant failover** | Certain features (SFTP, NFS 3.0, etc.) block GRS/GZRS failover. | - -### Data Factory - -| Paired With | Constraint | -|-------------|------------| -| **Storage Account** | Linked service requires `Storage Blob Data Contributor` role on the storage account for the ADF managed identity. For ADLS Gen2, also requires `Storage Blob Data Reader` at minimum. | -| **Key Vault** | For CMK encryption, Key Vault must have `enableSoftDelete: true` and `enablePurgeProtection: true`. ADF managed identity needs `Key Vault Crypto Service Encryption User` role or equivalent access policy. | -| **Managed VNet** | When `managedVirtualNetworks` is configured, all outbound connections must use managed private endpoints (`factories/managedVirtualNetworks/managedPrivateEndpoints`). | -| **Private Endpoint** | When `publicNetworkAccess: 'Disabled'`, must create private endpoint to `dataFactory` sub-resource for studio access and pipeline connectivity. | -| **Purview** | Requires Microsoft Purview instance resource ID. ADF managed identity must have `Data Curator` role in Purview. | -| **Integration Runtime** | Self-hosted IR requires network line-of-sight to on-premises sources. Azure IR regional choice affects data residency. | - -### Synapse Workspace - -| Paired With | Constraint | -|-------------|------------| -| **ADLS Gen2 Storage Account** | **Required.** Storage account must have `isHnsEnabled: true` (hierarchical namespace / Data Lake Storage Gen2) and `kind: 'StorageV2'`. Synapse managed identity needs `Storage Blob Data Contributor` role on the storage account. | -| **Key Vault** | For CMK encryption, Key Vault must have `enableSoftDelete: true` and `enablePurgeProtection: true`. Synapse managed identity needs `Get`, `Unwrap Key`, and `Wrap Key` permissions. | -| **Managed VNet** | When `managedVirtualNetwork: 'default'`, all outbound connections require managed private endpoints. Set at creation time — cannot be changed after. | -| **Private Endpoint** | When `publicNetworkAccess: 'Disabled'`, create private endpoints for sub-resources: `Dev` (Studio), `Sql` (dedicated SQL), `SqlOnDemand` (serverless SQL). | -| **Purview** | Requires Microsoft Purview resource ID. Synapse managed identity needs appropriate Purview roles. | -| **VNet (compute subnet)** | `virtualNetworkProfile.computeSubnetId` must reference an existing subnet. The subnet must be delegated to `Microsoft.Synapse/workspaces` if required by the deployment model. | +# Data (Analytics) Pairing Constraints + +### Cosmos DB + +| Paired With | Constraint | +|-------------|------------| +| **Multi-region writes** | `consistencyPolicy.defaultConsistencyLevel` cannot be `Strong` when `enableMultipleWriteLocations: true`. | +| **Strong consistency** | Strong consistency with regions >5000 miles apart is blocked by default (requires support ticket to enable). Strong and Bounded Staleness reads cost 2× RU/s compared to Session/Consistent Prefix/Eventual. | +| **Serverless** | Cannot combine `EnableServerless` capability with multi-region writes or analytical store. Serverless is single-region only — cannot add regions. No shared throughput databases. Cannot provision throughput (auto-managed; settings return error). Merge partitions not available for serverless accounts. | +| **Free tier** | Only one free-tier account per subscription. Cannot combine with multi-region writes. | +| **VNet** | Set `isVirtualNetworkFilterEnabled: true` and configure `virtualNetworkRules[]` with subnet IDs. Subnets need `Microsoft.AzureCosmosDB` service endpoint. | +| **Private Endpoint** | Set `publicNetworkAccess: 'Disabled'` when using private endpoints exclusively. One Private DNS Zone record per DNS name — multiple private endpoints in different regions need separate Private DNS Zones. | +| **Key Vault (CMK)** | Requires `keyVaultKeyUri` in encryption config. Key Vault must be in same region. | +| **Merge Partitions** | Not available for serverless or multi-region write accounts. Single-region provisioned throughput only. | + +### Redis Cache + +| Paired With | Constraint | +|-------------|------------| +| **VNet** | Only Premium SKU supports VNet injection via `subnetId`. Basic/Standard use firewall rules only. | +| **VNet + Private Endpoint** | VNet injection and private endpoint are mutually exclusive — cannot use both on the same cache. | +| **Private Endpoint** | Available for Basic, Standard, Premium, and Enterprise tiers. Set `publicNetworkAccess: 'Disabled'` when using private endpoints. Premium with clustering supports max 1 private link; non-clustered supports up to 100. | +| **Clustering** | Only Premium SKU supports `shardCount`. Basic and Standard are single-node/two-node only. | +| **Persistence** | Only Premium SKU supports RDB/AOF persistence. Requires a storage account for RDB exports. | +| **Geo-replication** | Only Premium SKU. Primary and secondary must be Premium with same shard count. Passive geo-replication with private endpoints requires unlinking geo-replication first, adding private link, then re-linking. | +| **Zones** | Zone redundancy requires Premium SKU with multiple replicas. | +| **Tier Scaling** | Cannot scale down tiers (Enterprise → lower, Premium → Standard/Basic, Standard → Basic). Cannot scale between Enterprise and Enterprise Flash, or from Basic/Standard/Premium to Enterprise/Flash — must create a new cache. | +| **Enterprise/Flash** | Firewall rules and `publicNetworkAccess` flag are not available on Enterprise/Enterprise Flash tiers. | +| **Azure Lighthouse** | Azure Lighthouse + VNet injection is not supported. Use private links instead. | + +### Storage Account + +| Paired With | Constraint | +|-------------|------------| +| **Azure Functions** | Must use `StorageV2` or `Storage` kind. `BlobStorage`, `BlockBlobStorage`, `FileStorage` not supported (missing Queue/Table). | +| **Functions (Consumption plan)** | Cannot use network-secured storage (VNet rules). Only Premium/Dedicated plans support VNet-restricted storage. | +| **Functions (zone-redundant)** | Must use ZRS SKU (`Standard_ZRS`). LRS/GRS not sufficient. | +| **VM Boot Diagnostics** | Cannot use Premium storage or ZRS. Use `Standard_LRS` or `Standard_GRS`. Managed boot diagnostics (no storage account required) is also available. | +| **CMK Encryption** | Key Vault must have `enableSoftDelete: true` AND `enablePurgeProtection: true`. | +| **CMK at creation** | Requires user-assigned managed identity (system-assigned only works for existing accounts). | +| **Geo-redundant failover** | Certain features (SFTP, NFS 3.0, etc.) block GRS/GZRS failover. | + +### Data Factory + +| Paired With | Constraint | +|-------------|------------| +| **Storage Account** | Linked service requires `Storage Blob Data Contributor` role on the storage account for the ADF managed identity. For ADLS Gen2, also requires `Storage Blob Data Reader` at minimum. | +| **Key Vault** | For CMK encryption, Key Vault must have `enableSoftDelete: true` and `enablePurgeProtection: true`. ADF managed identity needs `Key Vault Crypto Service Encryption User` role or equivalent access policy. | +| **Managed VNet** | When `managedVirtualNetworks` is configured, all outbound connections must use managed private endpoints (`factories/managedVirtualNetworks/managedPrivateEndpoints`). | +| **Private Endpoint** | When `publicNetworkAccess: 'Disabled'`, must create private endpoint to `dataFactory` sub-resource for studio access and pipeline connectivity. | +| **Purview** | Requires Microsoft Purview instance resource ID. ADF managed identity must have `Data Curator` role in Purview. | +| **Integration Runtime** | Self-hosted IR requires network line-of-sight to on-premises sources. Azure IR regional choice affects data residency. | + +### Synapse Workspace + +| Paired With | Constraint | +|-------------|------------| +| **ADLS Gen2 Storage Account** | **Required.** Storage account must have `isHnsEnabled: true` (hierarchical namespace / Data Lake Storage Gen2) and `kind: 'StorageV2'`. Synapse managed identity needs `Storage Blob Data Contributor` role on the storage account. | +| **Key Vault** | For CMK encryption, Key Vault must have `enableSoftDelete: true` and `enablePurgeProtection: true`. Synapse managed identity needs `Get`, `Unwrap Key`, and `Wrap Key` permissions. | +| **Managed VNet** | When `managedVirtualNetwork: 'default'`, all outbound connections require managed private endpoints. Set at creation time — cannot be changed after. | +| **Private Endpoint** | When `publicNetworkAccess: 'Disabled'`, create private endpoints for sub-resources: `Dev` (Studio), `Sql` (dedicated SQL), `SqlOnDemand` (serverless SQL). | +| **Purview** | Requires Microsoft Purview resource ID. Synapse managed identity needs appropriate Purview roles. | +| **VNet (compute subnet)** | `virtualNetworkProfile.computeSubnetId` must reference an existing subnet. The subnet must be delegated to `Microsoft.Synapse/workspaces` if required by the deployment model. | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/data-relational.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/data-relational.md index 72c28e59..19794029 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/data-relational.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/data-relational.md @@ -1,48 +1,48 @@ -# Data (Relational) Pairing Constraints - -### SQL Server - -| Paired With | Constraint | -|-------------|------------| -| **SQL Database** | Databases are child resources — must reference this server as parent. | -| **Key Vault (TDE)** | Key Vault must have `enablePurgeProtection: true`. Must be in same Azure AD tenant. Server needs GET, WRAP KEY, UNWRAP KEY permissions on key. TDE protector setup fails if Key Vault soft-delete and purge-protection are not both enabled. | -| **Virtual Network** | Use `Microsoft.Sql/servers/virtualNetworkRules` to restrict access to specific subnets. Subnets need `Microsoft.Sql` service endpoint. | -| **Private Endpoint** | Set `publicNetworkAccess: 'Disabled'` when using private endpoints exclusively. | -| **Elastic Pool** | Databases using elastic pools reference `elasticPoolId` — server must host both pool and databases. Hyperscale elastic pools cannot be created from non-Hyperscale pools. | -| **Failover Group** | Both primary and secondary servers must exist. Databases to be replicated must belong to the primary server. Failover group from zone-redundant to non-zone-redundant Hyperscale elastic pool fails silently (geo-secondary shows "Seeding 0%"). | - -### SQL Database - -| Paired With | Constraint | -|-------------|------------| -| **SQL Server** | Must be deployed as child of the parent SQL Server. Location must match. | -| **Elastic Pool** | `elasticPoolId` must reference a pool on the same server. Cannot set `sku` when using elastic pool (it inherits pool SKU). | -| **Zone Redundancy** | Only available in `GeneralPurpose`, `BusinessCritical`, and `Hyperscale` tiers. Not available in DTU tiers. General Purpose zone redundancy is only available in selected regions. Hyperscale zone redundancy can only be set at creation — cannot modify after provisioning; must recreate via copy/restore/geo-replica. | -| **Serverless** | Only available in `GeneralPurpose` tier. SKU name uses `GP_S_Gen5_*` pattern. | -| **Hyperscale** | Reverse migration from Hyperscale to General Purpose is supported within 45 days of the original migration. Databases originally created as Hyperscale cannot reverse migrate. | -| **Hyperscale Elastic Pool** | Cannot be created from a non-Hyperscale pool. Cannot be converted to non-Hyperscale (one-way only). Named replicas cannot be added to Hyperscale elastic pools (`UnsupportedReplicationOperation`). Zone-redundant Hyperscale elastic pools require databases with ZRS/GZRS backup storage — cannot add LRS-backed databases. | -| **Failover Group** | Failover group from zone-redundant to non-zone-redundant Hyperscale elastic pool fails silently (geo-secondary shows "Seeding 0%"). | -| **Backup Redundancy** | `GeoZone` only available in select regions. `Local` not available in all regions. | - -### MySQL Flexible Server - -| Paired With | Constraint | -|-------------|------------| -| **VNet (private access)** | Requires a dedicated subnet delegated to `Microsoft.DBforMySQL/flexibleServers`. Subnet must have no other resources. | -| **Private DNS Zone** | For VNet-integrated (private access) servers, use the zone name `{name}.mysql.database.azure.com` (not `privatelink.*`). The `privatelink.mysql.database.azure.com` zone is used for Private Endpoint connectivity only. Provide `privateDnsZoneResourceId` and the DNS zone must be linked to the VNet. | -| **High Availability** | `ZoneRedundant` HA requires `GeneralPurpose` or `MemoryOptimized` tier. Not available with `Burstable`. | -| **Geo-Redundant Backup** | Must be enabled at server creation time. Cannot be changed after creation. Not available in all regions. | -| **Storage Auto-Grow** | Storage can only grow, never shrink. Enabled by default. | -| **Read Replicas** | Source server must have `backup.backupRetentionDays` > 1. Replica count limit: up to 10 replicas. | -| **Key Vault (CMK)** | Customer-managed keys require user-assigned managed identity and Key Vault with purge protection enabled. | - -### PostgreSQL Flexible Server - -| Paired With | Constraint | -|-------------|------------| -| **VNet (private access)** | Requires a dedicated subnet delegated to `Microsoft.DBforPostgreSQL/flexibleServers`. Subnet must have no other resources. | -| **Private DNS Zone** | For VNet-integrated (private access) servers, use the zone name `{name}.postgres.database.azure.com` (not `privatelink.*`). The `privatelink.postgres.database.azure.com` zone is used for Private Endpoint connectivity only. Provide `privateDnsZoneArmResourceId` and the DNS zone must be linked to the VNet. | -| **High Availability** | `ZoneRedundant` HA requires `GeneralPurpose` or `MemoryOptimized` tier. Not available with `Burstable`. | -| **Geo-Redundant Backup** | Not available in all regions. Cannot be enabled with VNet-integrated (private access) servers in some configurations. | -| **Storage Auto-Grow** | Storage can only grow, never shrink. Minimum increase is based on current size. | -| **Key Vault (CMK)** | Customer-managed keys require user-assigned managed identity and Key Vault with purge protection enabled. | +# Data (Relational) Pairing Constraints + +### SQL Server + +| Paired With | Constraint | +|-------------|------------| +| **SQL Database** | Databases are child resources — must reference this server as parent. | +| **Key Vault (TDE)** | Key Vault must have `enablePurgeProtection: true`. Must be in same Azure AD tenant. Server needs GET, WRAP KEY, UNWRAP KEY permissions on key. TDE protector setup fails if Key Vault soft-delete and purge-protection are not both enabled. | +| **Virtual Network** | Use `Microsoft.Sql/servers/virtualNetworkRules` to restrict access to specific subnets. Subnets need `Microsoft.Sql` service endpoint. | +| **Private Endpoint** | Set `publicNetworkAccess: 'Disabled'` when using private endpoints exclusively. | +| **Elastic Pool** | Databases using elastic pools reference `elasticPoolId` — server must host both pool and databases. Hyperscale elastic pools cannot be created from non-Hyperscale pools. | +| **Failover Group** | Both primary and secondary servers must exist. Databases to be replicated must belong to the primary server. Failover group from zone-redundant to non-zone-redundant Hyperscale elastic pool fails silently (geo-secondary shows "Seeding 0%"). | + +### SQL Database + +| Paired With | Constraint | +|-------------|------------| +| **SQL Server** | Must be deployed as child of the parent SQL Server. Location must match. | +| **Elastic Pool** | `elasticPoolId` must reference a pool on the same server. Cannot set `sku` when using elastic pool (it inherits pool SKU). | +| **Zone Redundancy** | Only available in `GeneralPurpose`, `BusinessCritical`, and `Hyperscale` tiers. Not available in DTU tiers. General Purpose zone redundancy is only available in selected regions. Hyperscale zone redundancy can only be set at creation — cannot modify after provisioning; must recreate via copy/restore/geo-replica. | +| **Serverless** | Only available in `GeneralPurpose` tier. SKU name uses `GP_S_Gen5_*` pattern. | +| **Hyperscale** | Reverse migration from Hyperscale to General Purpose is supported within 45 days of the original migration. Databases originally created as Hyperscale cannot reverse migrate. | +| **Hyperscale Elastic Pool** | Cannot be created from a non-Hyperscale pool. Cannot be converted to non-Hyperscale (one-way only). Named replicas cannot be added to Hyperscale elastic pools (`UnsupportedReplicationOperation`). Zone-redundant Hyperscale elastic pools require databases with ZRS/GZRS backup storage — cannot add LRS-backed databases. | +| **Failover Group** | Failover group from zone-redundant to non-zone-redundant Hyperscale elastic pool fails silently (geo-secondary shows "Seeding 0%"). | +| **Backup Redundancy** | `GeoZone` only available in select regions. `Local` not available in all regions. | + +### MySQL Flexible Server + +| Paired With | Constraint | +|-------------|------------| +| **VNet (private access)** | Requires a dedicated subnet delegated to `Microsoft.DBforMySQL/flexibleServers`. Subnet must have no other resources. | +| **Private DNS Zone** | For VNet-integrated (private access) servers, use the zone name `{name}.mysql.database.azure.com` (not `privatelink.*`). The `privatelink.mysql.database.azure.com` zone is used for Private Endpoint connectivity only. Provide `privateDnsZoneResourceId` and the DNS zone must be linked to the VNet. | +| **High Availability** | `ZoneRedundant` HA requires `GeneralPurpose` or `MemoryOptimized` tier. Not available with `Burstable`. | +| **Geo-Redundant Backup** | Must be enabled at server creation time. Cannot be changed after creation. Not available in all regions. | +| **Storage Auto-Grow** | Storage can only grow, never shrink. Enabled by default. | +| **Read Replicas** | Source server must have `backup.backupRetentionDays` > 1. Replica count limit: up to 10 replicas. | +| **Key Vault (CMK)** | Customer-managed keys require user-assigned managed identity and Key Vault with purge protection enabled. | + +### PostgreSQL Flexible Server + +| Paired With | Constraint | +|-------------|------------| +| **VNet (private access)** | Requires a dedicated subnet delegated to `Microsoft.DBforPostgreSQL/flexibleServers`. Subnet must have no other resources. | +| **Private DNS Zone** | For VNet-integrated (private access) servers, use the zone name `{name}.postgres.database.azure.com` (not `privatelink.*`). The `privatelink.postgres.database.azure.com` zone is used for Private Endpoint connectivity only. Provide `privateDnsZoneArmResourceId` and the DNS zone must be linked to the VNet. | +| **High Availability** | `ZoneRedundant` HA requires `GeneralPurpose` or `MemoryOptimized` tier. Not available with `Burstable`. | +| **Geo-Redundant Backup** | Not available in all regions. Cannot be enabled with VNet-integrated (private access) servers in some configurations. | +| **Storage Auto-Grow** | Storage can only grow, never shrink. Minimum increase is based on current size. | +| **Key Vault (CMK)** | Customer-managed keys require user-assigned managed identity and Key Vault with purge protection enabled. | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/messaging.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/messaging.md index b89c95a8..f44a1f0c 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/messaging.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/messaging.md @@ -1,36 +1,36 @@ -# Messaging Pairing Constraints - -### Event Grid Topic - -| Paired With | Constraint | -|-------------|------------| -| **Event Subscriptions** | Subscriptions are child resources. Delivery endpoints include: Webhook, Azure Function, Event Hub, Service Bus Queue/Topic, Storage Queue, Hybrid Connection. | -| **Private Endpoint** | Only available with `Premium` SKU. Set `publicNetworkAccess: 'Disabled'` when using private endpoints exclusively. | -| **Managed Identity** | Required for dead-letter destinations and delivery to Azure resources that require authentication (Event Hub, Service Bus, Storage). | -| **Function App** | Use Event Grid trigger binding. Subscription endpoint type is `AzureFunction`. Function must have Event Grid extension registered. | -| **Event Hub** | Subscription endpoint type is `EventHub`. Provide the Event Hub resource ID. Requires managed identity or connection string. | -| **Storage Queue** | Subscription endpoint type is `StorageQueue`. Provide storage account ID and queue name. | -| **Dead Letter** | Dead-letter destination must be a Storage blob container. Requires managed identity or storage key for access. | - -### Event Hub - -| Paired With | Constraint | -|-------------|------------| -| **VNet** | Standard, Premium, and Dedicated SKUs support VNet service endpoints and private endpoints. | -| **Zone Redundancy** | Available in Standard (with ≥4 TU recommended) and Premium. | -| **Kafka** | Kafka protocol support available in Standard and Premium only (not Basic). | -| **Capture** | Event capture to Storage/Data Lake available in Standard and Premium only. | -| **Consumer Groups** | Basic: 1 consumer group. Standard: 20. Premium: 100. Dedicated: 1,000. | -| **Retention** | Basic: 1 day. Standard: 1–7 days. Premium: up to 90 days. | -| **Function App** | Event Hub trigger uses connection string or managed identity. Set `EventHubConnection` in app settings. | - -### Service Bus - -| Paired With | Constraint | -|-------------|------------| -| **Topics** | Only Standard and Premium SKUs support topics and subscriptions. Basic supports queues only. | -| **VNet** | Only Premium SKU supports VNet service endpoints and private endpoints. | -| **Zone Redundancy** | Only Premium SKU supports zone redundancy. | -| **Partitioning** | Premium messaging partitions cannot be changed after creation. | -| **Message Size** | Basic/Standard: max 256 KB. Premium: max 100 MB. Plan accordingly for large payloads. | -| **Function App** | Service Bus trigger uses connection string or managed identity. Set `ServiceBusConnection` in app settings. | +# Messaging Pairing Constraints + +### Event Grid Topic + +| Paired With | Constraint | +|-------------|------------| +| **Event Subscriptions** | Subscriptions are child resources. Delivery endpoints include: Webhook, Azure Function, Event Hub, Service Bus Queue/Topic, Storage Queue, Hybrid Connection. | +| **Private Endpoint** | Only available with `Premium` SKU. Set `publicNetworkAccess: 'Disabled'` when using private endpoints exclusively. | +| **Managed Identity** | Required for dead-letter destinations and delivery to Azure resources that require authentication (Event Hub, Service Bus, Storage). | +| **Function App** | Use Event Grid trigger binding. Subscription endpoint type is `AzureFunction`. Function must have Event Grid extension registered. | +| **Event Hub** | Subscription endpoint type is `EventHub`. Provide the Event Hub resource ID. Requires managed identity or connection string. | +| **Storage Queue** | Subscription endpoint type is `StorageQueue`. Provide storage account ID and queue name. | +| **Dead Letter** | Dead-letter destination must be a Storage blob container. Requires managed identity or storage key for access. | + +### Event Hub + +| Paired With | Constraint | +|-------------|------------| +| **VNet** | Standard, Premium, and Dedicated SKUs support VNet service endpoints and private endpoints. | +| **Zone Redundancy** | Available in Standard (with ≥4 TU recommended) and Premium. | +| **Kafka** | Kafka protocol support available in Standard and Premium only (not Basic). | +| **Capture** | Event capture to Storage/Data Lake available in Standard and Premium only. | +| **Consumer Groups** | Basic: 1 consumer group. Standard: 20. Premium: 100. Dedicated: 1,000. | +| **Retention** | Basic: 1 day. Standard: 1–7 days. Premium: up to 90 days. | +| **Function App** | Event Hub trigger uses connection string or managed identity. Set `EventHubConnection` in app settings. | + +### Service Bus + +| Paired With | Constraint | +|-------------|------------| +| **Topics** | Only Standard and Premium SKUs support topics and subscriptions. Basic supports queues only. | +| **VNet** | Only Premium SKU supports VNet service endpoints and private endpoints. | +| **Zone Redundancy** | Only Premium SKU supports zone redundancy. | +| **Partitioning** | Premium messaging partitions cannot be changed after creation. | +| **Message Size** | Basic/Standard: max 256 KB. Premium: max 100 MB. Plan accordingly for large payloads. | +| **Function App** | Service Bus trigger uses connection string or managed identity. Set `ServiceBusConnection` in app settings. | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/monitoring.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/monitoring.md index c66dcdf1..147a5a07 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/monitoring.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/monitoring.md @@ -1,22 +1,22 @@ -# Monitoring Pairing Constraints - -### Application Insights - -| Paired With | Constraint | -|-------------|------------| -| **Log Analytics** | Workspace-based App Insights (recommended) requires `WorkspaceResourceId`. Classic (standalone) is being phased out. | -| **Function App** | Set `APPLICATIONINSIGHTS_CONNECTION_STRING` or `APPINSIGHTS_INSTRUMENTATIONKEY` in function app settings. | -| **App Service** | Set `APPLICATIONINSIGHTS_CONNECTION_STRING` in app settings. Enable auto-instrumentation for supported runtimes. | -| **AKS** | Use Container Insights (different from App Insights) for cluster-level monitoring. App Insights used for application-level telemetry. | -| **Private Link** | Use Azure Monitor Private Link Scope (AMPLS) to restrict ingestion/query to private networks. | -| **Retention** | If workspace-based, retention is governed by the Log Analytics workspace. Component-level retention acts as an override. | - -### Log Analytics - -| Paired With | Constraint | -|-------------|------------| -| **Application Insights** | App Insights `WorkspaceResourceId` must reference this workspace. Both should be in the same region for optimal performance. | -| **AKS (Container Insights)** | AKS `omsagent` addon references workspace via `logAnalyticsWorkspaceResourceID`. | -| **Diagnostic Settings** | Multiple resources can send diagnostics to the same workspace. Configure via `Microsoft.Insights/diagnosticSettings` on each resource. | -| **Retention** | Free tier is limited to 7-day retention. PerGB2018 supports 30–730 days. Archive tier available for longer retention. | -| **Private Link** | Use Azure Monitor Private Link Scope (AMPLS) for private ingestion/query. A workspace can be linked to up to 100 AMPLS resources (a VNet can connect to only one AMPLS). | +# Monitoring Pairing Constraints + +### Application Insights + +| Paired With | Constraint | +|-------------|------------| +| **Log Analytics** | Workspace-based App Insights (recommended) requires `WorkspaceResourceId`. Classic (standalone) is being phased out. | +| **Function App** | Set `APPLICATIONINSIGHTS_CONNECTION_STRING` or `APPINSIGHTS_INSTRUMENTATIONKEY` in function app settings. | +| **App Service** | Set `APPLICATIONINSIGHTS_CONNECTION_STRING` in app settings. Enable auto-instrumentation for supported runtimes. | +| **AKS** | Use Container Insights (different from App Insights) for cluster-level monitoring. App Insights used for application-level telemetry. | +| **Private Link** | Use Azure Monitor Private Link Scope (AMPLS) to restrict ingestion/query to private networks. | +| **Retention** | If workspace-based, retention is governed by the Log Analytics workspace. Component-level retention acts as an override. | + +### Log Analytics + +| Paired With | Constraint | +|-------------|------------| +| **Application Insights** | App Insights `WorkspaceResourceId` must reference this workspace. Both should be in the same region for optimal performance. | +| **AKS (Container Insights)** | AKS `omsagent` addon references workspace via `logAnalyticsWorkspaceResourceID`. | +| **Diagnostic Settings** | Multiple resources can send diagnostics to the same workspace. Configure via `Microsoft.Insights/diagnosticSettings` on each resource. | +| **Retention** | Free tier is limited to 7-day retention. PerGB2018 supports 30–730 days. Archive tier available for longer retention. | +| **Private Link** | Use Azure Monitor Private Link Scope (AMPLS) for private ingestion/query. A workspace can be linked to up to 100 AMPLS resources (a VNet can connect to only one AMPLS). | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/networking-core.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/networking-core.md index 8ea36abd..68cfd674 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/networking-core.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/networking-core.md @@ -1,85 +1,85 @@ -# Networking (Core) Pairing Constraints - -### Virtual Network - -| Paired With | Constraint | -|-------------|------------| -| **Subnets** | Address prefixes of all subnets must fall within the VNet address space. Subnet CIDRs cannot overlap. | -| **VNet Peering** | Peered VNets cannot have overlapping address spaces. | -| **Azure Firewall** | Requires a subnet named exactly `AzureFirewallSubnet` with minimum /26 prefix. | -| **Azure Bastion** | Requires a subnet named exactly `AzureBastionSubnet` with minimum /26 prefix (recommended /26). | -| **VPN Gateway** | Requires a subnet named exactly `GatewaySubnet` with minimum /27 prefix (recommended /27). | -| **Application Gateway** | Requires a dedicated subnet (no mandatory name, but must not contain other resource types). | -| **AKS** | AKS subnet must have enough IP addresses for nodes + pods. With Azure CNI, each node reserves IPs for max pods. | - -### Subnet - -| Paired With | Constraint | -|-------------|------------| -| **NSG** | Cannot attach NSG to `GatewaySubnet` — NSGs are not supported for either VPN or ExpressRoute gateways. NSG on `AzureBastionSubnet` requires specific required rules. | -| **Delegations** | A subnet can only be delegated to one service. Delegated subnets cannot host other resource types. | -| **Service Endpoints** | Must match the service being accessed (e.g., `Microsoft.Sql` for SQL Server VNet rules). | -| **Private Endpoints** | Set `privateEndpointNetworkPolicies: 'Enabled'` to apply NSG/route table to private endpoints (default is `Disabled`). | -| **AKS** | AKS subnet needs enough IPs for all nodes + pods. Cannot be delegated or have conflicting service endpoints. | -| **Application Gateway** | Dedicated subnet required — cannot coexist with other resources except other App Gateways. Cannot mix v1 and v2 App Gateway SKUs on the same subnet. | -| **Azure Firewall** | Subnet must be named `AzureFirewallSubnet`, minimum /26. Cannot have other resources. | -| **App Service VNet Integration** | Subnet must be delegated to `Microsoft.Web/serverFarms`. Minimum size /28 (or /26 for multi-plan subnet join). This subnet must be different from any subnet used for App Service Private Endpoints. | -| **GatewaySubnet UDR** | Do not apply UDR with `0.0.0.0/0` next hop on `GatewaySubnet`. ExpressRoute gateways require management controller access. BGP route propagation must remain enabled on `GatewaySubnet`. | - -### NSG - -| Paired With | Constraint | -|-------------|------------| -| **GatewaySubnet** | NSGs are not supported on `GatewaySubnet`. Associating an NSG may cause VPN and ExpressRoute gateways to stop functioning. | -| **AzureBastionSubnet** | NSG on Bastion subnet requires specific inbound/outbound rules (see [Azure Bastion NSG](https://learn.microsoft.com/azure/bastion/bastion-nsg)). | -| **Application Gateway** | NSG on App Gateway subnet must allow `GatewayManager` service tag on ports `65200–65535` (v2) and health probe traffic. | -| **Load Balancer** | Must allow `AzureLoadBalancer` service tag for health probes. Standard LB requires NSG — it is secure by default and blocks inbound traffic without an NSG. | -| **Virtual Network** | NSG is associated to subnets, not directly to VNets. Each subnet can have at most one NSG. | - -### Route Table - -| Paired With | Constraint | -|-------------|------------| -| **Subnet** | Route table is associated on the subnet side: set `subnet.properties.routeTable.id` to the route table resource ID. Each subnet can have at most one route table. | -| **Azure Firewall** | For forced tunneling, create a default route (`0.0.0.0/0`) with `nextHopType: 'VirtualAppliance'` pointing to the firewall private IP. | -| **VPN Gateway** | Set `disableBgpRoutePropagation: true` to prevent BGP routes from overriding UDRs on the subnet. | -| **GatewaySubnet** | UDRs on `GatewaySubnet` have restrictions — cannot use `0.0.0.0/0` route pointing to a virtual appliance. | -| **AKS** | AKS subnets with UDRs require careful route design. Must allow traffic to Azure management APIs. `kubenet` and `Azure CNI` have different routing requirements. | -| **Virtual Appliance** | `nextHopIpAddress` must be a reachable private IP in the same VNet or a peered VNet. The appliance NIC must have `enableIPForwarding: true`. | - -### Network Interface - -| Paired With | Constraint | -|-------------|------------| -| **Virtual Machine** | Each VM requires at least one NIC. NIC must be in the same region and subscription as the VM. | -| **Subnet** | NIC must reference a subnet. The subnet determines the VNet, NSG, and route table that apply. | -| **NSG** | NSG can be associated at the NIC level or at the subnet level (or both). NIC-level NSG is evaluated after subnet-level NSG. | -| **Public IP** | Public IP and NIC must be in the same region. When associated with a Load Balancer, Public IP SKU must match the LB SKU (Basic with Basic, Standard with Standard). | -| **Load Balancer** | NIC IP configuration can reference `loadBalancerBackendAddressPools` and `loadBalancerInboundNatRules`. Load balancer and NIC must be in the same VNet. | -| **Accelerated Networking** | Not all VM sizes support accelerated networking. Must verify VM size compatibility. | -| **VM Scale Set** | NICs for VMSS instances are managed by the scale set — do not create standalone NICs for VMSS. | -| **Application Gateway** | NIC IP configuration can reference `applicationGatewayBackendAddressPools`. | - -### Public IP - -| Paired With | Constraint | -|-------------|------------| -| **Standard SKU** | Must use `Static` allocation method. `Dynamic` only works with Basic SKU. | -| **Load Balancer** | Public IP SKU must match Load Balancer SKU (Standard ↔ Standard, Basic ↔ Basic). | -| **Application Gateway** | Standard_v2 App Gateway requires Standard SKU public IP with Static allocation. | -| **Azure Bastion** | Requires Standard SKU with Static allocation. | -| **VPN Gateway** | Basic VPN Gateway SKU requires Basic public IP. Standard+ gateway SKUs require Standard public IP. | -| **Azure Firewall** | Requires Standard SKU with Static allocation. | -| **Zones** | Standard SKU is zone-redundant by default. Specify `zones` only to pin to specific zone(s). | - -### NAT Gateway - -| Paired With | Constraint | -|-------------|------------| -| **Subnet** | NAT Gateway is associated on the subnet side: set `subnet.properties.natGateway.id` to the NAT Gateway resource ID. A subnet can have at most one NAT Gateway. | -| **Public IP** | Public IP must use `Standard` SKU and `Static` allocation. Public IP and NAT Gateway must be in the same region. | -| **Public IP Prefix** | Public IP prefix must use `Standard` SKU. Provides contiguous outbound IPs. | -| **Availability Zones** | NAT Gateway can be zonal (pinned to one zone) or non-zonal. Public IPs must match the same zone or be zone-redundant. | -| **Load Balancer** | NAT Gateway takes precedence over outbound rules of a Standard Load Balancer when both are on the same subnet. | -| **VPN Gateway / ExpressRoute** | `GatewaySubnet` does not support NAT Gateway association. | -| **Azure Firewall** | NAT Gateway can be associated with the `AzureFirewallSubnet` for deterministic outbound IPs in SNAT scenarios. | +# Networking (Core) Pairing Constraints + +### Virtual Network + +| Paired With | Constraint | +|-------------|------------| +| **Subnets** | Address prefixes of all subnets must fall within the VNet address space. Subnet CIDRs cannot overlap. | +| **VNet Peering** | Peered VNets cannot have overlapping address spaces. | +| **Azure Firewall** | Requires a subnet named exactly `AzureFirewallSubnet` with minimum /26 prefix. | +| **Azure Bastion** | Requires a subnet named exactly `AzureBastionSubnet` with minimum /26 prefix (recommended /26). | +| **VPN Gateway** | Requires a subnet named exactly `GatewaySubnet` with minimum /27 prefix (recommended /27). | +| **Application Gateway** | Requires a dedicated subnet (no mandatory name, but must not contain other resource types). | +| **AKS** | AKS subnet must have enough IP addresses for nodes + pods. With Azure CNI, each node reserves IPs for max pods. | + +### Subnet + +| Paired With | Constraint | +|-------------|------------| +| **NSG** | Cannot attach NSG to `GatewaySubnet` — NSGs are not supported for either VPN or ExpressRoute gateways. NSG on `AzureBastionSubnet` requires specific required rules. | +| **Delegations** | A subnet can only be delegated to one service. Delegated subnets cannot host other resource types. | +| **Service Endpoints** | Must match the service being accessed (e.g., `Microsoft.Sql` for SQL Server VNet rules). | +| **Private Endpoints** | Set `privateEndpointNetworkPolicies: 'Enabled'` to apply NSG/route table to private endpoints (default is `Disabled`). | +| **AKS** | AKS subnet needs enough IPs for all nodes + pods. Cannot be delegated or have conflicting service endpoints. | +| **Application Gateway** | Dedicated subnet required — cannot coexist with other resources except other App Gateways. Cannot mix v1 and v2 App Gateway SKUs on the same subnet. | +| **Azure Firewall** | Subnet must be named `AzureFirewallSubnet`, minimum /26. Cannot have other resources. | +| **App Service VNet Integration** | Subnet must be delegated to `Microsoft.Web/serverFarms`. Minimum size /28 (or /26 for multi-plan subnet join). This subnet must be different from any subnet used for App Service Private Endpoints. | +| **GatewaySubnet UDR** | Do not apply UDR with `0.0.0.0/0` next hop on `GatewaySubnet`. ExpressRoute gateways require management controller access. BGP route propagation must remain enabled on `GatewaySubnet`. | + +### NSG + +| Paired With | Constraint | +|-------------|------------| +| **GatewaySubnet** | NSGs are not supported on `GatewaySubnet`. Associating an NSG may cause VPN and ExpressRoute gateways to stop functioning. | +| **AzureBastionSubnet** | NSG on Bastion subnet requires specific inbound/outbound rules (see [Azure Bastion NSG](https://learn.microsoft.com/azure/bastion/bastion-nsg)). | +| **Application Gateway** | NSG on App Gateway subnet must allow `GatewayManager` service tag on ports `65200–65535` (v2) and health probe traffic. | +| **Load Balancer** | Must allow `AzureLoadBalancer` service tag for health probes. Standard LB requires NSG — it is secure by default and blocks inbound traffic without an NSG. | +| **Virtual Network** | NSG is associated to subnets, not directly to VNets. Each subnet can have at most one NSG. | + +### Route Table + +| Paired With | Constraint | +|-------------|------------| +| **Subnet** | Route table is associated on the subnet side: set `subnet.properties.routeTable.id` to the route table resource ID. Each subnet can have at most one route table. | +| **Azure Firewall** | For forced tunneling, create a default route (`0.0.0.0/0`) with `nextHopType: 'VirtualAppliance'` pointing to the firewall private IP. | +| **VPN Gateway** | Set `disableBgpRoutePropagation: true` to prevent BGP routes from overriding UDRs on the subnet. | +| **GatewaySubnet** | UDRs on `GatewaySubnet` have restrictions — cannot use `0.0.0.0/0` route pointing to a virtual appliance. | +| **AKS** | AKS subnets with UDRs require careful route design. Must allow traffic to Azure management APIs. `kubenet` and `Azure CNI` have different routing requirements. | +| **Virtual Appliance** | `nextHopIpAddress` must be a reachable private IP in the same VNet or a peered VNet. The appliance NIC must have `enableIPForwarding: true`. | + +### Network Interface + +| Paired With | Constraint | +|-------------|------------| +| **Virtual Machine** | Each VM requires at least one NIC. NIC must be in the same region and subscription as the VM. | +| **Subnet** | NIC must reference a subnet. The subnet determines the VNet, NSG, and route table that apply. | +| **NSG** | NSG can be associated at the NIC level or at the subnet level (or both). NIC-level NSG is evaluated after subnet-level NSG. | +| **Public IP** | Public IP and NIC must be in the same region. When associated with a Load Balancer, Public IP SKU must match the LB SKU (Basic with Basic, Standard with Standard). | +| **Load Balancer** | NIC IP configuration can reference `loadBalancerBackendAddressPools` and `loadBalancerInboundNatRules`. Load balancer and NIC must be in the same VNet. | +| **Accelerated Networking** | Not all VM sizes support accelerated networking. Must verify VM size compatibility. | +| **VM Scale Set** | NICs for VMSS instances are managed by the scale set — do not create standalone NICs for VMSS. | +| **Application Gateway** | NIC IP configuration can reference `applicationGatewayBackendAddressPools`. | + +### Public IP + +| Paired With | Constraint | +|-------------|------------| +| **Standard SKU** | Must use `Static` allocation method. `Dynamic` only works with Basic SKU. | +| **Load Balancer** | Public IP SKU must match Load Balancer SKU (Standard ↔ Standard, Basic ↔ Basic). | +| **Application Gateway** | Standard_v2 App Gateway requires Standard SKU public IP with Static allocation. | +| **Azure Bastion** | Requires Standard SKU with Static allocation. | +| **VPN Gateway** | Basic VPN Gateway SKU requires Basic public IP. Standard+ gateway SKUs require Standard public IP. | +| **Azure Firewall** | Requires Standard SKU with Static allocation. | +| **Zones** | Standard SKU is zone-redundant by default. Specify `zones` only to pin to specific zone(s). | + +### NAT Gateway + +| Paired With | Constraint | +|-------------|------------| +| **Subnet** | NAT Gateway is associated on the subnet side: set `subnet.properties.natGateway.id` to the NAT Gateway resource ID. A subnet can have at most one NAT Gateway. | +| **Public IP** | Public IP must use `Standard` SKU and `Static` allocation. Public IP and NAT Gateway must be in the same region. | +| **Public IP Prefix** | Public IP prefix must use `Standard` SKU. Provides contiguous outbound IPs. | +| **Availability Zones** | NAT Gateway can be zonal (pinned to one zone) or non-zonal. Public IPs must match the same zone or be zone-redundant. | +| **Load Balancer** | NAT Gateway takes precedence over outbound rules of a Standard Load Balancer when both are on the same subnet. | +| **VPN Gateway / ExpressRoute** | `GatewaySubnet` does not support NAT Gateway association. | +| **Azure Firewall** | NAT Gateway can be associated with the `AzureFirewallSubnet` for deterministic outbound IPs in SNAT scenarios. | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/security.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/security.md index 312d52cf..48124824 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/security.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/constraints/security.md @@ -1,26 +1,26 @@ -# Security Pairing Constraints - -### Key Vault - -| Paired With | Constraint | -|-------------|------------| -| **Storage Account (CMK)** | Must have `enableSoftDelete: true` AND `enablePurgeProtection: true`. | -| **Storage Account (CMK at creation)** | Storage must use user-assigned managed identity — system-assigned only works for existing accounts. | -| **SQL Server (TDE)** | Must enable `enablePurgeProtection`. Key Vault and SQL Server must be in the same Azure AD tenant. | -| **AKS (secrets)** | Use `enableRbacAuthorization: true` with Azure RBAC for secrets access. AKS needs `azureKeyvaultSecretsProvider` addon. | -| **Disk Encryption** | Must set `enabledForDiskEncryption: true`. Premium SKU required for HSM-protected keys. | -| **Private Endpoint** | Set `publicNetworkAccess: 'Disabled'` and `networkAcls.defaultAction: 'Deny'` when using private endpoints. | -| **CMK Firewall** | When any Azure service uses CMK from Key Vault, the Key Vault firewall must enable "Allow trusted Microsoft services to bypass this firewall" — unless using private endpoints to Key Vault. | -| **CMK Key Type** | Key must be RSA or RSA-HSM, 2048/3072/4096-bit. Other key types are not supported for customer-managed keys. | -| **CMK Cross-Tenant** | Key Vault and consuming service must be in the same Azure AD tenant. Cross-tenant CMK requires separate configuration. | - -### Managed Identity - -| Paired With | Constraint | -|-------------|------------| -| **Any Resource (identity assignment)** | Reference the identity resource ID in the resource's `identity.userAssignedIdentities` object as `{ '${managedIdentity.id}': {} }`. | -| **Key Vault (CMK)** | Storage accounts using CMK at creation require a user-assigned identity — system-assigned only works for existing accounts. | -| **Container Registry (ACR pull)** | Assign `AcrPull` role to the identity's `principalId`. Reference the identity in the pulling resource (AKS, Container App, etc.). | -| **AKS (workload identity)** | Create a federated identity credential on the managed identity. Map it to a Kubernetes service account via OIDC issuer. | -| **Role Assignments** | Use `properties.principalId` with `principalType: 'ServicePrincipal'` in `Microsoft.Authorization/roleAssignments`. | -| **Function App / App Service** | Set `identity.type` to `'UserAssigned'` and reference the identity resource ID. Use for Key Vault references, storage access, etc. | +# Security Pairing Constraints + +### Key Vault + +| Paired With | Constraint | +|-------------|------------| +| **Storage Account (CMK)** | Must have `enableSoftDelete: true` AND `enablePurgeProtection: true`. | +| **Storage Account (CMK at creation)** | Storage must use user-assigned managed identity — system-assigned only works for existing accounts. | +| **SQL Server (TDE)** | Must enable `enablePurgeProtection`. Key Vault and SQL Server must be in the same Azure AD tenant. | +| **AKS (secrets)** | Use `enableRbacAuthorization: true` with Azure RBAC for secrets access. AKS needs `azureKeyvaultSecretsProvider` addon. | +| **Disk Encryption** | Must set `enabledForDiskEncryption: true`. Premium SKU required for HSM-protected keys. | +| **Private Endpoint** | Set `publicNetworkAccess: 'Disabled'` and `networkAcls.defaultAction: 'Deny'` when using private endpoints. | +| **CMK Firewall** | When any Azure service uses CMK from Key Vault, the Key Vault firewall must enable "Allow trusted Microsoft services to bypass this firewall" — unless using private endpoints to Key Vault. | +| **CMK Key Type** | Key must be RSA or RSA-HSM, 2048/3072/4096-bit. Other key types are not supported for customer-managed keys. | +| **CMK Cross-Tenant** | Key Vault and consuming service must be in the same Azure AD tenant. Cross-tenant CMK requires separate configuration. | + +### Managed Identity + +| Paired With | Constraint | +|-------------|------------| +| **Any Resource (identity assignment)** | Reference the identity resource ID in the resource's `identity.userAssignedIdentities` object as `{ '${managedIdentity.id}': {} }`. | +| **Key Vault (CMK)** | Storage accounts using CMK at creation require a user-assigned identity — system-assigned only works for existing accounts. | +| **Container Registry (ACR pull)** | Assign `AcrPull` role to the identity's `principalId`. Reference the identity in the pulling resource (AKS, Container App, etc.). | +| **AKS (workload identity)** | Create a federated identity credential on the managed identity. Map it to a Kubernetes service account via OIDC issuer. | +| **Role Assignments** | Use `properties.principalId` with `principalType: 'ServicePrincipal'` in `Microsoft.Authorization/roleAssignments`. | +| **Function App / App Service** | Set `identity.type` to `'UserAssigned'` and reference the identity resource ID. Use for Key Vault references, storage access, etc. | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/README.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/README.md index 94063996..4a4be6a0 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/README.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/README.md @@ -1,27 +1,27 @@ -# Resource Reference Index - -## Region Categories - -Resources fall into region-scoped or global categories. Check individual files for details. - -## Globally-Unique Names - -Some resources require globally unique names (e.g., Storage Accounts, Key Vaults). See individual category files. - -## Shared ARM Types - -Resources share the `Microsoft.*` ARM type namespace. Each file documents the relevant ARM types and documentation URLs. - -| Sub-Category | File | Resources | -|-------------|------|-----------| -| AI & ML | [ai-ml.md](ai-ml.md) | AI Search, Cognitive Services, ML Workspace | -| Compute (PaaS) | [compute-apps.md](compute-apps.md) | App Service, ASP, Function App, Container App, CAE, ACR, Static Web App | -| Compute (IaaS) | [compute-infra.md](compute-infra.md) | AKS, VM, VMSS, Availability Set, Managed Disk | -| Data (Relational) | [data-relational.md](data-relational.md) | SQL Server, SQL Database, MySQL, PostgreSQL | -| Data (Analytics) | [data-analytics.md](data-analytics.md) | Cosmos DB, Redis, Storage, Data Factory, Synapse | -| Messaging | [messaging.md](messaging.md) | Event Grid, Event Hub, Service Bus | -| Monitoring | [monitoring.md](monitoring.md) | App Insights, Log Analytics | -| Networking (Core) | [networking-core.md](networking-core.md) | VNet, Subnet, NSG, Route Table, NIC, Public IP, NAT Gateway | -| Networking (Traffic) | [networking-traffic.md](networking-traffic.md) | App Gateway, Front Door, LB, APIM | -| Networking (Connectivity) | [networking-connectivity.md](networking-connectivity.md) | Bastion, Firewall, VPN GW, DNS, Private DNS, PE | -| Security | [security.md](security.md) | Key Vault, Managed Identity | +# Resource Reference Index + +## Region Categories + +Resources fall into region-scoped or global categories. Check individual files for details. + +## Globally-Unique Names + +Some resources require globally unique names (e.g., Storage Accounts, Key Vaults). See individual category files. + +## Shared ARM Types + +Resources share the `Microsoft.*` ARM type namespace. Each file documents the relevant ARM types and documentation URLs. + +| Sub-Category | File | Resources | +|-------------|------|-----------| +| AI & ML | [ai-ml.md](ai-ml.md) | AI Search, Cognitive Services, ML Workspace | +| Compute (PaaS) | [compute-apps.md](compute-apps.md) | App Service, ASP, Function App, Container App, CAE, ACR, Static Web App | +| Compute (IaaS) | [compute-infra.md](compute-infra.md) | AKS, VM, VMSS, Availability Set, Managed Disk | +| Data (Relational) | [data-relational.md](data-relational.md) | SQL Server, SQL Database, MySQL, PostgreSQL | +| Data (Analytics) | [data-analytics.md](data-analytics.md) | Cosmos DB, Redis, Storage, Data Factory, Synapse | +| Messaging | [messaging.md](messaging.md) | Event Grid, Event Hub, Service Bus | +| Monitoring | [monitoring.md](monitoring.md) | App Insights, Log Analytics | +| Networking (Core) | [networking-core.md](networking-core.md) | VNet, Subnet, NSG, Route Table, NIC, Public IP, NAT Gateway | +| Networking (Traffic) | [networking-traffic.md](networking-traffic.md) | App Gateway, Front Door, LB, APIM | +| Networking (Connectivity) | [networking-connectivity.md](networking-connectivity.md) | Bastion, Firewall, VPN GW, DNS, Private DNS, PE | +| Security | [security.md](security.md) | Key Vault, Managed Identity | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/ai-ml.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/ai-ml.md index 6717f1a8..b0da78bb 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/ai-ml.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/ai-ml.md @@ -1,15 +1,15 @@ -# AI & ML Resources - -| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | -|----------|----------|-------------|------------|--------------|--------| -| Cognitive Services | `Microsoft.CognitiveServices/accounts` | `2025-06-01` | varies by kind | Resource group | Mainstream | -| ML Workspace | `Microsoft.MachineLearningServices/workspaces` | `2025-06-01` | `mlw`/`hub`/`proj` | Resource group | Mainstream | -| AI Search | `Microsoft.Search/searchServices` | `2025-05-01` | `srch` | Global | Mainstream | - -## Documentation - -| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | -|----------|----------------|------------------|--------------|------------| -| Cognitive Services | [2025-06-01](https://learn.microsoft.com/azure/templates/microsoft.cognitiveservices/accounts?pivots=deployment-language-bicep) | [Custom subdomain names](https://learn.microsoft.com/azure/ai-services/cognitive-services-custom-subdomains) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftcognitiveservices) | [All API versions](https://learn.microsoft.com/azure/templates/microsoft.cognitiveservices/allversions) | -| ML Workspace | [2025-06-01](https://learn.microsoft.com/azure/templates/microsoft.machinelearningservices/workspaces?pivots=deployment-language-bicep) | [ML Services](https://learn.microsoft.com/azure/templates/microsoft.machinelearningservices/allversions) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftmachinelearningservices) | [All API versions](https://learn.microsoft.com/azure/templates/microsoft.machinelearningservices/allversions) | -| AI Search | [2025-05-01](https://learn.microsoft.com/azure/templates/microsoft.search/searchservices?pivots=deployment-language-bicep) | [Service limits](https://learn.microsoft.com/azure/search/search-limits-quotas-capacity) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftsearch) | [All API versions](https://learn.microsoft.com/azure/templates/microsoft.search/allversions) | +# AI & ML Resources + +| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | +|----------|----------|-------------|------------|--------------|--------| +| Cognitive Services | `Microsoft.CognitiveServices/accounts` | `2025-06-01` | varies by kind | Resource group | Mainstream | +| ML Workspace | `Microsoft.MachineLearningServices/workspaces` | `2025-06-01` | `mlw`/`hub`/`proj` | Resource group | Mainstream | +| AI Search | `Microsoft.Search/searchServices` | `2025-05-01` | `srch` | Global | Mainstream | + +## Documentation + +| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | +|----------|----------------|------------------|--------------|------------| +| Cognitive Services | [2025-06-01](https://learn.microsoft.com/azure/templates/microsoft.cognitiveservices/accounts?pivots=deployment-language-bicep) | [Custom subdomain names](https://learn.microsoft.com/azure/ai-services/cognitive-services-custom-subdomains) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftcognitiveservices) | [All API versions](https://learn.microsoft.com/azure/templates/microsoft.cognitiveservices/allversions) | +| ML Workspace | [2025-06-01](https://learn.microsoft.com/azure/templates/microsoft.machinelearningservices/workspaces?pivots=deployment-language-bicep) | [ML Services](https://learn.microsoft.com/azure/templates/microsoft.machinelearningservices/allversions) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftmachinelearningservices) | [All API versions](https://learn.microsoft.com/azure/templates/microsoft.machinelearningservices/allversions) | +| AI Search | [2025-05-01](https://learn.microsoft.com/azure/templates/microsoft.search/searchservices?pivots=deployment-language-bicep) | [Service limits](https://learn.microsoft.com/azure/search/search-limits-quotas-capacity) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftsearch) | [All API versions](https://learn.microsoft.com/azure/templates/microsoft.search/allversions) | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/compute-apps.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/compute-apps.md index 53b008e6..7d6aa9b7 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/compute-apps.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/compute-apps.md @@ -1,23 +1,23 @@ -# Compute (PaaS) Resources - -| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | -|----------|----------|-------------|------------|--------------|--------| -| App Service | `Microsoft.Web/sites` | `2024-11-01` | `app` | Global | Mainstream | -| App Service Plan | `Microsoft.Web/serverfarms` | `2024-11-01` | `asp` | Resource group | Mainstream | -| Container App | `Microsoft.App/containerApps` | `2025-01-01` | `ca` | Environment | Strategic | -| Container Apps Environment | `Microsoft.App/managedEnvironments` | `2025-01-01` | `cae` | Resource group | Strategic | -| Container Registry | `Microsoft.ContainerRegistry/registries` | `2025-04-01` | `cr` | Global | Mainstream | -| Function App | `Microsoft.Web/sites` | `2024-11-01` | `func` | Global | Mainstream | -| Static Web App | `Microsoft.Web/staticSites` | `2024-11-01` | `stapp` | Resource group | Mainstream | - -## Documentation - -| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | -|----------|----------------|------------------|--------------|------------| -| App Service | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.web/sites?pivots=deployment-language-bicep) | [App Service overview](https://learn.microsoft.com/azure/app-service/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftweb) | [Hosting plans](https://learn.microsoft.com/azure/app-service/overview-hosting-plans) | -| App Service Plan | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.web/serverfarms?pivots=deployment-language-bicep) | [Plan overview](https://learn.microsoft.com/azure/app-service/overview-hosting-plans) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftweb) | [Pricing](https://azure.microsoft.com/en-us/pricing/details/app-service/linux) | -| Container App | [2025-01-01](https://learn.microsoft.com/azure/templates/microsoft.app/containerapps?pivots=deployment-language-bicep) | [Container Apps overview](https://learn.microsoft.com/azure/container-apps/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftapp) | [Environments](https://learn.microsoft.com/azure/container-apps/environment) | -| Container Apps Environment | [2025-01-01](https://learn.microsoft.com/azure/templates/microsoft.app/managedenvironments?pivots=deployment-language-bicep) | [Environments overview](https://learn.microsoft.com/azure/container-apps/environment) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftapp) | [Workload profiles](https://learn.microsoft.com/azure/container-apps/workload-profiles-overview) | -| Container Registry | [2025-04-01](https://learn.microsoft.com/azure/templates/microsoft.containerregistry/registries?pivots=deployment-language-bicep) | [ACR overview](https://learn.microsoft.com/azure/container-registry/container-registry-intro) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftcontainerregistry) | [SKU tiers](https://learn.microsoft.com/azure/container-registry/container-registry-skus) | -| Function App | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.web/sites?pivots=deployment-language-bicep) | [Functions overview](https://learn.microsoft.com/azure/azure-functions/functions-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftweb) | [Hosting plans](https://learn.microsoft.com/azure/azure-functions/functions-scale) | -| Static Web App | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.web/staticsites?pivots=deployment-language-bicep) | [Static Web Apps overview](https://learn.microsoft.com/azure/static-web-apps/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftweb) | [Hosting plans](https://learn.microsoft.com/azure/static-web-apps/plans) | +# Compute (PaaS) Resources + +| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | +|----------|----------|-------------|------------|--------------|--------| +| App Service | `Microsoft.Web/sites` | `2024-11-01` | `app` | Global | Mainstream | +| App Service Plan | `Microsoft.Web/serverfarms` | `2024-11-01` | `asp` | Resource group | Mainstream | +| Container App | `Microsoft.App/containerApps` | `2025-01-01` | `ca` | Environment | Strategic | +| Container Apps Environment | `Microsoft.App/managedEnvironments` | `2025-01-01` | `cae` | Resource group | Strategic | +| Container Registry | `Microsoft.ContainerRegistry/registries` | `2025-04-01` | `cr` | Global | Mainstream | +| Function App | `Microsoft.Web/sites` | `2024-11-01` | `func` | Global | Mainstream | +| Static Web App | `Microsoft.Web/staticSites` | `2024-11-01` | `stapp` | Resource group | Mainstream | + +## Documentation + +| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | +|----------|----------------|------------------|--------------|------------| +| App Service | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.web/sites?pivots=deployment-language-bicep) | [App Service overview](https://learn.microsoft.com/azure/app-service/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftweb) | [Hosting plans](https://learn.microsoft.com/azure/app-service/overview-hosting-plans) | +| App Service Plan | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.web/serverfarms?pivots=deployment-language-bicep) | [Plan overview](https://learn.microsoft.com/azure/app-service/overview-hosting-plans) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftweb) | [Pricing](https://azure.microsoft.com/en-us/pricing/details/app-service/linux) | +| Container App | [2025-01-01](https://learn.microsoft.com/azure/templates/microsoft.app/containerapps?pivots=deployment-language-bicep) | [Container Apps overview](https://learn.microsoft.com/azure/container-apps/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftapp) | [Environments](https://learn.microsoft.com/azure/container-apps/environment) | +| Container Apps Environment | [2025-01-01](https://learn.microsoft.com/azure/templates/microsoft.app/managedenvironments?pivots=deployment-language-bicep) | [Environments overview](https://learn.microsoft.com/azure/container-apps/environment) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftapp) | [Workload profiles](https://learn.microsoft.com/azure/container-apps/workload-profiles-overview) | +| Container Registry | [2025-04-01](https://learn.microsoft.com/azure/templates/microsoft.containerregistry/registries?pivots=deployment-language-bicep) | [ACR overview](https://learn.microsoft.com/azure/container-registry/container-registry-intro) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftcontainerregistry) | [SKU tiers](https://learn.microsoft.com/azure/container-registry/container-registry-skus) | +| Function App | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.web/sites?pivots=deployment-language-bicep) | [Functions overview](https://learn.microsoft.com/azure/azure-functions/functions-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftweb) | [Hosting plans](https://learn.microsoft.com/azure/azure-functions/functions-scale) | +| Static Web App | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.web/staticsites?pivots=deployment-language-bicep) | [Static Web Apps overview](https://learn.microsoft.com/azure/static-web-apps/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftweb) | [Hosting plans](https://learn.microsoft.com/azure/static-web-apps/plans) | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/compute-infra.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/compute-infra.md index 082161b2..7bec7ae9 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/compute-infra.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/compute-infra.md @@ -1,19 +1,19 @@ -# Compute (IaaS) Resources - -| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | -|----------|----------|-------------|------------|--------------|--------| -| AKS Cluster | `Microsoft.ContainerService/managedClusters` | `2025-05-01` | `aks` | Resource group | Foundational | -| Availability Set | `Microsoft.Compute/availabilitySets` | `2024-11-01` | `avail` | Resource group | Foundational | -| Managed Disk | `Microsoft.Compute/disks` | `2025-01-02` | `osdisk`/`disk` | Resource group | Foundational | -| Virtual Machine | `Microsoft.Compute/virtualMachines` | `2024-11-01` | `vm` | Resource group | Foundational | -| VM Scale Set | `Microsoft.Compute/virtualMachineScaleSets` | `2024-11-01` | `vmss` | Resource group | Foundational | - -## Documentation - -| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | -|----------|----------------|------------------|--------------|------------| -| AKS Cluster | [2025-05-01](https://learn.microsoft.com/azure/templates/microsoft.containerservice/managedclusters?pivots=deployment-language-bicep) | [AKS overview](https://learn.microsoft.com/azure/aks/intro-kubernetes) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftcontainerservice) | [Networking concepts](https://learn.microsoft.com/azure/aks/concepts-network) | -| Availability Set | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.compute/availabilitysets?pivots=deployment-language-bicep) | [Overview](https://learn.microsoft.com/azure/virtual-machines/availability-set-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftcompute) | — | -| Managed Disk | [2025-01-02](https://learn.microsoft.com/azure/templates/microsoft.compute/disks?pivots=deployment-language-bicep) | [Managed disks overview](https://learn.microsoft.com/azure/virtual-machines/managed-disks-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftcompute) | — | -| Virtual Machine | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.compute/virtualmachines?pivots=deployment-language-bicep) | [VMs overview](https://learn.microsoft.com/azure/virtual-machines/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftcompute) | [VM sizes](https://learn.microsoft.com/azure/virtual-machines/sizes/overview) | -| VM Scale Set | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.compute/virtualmachinescalesets?pivots=deployment-language-bicep) | [VMSS overview](https://learn.microsoft.com/azure/virtual-machine-scale-sets/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftcompute) | — | +# Compute (IaaS) Resources + +| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | +|----------|----------|-------------|------------|--------------|--------| +| AKS Cluster | `Microsoft.ContainerService/managedClusters` | `2025-05-01` | `aks` | Resource group | Foundational | +| Availability Set | `Microsoft.Compute/availabilitySets` | `2024-11-01` | `avail` | Resource group | Foundational | +| Managed Disk | `Microsoft.Compute/disks` | `2025-01-02` | `osdisk`/`disk` | Resource group | Foundational | +| Virtual Machine | `Microsoft.Compute/virtualMachines` | `2024-11-01` | `vm` | Resource group | Foundational | +| VM Scale Set | `Microsoft.Compute/virtualMachineScaleSets` | `2024-11-01` | `vmss` | Resource group | Foundational | + +## Documentation + +| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | +|----------|----------------|------------------|--------------|------------| +| AKS Cluster | [2025-05-01](https://learn.microsoft.com/azure/templates/microsoft.containerservice/managedclusters?pivots=deployment-language-bicep) | [AKS overview](https://learn.microsoft.com/azure/aks/intro-kubernetes) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftcontainerservice) | [Networking concepts](https://learn.microsoft.com/azure/aks/concepts-network) | +| Availability Set | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.compute/availabilitysets?pivots=deployment-language-bicep) | [Overview](https://learn.microsoft.com/azure/virtual-machines/availability-set-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftcompute) | — | +| Managed Disk | [2025-01-02](https://learn.microsoft.com/azure/templates/microsoft.compute/disks?pivots=deployment-language-bicep) | [Managed disks overview](https://learn.microsoft.com/azure/virtual-machines/managed-disks-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftcompute) | — | +| Virtual Machine | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.compute/virtualmachines?pivots=deployment-language-bicep) | [VMs overview](https://learn.microsoft.com/azure/virtual-machines/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftcompute) | [VM sizes](https://learn.microsoft.com/azure/virtual-machines/sizes/overview) | +| VM Scale Set | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.compute/virtualmachinescalesets?pivots=deployment-language-bicep) | [VMSS overview](https://learn.microsoft.com/azure/virtual-machine-scale-sets/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftcompute) | — | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/data-analytics.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/data-analytics.md index 7e457676..422fe926 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/data-analytics.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/data-analytics.md @@ -1,19 +1,19 @@ -# Data (Analytics) Resources - -| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | -|----------|----------|-------------|------------|--------------|--------| -| Cosmos DB | `Microsoft.DocumentDB/databaseAccounts` | `2025-04-15` | `cosmos` | Global | Foundational | -| Data Factory | `Microsoft.DataFactory/factories` | `2018-06-01` | `adf` | Global | Mainstream | -| Redis Cache | `Microsoft.Cache/redis` | `2024-11-01` | `redis` | Global | Mainstream | -| Storage Account | `Microsoft.Storage/storageAccounts` | `2025-01-01` | `st` | Global | Foundational | -| Synapse Workspace | `Microsoft.Synapse/workspaces` | `2021-06-01` | `synw` | Global | Strategic | - -## Documentation - -| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | -|----------|----------------|------------------|--------------|------------| -| Cosmos DB | [2025-04-15](https://learn.microsoft.com/azure/templates/microsoft.documentdb/databaseaccounts?pivots=deployment-language-bicep) | [Cosmos DB overview](https://learn.microsoft.com/azure/cosmos-db/introduction) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftdocumentdb) | [Consistency levels](https://learn.microsoft.com/azure/cosmos-db/consistency-levels) | -| Data Factory | [2018-06-01](https://learn.microsoft.com/azure/templates/microsoft.datafactory/factories?pivots=deployment-language-bicep) | [ADF overview](https://learn.microsoft.com/azure/data-factory/introduction) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftdatafactory) | [ADF naming rules](https://learn.microsoft.com/azure/data-factory/naming-rules) | -| Redis Cache | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.cache/redis?pivots=deployment-language-bicep) | [Redis overview](https://learn.microsoft.com/azure/azure-cache-for-redis/cache-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftcache) | [Service tiers](https://learn.microsoft.com/azure/azure-cache-for-redis/cache-overview#service-tiers) | -| Storage Account | [2025-01-01](https://learn.microsoft.com/azure/templates/microsoft.storage/storageaccounts?pivots=deployment-language-bicep) | [Storage overview](https://learn.microsoft.com/azure/storage/common/storage-account-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftstorage) | [Storage redundancy](https://learn.microsoft.com/azure/storage/common/storage-redundancy) | -| Synapse Workspace | [2021-06-01](https://learn.microsoft.com/azure/templates/microsoft.synapse/workspaces?pivots=deployment-language-bicep) | [Synapse overview](https://learn.microsoft.com/azure/synapse-analytics/overview-what-is) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftsynapse) | [All API versions](https://learn.microsoft.com/azure/templates/microsoft.synapse/allversions) | +# Data (Analytics) Resources + +| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | +|----------|----------|-------------|------------|--------------|--------| +| Cosmos DB | `Microsoft.DocumentDB/databaseAccounts` | `2025-04-15` | `cosmos` | Global | Foundational | +| Data Factory | `Microsoft.DataFactory/factories` | `2018-06-01` | `adf` | Global | Mainstream | +| Redis Cache | `Microsoft.Cache/redis` | `2024-11-01` | `redis` | Global | Mainstream | +| Storage Account | `Microsoft.Storage/storageAccounts` | `2025-01-01` | `st` | Global | Foundational | +| Synapse Workspace | `Microsoft.Synapse/workspaces` | `2021-06-01` | `synw` | Global | Strategic | + +## Documentation + +| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | +|----------|----------------|------------------|--------------|------------| +| Cosmos DB | [2025-04-15](https://learn.microsoft.com/azure/templates/microsoft.documentdb/databaseaccounts?pivots=deployment-language-bicep) | [Cosmos DB overview](https://learn.microsoft.com/azure/cosmos-db/introduction) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftdocumentdb) | [Consistency levels](https://learn.microsoft.com/azure/cosmos-db/consistency-levels) | +| Data Factory | [2018-06-01](https://learn.microsoft.com/azure/templates/microsoft.datafactory/factories?pivots=deployment-language-bicep) | [ADF overview](https://learn.microsoft.com/azure/data-factory/introduction) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftdatafactory) | [ADF naming rules](https://learn.microsoft.com/azure/data-factory/naming-rules) | +| Redis Cache | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.cache/redis?pivots=deployment-language-bicep) | [Redis overview](https://learn.microsoft.com/azure/azure-cache-for-redis/cache-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftcache) | [Service tiers](https://learn.microsoft.com/azure/azure-cache-for-redis/cache-overview#service-tiers) | +| Storage Account | [2025-01-01](https://learn.microsoft.com/azure/templates/microsoft.storage/storageaccounts?pivots=deployment-language-bicep) | [Storage overview](https://learn.microsoft.com/azure/storage/common/storage-account-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftstorage) | [Storage redundancy](https://learn.microsoft.com/azure/storage/common/storage-redundancy) | +| Synapse Workspace | [2021-06-01](https://learn.microsoft.com/azure/templates/microsoft.synapse/workspaces?pivots=deployment-language-bicep) | [Synapse overview](https://learn.microsoft.com/azure/synapse-analytics/overview-what-is) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftsynapse) | [All API versions](https://learn.microsoft.com/azure/templates/microsoft.synapse/allversions) | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/data-relational.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/data-relational.md index 5be0f2aa..95283895 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/data-relational.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/data-relational.md @@ -1,17 +1,17 @@ -# Data (Relational) Resources - -| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | -|----------|----------|-------------|------------|--------------|--------| -| SQL Server | `Microsoft.Sql/servers` | `2023-08-01` | `sql` | Global | Foundational | -| SQL Database | `Microsoft.Sql/servers/databases` | `2023-08-01` | `sqldb` | Parent server | Foundational | -| MySQL Flexible Server | `Microsoft.DBforMySQL/flexibleServers` | `2023-12-30` | `mysql` | Global | Mainstream | -| PostgreSQL Flexible Server | `Microsoft.DBforPostgreSQL/flexibleServers` | `2024-08-01` | `psql` | Global | Mainstream | - -## Documentation - -| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | -|----------|----------------|------------------|--------------|------------| -| SQL Server | [2023-08-01](https://learn.microsoft.com/azure/templates/microsoft.sql/servers?pivots=deployment-language-bicep) | [SQL Server overview](https://learn.microsoft.com/azure/azure-sql/database/sql-database-paas-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftsql) | [TDE with Key Vault](https://learn.microsoft.com/azure/azure-sql/database/transparent-data-encryption-byok-overview) | -| SQL Database | [2023-08-01](https://learn.microsoft.com/azure/templates/microsoft.sql/servers/databases?pivots=deployment-language-bicep) | [SQL Database overview](https://learn.microsoft.com/azure/azure-sql/database/sql-database-paas-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftsql) | [DTU vs vCore](https://learn.microsoft.com/azure/azure-sql/database/purchasing-models) | -| MySQL Flexible Server | [2023-12-30](https://learn.microsoft.com/azure/templates/microsoft.dbformysql/flexibleservers?pivots=deployment-language-bicep) | [MySQL overview](https://learn.microsoft.com/azure/mysql/flexible-server/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftdbformysql) | [Compute and storage](https://learn.microsoft.com/azure/mysql/flexible-server/concepts-compute-storage) | -| PostgreSQL Flexible Server | [2024-08-01](https://learn.microsoft.com/azure/templates/microsoft.dbforpostgresql/flexibleservers?pivots=deployment-language-bicep) | [PostgreSQL overview](https://learn.microsoft.com/azure/postgresql/flexible-server/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftdbforpostgresql) | [Compute and storage](https://learn.microsoft.com/azure/postgresql/flexible-server/concepts-compute-storage) | +# Data (Relational) Resources + +| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | +|----------|----------|-------------|------------|--------------|--------| +| SQL Server | `Microsoft.Sql/servers` | `2023-08-01` | `sql` | Global | Foundational | +| SQL Database | `Microsoft.Sql/servers/databases` | `2023-08-01` | `sqldb` | Parent server | Foundational | +| MySQL Flexible Server | `Microsoft.DBforMySQL/flexibleServers` | `2023-12-30` | `mysql` | Global | Mainstream | +| PostgreSQL Flexible Server | `Microsoft.DBforPostgreSQL/flexibleServers` | `2024-08-01` | `psql` | Global | Mainstream | + +## Documentation + +| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | +|----------|----------------|------------------|--------------|------------| +| SQL Server | [2023-08-01](https://learn.microsoft.com/azure/templates/microsoft.sql/servers?pivots=deployment-language-bicep) | [SQL Server overview](https://learn.microsoft.com/azure/azure-sql/database/sql-database-paas-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftsql) | [TDE with Key Vault](https://learn.microsoft.com/azure/azure-sql/database/transparent-data-encryption-byok-overview) | +| SQL Database | [2023-08-01](https://learn.microsoft.com/azure/templates/microsoft.sql/servers/databases?pivots=deployment-language-bicep) | [SQL Database overview](https://learn.microsoft.com/azure/azure-sql/database/sql-database-paas-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftsql) | [DTU vs vCore](https://learn.microsoft.com/azure/azure-sql/database/purchasing-models) | +| MySQL Flexible Server | [2023-12-30](https://learn.microsoft.com/azure/templates/microsoft.dbformysql/flexibleservers?pivots=deployment-language-bicep) | [MySQL overview](https://learn.microsoft.com/azure/mysql/flexible-server/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftdbformysql) | [Compute and storage](https://learn.microsoft.com/azure/mysql/flexible-server/concepts-compute-storage) | +| PostgreSQL Flexible Server | [2024-08-01](https://learn.microsoft.com/azure/templates/microsoft.dbforpostgresql/flexibleservers?pivots=deployment-language-bicep) | [PostgreSQL overview](https://learn.microsoft.com/azure/postgresql/flexible-server/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftdbforpostgresql) | [Compute and storage](https://learn.microsoft.com/azure/postgresql/flexible-server/concepts-compute-storage) | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/messaging.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/messaging.md index 338822cc..722dfa88 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/messaging.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/messaging.md @@ -1,15 +1,15 @@ -# Messaging Resources - -| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | -|----------|----------|-------------|------------|--------------|--------| -| Event Grid Topic | `Microsoft.EventGrid/topics` | `2025-02-15` | `evgt` | Region | Mainstream | -| Event Hub | `Microsoft.EventHub/namespaces` | `2024-01-01` | `evhns` | Global | Foundational | -| Service Bus | `Microsoft.ServiceBus/namespaces` | `2024-01-01` | `sbns` | Global | Foundational | - -## Documentation - -| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | -|----------|----------------|------------------|--------------|------------| -| Event Grid Topic | [2025-02-15](https://learn.microsoft.com/azure/templates/microsoft.eventgrid/topics?pivots=deployment-language-bicep) | [Event Grid overview](https://learn.microsoft.com/azure/event-grid/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsofteventgrid) | [Security and auth](https://learn.microsoft.com/azure/event-grid/security-authentication) | -| Event Hub | [2024-01-01](https://learn.microsoft.com/azure/templates/microsoft.eventhub/namespaces?pivots=deployment-language-bicep) | [Event Hubs overview](https://learn.microsoft.com/azure/event-hubs/event-hubs-about) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsofteventhub) | [Event Hubs tiers](https://learn.microsoft.com/azure/event-hubs/event-hubs-quotas) | -| Service Bus | [2024-01-01](https://learn.microsoft.com/azure/templates/microsoft.servicebus/namespaces?pivots=deployment-language-bicep) | [Service Bus overview](https://learn.microsoft.com/azure/service-bus-messaging/service-bus-messaging-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftservicebus) | [Service Bus tiers](https://learn.microsoft.com/azure/service-bus-messaging/service-bus-premium-messaging) | +# Messaging Resources + +| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | +|----------|----------|-------------|------------|--------------|--------| +| Event Grid Topic | `Microsoft.EventGrid/topics` | `2025-02-15` | `evgt` | Region | Mainstream | +| Event Hub | `Microsoft.EventHub/namespaces` | `2024-01-01` | `evhns` | Global | Foundational | +| Service Bus | `Microsoft.ServiceBus/namespaces` | `2024-01-01` | `sbns` | Global | Foundational | + +## Documentation + +| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | +|----------|----------------|------------------|--------------|------------| +| Event Grid Topic | [2025-02-15](https://learn.microsoft.com/azure/templates/microsoft.eventgrid/topics?pivots=deployment-language-bicep) | [Event Grid overview](https://learn.microsoft.com/azure/event-grid/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsofteventgrid) | [Security and auth](https://learn.microsoft.com/azure/event-grid/security-authentication) | +| Event Hub | [2024-01-01](https://learn.microsoft.com/azure/templates/microsoft.eventhub/namespaces?pivots=deployment-language-bicep) | [Event Hubs overview](https://learn.microsoft.com/azure/event-hubs/event-hubs-about) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsofteventhub) | [Event Hubs tiers](https://learn.microsoft.com/azure/event-hubs/event-hubs-quotas) | +| Service Bus | [2024-01-01](https://learn.microsoft.com/azure/templates/microsoft.servicebus/namespaces?pivots=deployment-language-bicep) | [Service Bus overview](https://learn.microsoft.com/azure/service-bus-messaging/service-bus-messaging-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftservicebus) | [Service Bus tiers](https://learn.microsoft.com/azure/service-bus-messaging/service-bus-premium-messaging) | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/monitoring.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/monitoring.md index 094a04f7..adf88dcf 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/monitoring.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/monitoring.md @@ -1,13 +1,13 @@ -# Monitoring Resources - -| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | -|----------|----------|-------------|------------|--------------|--------| -| Application Insights | `Microsoft.Insights/components` | `2020-02-02` | `appi` | Resource group | Mainstream | -| Log Analytics | `Microsoft.OperationalInsights/workspaces` | `2025-02-01` | `log` | Resource group | Mainstream | - -## Documentation - -| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | -|----------|----------------|------------------|--------------|------------| -| Application Insights | [2020-02-02](https://learn.microsoft.com/azure/templates/microsoft.insights/components?pivots=deployment-language-bicep) | [App Insights overview](https://learn.microsoft.com/azure/azure-monitor/app/app-insights-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftinsights) | [Workspace-based](https://learn.microsoft.com/azure/azure-monitor/app/convert-classic-resource) | -| Log Analytics | [2025-02-01](https://learn.microsoft.com/azure/templates/microsoft.operationalinsights/workspaces?pivots=deployment-language-bicep) | [Log Analytics overview](https://learn.microsoft.com/azure/azure-monitor/logs/log-analytics-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftoperationalinsights) | [Pricing](https://learn.microsoft.com/azure/azure-monitor/logs/cost-logs) | +# Monitoring Resources + +| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | +|----------|----------|-------------|------------|--------------|--------| +| Application Insights | `Microsoft.Insights/components` | `2020-02-02` | `appi` | Resource group | Mainstream | +| Log Analytics | `Microsoft.OperationalInsights/workspaces` | `2025-02-01` | `log` | Resource group | Mainstream | + +## Documentation + +| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | +|----------|----------------|------------------|--------------|------------| +| Application Insights | [2020-02-02](https://learn.microsoft.com/azure/templates/microsoft.insights/components?pivots=deployment-language-bicep) | [App Insights overview](https://learn.microsoft.com/azure/azure-monitor/app/app-insights-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftinsights) | [Workspace-based](https://learn.microsoft.com/azure/azure-monitor/app/convert-classic-resource) | +| Log Analytics | [2025-02-01](https://learn.microsoft.com/azure/templates/microsoft.operationalinsights/workspaces?pivots=deployment-language-bicep) | [Log Analytics overview](https://learn.microsoft.com/azure/azure-monitor/logs/log-analytics-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftoperationalinsights) | [Pricing](https://learn.microsoft.com/azure/azure-monitor/logs/cost-logs) | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/networking-core.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/networking-core.md index a6bb51e9..78dd8fe5 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/networking-core.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/networking-core.md @@ -1,23 +1,23 @@ -# Networking (Core) Resources - -| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | -|----------|----------|-------------|------------|--------------|--------| -| Virtual Network | `Microsoft.Network/virtualNetworks` | `2024-07-01` | `vnet` | Resource group | Foundational | -| Subnet | `Microsoft.Network/virtualNetworks/subnets` | `2024-07-01` | `snet` | Parent VNet | Foundational | -| NSG | `Microsoft.Network/networkSecurityGroups` | `2024-07-01` | `nsg` | Resource group | Foundational | -| Route Table | `Microsoft.Network/routeTables` | `2024-07-01` | `rt` | Resource group | Foundational | -| Network Interface | `Microsoft.Network/networkInterfaces` | `2024-07-01` | `nic` | Resource group | Foundational | -| Public IP | `Microsoft.Network/publicIPAddresses` | `2024-07-01` | `pip` | Resource group | Foundational | -| NAT Gateway | `Microsoft.Network/natGateways` | `2024-07-01` | `ng` | Resource group | Foundational | - -## Documentation - -| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | -|----------|----------------|------------------|--------------|------------| -| Virtual Network | [2024-07-01](https://learn.microsoft.com/azure/templates/microsoft.network/virtualnetworks?pivots=deployment-language-bicep) | [VNet overview](https://learn.microsoft.com/azure/virtual-network/virtual-networks-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftnetwork) | [VNet planning](https://learn.microsoft.com/azure/virtual-network/virtual-network-vnet-plan-design-arm) | -| Subnet | [2024-07-01](https://learn.microsoft.com/azure/templates/microsoft.network/virtualnetworks/subnets?pivots=deployment-language-bicep) | [Subnets](https://learn.microsoft.com/azure/virtual-network/virtual-network-manage-subnet) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftnetwork) | [Subnet delegation](https://learn.microsoft.com/azure/virtual-network/subnet-delegation-overview) | -| NSG | [2024-07-01](https://learn.microsoft.com/azure/templates/microsoft.network/networksecuritygroups?pivots=deployment-language-bicep) | [NSG overview](https://learn.microsoft.com/azure/virtual-network/network-security-groups-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftnetwork) | [Security rules](https://learn.microsoft.com/azure/virtual-network/network-security-group-how-it-works) | -| Route Table | [2024-07-01](https://learn.microsoft.com/azure/templates/microsoft.network/routetables?pivots=deployment-language-bicep) | [Traffic routing](https://learn.microsoft.com/azure/virtual-network/virtual-networks-udr-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftnetwork) | [Forced tunneling](https://learn.microsoft.com/azure/vpn-gateway/vpn-gateway-forced-tunneling-rm) | -| Network Interface | [2024-07-01](https://learn.microsoft.com/azure/templates/microsoft.network/networkinterfaces?pivots=deployment-language-bicep) | [NIC overview](https://learn.microsoft.com/azure/virtual-network/virtual-network-network-interface) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftnetwork) | [Accelerated networking](https://learn.microsoft.com/azure/virtual-network/accelerated-networking-overview) | -| Public IP | [2024-07-01](https://learn.microsoft.com/azure/templates/microsoft.network/publicipaddresses?pivots=deployment-language-bicep) | [Public IP overview](https://learn.microsoft.com/azure/virtual-network/ip-services/public-ip-addresses) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftnetwork) | [Basic SKU retirement](https://learn.microsoft.com/azure/virtual-network/ip-services/public-ip-basic-upgrade-guidance) | -| NAT Gateway | [2024-07-01](https://learn.microsoft.com/azure/templates/microsoft.network/natgateways?pivots=deployment-language-bicep) | [NAT Gateway overview](https://learn.microsoft.com/azure/nat-gateway/nat-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftnetwork) | [Availability zones](https://learn.microsoft.com/azure/nat-gateway/nat-availability-zones) | +# Networking (Core) Resources + +| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | +|----------|----------|-------------|------------|--------------|--------| +| Virtual Network | `Microsoft.Network/virtualNetworks` | `2024-07-01` | `vnet` | Resource group | Foundational | +| Subnet | `Microsoft.Network/virtualNetworks/subnets` | `2024-07-01` | `snet` | Parent VNet | Foundational | +| NSG | `Microsoft.Network/networkSecurityGroups` | `2024-07-01` | `nsg` | Resource group | Foundational | +| Route Table | `Microsoft.Network/routeTables` | `2024-07-01` | `rt` | Resource group | Foundational | +| Network Interface | `Microsoft.Network/networkInterfaces` | `2024-07-01` | `nic` | Resource group | Foundational | +| Public IP | `Microsoft.Network/publicIPAddresses` | `2024-07-01` | `pip` | Resource group | Foundational | +| NAT Gateway | `Microsoft.Network/natGateways` | `2024-07-01` | `ng` | Resource group | Foundational | + +## Documentation + +| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | +|----------|----------------|------------------|--------------|------------| +| Virtual Network | [2024-07-01](https://learn.microsoft.com/azure/templates/microsoft.network/virtualnetworks?pivots=deployment-language-bicep) | [VNet overview](https://learn.microsoft.com/azure/virtual-network/virtual-networks-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftnetwork) | [VNet planning](https://learn.microsoft.com/azure/virtual-network/virtual-network-vnet-plan-design-arm) | +| Subnet | [2024-07-01](https://learn.microsoft.com/azure/templates/microsoft.network/virtualnetworks/subnets?pivots=deployment-language-bicep) | [Subnets](https://learn.microsoft.com/azure/virtual-network/virtual-network-manage-subnet) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftnetwork) | [Subnet delegation](https://learn.microsoft.com/azure/virtual-network/subnet-delegation-overview) | +| NSG | [2024-07-01](https://learn.microsoft.com/azure/templates/microsoft.network/networksecuritygroups?pivots=deployment-language-bicep) | [NSG overview](https://learn.microsoft.com/azure/virtual-network/network-security-groups-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftnetwork) | [Security rules](https://learn.microsoft.com/azure/virtual-network/network-security-group-how-it-works) | +| Route Table | [2024-07-01](https://learn.microsoft.com/azure/templates/microsoft.network/routetables?pivots=deployment-language-bicep) | [Traffic routing](https://learn.microsoft.com/azure/virtual-network/virtual-networks-udr-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftnetwork) | [Forced tunneling](https://learn.microsoft.com/azure/vpn-gateway/vpn-gateway-forced-tunneling-rm) | +| Network Interface | [2024-07-01](https://learn.microsoft.com/azure/templates/microsoft.network/networkinterfaces?pivots=deployment-language-bicep) | [NIC overview](https://learn.microsoft.com/azure/virtual-network/virtual-network-network-interface) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftnetwork) | [Accelerated networking](https://learn.microsoft.com/azure/virtual-network/accelerated-networking-overview) | +| Public IP | [2024-07-01](https://learn.microsoft.com/azure/templates/microsoft.network/publicipaddresses?pivots=deployment-language-bicep) | [Public IP overview](https://learn.microsoft.com/azure/virtual-network/ip-services/public-ip-addresses) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftnetwork) | [Basic SKU retirement](https://learn.microsoft.com/azure/virtual-network/ip-services/public-ip-basic-upgrade-guidance) | +| NAT Gateway | [2024-07-01](https://learn.microsoft.com/azure/templates/microsoft.network/natgateways?pivots=deployment-language-bicep) | [NAT Gateway overview](https://learn.microsoft.com/azure/nat-gateway/nat-overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftnetwork) | [Availability zones](https://learn.microsoft.com/azure/nat-gateway/nat-availability-zones) | diff --git a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/security.md b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/security.md index 74aaa489..7ab9e1a8 100644 --- a/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/security.md +++ b/.github/plugins/azure-skills/skills/azure-enterprise-infra-planner/references/resources/security.md @@ -1,13 +1,13 @@ -# Security Resources - -| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | -|----------|----------|-------------|------------|--------------|--------| -| Key Vault | `Microsoft.KeyVault/vaults` | `2024-11-01` | `kv` | Global | Foundational | -| Managed Identity | `Microsoft.ManagedIdentity/userAssignedIdentities` | `2024-11-30` | `id` | Resource group | Foundational | - -## Documentation - -| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | -|----------|----------------|------------------|--------------|------------| -| Key Vault | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.keyvault/vaults?pivots=deployment-language-bicep) | [Key Vault overview](https://learn.microsoft.com/azure/key-vault/general/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftkeyvault) | [Soft-delete](https://learn.microsoft.com/azure/key-vault/general/soft-delete-overview) | -| Managed Identity | [2024-11-30](https://learn.microsoft.com/azure/templates/microsoft.managedidentity/userassignedidentities?pivots=deployment-language-bicep) | [Managed identities](https://learn.microsoft.com/entra/identity/managed-identities-azure-resources/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftmanagedidentity) | [Workload identity federation](https://learn.microsoft.com/entra/workload-id/workload-identity-federation) | +# Security Resources + +| Resource | ARM Type | API Version | CAF Prefix | Naming Scope | Region | +|----------|----------|-------------|------------|--------------|--------| +| Key Vault | `Microsoft.KeyVault/vaults` | `2024-11-01` | `kv` | Global | Foundational | +| Managed Identity | `Microsoft.ManagedIdentity/userAssignedIdentities` | `2024-11-30` | `id` | Resource group | Foundational | + +## Documentation + +| Resource | Bicep Reference | Service Overview | Naming Rules | Additional | +|----------|----------------|------------------|--------------|------------| +| Key Vault | [2024-11-01](https://learn.microsoft.com/azure/templates/microsoft.keyvault/vaults?pivots=deployment-language-bicep) | [Key Vault overview](https://learn.microsoft.com/azure/key-vault/general/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftkeyvault) | [Soft-delete](https://learn.microsoft.com/azure/key-vault/general/soft-delete-overview) | +| Managed Identity | [2024-11-30](https://learn.microsoft.com/azure/templates/microsoft.managedidentity/userassignedidentities?pivots=deployment-language-bicep) | [Managed identities](https://learn.microsoft.com/entra/identity/managed-identities-azure-resources/overview) | [Naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules#microsoftmanagedidentity) | [Workload identity federation](https://learn.microsoft.com/entra/workload-id/workload-identity-federation) | diff --git a/.github/plugins/azure-skills/skills/azure-prepare/SKILL.md b/.github/plugins/azure-skills/skills/azure-prepare/SKILL.md index c1f94b2f..15e9ea97 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/SKILL.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/SKILL.md @@ -4,7 +4,7 @@ description: "Prepare azd-based Azure projects for deployment: generates azure.y license: MIT metadata: author: Microsoft - version: "1.3.4" + version: "1.3.5" --- # Azure Prepare diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/architecture.md b/.github/plugins/azure-skills/skills/azure-prepare/references/architecture.md index cea8f79d..42951a84 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/architecture.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/architecture.md @@ -1,107 +1,107 @@ -# Architecture Planning - -Select hosting stack and map components to Azure services. - -## Stack Selection - -| Stack | Best For | Azure Services | -| --------------- | ------------------------------------------------------ | --------------------------------- | -| **Containers** | Docker experience, complex dependencies, microservices | Container Apps, AKS, ACR | -| **Serverless** | Event-driven, variable traffic, cost optimization | Functions, Logic Apps, Event Grid | -| **App Service** | Traditional web apps, PaaS preference | App Service, Static Web Apps | - -### Decision Factors - -| Factor | Containers | Serverless | App Service | -| ------------------------ | :--------: | :--------------------------: | :---------: | -| Docker experience | ✓✓ | | | -| Event-driven | ✓ | ✓✓ | | -| Variable traffic | | ✓✓ | ✓ | -| Complex dependencies | ✓✓ | | ✓ | -| Long-running processes | ✓✓ | ✓ (Durable Functions) | ✓ | -| Workflow / orchestration | | ✓✓ (Durable Functions + DTS) | | -| Minimal ops overhead | | ✓✓ | ✓ | - -### Container Hosting: Container Apps vs AKS - -| Factor | Container Apps | AKS | -| ------------------------- | :-------------------------: | :---------------------------------: | -| **Scale to zero** | ✓✓ | | -| **Kubernetes API access** | | ✓✓ | -| **Custom operators/CRDs** | | ✓✓ | -| **Service mesh** | Dapr (built-in) | Istio | -| **Networking/dataplane** | Managed platform defaults | Azure CNI powered by Cilium | -| **GPU workloads** | | ✓✓ | -| **Best for** | Microservices, event-driven | Full K8s control, complex workloads | - -#### When to Use Container Apps - -- Microservices without Kubernetes complexity -- Event-driven workloads (KEDA built-in) -- Need scale-to-zero for cost optimization -- Teams without Kubernetes expertise - -#### When to Use AKS - -- Need Kubernetes API/kubectl access -- Require custom operators or CRDs -- Service mesh requirements (Istio, Linkerd) -- GPU/ML workloads -- Complex networking or multi-tenant architectures - -> **AKS Planning:** For AKS SKU selection (Automatic vs Standard), networking, identity, scaling, and security configuration, invoke the **azure-kubernetes** skill. - -## Service Mapping - -### Hosting - -| Component Type | Primary Service | Alternatives | -| ------------------------ | ----------------- | ------------------------------------------------ | -| SPA Frontend | Static Web Apps | Blob + CDN | -| SSR Web App | Container Apps | App Service, AKS | -| REST/GraphQL API | Container Apps | App Service, Functions, AKS | -| Background Worker | Container Apps | Functions, AKS | -| Scheduled Task | Functions (Timer) | Container Apps Jobs, Kubernetes CronJob (on AKS) | -| Event Processor | Functions | Container Apps, AKS + KEDA | -| Microservices (full K8s) | AKS | Container Apps | -| GPU/ML Workloads | AKS | Azure ML | - -### Data - -| Need | Primary | Reference | Alternatives | -| ---------- | ------------ | ----------------------------------------------- | ----------------- | -| Relational | Azure SQL | [SQL Database](services/sql-database/README.md) | PostgreSQL, MySQL | -| Document | Cosmos DB | [Cosmos DB](services/cosmos-db/README.md) | MongoDB | -| Cache | Redis Cache | | | -| Files | Blob Storage | [Storage](services/storage/README.md) | Files Storage | -| Search | AI Search | | | - -### Integration - -| Need | Service | -| ------------- | ----------- | -| Message Queue | Service Bus | -| Pub/Sub | Event Grid | -| Streaming | Event Hubs | - -### Workflow & Orchestration - -| Need | Service | Notes | -| ----------------------------------- | ---------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Multi-step workflow / orchestration | **Durable Functions + Durable Task Scheduler** | DTS is the **required** managed backend for Durable Functions. Do NOT use Azure Storage or MSSQL backends. See [durable.md](services/functions/durable.md). | -| Low-code / visual workflow | Logic Apps | For integration-heavy, low-code scenarios | - -### Supporting (Always Include) - -| Service | Purpose | -| -------------------- | ----------------------- | -| Log Analytics | Centralized logging | -| Application Insights | Monitoring, APM | -| Key Vault | Secrets management | -| Managed Identity | Service-to-service auth | - ---- - -## Document Architecture - -Record selections in `.azure/deployment-plan.md` with rationale for each choice. +# Architecture Planning + +Select hosting stack and map components to Azure services. + +## Stack Selection + +| Stack | Best For | Azure Services | +| --------------- | ------------------------------------------------------ | --------------------------------- | +| **Containers** | Docker experience, complex dependencies, microservices | Container Apps, AKS, ACR | +| **Serverless** | Event-driven, variable traffic, cost optimization | Functions, Logic Apps, Event Grid | +| **App Service** | Traditional web apps, PaaS preference | App Service, Static Web Apps | + +### Decision Factors + +| Factor | Containers | Serverless | App Service | +| ------------------------ | :--------: | :--------------------------: | :---------: | +| Docker experience | ✓✓ | | | +| Event-driven | ✓ | ✓✓ | | +| Variable traffic | | ✓✓ | ✓ | +| Complex dependencies | ✓✓ | | ✓ | +| Long-running processes | ✓✓ | ✓ (Durable Functions) | ✓ | +| Workflow / orchestration | | ✓✓ (Durable Functions + DTS) | | +| Minimal ops overhead | | ✓✓ | ✓ | + +### Container Hosting: Container Apps vs AKS + +| Factor | Container Apps | AKS | +| ------------------------- | :-------------------------: | :---------------------------------: | +| **Scale to zero** | ✓✓ | | +| **Kubernetes API access** | | ✓✓ | +| **Custom operators/CRDs** | | ✓✓ | +| **Service mesh** | Dapr (built-in) | Istio | +| **Networking/dataplane** | Managed platform defaults | Azure CNI powered by Cilium | +| **GPU workloads** | | ✓✓ | +| **Best for** | Microservices, event-driven | Full K8s control, complex workloads | + +#### When to Use Container Apps + +- Microservices without Kubernetes complexity +- Event-driven workloads (KEDA built-in) +- Need scale-to-zero for cost optimization +- Teams without Kubernetes expertise + +#### When to Use AKS + +- Need Kubernetes API/kubectl access +- Require custom operators or CRDs +- Service mesh requirements (Istio, Linkerd) +- GPU/ML workloads +- Complex networking or multi-tenant architectures + +> **AKS Planning:** For AKS SKU selection (Automatic vs Standard), networking, identity, scaling, and security configuration, invoke the **azure-kubernetes** skill. + +## Service Mapping + +### Hosting + +| Component Type | Primary Service | Alternatives | +| ------------------------ | ----------------- | ------------------------------------------------ | +| SPA Frontend | Static Web Apps | Blob + CDN | +| SSR Web App | Container Apps | App Service, AKS | +| REST/GraphQL API | Container Apps | App Service, Functions, AKS | +| Background Worker | Container Apps | Functions, AKS | +| Scheduled Task | Functions (Timer) | Container Apps Jobs, Kubernetes CronJob (on AKS) | +| Event Processor | Functions | Container Apps, AKS + KEDA | +| Microservices (full K8s) | AKS | Container Apps | +| GPU/ML Workloads | AKS | Azure ML | + +### Data + +| Need | Primary | Reference | Alternatives | +| ---------- | ------------ | ----------------------------------------------- | ----------------- | +| Relational | Azure SQL | [SQL Database](services/sql-database/README.md) | PostgreSQL, MySQL | +| Document | Cosmos DB | [Cosmos DB](services/cosmos-db/README.md) | MongoDB | +| Cache | Redis Cache | | | +| Files | Blob Storage | [Storage](services/storage/README.md) | Files Storage | +| Search | AI Search | | | + +### Integration + +| Need | Service | +| ------------- | ----------- | +| Message Queue | Service Bus | +| Pub/Sub | Event Grid | +| Streaming | Event Hubs | + +### Workflow & Orchestration + +| Need | Service | Notes | +| ----------------------------------- | ---------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Multi-step workflow / orchestration | **Durable Functions + Durable Task Scheduler** | DTS is the **required** managed backend for Durable Functions. Do NOT use Azure Storage or MSSQL backends. See [durable.md](services/functions/durable.md). | +| Low-code / visual workflow | Logic Apps | For integration-heavy, low-code scenarios | + +### Supporting (Always Include) + +| Service | Purpose | +| -------------------- | ----------------------- | +| Log Analytics | Centralized logging | +| Application Insights | Monitoring, APM | +| Key Vault | Secrets management | +| Managed Identity | Service-to-service auth | + +--- + +## Document Architecture + +Record selections in `.azure/deployment-plan.md` with rationale for each choice. diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/azure-context.md b/.github/plugins/azure-skills/skills/azure-prepare/references/azure-context.md index 8d20a543..29eb3b0c 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/azure-context.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/azure-context.md @@ -1,185 +1,185 @@ -# Azure Context (Subscription & Location) - -Detect and confirm Azure subscription and location before generating artifacts. Run region capacity check for customer selected location - ---- - -## Step 1: Check for Existing AZD Environment - -If the project already uses AZD, check for an existing environment with values already set: - -```bash -azd env list -``` - -**If an environment is selected** (marked with `*`), check its values: - -```bash -azd env get-values -``` - -If `AZURE_SUBSCRIPTION_ID` and `AZURE_LOCATION` are already set, use `ask_user` to confirm reuse: - -``` -Question: "I found an existing AZD environment with these settings. Would you like to continue with them?" - - Environment: {env-name} - Subscription: {subscription-name} ({subscription-id}) - Location: {location} - -Choices: [ - "Yes, use these settings (Recommended)", - "No, let me choose different settings" -] -``` - -If user confirms → skip to **Record in Plan**. Otherwise → continue to Step 2. - ---- - -## Step 2: Detect Defaults - -Check for user-configured defaults: - -```bash -azd config get defaults -``` - -Returns JSON with any configured defaults: -```json -{ - "subscription": "25fd0362-aa79-488b-b37b-d6e892009fdf", - "location": "eastus2" -} -``` - -Use these as **recommended** values if present. - -If no defaults, fall back to az CLI: -```bash -az account show --query "{name:name, id:id}" -o json -``` - -## Step 3: Confirm Subscription with User - -Use `ask_user` with the **actual subscription name and ID**: - -✅ **Correct:** -``` -Question: "Which Azure subscription would you like to deploy to?" -Choices: [ - "Use current: jongdevdiv (25fd0362-aa79-488b-b37b-d6e892009fdf) (Recommended)", - "Let me specify a different subscription" -] -``` - -❌ **Wrong** (never do this): -``` -Choices: [ - "Use default subscription", // ← Does not show actual name - "Let me specify" -] -``` - -If user wants a different subscription: -```bash -az account list --output table -``` - ---- - -## Step 4: Confirm Location with User - -1. Consult [Region Availability](region-availability.md) for services with limited availability -2. Present only regions that support ALL selected services -3. Use `ask_user`: -4. After customer selected region, do provisioning limit check, consult [Resource Limits and Quotas](resources-limits-quotas.md). For this also invoke azure-quotas - -``` -Question: "Which Azure region would you like to deploy to?" -Based on your architecture ({list services}), these regions support all services: -Choices: [ - "eastus2 (Recommended)", - "westus2", - "westeurope" -] -``` - -⚠️ Do NOT include regions that don't support all services — deployment will fail. - ---- -## Step 5: Check Resource Provisioning Limits - -1. **List resource types and quantities** that will be deployed from the planned architecture (e.g., 2x Standard D4s v3 VMs, 1x VNet, 3x Storage Accounts) - -2. **Determine limits for each resource type** using the user-selected subscription and region: - - Reference [./resources-limits-quotas.md](./resources-limits-quotas.md) for documented limits - - Use **azure-quotas** skill to check current quotas and usage for the selected subscription and region - - If `az quota list` returns `BadRequest` error, the resource provider doesn't support quota API - -3. **For resources that don't support quota API** (e.g., Microsoft.DocumentDB, or when you get `BadRequest` from `az quota list`): - - Invoke **azure-resource-lookup** skill to count existing deployments of that resource type in the selected subscription and region - - Use the count to calculate: `Total After Deployment = Current Count + Planned Deployment` - - Reference [Azure service limits documentation](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/azure-subscription-service-limits) for the limit value - - Document in provisioning checklist as "Fetched from: azure-resource-lookup + Official docs" - -4. **Validate deployment capacity**: - - Compare planned deployment quantities against available quota (limit - current usage) - - If **insufficient capacity** is found, notify the customer and return to **Step 4** to select a different region - - Use **azure-quotas** skill to compare capacity across multiple regions and recommend alternatives - -## Record in Plan - -After confirmation, record in `.azure/deployment-plan.md`: - -```markdown -## Azure Context -- **Subscription**: jongdevdiv (25fd0362-aa79-488b-b37b-d6e892009fdf) -- **Location**: eastus2 -``` - ---- - -## Step 6: Apply to AZD Environment - -> **⛔ CRITICAL for Aspire and azd projects**: After user confirms subscription and location, you **MUST** set these values in the azd environment immediately after running `azd init` or `azd env new`. Always use `--no-prompt` with these commands to prevent interactive prompts from blocking execution. -> -> **DO NOT** wait until validation or deployment. The Azure CLI and azd maintain separate configuration contexts. - -**For Aspire projects using `azd init --from-code`:** - -```bash -# 1. Run azd init -azd init --from-code -e --no-prompt - -# 2. IMMEDIATELY set the user-confirmed subscription -azd env set AZURE_SUBSCRIPTION_ID - -# 3. Set the location -azd env set AZURE_LOCATION - -# 4. Verify -azd env get-values -``` - -**For non-Aspire projects using `azd env new`:** - -```bash -# 1. Create environment -azd env new --no-prompt - -# 2. IMMEDIATELY set the user-confirmed subscription -azd env set AZURE_SUBSCRIPTION_ID - -# 3. Set the location -azd env set AZURE_LOCATION - -# 4. Verify -azd env get-values -``` - -**Why this is critical:** -- `az account show` returns the Azure CLI's default subscription -- `azd` maintains its own configuration with potentially different defaults -- If you don't set `AZURE_SUBSCRIPTION_ID` explicitly, azd will use its own default -- This can result in deploying to the wrong subscription despite user confirmation +# Azure Context (Subscription & Location) + +Detect and confirm Azure subscription and location before generating artifacts. Run region capacity check for customer selected location + +--- + +## Step 1: Check for Existing AZD Environment + +If the project already uses AZD, check for an existing environment with values already set: + +```bash +azd env list +``` + +**If an environment is selected** (marked with `*`), check its values: + +```bash +azd env get-values +``` + +If `AZURE_SUBSCRIPTION_ID` and `AZURE_LOCATION` are already set, use `ask_user` to confirm reuse: + +``` +Question: "I found an existing AZD environment with these settings. Would you like to continue with them?" + + Environment: {env-name} + Subscription: {subscription-name} ({subscription-id}) + Location: {location} + +Choices: [ + "Yes, use these settings (Recommended)", + "No, let me choose different settings" +] +``` + +If user confirms → skip to **Record in Plan**. Otherwise → continue to Step 2. + +--- + +## Step 2: Detect Defaults + +Check for user-configured defaults: + +```bash +azd config get defaults +``` + +Returns JSON with any configured defaults: +```json +{ + "subscription": "25fd0362-aa79-488b-b37b-d6e892009fdf", + "location": "eastus2" +} +``` + +Use these as **recommended** values if present. + +If no defaults, fall back to az CLI: +```bash +az account show --query "{name:name, id:id}" -o json +``` + +## Step 3: Confirm Subscription with User + +Use `ask_user` with the **actual subscription name and ID**: + +✅ **Correct:** +``` +Question: "Which Azure subscription would you like to deploy to?" +Choices: [ + "Use current: jongdevdiv (25fd0362-aa79-488b-b37b-d6e892009fdf) (Recommended)", + "Let me specify a different subscription" +] +``` + +❌ **Wrong** (never do this): +``` +Choices: [ + "Use default subscription", // ← Does not show actual name + "Let me specify" +] +``` + +If user wants a different subscription: +```bash +az account list --output table +``` + +--- + +## Step 4: Confirm Location with User + +1. Consult [Region Availability](region-availability.md) for services with limited availability +2. Present only regions that support ALL selected services +3. Use `ask_user`: +4. After customer selected region, do provisioning limit check, consult [Resource Limits and Quotas](resources-limits-quotas.md). For this also invoke azure-quotas + +``` +Question: "Which Azure region would you like to deploy to?" +Based on your architecture ({list services}), these regions support all services: +Choices: [ + "eastus2 (Recommended)", + "westus2", + "westeurope" +] +``` + +⚠️ Do NOT include regions that don't support all services — deployment will fail. + +--- +## Step 5: Check Resource Provisioning Limits + +1. **List resource types and quantities** that will be deployed from the planned architecture (e.g., 2x Standard D4s v3 VMs, 1x VNet, 3x Storage Accounts) + +2. **Determine limits for each resource type** using the user-selected subscription and region: + - Reference [./resources-limits-quotas.md](./resources-limits-quotas.md) for documented limits + - Use **azure-quotas** skill to check current quotas and usage for the selected subscription and region + - If `az quota list` returns `BadRequest` error, the resource provider doesn't support quota API + +3. **For resources that don't support quota API** (e.g., Microsoft.DocumentDB, or when you get `BadRequest` from `az quota list`): + - Invoke **azure-resource-lookup** skill to count existing deployments of that resource type in the selected subscription and region + - Use the count to calculate: `Total After Deployment = Current Count + Planned Deployment` + - Reference [Azure service limits documentation](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/azure-subscription-service-limits) for the limit value + - Document in provisioning checklist as "Fetched from: azure-resource-lookup + Official docs" + +4. **Validate deployment capacity**: + - Compare planned deployment quantities against available quota (limit - current usage) + - If **insufficient capacity** is found, notify the customer and return to **Step 4** to select a different region + - Use **azure-quotas** skill to compare capacity across multiple regions and recommend alternatives + +## Record in Plan + +After confirmation, record in `.azure/deployment-plan.md`: + +```markdown +## Azure Context +- **Subscription**: jongdevdiv (25fd0362-aa79-488b-b37b-d6e892009fdf) +- **Location**: eastus2 +``` + +--- + +## Step 6: Apply to AZD Environment + +> **⛔ CRITICAL for Aspire and azd projects**: After user confirms subscription and location, you **MUST** set these values in the azd environment immediately after running `azd init` or `azd env new`. Always use `--no-prompt` with these commands to prevent interactive prompts from blocking execution. +> +> **DO NOT** wait until validation or deployment. The Azure CLI and azd maintain separate configuration contexts. + +**For Aspire projects using `azd init --from-code`:** + +```bash +# 1. Run azd init +azd init --from-code -e --no-prompt + +# 2. IMMEDIATELY set the user-confirmed subscription +azd env set AZURE_SUBSCRIPTION_ID + +# 3. Set the location +azd env set AZURE_LOCATION + +# 4. Verify +azd env get-values +``` + +**For non-Aspire projects using `azd env new`:** + +```bash +# 1. Create environment +azd env new --no-prompt + +# 2. IMMEDIATELY set the user-confirmed subscription +azd env set AZURE_SUBSCRIPTION_ID + +# 3. Set the location +azd env set AZURE_LOCATION + +# 4. Verify +azd env get-values +``` + +**Why this is critical:** +- `az account show` returns the Azure CLI's default subscription +- `azd` maintains its own configuration with potentially different defaults +- If you don't set `AZURE_SUBSCRIPTION_ID` explicitly, azd will use its own default +- This can result in deploying to the wrong subscription despite user confirmation diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/generate.md b/.github/plugins/azure-skills/skills/azure-prepare/references/generate.md index fb2917fb..39b11d71 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/generate.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/generate.md @@ -1,128 +1,128 @@ -# Artifact Generation - -Generate infrastructure and configuration files based on selected recipe. - -## ⛔ CRITICAL: Check for .NET Aspire Projects FIRST - -**MANDATORY: Before generating any files, detect .NET Aspire projects:** - -```bash -# Method 1: Find AppHost project files -find . -name "*.AppHost.csproj" -o -name "*AppHost.csproj" - -# Method 2: Search for Aspire packages -grep -r "Aspire\.Hosting\|Aspire\.AppHost\.Sdk" . --include="*.csproj" -``` - -**If Aspire is detected:** -1. ⛔ **STOP** - Do NOT manually create `azure.yaml` -2. ⛔ **STOP** - Do NOT manually create `infra/` files -3. ✅ **USE** - `azd init --from-code -e ` instead -4. 📖 **READ** - [aspire.md](aspire.md) and [recipes/azd/aspire.md](recipes/azd/aspire.md) for complete guidance - -**Why this is critical:** -- Aspire AppHost auto-generates infrastructure from code -- Manual `azure.yaml` without `services` section causes "infra\main.bicep not found" error -- `azd init --from-code` correctly detects AppHost and generates proper configuration - -> ⚠️ **Manually creating azure.yaml for Aspire projects is the most common deployment failure.** Always use `azd init --from-code`. - -## Check for Other Special Patterns - -After verifying the project is NOT Aspire, check for these patterns: - -| Pattern | Detection | Action | -|---------|-----------|--------| -| **Complex existing codebase** | Multiple services, existing structure | Consider `azd init --from-code` | -| **Existing azure.yaml** | File already present | MODIFY mode - update existing config | - -> **CRITICAL:** After running `azd init --from-code`, you **MUST** immediately set the user-confirmed subscription with `azd env set AZURE_SUBSCRIPTION_ID `. Do NOT skip this step. See [aspire.md](aspire.md) Step 3 for the complete sequence. - -## CRITICAL: Research Must Be Complete - -**DO NOT generate any files without first completing the [Research Components](research.md) step.** - -The research step loads service-specific references and invokes related skills to gather best practices. Apply all research findings to generated artifacts. - -## Research Checklist - -1. ✅ Completed [Research Components](research.md) step -2. ✅ Loaded all relevant `services/*.md` references -3. ✅ Invoked related skills for specialized guidance -4. ✅ Documented findings in `.azure/deployment-plan.md` - -## Generation Order - -| Order | Artifact | Notes | -|-------|----------|-------| -| 1 | Application config (azure.yaml) | AZD only—defines services and hosting | -| 2 | Application code scaffolding | Entry points, health endpoints, config | -| 3 | Dockerfiles | If containerized | -| 4 | Infrastructure (Bicep/Terraform) | IaC templates in `./infra/` | -| 5 | CI/CD pipelines | If requested | - -## Recipe-Specific Generation - -Load the appropriate recipe for detailed generation steps: - -| Recipe | Guide | -|--------|-------| -| AZD | [AZD Recipe](recipes/azd/README.md) | -| AZCLI | [AZCLI Recipe](recipes/azcli/README.md) | -| Bicep | [Bicep Recipe](recipes/bicep/README.md) | -| Terraform | [Terraform Recipe](recipes/terraform/README.md) | - -## Common Standards - -### File Structure - -``` -project-root/ -├── .azure/ -│ └── deployment-plan.md -├── infra/ -│ ├── main.bicep (or main.tf) -│ └── modules/ -├── src/ -│ └── / -│ └── Dockerfile -└── azure.yaml (AZD only) -``` - -### Directory Creation - -> ⚠️ **Warning:** The `create` tool fails with `Parent directory does not exist` when intermediate directories are missing. Always create the full directory tree before writing files. - -**Before creating nested files** (e.g., `src/frontend/src/App.jsx`), create all parent directories first: - -```bash -mkdir -p src/frontend/src src/api -``` - -- Use **absolute paths** in `mkdir -p` when the working directory may differ from the project root -- Create directories for **all components** in a single command before writing any files -- Do **not** rely on the `create` tool to create parent directories — it will not - -### Security Requirements - -- No hardcoded secrets -- Use Key Vault for sensitive values -- Managed Identity for service auth -- HTTPS only, TLS 1.2+ -- SQL Server Bicep MUST use Entra-only auth — omit `administratorLogin` and `administratorLoginPassword` entirely, including from conditional/ternary branches (see [services/sql-database/bicep.md](services/sql-database/bicep.md)). These property names must not appear anywhere in a generated `.bicep` file. -- **SQL + Managed Identity: MUST add postprovision hook** — ARM role assignments only grant control-plane access; you MUST also generate `scripts/grant-sql-access.sh` + `.ps1` and add a `postprovision` hook in `azure.yaml` to run T-SQL grants. See [services/sql-database/bicep.md](services/sql-database/bicep.md). -- **App Service Bicep: MUST include `azd-service-name` tag** — Every App Service `Microsoft.Web/sites` resource MUST have `tags: union(tags, { 'azd-service-name': serviceName })`. Without this tag, `azd deploy` cannot locate the resource. See [services/app-service/bicep.md](services/app-service/bicep.md). - -### Runtime Configuration - -Apply language-specific production settings for containerized apps: - -| Runtime | Reference | -|---------|-----------| -| Node.js/Express | [runtimes/nodejs.md](runtimes/nodejs.md) | - -## After Generation - -1. Update `.azure/deployment-plan.md` with generated file list -2. Run validation checks -3. Proceed to **azure-validate** skill +# Artifact Generation + +Generate infrastructure and configuration files based on selected recipe. + +## ⛔ CRITICAL: Check for .NET Aspire Projects FIRST + +**MANDATORY: Before generating any files, detect .NET Aspire projects:** + +```bash +# Method 1: Find AppHost project files +find . -name "*.AppHost.csproj" -o -name "*AppHost.csproj" + +# Method 2: Search for Aspire packages +grep -r "Aspire\.Hosting\|Aspire\.AppHost\.Sdk" . --include="*.csproj" +``` + +**If Aspire is detected:** +1. ⛔ **STOP** - Do NOT manually create `azure.yaml` +2. ⛔ **STOP** - Do NOT manually create `infra/` files +3. ✅ **USE** - `azd init --from-code -e ` instead +4. 📖 **READ** - [aspire.md](aspire.md) and [recipes/azd/aspire.md](recipes/azd/aspire.md) for complete guidance + +**Why this is critical:** +- Aspire AppHost auto-generates infrastructure from code +- Manual `azure.yaml` without `services` section causes "infra\main.bicep not found" error +- `azd init --from-code` correctly detects AppHost and generates proper configuration + +> ⚠️ **Manually creating azure.yaml for Aspire projects is the most common deployment failure.** Always use `azd init --from-code`. + +## Check for Other Special Patterns + +After verifying the project is NOT Aspire, check for these patterns: + +| Pattern | Detection | Action | +|---------|-----------|--------| +| **Complex existing codebase** | Multiple services, existing structure | Consider `azd init --from-code` | +| **Existing azure.yaml** | File already present | MODIFY mode - update existing config | + +> **CRITICAL:** After running `azd init --from-code`, you **MUST** immediately set the user-confirmed subscription with `azd env set AZURE_SUBSCRIPTION_ID `. Do NOT skip this step. See [aspire.md](aspire.md) Step 3 for the complete sequence. + +## CRITICAL: Research Must Be Complete + +**DO NOT generate any files without first completing the [Research Components](research.md) step.** + +The research step loads service-specific references and invokes related skills to gather best practices. Apply all research findings to generated artifacts. + +## Research Checklist + +1. ✅ Completed [Research Components](research.md) step +2. ✅ Loaded all relevant `services/*.md` references +3. ✅ Invoked related skills for specialized guidance +4. ✅ Documented findings in `.azure/deployment-plan.md` + +## Generation Order + +| Order | Artifact | Notes | +|-------|----------|-------| +| 1 | Application config (azure.yaml) | AZD only—defines services and hosting | +| 2 | Application code scaffolding | Entry points, health endpoints, config | +| 3 | Dockerfiles | If containerized | +| 4 | Infrastructure (Bicep/Terraform) | IaC templates in `./infra/` | +| 5 | CI/CD pipelines | If requested | + +## Recipe-Specific Generation + +Load the appropriate recipe for detailed generation steps: + +| Recipe | Guide | +|--------|-------| +| AZD | [AZD Recipe](recipes/azd/README.md) | +| AZCLI | [AZCLI Recipe](recipes/azcli/README.md) | +| Bicep | [Bicep Recipe](recipes/bicep/README.md) | +| Terraform | [Terraform Recipe](recipes/terraform/README.md) | + +## Common Standards + +### File Structure + +``` +project-root/ +├── .azure/ +│ └── deployment-plan.md +├── infra/ +│ ├── main.bicep (or main.tf) +│ └── modules/ +├── src/ +│ └── / +│ └── Dockerfile +└── azure.yaml (AZD only) +``` + +### Directory Creation + +> ⚠️ **Warning:** The `create` tool fails with `Parent directory does not exist` when intermediate directories are missing. Always create the full directory tree before writing files. + +**Before creating nested files** (e.g., `src/frontend/src/App.jsx`), create all parent directories first: + +```bash +mkdir -p src/frontend/src src/api +``` + +- Use **absolute paths** in `mkdir -p` when the working directory may differ from the project root +- Create directories for **all components** in a single command before writing any files +- Do **not** rely on the `create` tool to create parent directories — it will not + +### Security Requirements + +- No hardcoded secrets +- Use Key Vault for sensitive values +- Managed Identity for service auth +- HTTPS only, TLS 1.2+ +- SQL Server Bicep MUST use Entra-only auth — omit `administratorLogin` and `administratorLoginPassword` entirely, including from conditional/ternary branches (see [services/sql-database/bicep.md](services/sql-database/bicep.md)). These property names must not appear anywhere in a generated `.bicep` file. +- **SQL + Managed Identity: MUST add postprovision hook** — ARM role assignments only grant control-plane access; you MUST also generate `scripts/grant-sql-access.sh` + `.ps1` and add a `postprovision` hook in `azure.yaml` to run T-SQL grants. See [services/sql-database/bicep.md](services/sql-database/bicep.md). +- **App Service Bicep: MUST include `azd-service-name` tag** — Every App Service `Microsoft.Web/sites` resource MUST have `tags: union(tags, { 'azd-service-name': serviceName })`. Without this tag, `azd deploy` cannot locate the resource. See [services/app-service/bicep.md](services/app-service/bicep.md). + +### Runtime Configuration + +Apply language-specific production settings for containerized apps: + +| Runtime | Reference | +|---------|-----------| +| Node.js/Express | [runtimes/nodejs.md](runtimes/nodejs.md) | + +## After Generation + +1. Update `.azure/deployment-plan.md` with generated file list +2. Run validation checks +3. Proceed to **azure-validate** skill diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/recipe-selection.md b/.github/plugins/azure-skills/skills/azure-prepare/references/recipe-selection.md index 2168591a..df636ed7 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/recipe-selection.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/recipe-selection.md @@ -1,85 +1,85 @@ -# Recipe Selection - -Choose the deployment recipe based on project needs and existing tooling. - -## ⛔ Special Cases: Detect First - -**Before selecting a recipe, check for these special project types:** - -| Project Type | Detection | Recipe Selection | -|--------------|-----------|------------------| -| **.NET Aspire** | `*.AppHost.csproj` or `Aspire.Hosting` package | **AZD (auto via `azd init --from-code`)** → [aspire.md](aspire.md) | - -> 💡 **Tip:** .NET Aspire projects always use AZD recipe with auto-generated configuration. Do not manually select recipe or create artifacts. - -## Quick Decision - -**Default: AZD** unless specific requirements indicate otherwise. - -> 💡 **Tip:** azd supports both Bicep and Terraform as IaC providers. When Terraform is mentioned for Azure deployment, **default to azd+Terraform** for the best developer experience. - -## Decision Criteria - -| Choose | When | -|--------|------| -| **AZD (Bicep)** | New projects, multi-service apps, want simplest deployment (`azd up`) | -| **AZD (Terraform)** | **DEFAULT for Terraform** - Want Terraform IaC + azd simplicity, Azure deployment with Terraform | -| **AZCLI** | Existing az scripts, need imperative control, custom pipelines, AKS | -| **Bicep** | IaC-first approach, no CLI wrapper needed, direct ARM deployment | -| **Terraform** | Multi-cloud deployments (non-Azure-first), complex TF workflows incompatible with azd, explicitly requested | - -## Auto-Detection - -| Found in Workspace | Suggested Recipe | -|--------------------|------------------| -| `azure.yaml` with `infra.provider: terraform` | AZD (Terraform) | -| `azure.yaml` (Bicep or no provider specified) | AZD (Bicep) | -| `*.tf` files (no azure.yaml) | **AZD (Terraform) - DEFAULT** (unless multi-cloud) | -| `infra/*.bicep` (no azure.yaml) | Bicep or AZCLI | -| Existing `az` scripts | AZCLI | -| None | AZD (Bicep) - default | - -## Recipe Comparison - -| Feature | AZD (Bicep) | AZD (Terraform) | AZCLI | Bicep | Terraform | -|---------|-------------|-----------------|-------|-------|-----------| -| Config file | azure.yaml | azure.yaml + *.tf | scripts | *.bicep | *.tf | -| IaC language | Bicep | Terraform | N/A | Bicep | Terraform | -| Deploy command | `azd up` | `azd up` | `az` commands | `az deployment` | `terraform apply` | -| Dockerfile gen | Auto | Auto | Manual | Manual | Manual | -| Environment mgmt | Built-in | Built-in | Manual | Manual | Workspaces | -| CI/CD gen | Built-in | Built-in | Manual | Manual | Manual | -| Multi-cloud | No | Yes | No | No | Yes | -| Learning curve | Low | Low-Medium | Medium | Medium | Medium | - -## Record Selection - -Document in `.azure/deployment-plan.md`: - -```markdown -## Recipe: AZD (Terraform) - -**Rationale:** -- Team has Terraform expertise -- Want multi-cloud IaC flexibility -- But prefer azd's simple deployment workflow -- Multi-service app (API + Web) -``` - -Or for pure Terraform: - -```markdown -## Recipe: Terraform - -**Rationale:** -- Multi-cloud deployment (AWS + Azure) -- Complex Terraform modules incompatible with azd conventions -- Existing Terraform CI/CD pipeline -``` - -## Recipe References - -- [AZD Recipe](recipes/azd/README.md) -- [AZCLI Recipe](recipes/azcli/README.md) -- [Bicep Recipe](recipes/bicep/README.md) -- [Terraform Recipe](recipes/terraform/README.md) +# Recipe Selection + +Choose the deployment recipe based on project needs and existing tooling. + +## ⛔ Special Cases: Detect First + +**Before selecting a recipe, check for these special project types:** + +| Project Type | Detection | Recipe Selection | +|--------------|-----------|------------------| +| **.NET Aspire** | `*.AppHost.csproj` or `Aspire.Hosting` package | **AZD (auto via `azd init --from-code`)** → [aspire.md](aspire.md) | + +> 💡 **Tip:** .NET Aspire projects always use AZD recipe with auto-generated configuration. Do not manually select recipe or create artifacts. + +## Quick Decision + +**Default: AZD** unless specific requirements indicate otherwise. + +> 💡 **Tip:** azd supports both Bicep and Terraform as IaC providers. When Terraform is mentioned for Azure deployment, **default to azd+Terraform** for the best developer experience. + +## Decision Criteria + +| Choose | When | +|--------|------| +| **AZD (Bicep)** | New projects, multi-service apps, want simplest deployment (`azd up`) | +| **AZD (Terraform)** | **DEFAULT for Terraform** - Want Terraform IaC + azd simplicity, Azure deployment with Terraform | +| **AZCLI** | Existing az scripts, need imperative control, custom pipelines, AKS | +| **Bicep** | IaC-first approach, no CLI wrapper needed, direct ARM deployment | +| **Terraform** | Multi-cloud deployments (non-Azure-first), complex TF workflows incompatible with azd, explicitly requested | + +## Auto-Detection + +| Found in Workspace | Suggested Recipe | +|--------------------|------------------| +| `azure.yaml` with `infra.provider: terraform` | AZD (Terraform) | +| `azure.yaml` (Bicep or no provider specified) | AZD (Bicep) | +| `*.tf` files (no azure.yaml) | **AZD (Terraform) - DEFAULT** (unless multi-cloud) | +| `infra/*.bicep` (no azure.yaml) | Bicep or AZCLI | +| Existing `az` scripts | AZCLI | +| None | AZD (Bicep) - default | + +## Recipe Comparison + +| Feature | AZD (Bicep) | AZD (Terraform) | AZCLI | Bicep | Terraform | +|---------|-------------|-----------------|-------|-------|-----------| +| Config file | azure.yaml | azure.yaml + *.tf | scripts | *.bicep | *.tf | +| IaC language | Bicep | Terraform | N/A | Bicep | Terraform | +| Deploy command | `azd up` | `azd up` | `az` commands | `az deployment` | `terraform apply` | +| Dockerfile gen | Auto | Auto | Manual | Manual | Manual | +| Environment mgmt | Built-in | Built-in | Manual | Manual | Workspaces | +| CI/CD gen | Built-in | Built-in | Manual | Manual | Manual | +| Multi-cloud | No | Yes | No | No | Yes | +| Learning curve | Low | Low-Medium | Medium | Medium | Medium | + +## Record Selection + +Document in `.azure/deployment-plan.md`: + +```markdown +## Recipe: AZD (Terraform) + +**Rationale:** +- Team has Terraform expertise +- Want multi-cloud IaC flexibility +- But prefer azd's simple deployment workflow +- Multi-service app (API + Web) +``` + +Or for pure Terraform: + +```markdown +## Recipe: Terraform + +**Rationale:** +- Multi-cloud deployment (AWS + Azure) +- Complex Terraform modules incompatible with azd conventions +- Existing Terraform CI/CD pipeline +``` + +## Recipe References + +- [AZD Recipe](recipes/azd/README.md) +- [AZCLI Recipe](recipes/azcli/README.md) +- [Bicep Recipe](recipes/bicep/README.md) +- [Terraform Recipe](recipes/terraform/README.md) diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azcli/README.md b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azcli/README.md index deab66f9..cbd00034 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azcli/README.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azcli/README.md @@ -1,78 +1,78 @@ -# AZCLI Recipe - -Azure CLI workflow for imperative Azure deployments. - -## When to Use - -- Existing az scripts in project -- Need imperative control over deployment -- Custom deployment pipelines -- AKS deployments -- Direct resource manipulation - -## Before Generation - -**REQUIRED: Research best practices before generating any files.** - -| Artifact | Research Action | -|----------|-----------------| -| Bicep files | Call `mcp_bicep_get_bicep_best_practices` | -| Bicep modules | Call `mcp_bicep_list_avm_metadata` and follow [AVM module order](../azd/iac-rules.md#avm-module-selection-order-mandatory) | -| Azure CLI commands | Call `activate_azure_cli_management_tools` | -| Azure best practices | Call `mcp_azure_mcp_get_azure_bestpractices` | - -## Generation Steps - -### 1. Generate Infrastructure (Bicep) - -Create Bicep templates in `./infra/`. - -**Structure:** -``` -infra/ -├── main.bicep -├── main.parameters.json -└── modules/ - └── *.bicep -``` - -### 2. Generate Deployment Scripts - -Create deployment scripts for provisioning. - -→ [scripts.md](scripts.md) - -### 3. Generate Dockerfiles (if containerized) - -Manual Dockerfile creation required. - -## Output Checklist - -| Artifact | Path | -|----------|------| -| Main Bicep | `./infra/main.bicep` | -| Parameters | `./infra/main.parameters.json` | -| Modules | `./infra/modules/*.bicep` | -| Deploy script | `./scripts/deploy.sh` or `deploy.ps1` | -| Dockerfiles | `src//Dockerfile` | - -## Deployment Commands - -See [commands.md](commands.md) for common patterns. - -## Naming Convention - -Resources: `{prefix}{token}{instance}` -- Alphanumeric only, no special characters -- Prefix ≤3 chars (e.g., `kv` for Key Vault) -- Token = 5 char random string -- Total ≤32 characters - -## References - -- [Deployment Commands](commands.md) -- [Deployment Scripts](scripts.md) - -## Next - -→ Update `.azure/deployment-plan.md` → **azure-validate** +# AZCLI Recipe + +Azure CLI workflow for imperative Azure deployments. + +## When to Use + +- Existing az scripts in project +- Need imperative control over deployment +- Custom deployment pipelines +- AKS deployments +- Direct resource manipulation + +## Before Generation + +**REQUIRED: Research best practices before generating any files.** + +| Artifact | Research Action | +|----------|-----------------| +| Bicep files | Call `mcp_bicep_get_bicep_best_practices` | +| Bicep modules | Call `mcp_bicep_list_avm_metadata` and follow [AVM module order](../azd/iac-rules.md#avm-module-selection-order-mandatory) | +| Azure CLI commands | Call `activate_azure_cli_management_tools` | +| Azure best practices | Call `mcp_azure_mcp_get_azure_bestpractices` | + +## Generation Steps + +### 1. Generate Infrastructure (Bicep) + +Create Bicep templates in `./infra/`. + +**Structure:** +``` +infra/ +├── main.bicep +├── main.parameters.json +└── modules/ + └── *.bicep +``` + +### 2. Generate Deployment Scripts + +Create deployment scripts for provisioning. + +→ [scripts.md](scripts.md) + +### 3. Generate Dockerfiles (if containerized) + +Manual Dockerfile creation required. + +## Output Checklist + +| Artifact | Path | +|----------|------| +| Main Bicep | `./infra/main.bicep` | +| Parameters | `./infra/main.parameters.json` | +| Modules | `./infra/modules/*.bicep` | +| Deploy script | `./scripts/deploy.sh` or `deploy.ps1` | +| Dockerfiles | `src//Dockerfile` | + +## Deployment Commands + +See [commands.md](commands.md) for common patterns. + +## Naming Convention + +Resources: `{prefix}{token}{instance}` +- Alphanumeric only, no special characters +- Prefix ≤3 chars (e.g., `kv` for Key Vault) +- Token = 5 char random string +- Total ≤32 characters + +## References + +- [Deployment Commands](commands.md) +- [Deployment Scripts](scripts.md) + +## Next + +→ Update `.azure/deployment-plan.md` → **azure-validate** diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azcli/commands.md b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azcli/commands.md index aab2ead7..11f4dd7d 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azcli/commands.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azcli/commands.md @@ -1,91 +1,91 @@ -# Azure CLI Commands - -Common az commands for deployment workflows. - -## Resource Group - -```bash -# Create -az group create --name --location - -# Delete -az group delete --name --yes --no-wait -``` - -## Container Registry - -```bash -# Create -az acr create --name --resource-group --sku Basic - -# Login -az acr login --name - -# Build and push -az acr build --registry --image . -``` - -## Container Apps - -```bash -# Create environment -az containerapp env create \ - --name \ - --resource-group \ - --location - -# Deploy app -az containerapp create \ - --name \ - --resource-group \ - --environment \ - --image .azurecr.io/ \ - --target-port 8080 \ - --ingress external -``` - -## App Service - -```bash -# Create plan -az appservice plan create \ - --name \ - --resource-group \ - --sku B1 --is-linux - -# Create web app -az webapp create \ - --name \ - --resource-group \ - --plan \ - --runtime "NODE:22-lts" -``` - -## Functions - -```bash -# Create function app -az functionapp create \ - --name \ - --resource-group \ - --storage-account \ - --consumption-plan-location \ - --runtime node \ - --functions-version 4 -``` - -## Key Vault - -```bash -# Create -az keyvault create \ - --name \ - --resource-group \ - --location - -# Set secret -az keyvault secret set \ - --vault-name \ - --name \ - --value -``` +# Azure CLI Commands + +Common az commands for deployment workflows. + +## Resource Group + +```bash +# Create +az group create --name --location + +# Delete +az group delete --name --yes --no-wait +``` + +## Container Registry + +```bash +# Create +az acr create --name --resource-group --sku Basic + +# Login +az acr login --name + +# Build and push +az acr build --registry --image . +``` + +## Container Apps + +```bash +# Create environment +az containerapp env create \ + --name \ + --resource-group \ + --location + +# Deploy app +az containerapp create \ + --name \ + --resource-group \ + --environment \ + --image .azurecr.io/ \ + --target-port 8080 \ + --ingress external +``` + +## App Service + +```bash +# Create plan +az appservice plan create \ + --name \ + --resource-group \ + --sku B1 --is-linux + +# Create web app +az webapp create \ + --name \ + --resource-group \ + --plan \ + --runtime "NODE:22-lts" +``` + +## Functions + +```bash +# Create function app +az functionapp create \ + --name \ + --resource-group \ + --storage-account \ + --consumption-plan-location \ + --runtime node \ + --functions-version 4 +``` + +## Key Vault + +```bash +# Create +az keyvault create \ + --name \ + --resource-group \ + --location + +# Set secret +az keyvault secret set \ + --vault-name \ + --name \ + --value +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azcli/scripts.md b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azcli/scripts.md index abe4f00d..69cec6e2 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azcli/scripts.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azcli/scripts.md @@ -1,91 +1,91 @@ -# Deployment Scripts - -Script templates for AZCLI deployments. - -## Bash Script - -```bash -#!/bin/bash -set -euo pipefail - -# Configuration -RESOURCE_GROUP="${RESOURCE_GROUP:-rg-myapp}" -LOCATION="${LOCATION:-eastus2}" -ENVIRONMENT="${ENVIRONMENT:-dev}" - -echo "Deploying to $ENVIRONMENT environment..." - -# Create resource group -az group create \ - --name "$RESOURCE_GROUP" \ - --location "$LOCATION" \ - --tags environment="$ENVIRONMENT" - -# Deploy infrastructure -az deployment group create \ - --resource-group "$RESOURCE_GROUP" \ - --template-file ./infra/main.bicep \ - --parameters ./infra/main.parameters.json \ - --parameters environmentName="$ENVIRONMENT" - -# Get outputs -ACR_NAME=$(az deployment group show \ - --resource-group "$RESOURCE_GROUP" \ - --name main \ - --query properties.outputs.acrName.value -o tsv) - -# Build and push containers -az acr login --name "$ACR_NAME" -az acr build --registry "$ACR_NAME" --image api:latest ./src/api - -echo "Deployment complete!" -``` - -## PowerShell Script - -```powershell -#Requires -Version 7.0 -$ErrorActionPreference = "Stop" - -# Configuration -$ResourceGroup = $env:RESOURCE_GROUP ?? "rg-myapp" -$Location = $env:LOCATION ?? "eastus2" -$Environment = $env:ENVIRONMENT ?? "dev" - -Write-Host "Deploying to $Environment environment..." - -# Create resource group -az group create ` - --name $ResourceGroup ` - --location $Location ` - --tags environment=$Environment - -# Deploy infrastructure -az deployment group create ` - --resource-group $ResourceGroup ` - --template-file ./infra/main.bicep ` - --parameters ./infra/main.parameters.json ` - --parameters environmentName=$Environment - -# Get outputs -$AcrName = az deployment group show ` - --resource-group $ResourceGroup ` - --name main ` - --query properties.outputs.acrName.value -o tsv - -# Build and push containers -az acr login --name $AcrName -az acr build --registry $AcrName --image api:latest ./src/api - -Write-Host "Deployment complete!" -``` - -## Script Best Practices - -| Practice | Description | -|----------|-------------| -| Fail fast | `set -euo pipefail` (bash) or `$ErrorActionPreference = "Stop"` (pwsh) | -| Use variables | Environment-based configuration | -| Idempotent | Safe to run multiple times | -| Output logging | Clear progress messages | -| Error handling | Capture and report failures | +# Deployment Scripts + +Script templates for AZCLI deployments. + +## Bash Script + +```bash +#!/bin/bash +set -euo pipefail + +# Configuration +RESOURCE_GROUP="${RESOURCE_GROUP:-rg-myapp}" +LOCATION="${LOCATION:-eastus2}" +ENVIRONMENT="${ENVIRONMENT:-dev}" + +echo "Deploying to $ENVIRONMENT environment..." + +# Create resource group +az group create \ + --name "$RESOURCE_GROUP" \ + --location "$LOCATION" \ + --tags environment="$ENVIRONMENT" + +# Deploy infrastructure +az deployment group create \ + --resource-group "$RESOURCE_GROUP" \ + --template-file ./infra/main.bicep \ + --parameters ./infra/main.parameters.json \ + --parameters environmentName="$ENVIRONMENT" + +# Get outputs +ACR_NAME=$(az deployment group show \ + --resource-group "$RESOURCE_GROUP" \ + --name main \ + --query properties.outputs.acrName.value -o tsv) + +# Build and push containers +az acr login --name "$ACR_NAME" +az acr build --registry "$ACR_NAME" --image api:latest ./src/api + +echo "Deployment complete!" +``` + +## PowerShell Script + +```powershell +#Requires -Version 7.0 +$ErrorActionPreference = "Stop" + +# Configuration +$ResourceGroup = $env:RESOURCE_GROUP ?? "rg-myapp" +$Location = $env:LOCATION ?? "eastus2" +$Environment = $env:ENVIRONMENT ?? "dev" + +Write-Host "Deploying to $Environment environment..." + +# Create resource group +az group create ` + --name $ResourceGroup ` + --location $Location ` + --tags environment=$Environment + +# Deploy infrastructure +az deployment group create ` + --resource-group $ResourceGroup ` + --template-file ./infra/main.bicep ` + --parameters ./infra/main.parameters.json ` + --parameters environmentName=$Environment + +# Get outputs +$AcrName = az deployment group show ` + --resource-group $ResourceGroup ` + --name main ` + --query properties.outputs.acrName.value -o tsv + +# Build and push containers +az acr login --name $AcrName +az acr build --registry $AcrName --image api:latest ./src/api + +Write-Host "Deployment complete!" +``` + +## Script Best Practices + +| Practice | Description | +|----------|-------------| +| Fail fast | `set -euo pipefail` (bash) or `$ErrorActionPreference = "Stop"` (pwsh) | +| Use variables | Environment-based configuration | +| Idempotent | Safe to run multiple times | +| Output logging | Clear progress messages | +| Error handling | Capture and report failures | diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azd/README.md b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azd/README.md index 98a7a5ad..c61b54b7 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azd/README.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azd/README.md @@ -1,104 +1,104 @@ -# AZD Recipe - -Azure Developer CLI workflow for preparing Azure deployments. - -## When to Use - -- New projects, multi-service apps, want `azd up` -- Need environment management, auto-generated CI/CD -- Team prefers simplified deployment workflow - -> 💡 **Tip:** azd supports both Bicep and Terraform as IaC providers. Choose based on your team's expertise and requirements. - -## IaC Provider Options - -| Provider | Use When | -|----------|----------| -| **Bicep** (default) | Azure-only, no existing IaC, want simplest setup | -| **Terraform** | Multi-cloud IaC, existing TF expertise, want azd simplicity | - -**For Terraform with azd:** See [terraform.md](terraform.md) - -## Before Generation - -**REQUIRED: Research best practices before generating any files.** - -### Check for Existing Codebase Patterns - -**⚠️ CRITICAL: For existing codebases with special patterns, use `azd init --from-code -e ` instead of manual generation.** - -| Pattern | Detection | Action | -|---------|-----------|--------| -| **.NET Aspire** | `*.AppHost.csproj` or `Aspire.Hosting` package | Use `azd init --from-code -e ` → [aspire.md](../../aspire.md) | -| **Existing azure.yaml** | `azure.yaml` present | MODIFY mode - update existing config | -| **New project** | No azure.yaml, no special patterns | Manual generation (steps below) | - -> 💡 **Note:** The `-e ` flag is **required** when running `azd init --from-code` in non-interactive environments (agents, CI/CD pipelines). Without it, the command will fail with a prompt error. - -### References for Manual Generation - -| Artifact | Reference | -|----------|-----------| -| azure.yaml | [Schema Guide](azure-yaml.md) | -| .NET Aspire projects | [Aspire Guide](../../aspire.md) | -| Terraform with azd | [Terraform Guide](terraform.md) | -| AZD IAC rules | [IAC Rules](iac-rules.md) | -| Azure Functions templates | [Templates](../../services/functions/templates/README.md) | -| Bicep best practices | `mcp_bicep_get_bicep_best_practices` | -| Bicep resource schema | `mcp_bicep_get_az_resource_type_schema` | -| Azure Verified Modules | `mcp_bicep_list_avm_metadata` + [AVM module order](iac-rules.md#avm-module-selection-order-mandatory) | -| Terraform best practices | `mcp_azure_mcp_azureterraformbestpractices` | -| Dockerfiles | [Docker Guide](docker.md) | - -## Generation Steps - -### For Bicep (default) - -| # | Artifact | Reference | -|---|----------|-----------| -| 1 | azure.yaml | [Schema Guide](azure-yaml.md) | -| 2 | Application code | Entry points, health endpoints, config | -| 3 | Dockerfiles | [Docker Guide](docker.md) (if containerized) | -| 4 | Infrastructure | `./infra/main.bicep` + modules per [IAC Rules](iac-rules.md) | - -### For Terraform - -| # | Artifact | Reference | -|---|----------|-----------| -| 1 | azure.yaml with `infra.provider: terraform` | [Terraform Guide](terraform.md) | -| 2 | Application code | Entry points, health endpoints, config | -| 3 | Dockerfiles | [Docker Guide](docker.md) (if containerized) | -| 4 | Terraform files | `./infra/*.tf` per [Terraform Guide](terraform.md) | - -## Outputs - -### For Bicep - -| Artifact | Path | -|----------|------| -| azure.yaml | `./azure.yaml` | -| App Code | `src//*` | -| Dockerfiles | `src//Dockerfile` (if containerized) | -| Infrastructure | `./infra/` (Bicep files) | - -### For Terraform - -| Artifact | Path | -|----------|------| -| azure.yaml | `./azure.yaml` (with `infra.provider: terraform`) | -| App Code | `src//*` | -| Dockerfiles | `src//Dockerfile` (if containerized) | -| Infrastructure | `./infra/` (Terraform files) | - -## References - -- [.NET Aspire Projects](../../aspire.md) -- [azure.yaml Schema](azure-yaml.md) -- [.NET Aspire Apps](aspire.md) -- [Terraform with AZD](terraform.md) -- [Docker Configuration](docker.md) -- [IAC Rules](iac-rules.md) - -## Next - -→ Update `.azure/deployment-plan.md` → **azure-validate** +# AZD Recipe + +Azure Developer CLI workflow for preparing Azure deployments. + +## When to Use + +- New projects, multi-service apps, want `azd up` +- Need environment management, auto-generated CI/CD +- Team prefers simplified deployment workflow + +> 💡 **Tip:** azd supports both Bicep and Terraform as IaC providers. Choose based on your team's expertise and requirements. + +## IaC Provider Options + +| Provider | Use When | +|----------|----------| +| **Bicep** (default) | Azure-only, no existing IaC, want simplest setup | +| **Terraform** | Multi-cloud IaC, existing TF expertise, want azd simplicity | + +**For Terraform with azd:** See [terraform.md](terraform.md) + +## Before Generation + +**REQUIRED: Research best practices before generating any files.** + +### Check for Existing Codebase Patterns + +**⚠️ CRITICAL: For existing codebases with special patterns, use `azd init --from-code -e ` instead of manual generation.** + +| Pattern | Detection | Action | +|---------|-----------|--------| +| **.NET Aspire** | `*.AppHost.csproj` or `Aspire.Hosting` package | Use `azd init --from-code -e ` → [aspire.md](../../aspire.md) | +| **Existing azure.yaml** | `azure.yaml` present | MODIFY mode - update existing config | +| **New project** | No azure.yaml, no special patterns | Manual generation (steps below) | + +> 💡 **Note:** The `-e ` flag is **required** when running `azd init --from-code` in non-interactive environments (agents, CI/CD pipelines). Without it, the command will fail with a prompt error. + +### References for Manual Generation + +| Artifact | Reference | +|----------|-----------| +| azure.yaml | [Schema Guide](azure-yaml.md) | +| .NET Aspire projects | [Aspire Guide](../../aspire.md) | +| Terraform with azd | [Terraform Guide](terraform.md) | +| AZD IAC rules | [IAC Rules](iac-rules.md) | +| Azure Functions templates | [Templates](../../services/functions/templates/README.md) | +| Bicep best practices | `mcp_bicep_get_bicep_best_practices` | +| Bicep resource schema | `mcp_bicep_get_az_resource_type_schema` | +| Azure Verified Modules | `mcp_bicep_list_avm_metadata` + [AVM module order](iac-rules.md#avm-module-selection-order-mandatory) | +| Terraform best practices | `mcp_azure_mcp_azureterraformbestpractices` | +| Dockerfiles | [Docker Guide](docker.md) | + +## Generation Steps + +### For Bicep (default) + +| # | Artifact | Reference | +|---|----------|-----------| +| 1 | azure.yaml | [Schema Guide](azure-yaml.md) | +| 2 | Application code | Entry points, health endpoints, config | +| 3 | Dockerfiles | [Docker Guide](docker.md) (if containerized) | +| 4 | Infrastructure | `./infra/main.bicep` + modules per [IAC Rules](iac-rules.md) | + +### For Terraform + +| # | Artifact | Reference | +|---|----------|-----------| +| 1 | azure.yaml with `infra.provider: terraform` | [Terraform Guide](terraform.md) | +| 2 | Application code | Entry points, health endpoints, config | +| 3 | Dockerfiles | [Docker Guide](docker.md) (if containerized) | +| 4 | Terraform files | `./infra/*.tf` per [Terraform Guide](terraform.md) | + +## Outputs + +### For Bicep + +| Artifact | Path | +|----------|------| +| azure.yaml | `./azure.yaml` | +| App Code | `src//*` | +| Dockerfiles | `src//Dockerfile` (if containerized) | +| Infrastructure | `./infra/` (Bicep files) | + +### For Terraform + +| Artifact | Path | +|----------|------| +| azure.yaml | `./azure.yaml` (with `infra.provider: terraform`) | +| App Code | `src//*` | +| Dockerfiles | `src//Dockerfile` (if containerized) | +| Infrastructure | `./infra/` (Terraform files) | + +## References + +- [.NET Aspire Projects](../../aspire.md) +- [azure.yaml Schema](azure-yaml.md) +- [.NET Aspire Apps](aspire.md) +- [Terraform with AZD](terraform.md) +- [Docker Configuration](docker.md) +- [IAC Rules](iac-rules.md) + +## Next + +→ Update `.azure/deployment-plan.md` → **azure-validate** diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azd/azure-yaml.md b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azd/azure-yaml.md index aaec0f40..8cad217c 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azd/azure-yaml.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azd/azure-yaml.md @@ -1,241 +1,241 @@ -# azure.yaml Generation - -> ⛔ **CRITICAL: Check for .NET Aspire projects FIRST** -> -> **DO NOT manually create azure.yaml for .NET Aspire projects.** If you detect: -> - Files ending with `*.AppHost.csproj` (e.g., `MyApp.AppHost.csproj`) -> - `Aspire.Hosting` or `Aspire.AppHost.Sdk` in `.csproj` files -> -> **STOP and use `azd init --from-code` instead.** See [aspire.md](aspire.md) for details. - -Create `azure.yaml` in project root for AZD. - -## Structure - -### Basic (Bicep - default) - -```yaml -name: -metadata: - template: azd-init - -services: - : - project: - language: - host: -``` - -### With Terraform Provider - -```yaml -name: -metadata: - template: azd-init - -# Specify Terraform as IaC provider -infra: - provider: terraform - path: ./infra - -services: - : - project: - language: - host: -``` - -> 💡 **Tip:** Omit `infra` section to use Bicep (default). Add `infra.provider: terraform` to use Terraform. See [terraform.md](terraform.md) for details. - -## Host Types - -| Host | Azure Service | Use For | -|------|---------------|---------| -| `containerapp` | Container Apps | APIs, microservices, workers | -| `appservice` | App Service | Traditional web apps | -| `function` | Azure Functions | Serverless functions | -| `staticwebapp` | Static Web Apps | SPAs, static sites | -| `aks` | AKS | Kubernetes workloads | - -## Examples - -### Container App with Bicep (default) - -```yaml -name: myapp - -services: - api: - project: ./src/api - language: python - host: containerapp - docker: - path: ./src/api/Dockerfile -``` - -### Container App with Terraform - -```yaml -name: myapp - -infra: - provider: terraform - path: ./infra - -services: - api: - project: ./src/api - language: python - host: containerapp - docker: - path: ./src/api/Dockerfile -``` - -### Container App with Custom Docker Context - -When a **non-Aspire** project has a Dockerfile that expects files relative to a specific directory: - -```yaml -name: myapp - -services: - ginapp: - project: . - host: containerapp - image: ginapp - docker: - path: ginapp/Dockerfile - context: ginapp -``` - -> 💡 **Tip:** The `context` field specifies the Docker build context directory. This is crucial for: -> - Dockerfiles with `COPY` commands expecting files relative to a subdirectory -> - Multi-service repos where each service has its own context - -> ⚠️ **Aspire projects:** Do NOT manually add per-service entries with `docker.context` for Aspire `AddDockerfile()` resources. Aspire handles container builds at runtime through the AppHost. The generated `azure.yaml` should contain only a single `app` service pointing to the AppHost. See [aspire.md](aspire.md) for details. - -> ⚠️ **Language Field:** When using the `docker` section, the `language` field should be **omitted** or set to the language that azd will use for framework-specific behaviors. For containerized apps with custom Dockerfiles, the language is not used by azd since the build is handled by Docker. Only include `language` if you need azd to perform additional framework-specific actions beyond Docker build. - -### Azure Functions - -```yaml -services: - functions: - project: ./src/functions - language: js - host: function -``` - -### Static Web App (with framework build) - -For React, Vue, Angular, Next.js, etc. that require `npm run build`: - -```yaml -services: - web: - project: ./src/web # folder containing package.json - language: js # triggers: npm install && npm run build - host: staticwebapp - dist: dist # build output folder (e.g., dist, build, out) -``` - -### Static Web App (pure HTML/CSS - no build) - -For pure HTML sites without a framework build step: - -**Static files in subfolder (recommended):** -```yaml -services: - web: - project: ./src/web # folder containing index.html - host: staticwebapp - dist: . # works when project != root -``` - -**Static files in root - requires build script:** - -> ⚠️ **SWA CLI Limitation:** When `project: .`, you cannot use `dist: .`. Files must be copied to a separate output folder. - -Add a minimal `package.json` with a build script: -```json -{ - "scripts": { - "build": "node -e \"require('fs').mkdirSync('public',{recursive:true});require('fs').readdirSync('.').filter(f=>/\\.(html|css|js|png|jpe?g|gif|svg|ico|json|xml|txt|webmanifest|map)$/i.test(f)).forEach(f=>require('fs').copyFileSync(f,'public/'+f))\"" - } -} -``` - -Then configure azure.yaml with `language: js` to trigger the build: -```yaml -services: - web: - project: . - language: js # triggers npm install && npm run build - host: staticwebapp - dist: public -``` - -### SWA Project Structure Detection - -| Layout | Configuration | -|--------|---------------| -| Static in root | `project: .`, `language: js`, `dist: public` + package.json build script | -| Framework in root | `project: .`, `language: js`, `dist: ` | -| Static in subfolder | `project: ./path`, `dist: .` | -| Framework in subfolder | `project: ./path`, `language: js`, `dist: ` | - -> **Key rules:** -> - `dist` is **relative to `project`** path -> - **SWA CLI limitation**: When `project: .`, cannot use `dist: .` - must use a distinct folder -> - For static files in root, add `package.json` with build script to copy files to dist folder -> - Use `language: js` to trigger npm build even for pure static sites in root -> - `language: html` and `language: static` are **NOT valid** - will fail - -### SWA Bicep Requirement - -Bicep must include the `azd-service-name` tag: -```bicep -resource staticWebApp 'Microsoft.Web/staticSites@2022-09-01' = { - name: name - location: location - tags: union(tags, { 'azd-service-name': 'web' })} -``` -} -``` - -### App Service - -```yaml -services: - api: - project: ./src/api - language: dotnet - host: appservice -``` - -## Hooks (Optional) - -```yaml -hooks: - preprovision: - shell: sh - run: ./scripts/setup.sh - postprovision: - shell: sh - run: ./scripts/seed-data.sh -``` - -## Valid Values - -| Field | Options | -|-------|---------| -| `language` | python, js, ts, java, dotnet, go (omit for staticwebapp without build) | -| `host` | containerapp, appservice, function, staticwebapp, aks | -| `docker.path` | Path to Dockerfile (relative to project root) | -| `docker.context` | Docker build context directory (optional, defaults to directory containing Dockerfile) | - -> 💡 **Docker Context:** When `docker.context` is omitted, azd uses the directory containing the Dockerfile as the build context. Specify `context` explicitly when the Dockerfile expects files from a different directory. - -## Output - -- `./azure.yaml` +# azure.yaml Generation + +> ⛔ **CRITICAL: Check for .NET Aspire projects FIRST** +> +> **DO NOT manually create azure.yaml for .NET Aspire projects.** If you detect: +> - Files ending with `*.AppHost.csproj` (e.g., `MyApp.AppHost.csproj`) +> - `Aspire.Hosting` or `Aspire.AppHost.Sdk` in `.csproj` files +> +> **STOP and use `azd init --from-code` instead.** See [aspire.md](aspire.md) for details. + +Create `azure.yaml` in project root for AZD. + +## Structure + +### Basic (Bicep - default) + +```yaml +name: +metadata: + template: azd-init + +services: + : + project: + language: + host: +``` + +### With Terraform Provider + +```yaml +name: +metadata: + template: azd-init + +# Specify Terraform as IaC provider +infra: + provider: terraform + path: ./infra + +services: + : + project: + language: + host: +``` + +> 💡 **Tip:** Omit `infra` section to use Bicep (default). Add `infra.provider: terraform` to use Terraform. See [terraform.md](terraform.md) for details. + +## Host Types + +| Host | Azure Service | Use For | +|------|---------------|---------| +| `containerapp` | Container Apps | APIs, microservices, workers | +| `appservice` | App Service | Traditional web apps | +| `function` | Azure Functions | Serverless functions | +| `staticwebapp` | Static Web Apps | SPAs, static sites | +| `aks` | AKS | Kubernetes workloads | + +## Examples + +### Container App with Bicep (default) + +```yaml +name: myapp + +services: + api: + project: ./src/api + language: python + host: containerapp + docker: + path: ./src/api/Dockerfile +``` + +### Container App with Terraform + +```yaml +name: myapp + +infra: + provider: terraform + path: ./infra + +services: + api: + project: ./src/api + language: python + host: containerapp + docker: + path: ./src/api/Dockerfile +``` + +### Container App with Custom Docker Context + +When a **non-Aspire** project has a Dockerfile that expects files relative to a specific directory: + +```yaml +name: myapp + +services: + ginapp: + project: . + host: containerapp + image: ginapp + docker: + path: ginapp/Dockerfile + context: ginapp +``` + +> 💡 **Tip:** The `context` field specifies the Docker build context directory. This is crucial for: +> - Dockerfiles with `COPY` commands expecting files relative to a subdirectory +> - Multi-service repos where each service has its own context + +> ⚠️ **Aspire projects:** Do NOT manually add per-service entries with `docker.context` for Aspire `AddDockerfile()` resources. Aspire handles container builds at runtime through the AppHost. The generated `azure.yaml` should contain only a single `app` service pointing to the AppHost. See [aspire.md](aspire.md) for details. + +> ⚠️ **Language Field:** When using the `docker` section, the `language` field should be **omitted** or set to the language that azd will use for framework-specific behaviors. For containerized apps with custom Dockerfiles, the language is not used by azd since the build is handled by Docker. Only include `language` if you need azd to perform additional framework-specific actions beyond Docker build. + +### Azure Functions + +```yaml +services: + functions: + project: ./src/functions + language: js + host: function +``` + +### Static Web App (with framework build) + +For React, Vue, Angular, Next.js, etc. that require `npm run build`: + +```yaml +services: + web: + project: ./src/web # folder containing package.json + language: js # triggers: npm install && npm run build + host: staticwebapp + dist: dist # build output folder (e.g., dist, build, out) +``` + +### Static Web App (pure HTML/CSS - no build) + +For pure HTML sites without a framework build step: + +**Static files in subfolder (recommended):** +```yaml +services: + web: + project: ./src/web # folder containing index.html + host: staticwebapp + dist: . # works when project != root +``` + +**Static files in root - requires build script:** + +> ⚠️ **SWA CLI Limitation:** When `project: .`, you cannot use `dist: .`. Files must be copied to a separate output folder. + +Add a minimal `package.json` with a build script: +```json +{ + "scripts": { + "build": "node -e \"require('fs').mkdirSync('public',{recursive:true});require('fs').readdirSync('.').filter(f=>/\\.(html|css|js|png|jpe?g|gif|svg|ico|json|xml|txt|webmanifest|map)$/i.test(f)).forEach(f=>require('fs').copyFileSync(f,'public/'+f))\"" + } +} +``` + +Then configure azure.yaml with `language: js` to trigger the build: +```yaml +services: + web: + project: . + language: js # triggers npm install && npm run build + host: staticwebapp + dist: public +``` + +### SWA Project Structure Detection + +| Layout | Configuration | +|--------|---------------| +| Static in root | `project: .`, `language: js`, `dist: public` + package.json build script | +| Framework in root | `project: .`, `language: js`, `dist: ` | +| Static in subfolder | `project: ./path`, `dist: .` | +| Framework in subfolder | `project: ./path`, `language: js`, `dist: ` | + +> **Key rules:** +> - `dist` is **relative to `project`** path +> - **SWA CLI limitation**: When `project: .`, cannot use `dist: .` - must use a distinct folder +> - For static files in root, add `package.json` with build script to copy files to dist folder +> - Use `language: js` to trigger npm build even for pure static sites in root +> - `language: html` and `language: static` are **NOT valid** - will fail + +### SWA Bicep Requirement + +Bicep must include the `azd-service-name` tag: +```bicep +resource staticWebApp 'Microsoft.Web/staticSites@2022-09-01' = { + name: name + location: location + tags: union(tags, { 'azd-service-name': 'web' })} +``` +} +``` + +### App Service + +```yaml +services: + api: + project: ./src/api + language: dotnet + host: appservice +``` + +## Hooks (Optional) + +```yaml +hooks: + preprovision: + shell: sh + run: ./scripts/setup.sh + postprovision: + shell: sh + run: ./scripts/seed-data.sh +``` + +## Valid Values + +| Field | Options | +|-------|---------| +| `language` | python, js, ts, java, dotnet, go (omit for staticwebapp without build) | +| `host` | containerapp, appservice, function, staticwebapp, aks | +| `docker.path` | Path to Dockerfile (relative to project root) | +| `docker.context` | Docker build context directory (optional, defaults to directory containing Dockerfile) | + +> 💡 **Docker Context:** When `docker.context` is omitted, azd uses the directory containing the Dockerfile as the build context. Specify `context` explicitly when the Dockerfile expects files from a different directory. + +## Output + +- `./azure.yaml` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azd/docker.md b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azd/docker.md index 2856da67..ecc8d14b 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azd/docker.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/azd/docker.md @@ -1,116 +1,116 @@ -# Dockerfile Generation - -Create Dockerfiles for containerized services. - -## When to Containerize - -| Include | Exclude | -|---------|---------| -| APIs, microservices | Static websites (use Static Web Apps) | -| Web apps (SSR) | Azure Functions (native deploy) | -| Background workers | Database services | -| Message processors | Logic Apps | - -## Templates by Language - -### Node.js - -```dockerfile -FROM node:22-slim -WORKDIR /app -COPY package*.json ./ -RUN npm ci --only=production -COPY . . -EXPOSE 3000 -CMD ["node", "index.js"] -``` - -### Python - -```dockerfile -FROM python:3.13-slim -WORKDIR /app -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt -COPY . . -EXPOSE 8000 -CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] -``` - -### .NET - -```dockerfile -FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS base -WORKDIR /app -EXPOSE 8080 - -FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build -WORKDIR /src -COPY ["*.csproj", "./"] -RUN dotnet restore -COPY . . -RUN dotnet publish -c Release -o /app/publish - -FROM base AS final -WORKDIR /app -COPY --from=build /app/publish . -ENTRYPOINT ["dotnet", "App.dll"] -``` - -### Java - -```dockerfile -FROM eclipse-temurin:21-jdk-alpine AS build -WORKDIR /app -COPY . . -RUN ./mvnw package -DskipTests - -FROM eclipse-temurin:21-jre-alpine -WORKDIR /app -COPY --from=build /app/target/*.jar app.jar -EXPOSE 8080 -ENTRYPOINT ["java", "-jar", "app.jar"] -``` - -### Go - -```dockerfile -FROM golang:1.22-alpine AS build -WORKDIR /app -COPY go.* ./ -RUN go mod download -COPY . . -RUN CGO_ENABLED=0 go build -o main . - -FROM alpine:latest -WORKDIR /app -COPY --from=build /app/main . -EXPOSE 8080 -CMD ["./main"] -``` - -## .dockerignore - -``` -.git -node_modules -__pycache__ -*.pyc -.env -.azure -``` - -## Best Practices - -- Use slim/alpine base images -- Multi-stage builds for compiled languages -- Non-root user when possible -- Include health check endpoint in app - -## Runtime-Specific Configuration - -For production settings specific to each runtime: - -| Runtime | Reference | -|---------|-----------| -| Node.js/Express | [runtimes/nodejs.md](../../runtimes/nodejs.md) | +# Dockerfile Generation + +Create Dockerfiles for containerized services. + +## When to Containerize + +| Include | Exclude | +|---------|---------| +| APIs, microservices | Static websites (use Static Web Apps) | +| Web apps (SSR) | Azure Functions (native deploy) | +| Background workers | Database services | +| Message processors | Logic Apps | + +## Templates by Language + +### Node.js + +```dockerfile +FROM node:22-slim +WORKDIR /app +COPY package*.json ./ +RUN npm ci --only=production +COPY . . +EXPOSE 3000 +CMD ["node", "index.js"] +``` + +### Python + +```dockerfile +FROM python:3.13-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . +EXPOSE 8000 +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] +``` + +### .NET + +```dockerfile +FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS base +WORKDIR /app +EXPOSE 8080 + +FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +WORKDIR /src +COPY ["*.csproj", "./"] +RUN dotnet restore +COPY . . +RUN dotnet publish -c Release -o /app/publish + +FROM base AS final +WORKDIR /app +COPY --from=build /app/publish . +ENTRYPOINT ["dotnet", "App.dll"] +``` + +### Java + +```dockerfile +FROM eclipse-temurin:21-jdk-alpine AS build +WORKDIR /app +COPY . . +RUN ./mvnw package -DskipTests + +FROM eclipse-temurin:21-jre-alpine +WORKDIR /app +COPY --from=build /app/target/*.jar app.jar +EXPOSE 8080 +ENTRYPOINT ["java", "-jar", "app.jar"] +``` + +### Go + +```dockerfile +FROM golang:1.22-alpine AS build +WORKDIR /app +COPY go.* ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 go build -o main . + +FROM alpine:latest +WORKDIR /app +COPY --from=build /app/main . +EXPOSE 8080 +CMD ["./main"] +``` + +## .dockerignore + +``` +.git +node_modules +__pycache__ +*.pyc +.env +.azure +``` + +## Best Practices + +- Use slim/alpine base images +- Multi-stage builds for compiled languages +- Non-root user when possible +- Include health check endpoint in app + +## Runtime-Specific Configuration + +For production settings specific to each runtime: + +| Runtime | Reference | +|---------|-----------| +| Node.js/Express | [runtimes/nodejs.md](../../runtimes/nodejs.md) | diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/bicep/README.md b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/bicep/README.md index 4ce274b3..33557a5c 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/bicep/README.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/bicep/README.md @@ -1,61 +1,61 @@ -# Bicep Recipe - -Standalone Bicep workflow (without AZD). - -## When to Use - -- IaC-first approach -- No CLI wrapper needed -- Direct ARM deployment control -- Existing Bicep modules to reuse -- Custom deployment orchestration - -## Before Generation - -**REQUIRED: Research best practices before generating any files.** - -| Artifact | Research Action | -|----------|-----------------| -| Bicep files | Call `mcp_bicep_get_bicep_best_practices` | -| Bicep modules | Call `mcp_bicep_list_avm_metadata` and follow [AVM module order](../azd/iac-rules.md#avm-module-selection-order-mandatory) | -| Resource schemas | Use `activate_azure_resource_schema_tools` if needed | - -## Generation Steps - -### 1. Generate Infrastructure - -Create Bicep templates in `./infra/`. - -→ [patterns.md](patterns.md) - -**Structure:** -``` -infra/ -├── main.bicep -├── main.parameters.json -└── modules/ - ├── container-app.bicep - ├── storage.bicep - └── ... -``` - -### 2. Generate Dockerfiles (if containerized) - -Manual Dockerfile creation required. - -## Output Checklist - -| Artifact | Path | -|----------|------| -| Main Bicep | `./infra/main.bicep` | -| Parameters | `./infra/main.parameters.json` | -| Modules | `./infra/modules/*.bicep` | -| Dockerfiles | `src//Dockerfile` | - -## References - -- [Bicep Patterns](patterns.md) - -## Next - -→ Update `.azure/deployment-plan.md` → **azure-validate** +# Bicep Recipe + +Standalone Bicep workflow (without AZD). + +## When to Use + +- IaC-first approach +- No CLI wrapper needed +- Direct ARM deployment control +- Existing Bicep modules to reuse +- Custom deployment orchestration + +## Before Generation + +**REQUIRED: Research best practices before generating any files.** + +| Artifact | Research Action | +|----------|-----------------| +| Bicep files | Call `mcp_bicep_get_bicep_best_practices` | +| Bicep modules | Call `mcp_bicep_list_avm_metadata` and follow [AVM module order](../azd/iac-rules.md#avm-module-selection-order-mandatory) | +| Resource schemas | Use `activate_azure_resource_schema_tools` if needed | + +## Generation Steps + +### 1. Generate Infrastructure + +Create Bicep templates in `./infra/`. + +→ [patterns.md](patterns.md) + +**Structure:** +``` +infra/ +├── main.bicep +├── main.parameters.json +└── modules/ + ├── container-app.bicep + ├── storage.bicep + └── ... +``` + +### 2. Generate Dockerfiles (if containerized) + +Manual Dockerfile creation required. + +## Output Checklist + +| Artifact | Path | +|----------|------| +| Main Bicep | `./infra/main.bicep` | +| Parameters | `./infra/main.parameters.json` | +| Modules | `./infra/modules/*.bicep` | +| Dockerfiles | `src//Dockerfile` | + +## References + +- [Bicep Patterns](patterns.md) + +## Next + +→ Update `.azure/deployment-plan.md` → **azure-validate** diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/bicep/patterns.md b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/bicep/patterns.md index c18d8df2..482e20b9 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/bicep/patterns.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/bicep/patterns.md @@ -1,138 +1,138 @@ -# Bicep Patterns - -Common patterns for Bicep infrastructure templates. - -## File Structure - -``` -infra/ -├── main.bicep # Entry point (subscription scope) -├── main.parameters.json # Parameter values -└── modules/ - ├── resources.bicep # Base resources - ├── container-app.bicep # Container App module - └── ... -``` - -## main.bicep Template - -```bicep -targetScope = 'subscription' - -@minLength(1) -@maxLength(64) -param environmentName string - -@minLength(1) -param location string - -var tags = { environment: environmentName } - -resource rg 'Microsoft.Resources/resourceGroups@2023-07-01' = { - name: 'rg-${environmentName}' - location: location - tags: tags -} - -module resources './modules/resources.bicep' = { - name: 'resources' - scope: rg - params: { - location: location - environmentName: environmentName - tags: tags - } -} - -output resourceGroupName string = rg.name -``` - -## main.parameters.json - -> ⚠️ **Warning:** This file uses ARM JSON syntax. Do **not** use `.bicepparam` syntax (`using`, `param`, `readEnvironmentVariable()`) in this file — `azd` will fail with a JSON parse error. - -```json -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "environmentName": { "value": "${AZURE_ENV_NAME}" }, - "location": { "value": "${AZURE_LOCATION}" } - } -} -``` - -Use `azd env set` to supply values at deploy time: - -```bash -azd env set AZURE_ENV_NAME myapp-1234 -azd env set AZURE_LOCATION eastus2 -``` - -## Naming Convention - -```bicep -var resourceToken = uniqueString(subscription().id, resourceGroup().id, location) - -// Pattern: {prefix}{name}{token} -// Total ≤32 chars, alphanumeric only -var kvName = 'kv${environmentName}${resourceToken}' -var storName = 'stor${resourceToken}' - -// Container Registry: alphanumeric only (5-50 chars) -var acrName = replace('cr${environmentName}${resourceToken}', '-', '') -``` - -## Security Requirements - -| Requirement | Pattern | -|-------------|---------| -| No hardcoded secrets | Use Key Vault references | -| Managed Identity | `identity: { type: 'UserAssigned' }` | -| HTTPS only | `httpsOnly: true` | -| TLS 1.2+ | `minTlsVersion: '1.2'` | -| No public blob access | `allowBlobPublicAccess: false` | - -## Common Modules - -### Log Analytics - -```bicep -resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2022-10-01' = { - name: 'log-${resourceToken}' - location: location - properties: { - sku: { name: 'PerGB2018' } - retentionInDays: 30 - } -} -``` - -### Application Insights - -```bicep -resource appInsights 'Microsoft.Insights/components@2020-02-02' = { - name: 'appi-${resourceToken}' - location: location - kind: 'web' - properties: { - Application_Type: 'web' - WorkspaceResourceId: logAnalytics.id - } -} -``` - -### Key Vault - -```bicep -resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' = { - name: 'kv-${resourceToken}' - location: location - properties: { - sku: { family: 'A', name: 'standard' } - tenantId: subscription().tenantId - enableRbacAuthorization: true - } -} -``` - +# Bicep Patterns + +Common patterns for Bicep infrastructure templates. + +## File Structure + +``` +infra/ +├── main.bicep # Entry point (subscription scope) +├── main.parameters.json # Parameter values +└── modules/ + ├── resources.bicep # Base resources + ├── container-app.bicep # Container App module + └── ... +``` + +## main.bicep Template + +```bicep +targetScope = 'subscription' + +@minLength(1) +@maxLength(64) +param environmentName string + +@minLength(1) +param location string + +var tags = { environment: environmentName } + +resource rg 'Microsoft.Resources/resourceGroups@2023-07-01' = { + name: 'rg-${environmentName}' + location: location + tags: tags +} + +module resources './modules/resources.bicep' = { + name: 'resources' + scope: rg + params: { + location: location + environmentName: environmentName + tags: tags + } +} + +output resourceGroupName string = rg.name +``` + +## main.parameters.json + +> ⚠️ **Warning:** This file uses ARM JSON syntax. Do **not** use `.bicepparam` syntax (`using`, `param`, `readEnvironmentVariable()`) in this file — `azd` will fail with a JSON parse error. + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "environmentName": { "value": "${AZURE_ENV_NAME}" }, + "location": { "value": "${AZURE_LOCATION}" } + } +} +``` + +Use `azd env set` to supply values at deploy time: + +```bash +azd env set AZURE_ENV_NAME myapp-1234 +azd env set AZURE_LOCATION eastus2 +``` + +## Naming Convention + +```bicep +var resourceToken = uniqueString(subscription().id, resourceGroup().id, location) + +// Pattern: {prefix}{name}{token} +// Total ≤32 chars, alphanumeric only +var kvName = 'kv${environmentName}${resourceToken}' +var storName = 'stor${resourceToken}' + +// Container Registry: alphanumeric only (5-50 chars) +var acrName = replace('cr${environmentName}${resourceToken}', '-', '') +``` + +## Security Requirements + +| Requirement | Pattern | +|-------------|---------| +| No hardcoded secrets | Use Key Vault references | +| Managed Identity | `identity: { type: 'UserAssigned' }` | +| HTTPS only | `httpsOnly: true` | +| TLS 1.2+ | `minTlsVersion: '1.2'` | +| No public blob access | `allowBlobPublicAccess: false` | + +## Common Modules + +### Log Analytics + +```bicep +resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2022-10-01' = { + name: 'log-${resourceToken}' + location: location + properties: { + sku: { name: 'PerGB2018' } + retentionInDays: 30 + } +} +``` + +### Application Insights + +```bicep +resource appInsights 'Microsoft.Insights/components@2020-02-02' = { + name: 'appi-${resourceToken}' + location: location + kind: 'web' + properties: { + Application_Type: 'web' + WorkspaceResourceId: logAnalytics.id + } +} +``` + +### Key Vault + +```bicep +resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' = { + name: 'kv-${resourceToken}' + location: location + properties: { + sku: { family: 'A', name: 'standard' } + tenantId: subscription().tenantId + enableRbacAuthorization: true + } +} +``` + diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/terraform/README.md b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/terraform/README.md index 78da763a..f46913bb 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/terraform/README.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/terraform/README.md @@ -1,92 +1,92 @@ -# Terraform Recipe - -Terraform workflow for Azure deployments. - -> **⚠️ IMPORTANT: Consider azd+Terraform First** -> -> If you're deploying to Azure, you should **default to [azd with Terraform](../azd/terraform.md)** instead of pure Terraform. azd+Terraform gives you: -> - Terraform's IaC capabilities -> - Simple `azd up` deployment workflow -> - Built-in environment management -> - Automatic CI/CD pipeline generation -> - Service orchestration from azure.yaml -> -> → **See [azd+Terraform documentation](../azd/terraform.md)** ← - -## When to Use Pure Terraform (Without azd) - -Only use pure Terraform workflow when you have specific requirements that prevent using azd: - -- **Multi-cloud deployments** where Azure is not the primary target -- **Complex Terraform modules/workspaces** that are incompatible with azd conventions -- **Existing Terraform CI/CD** pipelines that are hard to migrate -- **Organization mandate** for pure Terraform workflow without any wrapper tools -- **Explicitly requested** by the user to use Terraform without azd - -## When to Use azd+Terraform Instead - -Use azd+Terraform (the default) when: - -- **Azure-first deployment** (even if you want multi-cloud IaC) -- Want **`azd up` simplicity** with Terraform IaC -- **Multi-service apps** needing orchestration -- Team wants to learn Terraform with a simpler workflow - -→ See [azd+Terraform documentation](../azd/terraform.md) - -## Before Generation - -**REQUIRED: Research best practices before generating any files.** - -| Artifact | Research Action | -|----------|-----------------| -| Terraform patterns | Call `mcp_azure_mcp_azureterraformbestpractices` | -| Azure best practices | Call `mcp_azure_mcp_get_azure_bestpractices` | - -## Generation Steps - -### 1. Generate Infrastructure - -Create Terraform files in `./infra/`. - -→ [patterns.md](patterns.md) - -**Structure:** -``` -infra/ -├── main.tf -├── variables.tf -├── outputs.tf -├── terraform.tfvars -├── backend.tf -└── modules/ - └── ... -``` - -### 2. Set Up State Backend - -Azure Storage for remote state. - -### 3. Generate Dockerfiles (if containerized) - -Manual Dockerfile creation required. - -## Output Checklist - -| Artifact | Path | -|----------|------| -| Main config | `./infra/main.tf` | -| Variables | `./infra/variables.tf` | -| Outputs | `./infra/outputs.tf` | -| Values | `./infra/terraform.tfvars` | -| Backend | `./infra/backend.tf` | -| Modules | `./infra/modules/` | -| Dockerfiles | `src//Dockerfile` | - -## References - -- [Terraform Patterns](patterns.md) - -## Next - -→ Update `.azure/deployment-plan.md` → **azure-validate** +# Terraform Recipe + +Terraform workflow for Azure deployments. + +> **⚠️ IMPORTANT: Consider azd+Terraform First** +> +> If you're deploying to Azure, you should **default to [azd with Terraform](../azd/terraform.md)** instead of pure Terraform. azd+Terraform gives you: +> - Terraform's IaC capabilities +> - Simple `azd up` deployment workflow +> - Built-in environment management +> - Automatic CI/CD pipeline generation +> - Service orchestration from azure.yaml +> +> → **See [azd+Terraform documentation](../azd/terraform.md)** ← + +## When to Use Pure Terraform (Without azd) + +Only use pure Terraform workflow when you have specific requirements that prevent using azd: + +- **Multi-cloud deployments** where Azure is not the primary target +- **Complex Terraform modules/workspaces** that are incompatible with azd conventions +- **Existing Terraform CI/CD** pipelines that are hard to migrate +- **Organization mandate** for pure Terraform workflow without any wrapper tools +- **Explicitly requested** by the user to use Terraform without azd + +## When to Use azd+Terraform Instead + +Use azd+Terraform (the default) when: + +- **Azure-first deployment** (even if you want multi-cloud IaC) +- Want **`azd up` simplicity** with Terraform IaC +- **Multi-service apps** needing orchestration +- Team wants to learn Terraform with a simpler workflow + +→ See [azd+Terraform documentation](../azd/terraform.md) + +## Before Generation + +**REQUIRED: Research best practices before generating any files.** + +| Artifact | Research Action | +|----------|-----------------| +| Terraform patterns | Call `mcp_azure_mcp_azureterraformbestpractices` | +| Azure best practices | Call `mcp_azure_mcp_get_azure_bestpractices` | + +## Generation Steps + +### 1. Generate Infrastructure + +Create Terraform files in `./infra/`. + +→ [patterns.md](patterns.md) + +**Structure:** +``` +infra/ +├── main.tf +├── variables.tf +├── outputs.tf +├── terraform.tfvars +├── backend.tf +└── modules/ + └── ... +``` + +### 2. Set Up State Backend + +Azure Storage for remote state. + +### 3. Generate Dockerfiles (if containerized) + +Manual Dockerfile creation required. + +## Output Checklist + +| Artifact | Path | +|----------|------| +| Main config | `./infra/main.tf` | +| Variables | `./infra/variables.tf` | +| Outputs | `./infra/outputs.tf` | +| Values | `./infra/terraform.tfvars` | +| Backend | `./infra/backend.tf` | +| Modules | `./infra/modules/` | +| Dockerfiles | `src//Dockerfile` | + +## References + +- [Terraform Patterns](patterns.md) + +## Next + +→ Update `.azure/deployment-plan.md` → **azure-validate** diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/terraform/patterns.md b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/terraform/patterns.md index 4e39e956..55bac09a 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/terraform/patterns.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/recipes/terraform/patterns.md @@ -1,163 +1,163 @@ -# Terraform Patterns - -Common patterns for Terraform Azure deployments. - -## File Structure - -``` -infra/ -├── main.tf # Main resources -├── variables.tf # Variable definitions -├── outputs.tf # Output values -├── terraform.tfvars # Variable values -├── backend.tf # State backend -└── modules/ - └── ... -``` - -## Provider Configuration - -```hcl -# backend.tf -terraform { - required_version = ">= 1.5.0" - - required_providers { - azurerm = { - source = "hashicorp/azurerm" - version = "~> 3.0" - } - azurecaf = { - source = "aztfmod/azurecaf" - version = "~> 1.2" - } - } - - backend "azurerm" { - resource_group_name = "rg-terraform-state" - storage_account_name = "tfstate" - container_name = "tfstate" - key = "app.terraform.tfstate" - } -} - -provider "azurerm" { - features {} -} -``` - -## Variables - -```hcl -# variables.tf -variable "environment" { - type = string - description = "Environment name" -} - -variable "location" { - type = string - description = "Azure region" - default = "eastus2" -} -``` - -## Main Configuration - -```hcl -# main.tf -resource "azurerm_resource_group" "main" { - name = "rg-${var.environment}" - location = var.location - tags = { environment = var.environment } -} - -module "app" { - source = "./modules/app" - resource_group_name = azurerm_resource_group.main.name - location = azurerm_resource_group.main.location - environment = var.environment -} -``` - -## Outputs - -```hcl -# outputs.tf -output "resource_group_name" { - value = azurerm_resource_group.main.name -} - -output "app_url" { - value = module.app.url -} -``` - -## Naming with Azure CAF - -```hcl -resource "azurecaf_name" "storage" { - name = var.environment - resource_type = "azurerm_storage_account" - random_length = 5 -} - -resource "azurerm_storage_account" "main" { - name = azurecaf_name.storage.result - # ... -} -``` - -## State Backend Setup - -```bash -# Create state storage -az group create --name rg-terraform-state --location eastus2 - -az storage account create \ - --name tfstate \ - --resource-group rg-terraform-state \ - --sku Standard_LRS - -az storage container create \ - --name tfstate \ - --account-name tfstate -``` - -## Security Requirements - -| Requirement | Pattern | -|-------------|---------| -| No hardcoded secrets | Use Key Vault data sources | -| Managed Identity | `identity { type = "UserAssigned" }` | -| State encryption | Azure Storage encryption | -| State locking | Azure Blob lease | - -## Container Apps with ACR - -> **⚠️ Two-Phase Deployment Required.** See [Container Apps Terraform Patterns](../../services/container-apps/terraform.md) for full details. - -Use a **public placeholder image** during initial provisioning. Never reference an ACR image that hasn't been built yet. Do **not** use `admin_enabled` on the ACR or add a `registry` block with `username`/`password_secret_name` — use managed identity instead. - -```hcl -resource "azurerm_container_app" "api" { - # ... - template { - container { - name = "api" - image = "mcr.microsoft.com/azuredocs/containerapps-helloworld:latest" - # ... - } - } - - # Prevent Terraform from reverting post-apply image and registry updates - lifecycle { - ignore_changes = [ - template[0].container[0].image, - registry, - ] - } -} -``` - -After `terraform apply`, build the real image and update the Container App via CLI — see the **azure-deploy** skill's `references/recipes/terraform/README.md` (Container Apps Two-Phase Deployment section). +# Terraform Patterns + +Common patterns for Terraform Azure deployments. + +## File Structure + +``` +infra/ +├── main.tf # Main resources +├── variables.tf # Variable definitions +├── outputs.tf # Output values +├── terraform.tfvars # Variable values +├── backend.tf # State backend +└── modules/ + └── ... +``` + +## Provider Configuration + +```hcl +# backend.tf +terraform { + required_version = ">= 1.5.0" + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.0" + } + azurecaf = { + source = "aztfmod/azurecaf" + version = "~> 1.2" + } + } + + backend "azurerm" { + resource_group_name = "rg-terraform-state" + storage_account_name = "tfstate" + container_name = "tfstate" + key = "app.terraform.tfstate" + } +} + +provider "azurerm" { + features {} +} +``` + +## Variables + +```hcl +# variables.tf +variable "environment" { + type = string + description = "Environment name" +} + +variable "location" { + type = string + description = "Azure region" + default = "eastus2" +} +``` + +## Main Configuration + +```hcl +# main.tf +resource "azurerm_resource_group" "main" { + name = "rg-${var.environment}" + location = var.location + tags = { environment = var.environment } +} + +module "app" { + source = "./modules/app" + resource_group_name = azurerm_resource_group.main.name + location = azurerm_resource_group.main.location + environment = var.environment +} +``` + +## Outputs + +```hcl +# outputs.tf +output "resource_group_name" { + value = azurerm_resource_group.main.name +} + +output "app_url" { + value = module.app.url +} +``` + +## Naming with Azure CAF + +```hcl +resource "azurecaf_name" "storage" { + name = var.environment + resource_type = "azurerm_storage_account" + random_length = 5 +} + +resource "azurerm_storage_account" "main" { + name = azurecaf_name.storage.result + # ... +} +``` + +## State Backend Setup + +```bash +# Create state storage +az group create --name rg-terraform-state --location eastus2 + +az storage account create \ + --name tfstate \ + --resource-group rg-terraform-state \ + --sku Standard_LRS + +az storage container create \ + --name tfstate \ + --account-name tfstate +``` + +## Security Requirements + +| Requirement | Pattern | +|-------------|---------| +| No hardcoded secrets | Use Key Vault data sources | +| Managed Identity | `identity { type = "UserAssigned" }` | +| State encryption | Azure Storage encryption | +| State locking | Azure Blob lease | + +## Container Apps with ACR + +> **⚠️ Two-Phase Deployment Required.** See [Container Apps Terraform Patterns](../../services/container-apps/terraform.md) for full details. + +Use a **public placeholder image** during initial provisioning. Never reference an ACR image that hasn't been built yet. Do **not** use `admin_enabled` on the ACR or add a `registry` block with `username`/`password_secret_name` — use managed identity instead. + +```hcl +resource "azurerm_container_app" "api" { + # ... + template { + container { + name = "api" + image = "mcr.microsoft.com/azuredocs/containerapps-helloworld:latest" + # ... + } + } + + # Prevent Terraform from reverting post-apply image and registry updates + lifecycle { + ignore_changes = [ + template[0].container[0].image, + registry, + ] + } +} +``` + +After `terraform apply`, build the real image and update the Container App via CLI — see the **azure-deploy** skill's `references/recipes/terraform/README.md` (Container Apps Two-Phase Deployment section). diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/requirements.md b/.github/plugins/azure-skills/skills/azure-prepare/references/requirements.md index 2fcb3062..9d6add1d 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/requirements.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/requirements.md @@ -1,73 +1,73 @@ -# Requirements Gathering - -Collect project requirements through conversation before making architecture decisions. - -## Categories - -### 1. Classification - -| Type | Description | Implications | -|------|-------------|--------------| -| POC | Proof of concept | Minimal infra, cost-optimized | -| Development | Internal tooling | Balanced, team-focused | -| Production | Customer-facing | Full reliability, monitoring | - -### 2. Scale - -| Scale | Users | Considerations | -|-------|-------|----------------| -| Small | <1K | Single region, basic SKUs | -| Medium | 1K-100K | Auto-scaling, multi-zone | -| Large | 100K+ | Multi-region, premium SKUs | - -### 3. Budget - -| Profile | Focus | -|---------|-------| -| Cost-Optimized | Minimize spend, lower SKUs | -| Balanced | Value for money, standard SKUs | -| Performance | Maximum capability, premium SKUs | - -### 4. Compliance - -| Requirement | Impact | -|-------------|--------| -| Data residency | Region constraints | -| Industry regulations | Security controls | -| Internal policies | Approval workflows | - -### 5. Subscription Policies - -After the user confirms a subscription, query Azure Policy assignments to discover enforcement constraints before making architecture decisions. - -``` -mcp_azure_mcp_policy(command: "policy_assignment_list", subscription: "") -``` - -| Policy Constraint | Impact | -|-------------------|--------| -| Blocked resource types or SKUs | Exclude from architecture | -| Required tags | Add to all Bicep/Terraform resources | -| Allowed regions | Restrict location choices | -| Network restrictions (e.g., no public endpoints) | Adjust networking and access patterns | -| Storage policies (e.g., deny shared key access) | Use policy-compliant auth | -| Naming conventions | Apply to resource naming | - -> ⚠️ **Warning:** Skipping this step can cause deployment failures when Azure Policy denies resource creation. Checking policies here prevents wasted work in architecture and generation phases. - -Record discovered policy constraints in `.azure/deployment-plan.md` under a **Policy Constraints** section so they feed into architecture decisions. - -## Gather via Conversation - -Use `ask_user` tool to confirm each of these with the user: - -1. Project classification (POC/Dev/Prod) -2. Expected scale -3. Budget constraints -4. Compliance requirements (including data residency preferences) -5. Subscription and policy constraints (confirm subscription, then check policies automatically) -6. Architecture preferences (if any) - -## Document in Plan - -Record all requirements in `.azure/deployment-plan.md` immediately after gathering. +# Requirements Gathering + +Collect project requirements through conversation before making architecture decisions. + +## Categories + +### 1. Classification + +| Type | Description | Implications | +|------|-------------|--------------| +| POC | Proof of concept | Minimal infra, cost-optimized | +| Development | Internal tooling | Balanced, team-focused | +| Production | Customer-facing | Full reliability, monitoring | + +### 2. Scale + +| Scale | Users | Considerations | +|-------|-------|----------------| +| Small | <1K | Single region, basic SKUs | +| Medium | 1K-100K | Auto-scaling, multi-zone | +| Large | 100K+ | Multi-region, premium SKUs | + +### 3. Budget + +| Profile | Focus | +|---------|-------| +| Cost-Optimized | Minimize spend, lower SKUs | +| Balanced | Value for money, standard SKUs | +| Performance | Maximum capability, premium SKUs | + +### 4. Compliance + +| Requirement | Impact | +|-------------|--------| +| Data residency | Region constraints | +| Industry regulations | Security controls | +| Internal policies | Approval workflows | + +### 5. Subscription Policies + +After the user confirms a subscription, query Azure Policy assignments to discover enforcement constraints before making architecture decisions. + +``` +mcp_azure_mcp_policy(command: "policy_assignment_list", subscription: "") +``` + +| Policy Constraint | Impact | +|-------------------|--------| +| Blocked resource types or SKUs | Exclude from architecture | +| Required tags | Add to all Bicep/Terraform resources | +| Allowed regions | Restrict location choices | +| Network restrictions (e.g., no public endpoints) | Adjust networking and access patterns | +| Storage policies (e.g., deny shared key access) | Use policy-compliant auth | +| Naming conventions | Apply to resource naming | + +> ⚠️ **Warning:** Skipping this step can cause deployment failures when Azure Policy denies resource creation. Checking policies here prevents wasted work in architecture and generation phases. + +Record discovered policy constraints in `.azure/deployment-plan.md` under a **Policy Constraints** section so they feed into architecture decisions. + +## Gather via Conversation + +Use `ask_user` tool to confirm each of these with the user: + +1. Project classification (POC/Dev/Prod) +2. Expected scale +3. Budget constraints +4. Compliance requirements (including data residency preferences) +5. Subscription and policy constraints (confirm subscription, then check policies automatically) +6. Architecture preferences (if any) + +## Document in Plan + +Record all requirements in `.azure/deployment-plan.md` immediately after gathering. diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/research.md b/.github/plugins/azure-skills/skills/azure-prepare/references/research.md index 0437987e..71f04d48 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/research.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/research.md @@ -1,142 +1,142 @@ -# Research Components - -After architecture planning, research each selected component to gather best practices before generating artifacts. - -## Process - -1. **Identify Components** — List all Azure services from architecture plan -2. **Load Service References** — For each service, load `services//README.md` first, then specific references as needed -3. **Check Resource Naming Rules** — For each resource type, check [resource naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules) for valid characters, length limits, and uniqueness scopes -4. **Load Recipe References** — Load the selected recipe's guide (e.g., [AZD](recipes/azd/README.md)) and its IAC rules, MCP best practices, and schema tools listed in its "Before Generation" table -5. **Check Region Availability** — Verify all selected services are available in the target region per [region-availability.md](region-availability.md) -6. **Check Provisioning Limits** — Invoke **azure-quotas** skill to validate that the selected subscription and region have sufficient quota/capacity for all planned resources. Complete [Step 6 of the plan template](plan-template.md#6-provisioning-limit-checklist) in two phases: (1) prepare resource inventory with deployment quantities, (2) fetch quotas and validate capacity using azure-quotas skill -7. **Load Runtime References** — For containerized apps, load language-specific production settings (e.g., [Node.js](runtimes/nodejs.md)) -8. **Invoke Related Skills** — For deeper guidance, invoke mapped skills from the table below -9. **Document Findings** — Record key insights in `.azure/deployment-plan.md` - -## Service-to-Reference Mapping - -| Azure Service | Reference | Related Skills | -|---------------|-----------|----------------| -| **Hosting** | | | -| Container Apps | [Container Apps](services/container-apps/README.md) | `azure-diagnostics`, `azure-observability`, `azure-nodejs-production` | -| App Service | [App Service](services/app-service/README.md) | `azure-diagnostics`, `azure-observability`, `azure-nodejs-production` | -| Azure Functions | [Functions](services/functions/README.md) | — | -| Static Web Apps | [Static Web Apps](services/static-web-apps/README.md) | — | -| AKS | [AKS](services/aks/README.md) | `azure-networking` | -| **Data** | | | -| Azure SQL | [SQL Database](services/sql-database/README.md) | — | -| Cosmos DB | [Cosmos DB](services/cosmos-db/README.md) | — | -| PostgreSQL | — | — | -| Storage (Blob/Files) | [Storage](services/storage/README.md) | `azure-storage` | -| **Messaging** | | | -| Service Bus | [Service Bus](services/service-bus/README.md) | — | -| Event Grid | [Event Grid](services/event-grid/README.md) | — | -| Event Hubs | — | — | -| **Integration** | | | -| API Management | [APIM](apim.md) | `azure-aigateway` (invoke for AI Gateway policies) | -| Logic Apps | [Logic Apps](services/logic-apps/README.md) | — | -| **Workflow & Orchestration** | | | -| Durable Functions | [Durable Functions](services/functions/durable.md), [Durable Task Scheduler](services/durable-task-scheduler/README.md) | — | -| Durable Task Scheduler | [Durable Task Scheduler](services/durable-task-scheduler/README.md) | — | -| **Security & Identity** | | | -| Key Vault | [Key Vault](services/key-vault/README.md) | `azure-keyvault-expiration-audit` | -| Managed Identity | — | `entra-app-registration` | -| **Observability** | | | -| Application Insights | [App Insights](services/app-insights/README.md) | `appinsights-instrumentation` (invoke for instrumentation) | -| Log Analytics | — | `azure-observability`, `azure-kusto` | -| **AI Services** | | | -| Azure OpenAI | [Foundry](services/foundry/README.md) | `microsoft-foundry` (invoke for AI patterns and model guidance) | -| AI Search | — | `azure-ai` (invoke for search configuration) | - -## Research Instructions - -### Step 1: Load Internal References (Progressive Loading) - -For each selected service, load the README.md first, then load specific files as needed: - -``` -Selected: Container Apps, Cosmos DB, Key Vault - -→ Load: services/container-apps/README.md (overview) - → If need Bicep: services/container-apps/bicep.md - → If need Terraform: services/container-apps/terraform.md - → If need scaling: services/container-apps/scaling.md - → If need health probes: services/container-apps/health-probes.md - -→ Load: services/cosmos-db/README.md (overview) - → If need partitioning: services/cosmos-db/partitioning.md - → If need SDK: services/cosmos-db/sdk.md - -→ Load: services/key-vault/README.md (overview) - → If need SDK: services/key-vault/sdk.md -``` - -### Step 2: Invoke Related Skills (When Deeper Guidance Needed) - -Invoke related skills for specialized scenarios: - -| Scenario | Action | -|----------|--------| -| Using Azure Functions | Stay in **azure-prepare** — load [selection.md](services/functions/templates/selection.md) → Follow [composition.md](services/functions/templates/recipes/composition.md) algorithm | -| PostgreSQL with passwordless auth | Handle directly without a separate skill | -| Need detailed security hardening | Handle directly with service-specific security guidance and platform best practices | -| Setting up App Insights instrumentation | `appinsights-instrumentation` | -| Building AI applications | `microsoft-foundry` | -| Estimate a planned deployment's cost | `cost-estimation` from the separately installed `azure-cost` plugin | - -**Skill/Reference Invocation Pattern:** - -For **Azure Functions**: -1. Load: [selection.md](services/functions/templates/selection.md) (decision tree) -2. Follow: [composition.md](services/functions/templates/recipes/composition.md) (algorithm) -3. Result: Base template + recipe composition (never synthesize IaC) - -For **PostgreSQL**: -1. Handle passwordless auth patterns directly without a separate skill - -### Step 3: Document in Plan - -Add research findings to `.azure/deployment-plan.md` under a `## Research Summary` section with source references and key insights per component. - -## Common Research Patterns - -### Web Application + API + Database (Cosmos DB) - -1. Load: [services/container-apps/README.md](services/container-apps/README.md) → [bicep.md](services/container-apps/bicep.md) or [terraform.md](services/container-apps/terraform.md), [scaling.md](services/container-apps/scaling.md) -2. Load: [services/cosmos-db/README.md](services/cosmos-db/README.md) → [partitioning.md](services/cosmos-db/partitioning.md) -3. Load: [services/key-vault/README.md](services/key-vault/README.md) -4. Invoke: `azure-observability` (monitoring setup) -5. Review service-specific security guidance directly before generation - -### Container Apps + API + SQL Database - -1. Load: [services/container-apps/README.md](services/container-apps/README.md) → [bicep.md](services/container-apps/bicep.md) or [terraform.md](services/container-apps/terraform.md), [scaling.md](services/container-apps/scaling.md) -2. Load: [services/sql-database/README.md](services/sql-database/README.md) → [bicep.md](services/sql-database/bicep.md), [auth.md](services/sql-database/auth.md) -3. Load: [services/key-vault/README.md](services/key-vault/README.md) -4. Review [auth.md](services/sql-database/auth.md) directly for Entra-only auth configuration - -### App Service + API + SQL Database - -1. Load: [services/app-service/README.md](services/app-service/README.md) → [bicep.md](services/app-service/bicep.md) -2. Load: [services/sql-database/README.md](services/sql-database/README.md) → [bicep.md](services/sql-database/bicep.md), [auth.md](services/sql-database/auth.md) -3. Load: [services/key-vault/README.md](services/key-vault/README.md) -4. Review [auth.md](services/sql-database/auth.md) directly for Entra-only auth configuration - -### Serverless Event-Driven - -1. Load: [services/functions/README.md](services/functions/README.md) (contains mandatory composition workflow) -2. Load: [services/event-grid/README.md](services/event-grid/README.md) or [services/service-bus/README.md](services/service-bus/README.md) (if using messaging) -3. Load: [services/storage/README.md](services/storage/README.md) (if using queues/blobs) -4. Invoke: `azure-observability` (distributed tracing) - -### AI Application - -1. Invoke: `microsoft-foundry` (AI patterns and best practices) -2. Load: [services/container-apps/README.md](services/container-apps/README.md) → [bicep.md](services/container-apps/bicep.md) or [terraform.md](services/container-apps/terraform.md) -3. Load: [services/cosmos-db/README.md](services/cosmos-db/README.md) → [partitioning.md](services/cosmos-db/partitioning.md) (vector storage) -4. Review Key Vault and Foundry references directly for API key management - -## After Research - -Proceed to **Generate Artifacts** step with research findings applied. +# Research Components + +After architecture planning, research each selected component to gather best practices before generating artifacts. + +## Process + +1. **Identify Components** — List all Azure services from architecture plan +2. **Load Service References** — For each service, load `services//README.md` first, then specific references as needed +3. **Check Resource Naming Rules** — For each resource type, check [resource naming rules](https://learn.microsoft.com/azure/azure-resource-manager/management/resource-name-rules) for valid characters, length limits, and uniqueness scopes +4. **Load Recipe References** — Load the selected recipe's guide (e.g., [AZD](recipes/azd/README.md)) and its IAC rules, MCP best practices, and schema tools listed in its "Before Generation" table +5. **Check Region Availability** — Verify all selected services are available in the target region per [region-availability.md](region-availability.md) +6. **Check Provisioning Limits** — Invoke **azure-quotas** skill to validate that the selected subscription and region have sufficient quota/capacity for all planned resources. Complete [Step 6 of the plan template](plan-template.md#6-provisioning-limit-checklist) in two phases: (1) prepare resource inventory with deployment quantities, (2) fetch quotas and validate capacity using azure-quotas skill +7. **Load Runtime References** — For containerized apps, load language-specific production settings (e.g., [Node.js](runtimes/nodejs.md)) +8. **Invoke Related Skills** — For deeper guidance, invoke mapped skills from the table below +9. **Document Findings** — Record key insights in `.azure/deployment-plan.md` + +## Service-to-Reference Mapping + +| Azure Service | Reference | Related Skills | +|---------------|-----------|----------------| +| **Hosting** | | | +| Container Apps | [Container Apps](services/container-apps/README.md) | `azure-diagnostics`, `azure-observability`, `azure-nodejs-production` | +| App Service | [App Service](services/app-service/README.md) | `azure-diagnostics`, `azure-observability`, `azure-nodejs-production` | +| Azure Functions | [Functions](services/functions/README.md) | — | +| Static Web Apps | [Static Web Apps](services/static-web-apps/README.md) | — | +| AKS | [AKS](services/aks/README.md) | `azure-networking` | +| **Data** | | | +| Azure SQL | [SQL Database](services/sql-database/README.md) | — | +| Cosmos DB | [Cosmos DB](services/cosmos-db/README.md) | — | +| PostgreSQL | — | — | +| Storage (Blob/Files) | [Storage](services/storage/README.md) | `azure-storage` | +| **Messaging** | | | +| Service Bus | [Service Bus](services/service-bus/README.md) | — | +| Event Grid | [Event Grid](services/event-grid/README.md) | — | +| Event Hubs | — | — | +| **Integration** | | | +| API Management | [APIM](apim.md) | `azure-aigateway` (invoke for AI Gateway policies) | +| Logic Apps | [Logic Apps](services/logic-apps/README.md) | — | +| **Workflow & Orchestration** | | | +| Durable Functions | [Durable Functions](services/functions/durable.md), [Durable Task Scheduler](services/durable-task-scheduler/README.md) | — | +| Durable Task Scheduler | [Durable Task Scheduler](services/durable-task-scheduler/README.md) | — | +| **Security & Identity** | | | +| Key Vault | [Key Vault](services/key-vault/README.md) | `azure-keyvault-expiration-audit` | +| Managed Identity | — | `entra-app-registration` | +| **Observability** | | | +| Application Insights | [App Insights](services/app-insights/README.md) | `appinsights-instrumentation` (invoke for instrumentation) | +| Log Analytics | — | `azure-observability`, `azure-kusto` | +| **AI Services** | | | +| Azure OpenAI | [Foundry](services/foundry/README.md) | `microsoft-foundry` (invoke for AI patterns and model guidance) | +| AI Search | — | `azure-ai` (invoke for search configuration) | + +## Research Instructions + +### Step 1: Load Internal References (Progressive Loading) + +For each selected service, load the README.md first, then load specific files as needed: + +``` +Selected: Container Apps, Cosmos DB, Key Vault + +→ Load: services/container-apps/README.md (overview) + → If need Bicep: services/container-apps/bicep.md + → If need Terraform: services/container-apps/terraform.md + → If need scaling: services/container-apps/scaling.md + → If need health probes: services/container-apps/health-probes.md + +→ Load: services/cosmos-db/README.md (overview) + → If need partitioning: services/cosmos-db/partitioning.md + → If need SDK: services/cosmos-db/sdk.md + +→ Load: services/key-vault/README.md (overview) + → If need SDK: services/key-vault/sdk.md +``` + +### Step 2: Invoke Related Skills (When Deeper Guidance Needed) + +Invoke related skills for specialized scenarios: + +| Scenario | Action | +|----------|--------| +| Using Azure Functions | Stay in **azure-prepare** — load [selection.md](services/functions/templates/selection.md) → Follow [composition.md](services/functions/templates/recipes/composition.md) algorithm | +| PostgreSQL with passwordless auth | Handle directly without a separate skill | +| Need detailed security hardening | Handle directly with service-specific security guidance and platform best practices | +| Setting up App Insights instrumentation | `appinsights-instrumentation` | +| Building AI applications | `microsoft-foundry` | +| Estimate a planned deployment's cost | `cost-estimation` from the separately installed `azure-cost` plugin | + +**Skill/Reference Invocation Pattern:** + +For **Azure Functions**: +1. Load: [selection.md](services/functions/templates/selection.md) (decision tree) +2. Follow: [composition.md](services/functions/templates/recipes/composition.md) (algorithm) +3. Result: Base template + recipe composition (never synthesize IaC) + +For **PostgreSQL**: +1. Handle passwordless auth patterns directly without a separate skill + +### Step 3: Document in Plan + +Add research findings to `.azure/deployment-plan.md` under a `## Research Summary` section with source references and key insights per component. + +## Common Research Patterns + +### Web Application + API + Database (Cosmos DB) + +1. Load: [services/container-apps/README.md](services/container-apps/README.md) → [bicep.md](services/container-apps/bicep.md) or [terraform.md](services/container-apps/terraform.md), [scaling.md](services/container-apps/scaling.md) +2. Load: [services/cosmos-db/README.md](services/cosmos-db/README.md) → [partitioning.md](services/cosmos-db/partitioning.md) +3. Load: [services/key-vault/README.md](services/key-vault/README.md) +4. Invoke: `azure-observability` (monitoring setup) +5. Review service-specific security guidance directly before generation + +### Container Apps + API + SQL Database + +1. Load: [services/container-apps/README.md](services/container-apps/README.md) → [bicep.md](services/container-apps/bicep.md) or [terraform.md](services/container-apps/terraform.md), [scaling.md](services/container-apps/scaling.md) +2. Load: [services/sql-database/README.md](services/sql-database/README.md) → [bicep.md](services/sql-database/bicep.md), [auth.md](services/sql-database/auth.md) +3. Load: [services/key-vault/README.md](services/key-vault/README.md) +4. Review [auth.md](services/sql-database/auth.md) directly for Entra-only auth configuration + +### App Service + API + SQL Database + +1. Load: [services/app-service/README.md](services/app-service/README.md) → [bicep.md](services/app-service/bicep.md) +2. Load: [services/sql-database/README.md](services/sql-database/README.md) → [bicep.md](services/sql-database/bicep.md), [auth.md](services/sql-database/auth.md) +3. Load: [services/key-vault/README.md](services/key-vault/README.md) +4. Review [auth.md](services/sql-database/auth.md) directly for Entra-only auth configuration + +### Serverless Event-Driven + +1. Load: [services/functions/README.md](services/functions/README.md) (contains mandatory composition workflow) +2. Load: [services/event-grid/README.md](services/event-grid/README.md) or [services/service-bus/README.md](services/service-bus/README.md) (if using messaging) +3. Load: [services/storage/README.md](services/storage/README.md) (if using queues/blobs) +4. Invoke: `azure-observability` (distributed tracing) + +### AI Application + +1. Invoke: `microsoft-foundry` (AI patterns and best practices) +2. Load: [services/container-apps/README.md](services/container-apps/README.md) → [bicep.md](services/container-apps/bicep.md) or [terraform.md](services/container-apps/terraform.md) +3. Load: [services/cosmos-db/README.md](services/cosmos-db/README.md) → [partitioning.md](services/cosmos-db/partitioning.md) (vector storage) +4. Review Key Vault and Foundry references directly for API key management + +## After Research + +Proceed to **Generate Artifacts** step with research findings applied. diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/scan.md b/.github/plugins/azure-skills/skills/azure-prepare/references/scan.md index cff26935..1ea4c85c 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/scan.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/scan.md @@ -1,89 +1,89 @@ -# Codebase Scan - -Analyze workspace to identify components, technologies, and dependencies. - -## Detection Patterns - -### Languages & Frameworks - -| File | Indicates | -|------|-----------| -| `package.json` | Node.js | -| `requirements.txt`, `pyproject.toml` | Python | -| `*.csproj`, `*.sln` | .NET | -| `pom.xml`, `build.gradle` | Java | -| `go.mod` | Go | - -### Component Types - -| Pattern | Component Type | -|---------|----------------| -| React/Vue/Angular in package.json | SPA Frontend | -| Only .html/.css/.js files, no package.json | Pure Static Site | -| Express/Fastify/Koa | API Service | -| Flask/FastAPI/Django | API Service | -| Next.js/Nuxt | SSR Web App | -| Celery/Bull/Agenda | Background Worker | -| azure-functions SDK | Azure Function | -| .AppHost.csproj or Aspire.Hosting package | .NET Aspire App | - -**Pure Static Site Detection:** -- No package.json, requirements.txt, or build configuration -- Contains only HTML, CSS, JavaScript, and asset files -- No framework dependencies (React, Vue, Angular, etc.) -- ⚠️ For pure static sites, do NOT add `language` field to azure.yaml to avoid triggering build steps - -### Existing Tooling - -| Found | Tooling | -|-------|---------| -| `azure.yaml` | AZD configured | -| `infra/*.bicep` | Bicep IaC | -| `infra/*.tf` | Terraform IaC | -| `Dockerfile` | Containerized | -| `.github/workflows/` | GitHub Actions CI/CD | -| `azure-pipelines.yml` | Azure DevOps CI/CD | - -### .NET Aspire Detection - -**.NET Aspire projects** are identified by: -- A project ending with `.AppHost.csproj` (e.g., `OrleansVoting.AppHost.csproj`) -- Reference to `Aspire.Hosting` or `Aspire.Hosting.AppHost` package in .csproj files -- Multiple .NET projects in a solution, typically including an AppHost orchestrator - -**When Aspire is detected:** -- Use `azd init --from-code -e ` instead of manual azure.yaml creation -- The `--from-code` flag automatically detects the AppHost and generates appropriate configuration -- The `-e` flag is **required** for non-interactive environments (agents, CI/CD) -- ⚠️ **CRITICAL:** If the AppHost contains `AddAzureFunctionsProject`, you **MUST** add `.WithEnvironment("AzureWebJobsSecretStorageType", "Files")` to the Functions builder chain BEFORE deployment. Without this, Functions will fail at startup with `Secret initialization from Blob storage failed`. See [aspire.md](aspire.md) Step 4b for the complete detection and fix procedure. -- See [aspire.md](aspire.md) for detailed Aspire-specific guidance - -## Output - -Document findings: - -```markdown -## Components - -| Component | Type | Technology | Path | -|-----------|------|------------|------| -| api | API Service | Node.js/Express | src/api | -| web | SPA | React | src/web | -| worker | Background | Python | src/worker | - -## Dependencies - -| Component | Depends On | Type | -|-----------|-----------|------| -| api | PostgreSQL | Database | -| web | api | HTTP | -| worker | Service Bus | Queue | - -## Existing Infrastructure - -| Item | Status | -|------|--------| -| azure.yaml | Not found | -| infra/ | Not found | -| Dockerfiles | Found: src/api/Dockerfile | -``` +# Codebase Scan + +Analyze workspace to identify components, technologies, and dependencies. + +## Detection Patterns + +### Languages & Frameworks + +| File | Indicates | +|------|-----------| +| `package.json` | Node.js | +| `requirements.txt`, `pyproject.toml` | Python | +| `*.csproj`, `*.sln` | .NET | +| `pom.xml`, `build.gradle` | Java | +| `go.mod` | Go | + +### Component Types + +| Pattern | Component Type | +|---------|----------------| +| React/Vue/Angular in package.json | SPA Frontend | +| Only .html/.css/.js files, no package.json | Pure Static Site | +| Express/Fastify/Koa | API Service | +| Flask/FastAPI/Django | API Service | +| Next.js/Nuxt | SSR Web App | +| Celery/Bull/Agenda | Background Worker | +| azure-functions SDK | Azure Function | +| .AppHost.csproj or Aspire.Hosting package | .NET Aspire App | + +**Pure Static Site Detection:** +- No package.json, requirements.txt, or build configuration +- Contains only HTML, CSS, JavaScript, and asset files +- No framework dependencies (React, Vue, Angular, etc.) +- ⚠️ For pure static sites, do NOT add `language` field to azure.yaml to avoid triggering build steps + +### Existing Tooling + +| Found | Tooling | +|-------|---------| +| `azure.yaml` | AZD configured | +| `infra/*.bicep` | Bicep IaC | +| `infra/*.tf` | Terraform IaC | +| `Dockerfile` | Containerized | +| `.github/workflows/` | GitHub Actions CI/CD | +| `azure-pipelines.yml` | Azure DevOps CI/CD | + +### .NET Aspire Detection + +**.NET Aspire projects** are identified by: +- A project ending with `.AppHost.csproj` (e.g., `OrleansVoting.AppHost.csproj`) +- Reference to `Aspire.Hosting` or `Aspire.Hosting.AppHost` package in .csproj files +- Multiple .NET projects in a solution, typically including an AppHost orchestrator + +**When Aspire is detected:** +- Use `azd init --from-code -e ` instead of manual azure.yaml creation +- The `--from-code` flag automatically detects the AppHost and generates appropriate configuration +- The `-e` flag is **required** for non-interactive environments (agents, CI/CD) +- ⚠️ **CRITICAL:** If the AppHost contains `AddAzureFunctionsProject`, you **MUST** add `.WithEnvironment("AzureWebJobsSecretStorageType", "Files")` to the Functions builder chain BEFORE deployment. Without this, Functions will fail at startup with `Secret initialization from Blob storage failed`. See [aspire.md](aspire.md) Step 4b for the complete detection and fix procedure. +- See [aspire.md](aspire.md) for detailed Aspire-specific guidance + +## Output + +Document findings: + +```markdown +## Components + +| Component | Type | Technology | Path | +|-----------|------|------------|------| +| api | API Service | Node.js/Express | src/api | +| web | SPA | React | src/web | +| worker | Background | Python | src/worker | + +## Dependencies + +| Component | Depends On | Type | +|-----------|-----------|------| +| api | PostgreSQL | Database | +| web | api | HTTP | +| worker | Service Bus | Queue | + +## Existing Infrastructure + +| Item | Status | +|------|--------| +| azure.yaml | Not found | +| infra/ | Not found | +| Dockerfiles | Found: src/api/Dockerfile | +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/aks/README.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/aks/README.md index 349ea5cf..a1190686 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/aks/README.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/aks/README.md @@ -1,47 +1,47 @@ -# Azure Kubernetes Service (AKS) - -Full Kubernetes orchestration for complex containerized workloads. - -## When to Use - -- Complex microservices requiring Kubernetes orchestration -- Teams with Kubernetes expertise -- Workloads needing fine-grained infrastructure control -- Multi-container pods with sidecars -- Custom networking requirements -- Hybrid/multi-cloud Kubernetes strategies - -## Service Type in azure.yaml - -```yaml -services: - my-service: - host: aks - project: ./src/my-service - docker: - path: ./Dockerfile - k8s: - deploymentPath: ./k8s -``` - -## Required Supporting Resources - -| Resource | Purpose | -|----------|---------| -| Container Registry | Image storage | -| Log Analytics Workspace | Monitoring | -| Virtual Network | Network isolation (optional) | -| Key Vault | Secrets management | - -## Node Pool Types - -| Pool | Purpose | -|------|---------| -| System | Cluster infrastructure, 3 nodes minimum | -| User | Application workloads, auto-scaling | - -## References - -- [Bicep Patterns](bicep.md) -- [K8s Manifests](manifests.md) -- [Add-ons](addons.md) +# Azure Kubernetes Service (AKS) + +Full Kubernetes orchestration for complex containerized workloads. + +## When to Use + +- Complex microservices requiring Kubernetes orchestration +- Teams with Kubernetes expertise +- Workloads needing fine-grained infrastructure control +- Multi-container pods with sidecars +- Custom networking requirements +- Hybrid/multi-cloud Kubernetes strategies + +## Service Type in azure.yaml + +```yaml +services: + my-service: + host: aks + project: ./src/my-service + docker: + path: ./Dockerfile + k8s: + deploymentPath: ./k8s +``` + +## Required Supporting Resources + +| Resource | Purpose | +|----------|---------| +| Container Registry | Image storage | +| Log Analytics Workspace | Monitoring | +| Virtual Network | Network isolation (optional) | +| Key Vault | Secrets management | + +## Node Pool Types + +| Pool | Purpose | +|------|---------| +| System | Cluster infrastructure, 3 nodes minimum | +| User | Application workloads, auto-scaling | + +## References + +- [Bicep Patterns](bicep.md) +- [K8s Manifests](manifests.md) +- [Add-ons](addons.md) diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/aks/addons.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/aks/addons.md index 0e61cc71..b38d1642 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/aks/addons.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/aks/addons.md @@ -1,58 +1,58 @@ -# AKS - Add-ons - -## Container Monitoring - -```bicep -addonProfiles: { - omsagent: { - enabled: true - config: { - logAnalyticsWorkspaceResourceID: logAnalytics.id - } - } -} -``` - -## Azure CNI Networking - -```bicep -networkProfile: { - networkPlugin: 'azure' - networkPolicy: 'calico' -} -``` - -## Azure Key Vault Provider - -```bicep -addonProfiles: { - azureKeyvaultSecretsProvider: { - enabled: true - config: { - enableSecretRotation: 'true' - } - } -} -``` - -## Application Gateway Ingress Controller - -```bicep -addonProfiles: { - ingressApplicationGateway: { - enabled: true - config: { - applicationGatewayId: appGateway.id - } - } -} -``` - -## Add-ons Summary - -| Add-on | Purpose | -|--------|---------| -| omsagent | Container Insights monitoring | -| azureKeyvaultSecretsProvider | Mount Key Vault secrets as volumes | -| ingressApplicationGateway | Application Gateway as ingress controller | -| azurepolicy | Azure Policy for Kubernetes | +# AKS - Add-ons + +## Container Monitoring + +```bicep +addonProfiles: { + omsagent: { + enabled: true + config: { + logAnalyticsWorkspaceResourceID: logAnalytics.id + } + } +} +``` + +## Azure CNI Networking + +```bicep +networkProfile: { + networkPlugin: 'azure' + networkPolicy: 'calico' +} +``` + +## Azure Key Vault Provider + +```bicep +addonProfiles: { + azureKeyvaultSecretsProvider: { + enabled: true + config: { + enableSecretRotation: 'true' + } + } +} +``` + +## Application Gateway Ingress Controller + +```bicep +addonProfiles: { + ingressApplicationGateway: { + enabled: true + config: { + applicationGatewayId: appGateway.id + } + } +} +``` + +## Add-ons Summary + +| Add-on | Purpose | +|--------|---------| +| omsagent | Container Insights monitoring | +| azureKeyvaultSecretsProvider | Mount Key Vault secrets as volumes | +| ingressApplicationGateway | Application Gateway as ingress controller | +| azurepolicy | Azure Policy for Kubernetes | diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/aks/bicep.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/aks/bicep.md index 39b339e5..383e3e2e 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/aks/bicep.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/aks/bicep.md @@ -1,92 +1,92 @@ -# AKS - Bicep Patterns - -## Cluster Resource - -```bicep -resource aks 'Microsoft.ContainerService/managedClusters@2023-07-01' = { - name: '${resourcePrefix}-aks-${uniqueHash}' - location: location - identity: { - type: 'SystemAssigned' - } - properties: { - dnsPrefix: '${resourcePrefix}-aks' - kubernetesVersion: '1.28' - agentPoolProfiles: [ - { - name: 'default' - count: 3 - vmSize: 'Standard_DS2_v2' - mode: 'System' - osType: 'Linux' - enableAutoScaling: true - minCount: 1 - maxCount: 5 - } - ] - networkProfile: { - networkPlugin: 'azure' - serviceCidr: '10.0.0.0/16' - dnsServiceIP: '10.0.0.10' - } - } -} -``` - -## ACR Pull Role Assignment - -```bicep -resource acrPullRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(aks.id, containerRegistry.id, 'acrpull') - scope: containerRegistry - properties: { - roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d') - principalId: aks.properties.identityProfile.kubeletidentity.objectId - principalType: 'ServicePrincipal' - } -} -``` - -## Node Pool Configuration - -### System Pool (Required) - -```bicep -{ - name: 'system' - count: 3 - vmSize: 'Standard_DS2_v2' - mode: 'System' - osType: 'Linux' -} -``` - -### User Pool (Workloads) - -```bicep -{ - name: 'workload' - count: 2 - vmSize: 'Standard_DS4_v2' - mode: 'User' - osType: 'Linux' - enableAutoScaling: true - minCount: 1 - maxCount: 10 -} -``` - -## Workload Identity - -```bicep -properties: { - oidcIssuerProfile: { - enabled: true - } - securityProfile: { - workloadIdentity: { - enabled: true - } - } -} -``` +# AKS - Bicep Patterns + +## Cluster Resource + +```bicep +resource aks 'Microsoft.ContainerService/managedClusters@2023-07-01' = { + name: '${resourcePrefix}-aks-${uniqueHash}' + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + dnsPrefix: '${resourcePrefix}-aks' + kubernetesVersion: '1.28' + agentPoolProfiles: [ + { + name: 'default' + count: 3 + vmSize: 'Standard_DS2_v2' + mode: 'System' + osType: 'Linux' + enableAutoScaling: true + minCount: 1 + maxCount: 5 + } + ] + networkProfile: { + networkPlugin: 'azure' + serviceCidr: '10.0.0.0/16' + dnsServiceIP: '10.0.0.10' + } + } +} +``` + +## ACR Pull Role Assignment + +```bicep +resource acrPullRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(aks.id, containerRegistry.id, 'acrpull') + scope: containerRegistry + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d') + principalId: aks.properties.identityProfile.kubeletidentity.objectId + principalType: 'ServicePrincipal' + } +} +``` + +## Node Pool Configuration + +### System Pool (Required) + +```bicep +{ + name: 'system' + count: 3 + vmSize: 'Standard_DS2_v2' + mode: 'System' + osType: 'Linux' +} +``` + +### User Pool (Workloads) + +```bicep +{ + name: 'workload' + count: 2 + vmSize: 'Standard_DS4_v2' + mode: 'User' + osType: 'Linux' + enableAutoScaling: true + minCount: 1 + maxCount: 10 +} +``` + +## Workload Identity + +```bicep +properties: { + oidcIssuerProfile: { + enabled: true + } + securityProfile: { + workloadIdentity: { + enabled: true + } + } +} +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/aks/manifests.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/aks/manifests.md index 1696eb04..54e38e44 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/aks/manifests.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/aks/manifests.md @@ -1,81 +1,81 @@ -# AKS - Kubernetes Manifests - -## Deployment - -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: my-service -spec: - replicas: 3 - selector: - matchLabels: - app: my-service - template: - metadata: - labels: - app: my-service - spec: - containers: - - name: my-service - image: myacr.azurecr.io/my-service:latest - ports: - - containerPort: 8080 - resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "256Mi" - cpu: "500m" - livenessProbe: - httpGet: - path: /health - port: 8080 - initialDelaySeconds: 10 - readinessProbe: - httpGet: - path: /ready - port: 8080 - initialDelaySeconds: 5 -``` - -## Service - -```yaml -apiVersion: v1 -kind: Service -metadata: - name: my-service -spec: - selector: - app: my-service - ports: - - port: 80 - targetPort: 8080 - type: ClusterIP -``` - -## Ingress - -```yaml -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: my-ingress - annotations: - kubernetes.io/ingress.class: azure/application-gateway -spec: - rules: - - host: myapp.example.com - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: my-service - port: - number: 80 -``` +# AKS - Kubernetes Manifests + +## Deployment + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-service +spec: + replicas: 3 + selector: + matchLabels: + app: my-service + template: + metadata: + labels: + app: my-service + spec: + containers: + - name: my-service + image: myacr.azurecr.io/my-service:latest + ports: + - containerPort: 8080 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "500m" + livenessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 10 + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 5 +``` + +## Service + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: my-service +spec: + selector: + app: my-service + ports: + - port: 80 + targetPort: 8080 + type: ClusterIP +``` + +## Ingress + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: my-ingress + annotations: + kubernetes.io/ingress.class: azure/application-gateway +spec: + rules: + - host: myapp.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: my-service + port: + number: 80 +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-insights/README.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-insights/README.md index 0c4364e1..0821b4ed 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-insights/README.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-insights/README.md @@ -1,32 +1,32 @@ -# App Insights - -Azure Application Insights for telemetry, monitoring, and APM. - -## When to Add - -- User wants observability/monitoring -- User mentions telemetry, tracing, or logging -- Production apps needing health visibility - -## Implementation - -> **→ Invoke the `appinsights-instrumentation` skill** -> -> This skill has detailed guides for: -> - Auto-instrumentation (ASP.NET Core on App Service) -> - Manual instrumentation (Node.js, Python, C#) -> - Bicep templates and CLI scripts - -## Quick Reference - -| Aspect | Value | -|--------|-------| -| Resource | `Microsoft.Insights/components` | -| Depends on | Log Analytics Workspace | -| SKU | PerGB2018 (consumption-based) | - -## Architecture Notes - -- Create in same resource group as the app -- Connect to centralized Log Analytics Workspace -- Use connection string (not instrumentation key) for new apps +# App Insights + +Azure Application Insights for telemetry, monitoring, and APM. + +## When to Add + +- User wants observability/monitoring +- User mentions telemetry, tracing, or logging +- Production apps needing health visibility + +## Implementation + +> **→ Invoke the `appinsights-instrumentation` skill** +> +> This skill has detailed guides for: +> - Auto-instrumentation (ASP.NET Core on App Service) +> - Manual instrumentation (Node.js, Python, C#) +> - Bicep templates and CLI scripts + +## Quick Reference + +| Aspect | Value | +|--------|-------| +| Resource | `Microsoft.Insights/components` | +| Depends on | Log Analytics Workspace | +| SKU | PerGB2018 (consumption-based) | + +## Architecture Notes + +- Create in same resource group as the app +- Connect to centralized Log Analytics Workspace +- Use connection string (not instrumentation key) for new apps diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-service/README.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-service/README.md index 4c1edb1c..1ff34a4b 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-service/README.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-service/README.md @@ -1,92 +1,92 @@ -# Azure App Service - -Hosting patterns and best practices for Azure App Service. - -## When to Use - -- Traditional web applications -- REST APIs without containerization -- .NET, Node.js, Python, Java, PHP applications -- When Docker is not required/desired -- When built-in deployment slots are needed - -## Service Type in azure.yaml - -```yaml -services: - my-web: - host: appservice - project: ./src/my-web -``` - -## Required Supporting Resources - -| Resource | Purpose | -|----------|---------| -| App Service Plan | Compute hosting | -| Application Insights | Monitoring | -| Key Vault | Secrets (optional) | - -## Runtime Stacks - -> 💡 **Tip:** Prefer Linux App Service Plans for Node.js, Python, and Java. Use Windows only when explicitly required (e.g. .NET Framework). See [Bicep Patterns](bicep.md) for Linux vs Windows configuration. - -### Linux (recommended) - -| Language | linuxFxVersion | -|----------|----------------| -| Node.js 24 | `NODE\|24-lts` | -| Node.js 22 | `NODE\|22-lts` | -| Python 3.14 | `PYTHON\|3.14` | -| .NET 10 | `DOTNETCORE\|10.0` | -| Java 21 | `JAVA\|21-java21` | - -### Windows - -| Language | Setting | -|----------|---------| -| Node.js | `WEBSITE_NODE_DEFAULT_VERSION: '~24'` (app setting) | -| .NET 10 | Built-in (no extra config) | - -## SKU Selection - -| SKU | Use Case | -|-----|----------| -| F1/D1 | Development/testing (free/shared) | -| B1-B3 | Small production, basic features | -| S1-S3 | Production with auto-scale, slots | -| P1v3-P3v3 | High-performance production | - -## Health Checks - -Always configure health check path: - -```bicep -siteConfig: { - healthCheckPath: '/health' -} -``` - -Endpoint should return 200 OK when healthy. - -## Templates - -For App Service templates with composable recipes (SQL, Cosmos DB, Auth, Redis), see [Template Selection](templates/selection.md). - -## Common Data Backends - -When pairing App Service with a data layer, load the corresponding service references: - -| Data Service | Reference | -| ------------ | ----------------------------------------- | -| Azure SQL | [SQL Database](../sql-database/README.md) | -| Cosmos DB | [Cosmos DB](../cosmos-db/README.md) | - -## References - -- [Bicep Patterns](bicep.md) -- [Deployment Slots](deployment-slots.md) -- [Auto-Scaling](scaling.md) -- [Networking](networking.md) -- [SKU Selection](sku-selection.md) -- [Custom Domains](custom-domains.md) +# Azure App Service + +Hosting patterns and best practices for Azure App Service. + +## When to Use + +- Traditional web applications +- REST APIs without containerization +- .NET, Node.js, Python, Java, PHP applications +- When Docker is not required/desired +- When built-in deployment slots are needed + +## Service Type in azure.yaml + +```yaml +services: + my-web: + host: appservice + project: ./src/my-web +``` + +## Required Supporting Resources + +| Resource | Purpose | +|----------|---------| +| App Service Plan | Compute hosting | +| Application Insights | Monitoring | +| Key Vault | Secrets (optional) | + +## Runtime Stacks + +> 💡 **Tip:** Prefer Linux App Service Plans for Node.js, Python, and Java. Use Windows only when explicitly required (e.g. .NET Framework). See [Bicep Patterns](bicep.md) for Linux vs Windows configuration. + +### Linux (recommended) + +| Language | linuxFxVersion | +|----------|----------------| +| Node.js 24 | `NODE\|24-lts` | +| Node.js 22 | `NODE\|22-lts` | +| Python 3.14 | `PYTHON\|3.14` | +| .NET 10 | `DOTNETCORE\|10.0` | +| Java 21 | `JAVA\|21-java21` | + +### Windows + +| Language | Setting | +|----------|---------| +| Node.js | `WEBSITE_NODE_DEFAULT_VERSION: '~24'` (app setting) | +| .NET 10 | Built-in (no extra config) | + +## SKU Selection + +| SKU | Use Case | +|-----|----------| +| F1/D1 | Development/testing (free/shared) | +| B1-B3 | Small production, basic features | +| S1-S3 | Production with auto-scale, slots | +| P1v3-P3v3 | High-performance production | + +## Health Checks + +Always configure health check path: + +```bicep +siteConfig: { + healthCheckPath: '/health' +} +``` + +Endpoint should return 200 OK when healthy. + +## Templates + +For App Service templates with composable recipes (SQL, Cosmos DB, Auth, Redis), see [Template Selection](templates/selection.md). + +## Common Data Backends + +When pairing App Service with a data layer, load the corresponding service references: + +| Data Service | Reference | +| ------------ | ----------------------------------------- | +| Azure SQL | [SQL Database](../sql-database/README.md) | +| Cosmos DB | [Cosmos DB](../cosmos-db/README.md) | + +## References + +- [Bicep Patterns](bicep.md) +- [Deployment Slots](deployment-slots.md) +- [Auto-Scaling](scaling.md) +- [Networking](networking.md) +- [SKU Selection](sku-selection.md) +- [Custom Domains](custom-domains.md) diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-service/bicep.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-service/bicep.md index 9165df60..8fd56750 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-service/bicep.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-service/bicep.md @@ -1,138 +1,138 @@ -# App Service Bicep Patterns - -## Linux vs Windows - -> ⚠️ **Warning:** `linuxFxVersion` and `reserved: true` are **Linux-only** properties. Setting `linuxFxVersion` on a Windows App Service Plan causes a deployment error: `LinuxFxVersion cannot be set for non-Linux App Service plans`. Omit both for Windows plans. - -| Property | Linux | Windows | -|----------|-------|---------| -| Plan `reserved` | `true` | omit (defaults to `false`) | -| Site `kind` | `'app,linux'` | `'app'` | -| Site `linuxFxVersion` | e.g. `'NODE\|20-lts'` | omit | -| Site `WEBSITE_NODE_DEFAULT_VERSION` | omit | e.g. `'~20'` | - -> 💡 **Tip:** When both frontend and API use Node.js, prefer a **single Linux App Service Plan** for both. This avoids mixed-platform complexity and keeps all services on one plan. - -## Linux App Service (Recommended for Node.js) - -> ⚠️ **REQUIRED: `azd-service-name` tag** — The `tags` property MUST include `union(tags, { 'azd-service-name': serviceName })` so that `azd deploy` can locate the resource. Without this tag, `azd deploy` fails with `resource not found: unable to find a resource tagged with 'azd-service-name: web'`. - -```bicep -resource appServicePlan 'Microsoft.Web/serverfarms@2022-09-01' = { - name: '${resourcePrefix}-plan-${uniqueHash}' - location: location - kind: 'linux' - sku: { - name: 'B1' - tier: 'Basic' - } - properties: { - reserved: true // Required for Linux - } -} - -resource webApp 'Microsoft.Web/sites@2022-09-01' = { - name: '${resourcePrefix}-${serviceName}-${uniqueHash}' - location: location - kind: 'app,linux' - tags: union(tags, { 'azd-service-name': serviceName }) // REQUIRED for azd deploy - properties: { - serverFarmId: appServicePlan.id - siteConfig: { - linuxFxVersion: 'NODE|24-lts' - alwaysOn: true - healthCheckPath: '/health' - appSettings: [ - { - name: 'SCM_DO_BUILD_DURING_DEPLOYMENT' - value: 'true' // Required for source-based Node.js deploys (azd deploy) - } - { - name: 'APPLICATIONINSIGHTS_CONNECTION_STRING' - value: applicationInsights.properties.ConnectionString - } - { - name: 'ApplicationInsightsAgent_EXTENSION_VERSION' - value: '~3' - } - ] - } - httpsOnly: true - } - identity: { - type: 'SystemAssigned' - } -} -``` - -## Windows App Service - -Use when the runtime requires Windows (e.g. .NET Framework) or when explicitly requested. - -```bicep -resource appServicePlan 'Microsoft.Web/serverfarms@2022-09-01' = { - name: '${resourcePrefix}-plan-${uniqueHash}' - location: location - sku: { - name: 'B1' - tier: 'Basic' - } - // Do NOT set reserved: true for Windows -} - -resource webApp 'Microsoft.Web/sites@2022-09-01' = { - name: '${resourcePrefix}-${serviceName}-${uniqueHash}' - location: location - kind: 'app' - tags: union(tags, { 'azd-service-name': serviceName }) // REQUIRED for azd deploy - properties: { - serverFarmId: appServicePlan.id - siteConfig: { - // Do NOT set linuxFxVersion for Windows plans - alwaysOn: true - healthCheckPath: '/health' - appSettings: [ - { - name: 'SCM_DO_BUILD_DURING_DEPLOYMENT' - value: 'true' // Required for source-based Node.js deploys (azd deploy) - } - { - name: 'WEBSITE_NODE_DEFAULT_VERSION' - value: '~20' - } - { - name: 'APPLICATIONINSIGHTS_CONNECTION_STRING' - value: applicationInsights.properties.ConnectionString - } - { - name: 'ApplicationInsightsAgent_EXTENSION_VERSION' - value: '~3' - } - ] - } - httpsOnly: true - } - identity: { - type: 'SystemAssigned' - } -} -``` - -## Node.js Build Configuration - -> ⚠️ **Warning:** For source-based Node.js deployments to App Service (e.g. `azd deploy`), you **must** set `SCM_DO_BUILD_DURING_DEPLOYMENT=true` so App Service runs `npm install` during deployment. Without this, the app fails at runtime with `Cannot find module` errors. - -This setting is already included in both the Linux and Windows examples above. - -## Key Vault Integration - -Reference secrets from Key Vault: - -```bicep -appSettings: [ - { - name: 'DATABASE_URL' - value: '@Microsoft.KeyVault(VaultName=${keyVault.name};SecretName=database-url)' - } -] -``` +# App Service Bicep Patterns + +## Linux vs Windows + +> ⚠️ **Warning:** `linuxFxVersion` and `reserved: true` are **Linux-only** properties. Setting `linuxFxVersion` on a Windows App Service Plan causes a deployment error: `LinuxFxVersion cannot be set for non-Linux App Service plans`. Omit both for Windows plans. + +| Property | Linux | Windows | +|----------|-------|---------| +| Plan `reserved` | `true` | omit (defaults to `false`) | +| Site `kind` | `'app,linux'` | `'app'` | +| Site `linuxFxVersion` | e.g. `'NODE\|20-lts'` | omit | +| Site `WEBSITE_NODE_DEFAULT_VERSION` | omit | e.g. `'~20'` | + +> 💡 **Tip:** When both frontend and API use Node.js, prefer a **single Linux App Service Plan** for both. This avoids mixed-platform complexity and keeps all services on one plan. + +## Linux App Service (Recommended for Node.js) + +> ⚠️ **REQUIRED: `azd-service-name` tag** — The `tags` property MUST include `union(tags, { 'azd-service-name': serviceName })` so that `azd deploy` can locate the resource. Without this tag, `azd deploy` fails with `resource not found: unable to find a resource tagged with 'azd-service-name: web'`. + +```bicep +resource appServicePlan 'Microsoft.Web/serverfarms@2022-09-01' = { + name: '${resourcePrefix}-plan-${uniqueHash}' + location: location + kind: 'linux' + sku: { + name: 'B1' + tier: 'Basic' + } + properties: { + reserved: true // Required for Linux + } +} + +resource webApp 'Microsoft.Web/sites@2022-09-01' = { + name: '${resourcePrefix}-${serviceName}-${uniqueHash}' + location: location + kind: 'app,linux' + tags: union(tags, { 'azd-service-name': serviceName }) // REQUIRED for azd deploy + properties: { + serverFarmId: appServicePlan.id + siteConfig: { + linuxFxVersion: 'NODE|24-lts' + alwaysOn: true + healthCheckPath: '/health' + appSettings: [ + { + name: 'SCM_DO_BUILD_DURING_DEPLOYMENT' + value: 'true' // Required for source-based Node.js deploys (azd deploy) + } + { + name: 'APPLICATIONINSIGHTS_CONNECTION_STRING' + value: applicationInsights.properties.ConnectionString + } + { + name: 'ApplicationInsightsAgent_EXTENSION_VERSION' + value: '~3' + } + ] + } + httpsOnly: true + } + identity: { + type: 'SystemAssigned' + } +} +``` + +## Windows App Service + +Use when the runtime requires Windows (e.g. .NET Framework) or when explicitly requested. + +```bicep +resource appServicePlan 'Microsoft.Web/serverfarms@2022-09-01' = { + name: '${resourcePrefix}-plan-${uniqueHash}' + location: location + sku: { + name: 'B1' + tier: 'Basic' + } + // Do NOT set reserved: true for Windows +} + +resource webApp 'Microsoft.Web/sites@2022-09-01' = { + name: '${resourcePrefix}-${serviceName}-${uniqueHash}' + location: location + kind: 'app' + tags: union(tags, { 'azd-service-name': serviceName }) // REQUIRED for azd deploy + properties: { + serverFarmId: appServicePlan.id + siteConfig: { + // Do NOT set linuxFxVersion for Windows plans + alwaysOn: true + healthCheckPath: '/health' + appSettings: [ + { + name: 'SCM_DO_BUILD_DURING_DEPLOYMENT' + value: 'true' // Required for source-based Node.js deploys (azd deploy) + } + { + name: 'WEBSITE_NODE_DEFAULT_VERSION' + value: '~20' + } + { + name: 'APPLICATIONINSIGHTS_CONNECTION_STRING' + value: applicationInsights.properties.ConnectionString + } + { + name: 'ApplicationInsightsAgent_EXTENSION_VERSION' + value: '~3' + } + ] + } + httpsOnly: true + } + identity: { + type: 'SystemAssigned' + } +} +``` + +## Node.js Build Configuration + +> ⚠️ **Warning:** For source-based Node.js deployments to App Service (e.g. `azd deploy`), you **must** set `SCM_DO_BUILD_DURING_DEPLOYMENT=true` so App Service runs `npm install` during deployment. Without this, the app fails at runtime with `Cannot find module` errors. + +This setting is already included in both the Linux and Windows examples above. + +## Key Vault Integration + +Reference secrets from Key Vault: + +```bicep +appSettings: [ + { + name: 'DATABASE_URL' + value: '@Microsoft.KeyVault(VaultName=${keyVault.name};SecretName=database-url)' + } +] +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-service/deployment-slots.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-service/deployment-slots.md index e34fff5d..c2f86613 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-service/deployment-slots.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-service/deployment-slots.md @@ -1,126 +1,126 @@ -# App Service Deployment Slots - -Zero-downtime deployments using staging slots. - -## Basic Staging Slot - -```bicep -resource stagingSlot 'Microsoft.Web/sites/slots@2022-09-01' = { - parent: webApp - name: 'staging' - location: location - properties: { - serverFarmId: appServicePlan.id - } -} -``` - -## Slot Requirements — App Service - -| SKU Tier | Slots Supported | -|----------|-----------------| -| Free/Shared | 0 | -| Basic | 0 | -| Standard | 5 | -| Premium | 20 | - -## Slot Requirements — Azure Functions - -> ⚠️ Slot support for Azure Functions varies by OS and hosting plan. - -| Hosting Plan | OS | Slots Supported | -|---|---|---| -| Flex Consumption (FC1) | Linux | ❌ 0 | -| Consumption (Y1) | **Windows** | ✅ 1 staging slot | -| Consumption (Y1) | Linux | ❌ 0 | -| Elastic Premium (EP1-EP3) | Windows or Linux | ✅ 20 slots | -| Dedicated (Standard+) | Windows or Linux | ✅ 5–20 slots | - -> 💡 **For Azure Functions requiring deployment slots:** -> - **Windows Consumption (Y1) supports 1 staging slot** — this is a supported platform capability. -> If you need it, use it. See the Bicep example below. -> - Recommendation for new projects: prefer **Elastic Premium (EP1+)** (no cold starts, VNet integration) -> or a **Dedicated plan (Standard+)**. Y1 cold starts can affect slot swap warm-up reliability. -> - **Linux Consumption and Flex Consumption do not support deployment slots.** - -### Windows Consumption Function App with Staging Slot (Bicep) - -```bicep -resource functionAppPlan 'Microsoft.Web/serverfarms@2022-09-01' = { - name: '${resourcePrefix}-funcplan-${uniqueHash}' - location: location - sku: { name: 'Y1', tier: 'Dynamic' } - // No 'reserved: true' — Windows Consumption -} - -resource functionApp 'Microsoft.Web/sites@2022-09-01' = { - name: '${resourcePrefix}-${serviceName}-${uniqueHash}' - location: location - kind: 'functionapp' // Windows (no 'linux' suffix) - identity: { type: 'SystemAssigned' } - properties: { - serverFarmId: functionAppPlan.id - httpsOnly: true - siteConfig: { - appSettings: [ - { name: 'WEBSITE_NODE_DEFAULT_VERSION', value: '~20' } - { name: 'FUNCTIONS_EXTENSION_VERSION', value: '~4' } - { name: 'FUNCTIONS_WORKER_RUNTIME', value: 'node' } - { name: 'WEBSITE_CONTENTAZUREFILECONNECTIONSTRING', value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};AccountKey=${storageAccount.listKeys().keys[0].value}' } - { name: 'WEBSITE_CONTENTSHARE', value: '${toLower(serviceName)}-prod' } - { name: 'AzureWebJobsStorage', value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};AccountKey=${storageAccount.listKeys().keys[0].value}' } - { name: 'APPLICATIONINSIGHTS_CONNECTION_STRING', value: applicationInsights.properties.ConnectionString } - ] - } - } -} - -// Staging slot — only 1 staging slot supported on Consumption -resource stagingSlot 'Microsoft.Web/sites/slots@2022-09-01' = { - parent: functionApp - name: 'staging' - location: location - kind: 'functionapp' - properties: { - serverFarmId: functionAppPlan.id - siteConfig: { - appSettings: [ - { name: 'WEBSITE_NODE_DEFAULT_VERSION', value: '~20' } - { name: 'FUNCTIONS_EXTENSION_VERSION', value: '~4' } - { name: 'FUNCTIONS_WORKER_RUNTIME', value: 'node' } - { name: 'WEBSITE_CONTENTAZUREFILECONNECTIONSTRING', value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};AccountKey=${storageAccount.listKeys().keys[0].value}' } - { name: 'WEBSITE_CONTENTSHARE', value: '${toLower(serviceName)}-staging' } // MUST differ from production - { name: 'AzureWebJobsStorage', value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};AccountKey=${storageAccount.listKeys().keys[0].value}' } - { name: 'APPLICATIONINSIGHTS_CONNECTION_STRING', value: applicationInsights.properties.ConnectionString } - ] - } - } -} -``` - -> ⚠️ `WEBSITE_CONTENTSHARE` **must be unique per slot** on Windows Consumption — each slot needs its own file share. -> Use slot-sticky settings (via `slotConfigNames`) for `WEBSITE_CONTENTSHARE` and `WEBSITE_CONTENTAZUREFILECONNECTIONSTRING` -> so these values do not swap with production. - -## Deployment Flow - -1. Deploy to staging slot -2. Warm up and test staging -3. Swap staging with production -4. Rollback by swapping again if needed - -## Slot Settings - -Configure settings that should not swap: - -```bicep -resource slotConfigNames 'Microsoft.Web/sites/config@2022-09-01' = { - parent: webApp - name: 'slotConfigNames' - properties: { - appSettingNames: [ - 'APPLICATIONINSIGHTS_CONNECTION_STRING' - ] - } -} -``` +# App Service Deployment Slots + +Zero-downtime deployments using staging slots. + +## Basic Staging Slot + +```bicep +resource stagingSlot 'Microsoft.Web/sites/slots@2022-09-01' = { + parent: webApp + name: 'staging' + location: location + properties: { + serverFarmId: appServicePlan.id + } +} +``` + +## Slot Requirements — App Service + +| SKU Tier | Slots Supported | +|----------|-----------------| +| Free/Shared | 0 | +| Basic | 0 | +| Standard | 5 | +| Premium | 20 | + +## Slot Requirements — Azure Functions + +> ⚠️ Slot support for Azure Functions varies by OS and hosting plan. + +| Hosting Plan | OS | Slots Supported | +|---|---|---| +| Flex Consumption (FC1) | Linux | ❌ 0 | +| Consumption (Y1) | **Windows** | ✅ 1 staging slot | +| Consumption (Y1) | Linux | ❌ 0 | +| Elastic Premium (EP1-EP3) | Windows or Linux | ✅ 20 slots | +| Dedicated (Standard+) | Windows or Linux | ✅ 5–20 slots | + +> 💡 **For Azure Functions requiring deployment slots:** +> - **Windows Consumption (Y1) supports 1 staging slot** — this is a supported platform capability. +> If you need it, use it. See the Bicep example below. +> - Recommendation for new projects: prefer **Elastic Premium (EP1+)** (no cold starts, VNet integration) +> or a **Dedicated plan (Standard+)**. Y1 cold starts can affect slot swap warm-up reliability. +> - **Linux Consumption and Flex Consumption do not support deployment slots.** + +### Windows Consumption Function App with Staging Slot (Bicep) + +```bicep +resource functionAppPlan 'Microsoft.Web/serverfarms@2022-09-01' = { + name: '${resourcePrefix}-funcplan-${uniqueHash}' + location: location + sku: { name: 'Y1', tier: 'Dynamic' } + // No 'reserved: true' — Windows Consumption +} + +resource functionApp 'Microsoft.Web/sites@2022-09-01' = { + name: '${resourcePrefix}-${serviceName}-${uniqueHash}' + location: location + kind: 'functionapp' // Windows (no 'linux' suffix) + identity: { type: 'SystemAssigned' } + properties: { + serverFarmId: functionAppPlan.id + httpsOnly: true + siteConfig: { + appSettings: [ + { name: 'WEBSITE_NODE_DEFAULT_VERSION', value: '~20' } + { name: 'FUNCTIONS_EXTENSION_VERSION', value: '~4' } + { name: 'FUNCTIONS_WORKER_RUNTIME', value: 'node' } + { name: 'WEBSITE_CONTENTAZUREFILECONNECTIONSTRING', value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};AccountKey=${storageAccount.listKeys().keys[0].value}' } + { name: 'WEBSITE_CONTENTSHARE', value: '${toLower(serviceName)}-prod' } + { name: 'AzureWebJobsStorage', value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};AccountKey=${storageAccount.listKeys().keys[0].value}' } + { name: 'APPLICATIONINSIGHTS_CONNECTION_STRING', value: applicationInsights.properties.ConnectionString } + ] + } + } +} + +// Staging slot — only 1 staging slot supported on Consumption +resource stagingSlot 'Microsoft.Web/sites/slots@2022-09-01' = { + parent: functionApp + name: 'staging' + location: location + kind: 'functionapp' + properties: { + serverFarmId: functionAppPlan.id + siteConfig: { + appSettings: [ + { name: 'WEBSITE_NODE_DEFAULT_VERSION', value: '~20' } + { name: 'FUNCTIONS_EXTENSION_VERSION', value: '~4' } + { name: 'FUNCTIONS_WORKER_RUNTIME', value: 'node' } + { name: 'WEBSITE_CONTENTAZUREFILECONNECTIONSTRING', value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};AccountKey=${storageAccount.listKeys().keys[0].value}' } + { name: 'WEBSITE_CONTENTSHARE', value: '${toLower(serviceName)}-staging' } // MUST differ from production + { name: 'AzureWebJobsStorage', value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};AccountKey=${storageAccount.listKeys().keys[0].value}' } + { name: 'APPLICATIONINSIGHTS_CONNECTION_STRING', value: applicationInsights.properties.ConnectionString } + ] + } + } +} +``` + +> ⚠️ `WEBSITE_CONTENTSHARE` **must be unique per slot** on Windows Consumption — each slot needs its own file share. +> Use slot-sticky settings (via `slotConfigNames`) for `WEBSITE_CONTENTSHARE` and `WEBSITE_CONTENTAZUREFILECONNECTIONSTRING` +> so these values do not swap with production. + +## Deployment Flow + +1. Deploy to staging slot +2. Warm up and test staging +3. Swap staging with production +4. Rollback by swapping again if needed + +## Slot Settings + +Configure settings that should not swap: + +```bicep +resource slotConfigNames 'Microsoft.Web/sites/config@2022-09-01' = { + parent: webApp + name: 'slotConfigNames' + properties: { + appSettingNames: [ + 'APPLICATIONINSIGHTS_CONNECTION_STRING' + ] + } +} +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-service/scaling.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-service/scaling.md index a78b1c9b..8c4f911d 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-service/scaling.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/app-service/scaling.md @@ -1,65 +1,65 @@ -# App Service Auto-scaling - -## Basic Auto-scale Configuration - -```bicep -resource autoScale 'Microsoft.Insights/autoscalesettings@2022-10-01' = { - name: '${webApp.name}-autoscale' - location: location - properties: { - targetResourceUri: appServicePlan.id - enabled: true - profiles: [ - { - name: 'Auto scale' - capacity: { - minimum: '1' - maximum: '10' - default: '1' - } - rules: [ - { - metricTrigger: { - metricName: 'CpuPercentage' - metricResourceUri: appServicePlan.id - timeGrain: 'PT1M' - statistic: 'Average' - timeWindow: 'PT5M' - timeAggregation: 'Average' - operator: 'GreaterThan' - threshold: 70 - } - scaleAction: { - direction: 'Increase' - type: 'ChangeCount' - value: '1' - cooldown: 'PT5M' - } - } - ] - } - ] - } -} -``` - -## Common Metrics - -| Metric | Use Case | -|--------|----------| -| CpuPercentage | CPU-bound workloads | -| MemoryPercentage | Memory-intensive apps | -| HttpQueueLength | Request queue depth | -| Requests | Request volume | - -## Recommendations - -| Workload | Min | Max | Metric | -|----------|-----|-----|--------| -| Production API | 2 | 10 | CPU + Requests | -| Dev/Test | 1 | 3 | CPU | -| High-traffic | 3 | 20 | HTTP Queue | - -## SKU Requirements - -Auto-scaling requires **Standard (S1+)** or **Premium** tier. +# App Service Auto-scaling + +## Basic Auto-scale Configuration + +```bicep +resource autoScale 'Microsoft.Insights/autoscalesettings@2022-10-01' = { + name: '${webApp.name}-autoscale' + location: location + properties: { + targetResourceUri: appServicePlan.id + enabled: true + profiles: [ + { + name: 'Auto scale' + capacity: { + minimum: '1' + maximum: '10' + default: '1' + } + rules: [ + { + metricTrigger: { + metricName: 'CpuPercentage' + metricResourceUri: appServicePlan.id + timeGrain: 'PT1M' + statistic: 'Average' + timeWindow: 'PT5M' + timeAggregation: 'Average' + operator: 'GreaterThan' + threshold: 70 + } + scaleAction: { + direction: 'Increase' + type: 'ChangeCount' + value: '1' + cooldown: 'PT5M' + } + } + ] + } + ] + } +} +``` + +## Common Metrics + +| Metric | Use Case | +|--------|----------| +| CpuPercentage | CPU-bound workloads | +| MemoryPercentage | Memory-intensive apps | +| HttpQueueLength | Request queue depth | +| Requests | Request volume | + +## Recommendations + +| Workload | Min | Max | Metric | +|----------|-----|-----|--------| +| Production API | 2 | 10 | CPU + Requests | +| Dev/Test | 1 | 3 | CPU | +| High-traffic | 3 | 20 | HTTP Queue | + +## SKU Requirements + +Auto-scaling requires **Standard (S1+)** or **Premium** tier. diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/container-apps/bicep.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/container-apps/bicep.md index c3582747..a75706ae 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/container-apps/bicep.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/container-apps/bicep.md @@ -1,134 +1,134 @@ -# Container Apps Bicep Patterns - -> **⚠️ Container Registry Naming:** If using Azure Container Registry, names must be alphanumeric only (5-50 characters). Use `replace()` to remove hyphens: `replace('cr${environmentName}${resourceSuffix}', '-', '')` - -> **⚠️ Two-Phase Deployment (Mandatory):** To avoid a circular dependency when scoping the AcrPull role assignment to a Bicep module, use the two-phase pattern below: -> - **Phase 1:** Deploy ACR and Container App with a public placeholder image and **no** `registries` block. -> - **Phase 2:** Deploy the AcrPull role assignment as a **separate module** using outputs from Phase 1. -> -> The Bicep template does **not** need a `registries` block. `azd deploy` handles the registry/identity link by calling `az containerapp registry set --server --identity system` via the Azure API before updating the container image. If you are not using AZD, you must run this command manually before switching the image to an ACR-hosted image; otherwise the app will hit image pull failures. - -## Phase 1: Container App Module (No Registry Link) - -```bicep -// Placeholder image allows provisioning before app image exists in ACR. -// No registries block in Bicep — azd deploy configures the registry/identity link -// (az containerapp registry set --identity system) and updates the image via the Azure API. -// If not using AZD, run that command manually before switching to an ACR-hosted image. -param containerImageName string = 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' - -resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { - name: appName - location: location - identity: { - type: 'SystemAssigned' - } - properties: { - environmentId: containerAppsEnvironment.id - configuration: { - ingress: { - external: true - targetPort: 8080 - transport: 'auto' - } - // No registries block in Bicep. azd deploy sets the registry/identity link via - // 'az containerapp registry set --identity system' before pushing the real image. - // Without this step (or its manual equivalent), the app will fail to pull from ACR. - } - template: { - containers: [ - { - name: serviceName - image: containerImageName - resources: { - cpu: json('0.5') - memory: '1Gi' - } - } - ] - } - } -} - -output systemAssignedMIPrincipalId string = containerApp.identity.principalId -``` - -## Phase 2: AcrPull Role Assignment Module (acr-pull-role.bicep) - -Place this in a **separate module file** so neither the ACR module nor the Container App module depends on it, eliminating the circular dependency. - -```bicep -// acr-pull-role.bicep -param acrName string -param principalId string - -resource containerRegistry 'Microsoft.ContainerRegistry/registries@2023-07-01' existing = { - name: acrName -} - -resource acrPullRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(containerRegistry.id, principalId, 'acrpull') - scope: containerRegistry - properties: { - roleDefinitionId: subscriptionResourceId( - 'Microsoft.Authorization/roleDefinitions', - '7f951dda-4ed3-4680-a7ca-43fe172d538d' - ) - principalId: principalId - principalType: 'ServicePrincipal' - } -} -``` - -> 💡 **Tip:** Always set `principalType: 'ServicePrincipal'` for managed identities. This avoids a Graph API lookup and speeds up role assignment propagation. - -## Wiring Phase 1 and Phase 2 in main.bicep - -```bicep -// Phase 1: ACR and Container App — neither module depends on the role assignment -module containerRegistry './modules/container-registry.bicep' = { - name: 'containerRegistry' - scope: rg - params: { /* ... */ } -} - -module api './modules/container-app.bicep' = { - name: 'api' - scope: rg - params: { - containerImageName: 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' - // No registries param in Bicep — azd deploy configures the registry/identity link - // and updates the image via the Azure API after provisioning. - /* ... */ - } -} - -// Phase 2: Role assignment depends on outputs of both Phase 1 modules, -// but neither Phase 1 module depends on this — no circular dependency. -module acrPullRole './modules/acr-pull-role.bicep' = { - name: 'acrPullRole' - scope: rg - params: { - acrName: containerRegistry.outputs.name - principalId: api.outputs.systemAssignedMIPrincipalId - } -} -``` - -## Container Apps Environment - -```bicep -resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' = { - name: '${resourcePrefix}-env' - location: location - properties: { - appLogsConfiguration: { - destination: 'log-analytics' - logAnalyticsConfiguration: { - customerId: logAnalyticsWorkspace.properties.customerId - sharedKey: logAnalyticsWorkspace.listKeys().primarySharedKey - } - } - } -} -``` +# Container Apps Bicep Patterns + +> **⚠️ Container Registry Naming:** If using Azure Container Registry, names must be alphanumeric only (5-50 characters). Use `replace()` to remove hyphens: `replace('cr${environmentName}${resourceSuffix}', '-', '')` + +> **⚠️ Two-Phase Deployment (Mandatory):** To avoid a circular dependency when scoping the AcrPull role assignment to a Bicep module, use the two-phase pattern below: +> - **Phase 1:** Deploy ACR and Container App with a public placeholder image and **no** `registries` block. +> - **Phase 2:** Deploy the AcrPull role assignment as a **separate module** using outputs from Phase 1. +> +> The Bicep template does **not** need a `registries` block. `azd deploy` handles the registry/identity link by calling `az containerapp registry set --server --identity system` via the Azure API before updating the container image. If you are not using AZD, you must run this command manually before switching the image to an ACR-hosted image; otherwise the app will hit image pull failures. + +## Phase 1: Container App Module (No Registry Link) + +```bicep +// Placeholder image allows provisioning before app image exists in ACR. +// No registries block in Bicep — azd deploy configures the registry/identity link +// (az containerapp registry set --identity system) and updates the image via the Azure API. +// If not using AZD, run that command manually before switching to an ACR-hosted image. +param containerImageName string = 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' + +resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { + name: appName + location: location + identity: { + type: 'SystemAssigned' + } + properties: { + environmentId: containerAppsEnvironment.id + configuration: { + ingress: { + external: true + targetPort: 8080 + transport: 'auto' + } + // No registries block in Bicep. azd deploy sets the registry/identity link via + // 'az containerapp registry set --identity system' before pushing the real image. + // Without this step (or its manual equivalent), the app will fail to pull from ACR. + } + template: { + containers: [ + { + name: serviceName + image: containerImageName + resources: { + cpu: json('0.5') + memory: '1Gi' + } + } + ] + } + } +} + +output systemAssignedMIPrincipalId string = containerApp.identity.principalId +``` + +## Phase 2: AcrPull Role Assignment Module (acr-pull-role.bicep) + +Place this in a **separate module file** so neither the ACR module nor the Container App module depends on it, eliminating the circular dependency. + +```bicep +// acr-pull-role.bicep +param acrName string +param principalId string + +resource containerRegistry 'Microsoft.ContainerRegistry/registries@2023-07-01' existing = { + name: acrName +} + +resource acrPullRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(containerRegistry.id, principalId, 'acrpull') + scope: containerRegistry + properties: { + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '7f951dda-4ed3-4680-a7ca-43fe172d538d' + ) + principalId: principalId + principalType: 'ServicePrincipal' + } +} +``` + +> 💡 **Tip:** Always set `principalType: 'ServicePrincipal'` for managed identities. This avoids a Graph API lookup and speeds up role assignment propagation. + +## Wiring Phase 1 and Phase 2 in main.bicep + +```bicep +// Phase 1: ACR and Container App — neither module depends on the role assignment +module containerRegistry './modules/container-registry.bicep' = { + name: 'containerRegistry' + scope: rg + params: { /* ... */ } +} + +module api './modules/container-app.bicep' = { + name: 'api' + scope: rg + params: { + containerImageName: 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' + // No registries param in Bicep — azd deploy configures the registry/identity link + // and updates the image via the Azure API after provisioning. + /* ... */ + } +} + +// Phase 2: Role assignment depends on outputs of both Phase 1 modules, +// but neither Phase 1 module depends on this — no circular dependency. +module acrPullRole './modules/acr-pull-role.bicep' = { + name: 'acrPullRole' + scope: rg + params: { + acrName: containerRegistry.outputs.name + principalId: api.outputs.systemAssignedMIPrincipalId + } +} +``` + +## Container Apps Environment + +```bicep +resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' = { + name: '${resourcePrefix}-env' + location: location + properties: { + appLogsConfiguration: { + destination: 'log-analytics' + logAnalyticsConfiguration: { + customerId: logAnalyticsWorkspace.properties.customerId + sharedKey: logAnalyticsWorkspace.listKeys().primarySharedKey + } + } + } +} +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/container-apps/environment.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/container-apps/environment.md index 39999a68..67803c5a 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/container-apps/environment.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/container-apps/environment.md @@ -1,61 +1,61 @@ -# Container Apps Environment Variables - -## Standard Environment Variables - -```bicep -env: [ - { - name: 'APPLICATIONINSIGHTS_CONNECTION_STRING' - value: applicationInsights.properties.ConnectionString - } - { - name: 'AZURE_CLIENT_ID' - value: managedIdentity.properties.clientId - } -] -``` - -## Secret References (Key Vault) - -Use secrets for sensitive values: - -```bicep -configuration: { - secrets: [ - { - name: 'database-url' - keyVaultUrl: 'https://myvault.vault.azure.net/secrets/database-url' - identity: managedIdentity.id - } - ] -} - -template: { - containers: [ - { - env: [ - { - name: 'DATABASE_URL' - secretRef: 'database-url' - } - ] - } - ] -} -``` - -## Common Variables - -| Variable | Source | Notes | -|----------|--------|-------| -| `APPLICATIONINSIGHTS_CONNECTION_STRING` | App Insights | Telemetry | -| `AZURE_CLIENT_ID` | Managed Identity | SDK auth | -| `DATABASE_URL` | Key Vault secret | Connection string | -| `REDIS_URL` | Key Vault secret | Cache connection | - -## Best Practices - -- Never hardcode secrets in Bicep -- Use Key Vault references for all sensitive values -- Use Managed Identity for authentication -- Set `AZURE_CLIENT_ID` for SDK-based auth +# Container Apps Environment Variables + +## Standard Environment Variables + +```bicep +env: [ + { + name: 'APPLICATIONINSIGHTS_CONNECTION_STRING' + value: applicationInsights.properties.ConnectionString + } + { + name: 'AZURE_CLIENT_ID' + value: managedIdentity.properties.clientId + } +] +``` + +## Secret References (Key Vault) + +Use secrets for sensitive values: + +```bicep +configuration: { + secrets: [ + { + name: 'database-url' + keyVaultUrl: 'https://myvault.vault.azure.net/secrets/database-url' + identity: managedIdentity.id + } + ] +} + +template: { + containers: [ + { + env: [ + { + name: 'DATABASE_URL' + secretRef: 'database-url' + } + ] + } + ] +} +``` + +## Common Variables + +| Variable | Source | Notes | +|----------|--------|-------| +| `APPLICATIONINSIGHTS_CONNECTION_STRING` | App Insights | Telemetry | +| `AZURE_CLIENT_ID` | Managed Identity | SDK auth | +| `DATABASE_URL` | Key Vault secret | Connection string | +| `REDIS_URL` | Key Vault secret | Cache connection | + +## Best Practices + +- Never hardcode secrets in Bicep +- Use Key Vault references for all sensitive values +- Use Managed Identity for authentication +- Set `AZURE_CLIENT_ID` for SDK-based auth diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/container-apps/health-probes.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/container-apps/health-probes.md index 0b0e730b..b14470d0 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/container-apps/health-probes.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/container-apps/health-probes.md @@ -1,68 +1,68 @@ -# Container Apps Health Probes - -Always configure health probes for production workloads. - -## Liveness Probe - -Detects if container is alive. Failure triggers restart. - -```bicep -probes: [ - { - type: 'liveness' - httpGet: { - path: '/health' - port: 8080 - } - initialDelaySeconds: 10 - periodSeconds: 30 - failureThreshold: 3 - } -] -``` - -## Readiness Probe - -Detects if container is ready to receive traffic. - -```bicep -probes: [ - { - type: 'readiness' - httpGet: { - path: '/ready' - port: 8080 - } - initialDelaySeconds: 5 - periodSeconds: 10 - failureThreshold: 3 - } -] -``` - -## Startup Probe - -For slow-starting containers. Delays other probes until startup succeeds. - -```bicep -probes: [ - { - type: 'startup' - httpGet: { - path: '/health' - port: 8080 - } - initialDelaySeconds: 0 - periodSeconds: 10 - failureThreshold: 30 // 30 * 10s = 5 min max startup - } -] -``` - -## Recommendations - -| Probe | Path | Initial Delay | Period | -|-------|------|---------------|--------| -| Liveness | `/health` | 10s | 30s | -| Readiness | `/ready` | 5s | 10s | -| Startup | `/health` | 0s | 10s | +# Container Apps Health Probes + +Always configure health probes for production workloads. + +## Liveness Probe + +Detects if container is alive. Failure triggers restart. + +```bicep +probes: [ + { + type: 'liveness' + httpGet: { + path: '/health' + port: 8080 + } + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + } +] +``` + +## Readiness Probe + +Detects if container is ready to receive traffic. + +```bicep +probes: [ + { + type: 'readiness' + httpGet: { + path: '/ready' + port: 8080 + } + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + } +] +``` + +## Startup Probe + +For slow-starting containers. Delays other probes until startup succeeds. + +```bicep +probes: [ + { + type: 'startup' + httpGet: { + path: '/health' + port: 8080 + } + initialDelaySeconds: 0 + periodSeconds: 10 + failureThreshold: 30 // 30 * 10s = 5 min max startup + } +] +``` + +## Recommendations + +| Probe | Path | Initial Delay | Period | +|-------|------|---------------|--------| +| Liveness | `/health` | 10s | 30s | +| Readiness | `/ready` | 5s | 10s | +| Startup | `/health` | 0s | 10s | diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/container-apps/scaling.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/container-apps/scaling.md index 8b5d100e..d749d534 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/container-apps/scaling.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/container-apps/scaling.md @@ -1,84 +1,84 @@ -# Container Apps Scaling Patterns - -## HTTP-based Scaling - -Best for APIs and web applications: - -```bicep -scale: { - minReplicas: 1 - maxReplicas: 10 - rules: [ - { - name: 'http-scaling' - http: { - metadata: { - concurrentRequests: '100' - } - } - } - ] -} -``` - -## Queue-based Scaling - -Best for background workers: - -```bicep -scale: { - minReplicas: 0 - maxReplicas: 30 - rules: [ - { - name: 'queue-scaling' - azureQueue: { - queueName: 'orders' - queueLength: 10 - auth: [ - { - secretRef: 'storage-connection' - triggerParameter: 'connection' - } - ] - } - } - ] -} -``` - -## Service Bus Scaling - -```bicep -scale: { - minReplicas: 0 - maxReplicas: 20 - rules: [ - { - name: 'servicebus-scaling' - custom: { - type: 'azure-servicebus' - metadata: { - queueName: 'myqueue' - messageCount: '5' - } - auth: [ - { - secretRef: 'servicebus-connection' - triggerParameter: 'connection' - } - ] - } - } - ] -} -``` - -## Recommendations - -| Workload | Min Replicas | Max Replicas | Rule Type | -|----------|--------------|--------------|-----------| -| Production API | 1 | 10-20 | HTTP | -| Dev/Test API | 0 | 5 | HTTP | -| Background Worker | 0 | 30+ | Queue/Event | -| Scheduled Job | 0 | 1 | KEDA cron | +# Container Apps Scaling Patterns + +## HTTP-based Scaling + +Best for APIs and web applications: + +```bicep +scale: { + minReplicas: 1 + maxReplicas: 10 + rules: [ + { + name: 'http-scaling' + http: { + metadata: { + concurrentRequests: '100' + } + } + } + ] +} +``` + +## Queue-based Scaling + +Best for background workers: + +```bicep +scale: { + minReplicas: 0 + maxReplicas: 30 + rules: [ + { + name: 'queue-scaling' + azureQueue: { + queueName: 'orders' + queueLength: 10 + auth: [ + { + secretRef: 'storage-connection' + triggerParameter: 'connection' + } + ] + } + } + ] +} +``` + +## Service Bus Scaling + +```bicep +scale: { + minReplicas: 0 + maxReplicas: 20 + rules: [ + { + name: 'servicebus-scaling' + custom: { + type: 'azure-servicebus' + metadata: { + queueName: 'myqueue' + messageCount: '5' + } + auth: [ + { + secretRef: 'servicebus-connection' + triggerParameter: 'connection' + } + ] + } + } + ] +} +``` + +## Recommendations + +| Workload | Min Replicas | Max Replicas | Rule Type | +|----------|--------------|--------------|-----------| +| Production API | 1 | 10-20 | HTTP | +| Dev/Test API | 0 | 5 | HTTP | +| Background Worker | 0 | 30+ | Queue/Event | +| Scheduled Job | 0 | 1 | KEDA cron | diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/cosmos-db/README.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/cosmos-db/README.md index 2040ef3e..c33c0b2b 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/cosmos-db/README.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/cosmos-db/README.md @@ -1,52 +1,52 @@ -# Azure Cosmos DB - -Globally distributed, multi-model database for low-latency data at scale. - -## When to Use - -- Global distribution requirements -- Multi-model data (document, graph, key-value) -- Variable and unpredictable throughput -- Low-latency reads/writes at scale -- Flexible schema requirements - -## Required Supporting Resources - -| Resource | Purpose | -|----------|---------| -| None required | Cosmos DB is fully managed | -| Key Vault | Store connection strings (recommended) | - -## Capacity Modes - -| Mode | Use Case | Billing | -|------|----------|---------| -| **Serverless** | Variable/low traffic, dev/test | Per request | -| **Provisioned** | Predictable workloads | Per RU/s | -| **Autoscale** | Variable but predictable peaks | Per max RU/s | - -## Consistency Levels - -| Level | Latency | Consistency | -|-------|---------|-------------| -| Strong | Highest | Linearizable | -| Bounded Staleness | High | Bounded | -| Session | Medium | Session-scoped | -| Consistent Prefix | Low | Prefix ordering | -| Eventual | Lowest | Eventually consistent | - -Recommendation: Use **Session** for most applications. - -## Environment Variables - -| Variable | Value | -|----------|-------| -| `COSMOS_CONNECTION_STRING` | Primary connection string (Key Vault reference) | -| `COSMOS_ENDPOINT` | Account endpoint URL | -| `COSMOS_DATABASE` | Database name | - -## References - -- [Bicep Patterns](bicep.md) -- [Partition Key Selection](partitioning.md) -- [SDK Connection Patterns](sdk.md) +# Azure Cosmos DB + +Globally distributed, multi-model database for low-latency data at scale. + +## When to Use + +- Global distribution requirements +- Multi-model data (document, graph, key-value) +- Variable and unpredictable throughput +- Low-latency reads/writes at scale +- Flexible schema requirements + +## Required Supporting Resources + +| Resource | Purpose | +|----------|---------| +| None required | Cosmos DB is fully managed | +| Key Vault | Store connection strings (recommended) | + +## Capacity Modes + +| Mode | Use Case | Billing | +|------|----------|---------| +| **Serverless** | Variable/low traffic, dev/test | Per request | +| **Provisioned** | Predictable workloads | Per RU/s | +| **Autoscale** | Variable but predictable peaks | Per max RU/s | + +## Consistency Levels + +| Level | Latency | Consistency | +|-------|---------|-------------| +| Strong | Highest | Linearizable | +| Bounded Staleness | High | Bounded | +| Session | Medium | Session-scoped | +| Consistent Prefix | Low | Prefix ordering | +| Eventual | Lowest | Eventually consistent | + +Recommendation: Use **Session** for most applications. + +## Environment Variables + +| Variable | Value | +|----------|-------| +| `COSMOS_CONNECTION_STRING` | Primary connection string (Key Vault reference) | +| `COSMOS_ENDPOINT` | Account endpoint URL | +| `COSMOS_DATABASE` | Database name | + +## References + +- [Bicep Patterns](bicep.md) +- [Partition Key Selection](partitioning.md) +- [SDK Connection Patterns](sdk.md) diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/cosmos-db/bicep.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/cosmos-db/bicep.md index c65ac0ea..fab19bc8 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/cosmos-db/bicep.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/cosmos-db/bicep.md @@ -1,108 +1,108 @@ -# Cosmos DB Bicep Patterns - -## Account - -```bicep -resource cosmosAccount 'Microsoft.DocumentDB/databaseAccounts@2023-04-15' = { - name: '${resourcePrefix}-cosmos-${uniqueHash}' - location: location - kind: 'GlobalDocumentDB' - properties: { - databaseAccountOfferType: 'Standard' - locations: [ - { - locationName: location - failoverPriority: 0 - isZoneRedundant: false - } - ] - consistencyPolicy: { - defaultConsistencyLevel: 'Session' - } - capabilities: [ - { - name: 'EnableServerless' - } - ] - } -} -``` - -## Database - -```bicep -resource cosmosDatabase 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases@2023-04-15' = { - parent: cosmosAccount - name: 'appdb' - properties: { - resource: { - id: 'appdb' - } - } -} -``` - -## Container - -```bicep -resource cosmosContainer 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers@2023-04-15' = { - parent: cosmosDatabase - name: 'items' - properties: { - resource: { - id: 'items' - partitionKey: { - paths: ['/partitionKey'] - kind: 'Hash' - } - indexingPolicy: { - indexingMode: 'consistent' - includedPaths: [ - { path: '/*' } - ] - } - } - } -} -``` - -## Autoscale Container - -```bicep -resource cosmosContainer 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers@2023-04-15' = { - parent: cosmosDatabase - name: 'items' - properties: { - resource: { - id: 'items' - partitionKey: { - paths: ['/partitionKey'] - kind: 'Hash' - } - } - options: { - autoscaleSettings: { - maxThroughput: 4000 - } - } - } -} -``` - -## Global Distribution - -```bicep -properties: { - locations: [ - { - locationName: 'East US' - failoverPriority: 0 - } - { - locationName: 'West US' - failoverPriority: 1 - } - ] - enableMultipleWriteLocations: true -} -``` +# Cosmos DB Bicep Patterns + +## Account + +```bicep +resource cosmosAccount 'Microsoft.DocumentDB/databaseAccounts@2023-04-15' = { + name: '${resourcePrefix}-cosmos-${uniqueHash}' + location: location + kind: 'GlobalDocumentDB' + properties: { + databaseAccountOfferType: 'Standard' + locations: [ + { + locationName: location + failoverPriority: 0 + isZoneRedundant: false + } + ] + consistencyPolicy: { + defaultConsistencyLevel: 'Session' + } + capabilities: [ + { + name: 'EnableServerless' + } + ] + } +} +``` + +## Database + +```bicep +resource cosmosDatabase 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases@2023-04-15' = { + parent: cosmosAccount + name: 'appdb' + properties: { + resource: { + id: 'appdb' + } + } +} +``` + +## Container + +```bicep +resource cosmosContainer 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers@2023-04-15' = { + parent: cosmosDatabase + name: 'items' + properties: { + resource: { + id: 'items' + partitionKey: { + paths: ['/partitionKey'] + kind: 'Hash' + } + indexingPolicy: { + indexingMode: 'consistent' + includedPaths: [ + { path: '/*' } + ] + } + } + } +} +``` + +## Autoscale Container + +```bicep +resource cosmosContainer 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers@2023-04-15' = { + parent: cosmosDatabase + name: 'items' + properties: { + resource: { + id: 'items' + partitionKey: { + paths: ['/partitionKey'] + kind: 'Hash' + } + } + options: { + autoscaleSettings: { + maxThroughput: 4000 + } + } + } +} +``` + +## Global Distribution + +```bicep +properties: { + locations: [ + { + locationName: 'East US' + failoverPriority: 0 + } + { + locationName: 'West US' + failoverPriority: 1 + } + ] + enableMultipleWriteLocations: true +} +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/cosmos-db/partitioning.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/cosmos-db/partitioning.md index d3f2ee5b..4cbd027b 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/cosmos-db/partitioning.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/cosmos-db/partitioning.md @@ -1,41 +1,41 @@ -# Cosmos DB Partition Key Selection - -## Good Partition Keys - -A good partition key should have: - -- **High cardinality** - Many distinct values -- **Even data distribution** - No hot partitions -- **Even request distribution** - Balanced workload -- **Used in most queries** - Enables efficient routing - -## Examples by Scenario - -| Scenario | Partition Key | Reason | -|----------|---------------|--------| -| User-centric data | `/userId` | Queries typically filter by user | -| Multi-tenant apps | `/tenantId` | Isolates tenant data | -| E-commerce orders | `/customerId` | Orders queried by customer | -| IoT telemetry | `/deviceId` | High cardinality, even distribution | - -## Hierarchical Partition Keys - -For complex scenarios, use hierarchical keys: - -```bicep -partitionKey: { - paths: ['/tenantId', '/userId'] - kind: 'MultiHash' -} -``` - -## Anti-Patterns - -Avoid these partition key choices: - -| Bad Choice | Problem | -|------------|---------| -| Timestamp | Creates hot partitions | -| Boolean values | Only 2 partitions | -| Low cardinality enums | Uneven distribution | -| Random GUID | Can't query efficiently | +# Cosmos DB Partition Key Selection + +## Good Partition Keys + +A good partition key should have: + +- **High cardinality** - Many distinct values +- **Even data distribution** - No hot partitions +- **Even request distribution** - Balanced workload +- **Used in most queries** - Enables efficient routing + +## Examples by Scenario + +| Scenario | Partition Key | Reason | +|----------|---------------|--------| +| User-centric data | `/userId` | Queries typically filter by user | +| Multi-tenant apps | `/tenantId` | Isolates tenant data | +| E-commerce orders | `/customerId` | Orders queried by customer | +| IoT telemetry | `/deviceId` | High cardinality, even distribution | + +## Hierarchical Partition Keys + +For complex scenarios, use hierarchical keys: + +```bicep +partitionKey: { + paths: ['/tenantId', '/userId'] + kind: 'MultiHash' +} +``` + +## Anti-Patterns + +Avoid these partition key choices: + +| Bad Choice | Problem | +|------------|---------| +| Timestamp | Creates hot partitions | +| Boolean values | Only 2 partitions | +| Low cardinality enums | Uneven distribution | +| Random GUID | Can't query efficiently | diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/cosmos-db/sdk.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/cosmos-db/sdk.md index 256787dd..74108804 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/cosmos-db/sdk.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/cosmos-db/sdk.md @@ -1,59 +1,59 @@ -# Cosmos DB SDK Connection Patterns - -## Node.js - -```javascript -const { CosmosClient } = require("@azure/cosmos"); - -const client = new CosmosClient(process.env.COSMOS_CONNECTION_STRING); -const database = client.database("appdb"); -const container = database.container("items"); - -// Query example -const { resources } = await container.items - .query("SELECT * FROM c WHERE c.userId = @userId", { - parameters: [{ name: "@userId", value: userId }] - }) - .fetchAll(); -``` - -## Python - -```python -from azure.cosmos import CosmosClient -import os - -client = CosmosClient.from_connection_string(os.environ["COSMOS_CONNECTION_STRING"]) -database = client.get_database_client("appdb") -container = database.get_container_client("items") - -# Query example -items = container.query_items( - query="SELECT * FROM c WHERE c.userId = @userId", - parameters=[{"name": "@userId", "value": user_id}] -) -``` - -## .NET - -```csharp -using Microsoft.Azure.Cosmos; - -var client = new CosmosClient(Environment.GetEnvironmentVariable("COSMOS_CONNECTION_STRING")); -var database = client.GetDatabase("appdb"); -var container = database.GetContainer("items"); - -// Query example -var query = new QueryDefinition("SELECT * FROM c WHERE c.userId = @userId") - .WithParameter("@userId", userId); -var iterator = container.GetItemQueryIterator(query); -``` - -## Best Practices - -| Practice | Reason | -|----------|--------| -| Reuse client instances | Connection pooling | -| Use parameterized queries | SQL injection prevention | -| Set appropriate timeouts | Handle transient failures | -| Enable diagnostics in dev | Debug RU consumption | +# Cosmos DB SDK Connection Patterns + +## Node.js + +```javascript +const { CosmosClient } = require("@azure/cosmos"); + +const client = new CosmosClient(process.env.COSMOS_CONNECTION_STRING); +const database = client.database("appdb"); +const container = database.container("items"); + +// Query example +const { resources } = await container.items + .query("SELECT * FROM c WHERE c.userId = @userId", { + parameters: [{ name: "@userId", value: userId }] + }) + .fetchAll(); +``` + +## Python + +```python +from azure.cosmos import CosmosClient +import os + +client = CosmosClient.from_connection_string(os.environ["COSMOS_CONNECTION_STRING"]) +database = client.get_database_client("appdb") +container = database.get_container_client("items") + +# Query example +items = container.query_items( + query="SELECT * FROM c WHERE c.userId = @userId", + parameters=[{"name": "@userId", "value": user_id}] +) +``` + +## .NET + +```csharp +using Microsoft.Azure.Cosmos; + +var client = new CosmosClient(Environment.GetEnvironmentVariable("COSMOS_CONNECTION_STRING")); +var database = client.GetDatabase("appdb"); +var container = database.GetContainer("items"); + +// Query example +var query = new QueryDefinition("SELECT * FROM c WHERE c.userId = @userId") + .WithParameter("@userId", userId); +var iterator = container.GetItemQueryIterator(query); +``` + +## Best Practices + +| Practice | Reason | +|----------|--------| +| Reuse client instances | Connection pooling | +| Use parameterized queries | SQL injection prevention | +| Set appropriate timeouts | Handle transient failures | +| Enable diagnostics in dev | Debug RU consumption | diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/event-grid/README.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/event-grid/README.md index 640d082e..1d30248a 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/event-grid/README.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/event-grid/README.md @@ -1,45 +1,45 @@ -# Azure Event Grid - -Serverless event routing for event-driven architectures. - -## When to Use - -- Event-driven architectures -- Reactive programming patterns -- Decoupled event routing -- Near real-time event delivery -- Fan-out to multiple subscribers - -## Required Supporting Resources - -| Resource | Purpose | -|----------|---------| -| None required | Event Grid is serverless | -| Key Vault | Store topic keys | - -## Event Sources - -| Type | Description | -|------|-------------| -| System Topics | Azure resource events (Storage, Key Vault, etc.) | -| Custom Topics | Your application events | -| Event Domains | Multi-tenant event management | - -## Event Schemas - -| Schema | Use Case | -|--------|----------| -| Event Grid Schema | Azure native format | -| CloudEvents 1.0 | CNCF standard, cross-platform | - -## Environment Variables - -| Variable | Value | -|----------|-------| -| `EVENTGRID_TOPIC_ENDPOINT` | Topic endpoint URL | -| `EVENTGRID_TOPIC_KEY` | Topic access key (Key Vault) | - -## References - -- [Bicep Patterns](bicep.md) -- [Subscriptions](subscriptions.md) +# Azure Event Grid + +Serverless event routing for event-driven architectures. + +## When to Use + +- Event-driven architectures +- Reactive programming patterns +- Decoupled event routing +- Near real-time event delivery +- Fan-out to multiple subscribers + +## Required Supporting Resources + +| Resource | Purpose | +|----------|---------| +| None required | Event Grid is serverless | +| Key Vault | Store topic keys | + +## Event Sources + +| Type | Description | +|------|-------------| +| System Topics | Azure resource events (Storage, Key Vault, etc.) | +| Custom Topics | Your application events | +| Event Domains | Multi-tenant event management | + +## Event Schemas + +| Schema | Use Case | +|--------|----------| +| Event Grid Schema | Azure native format | +| CloudEvents 1.0 | CNCF standard, cross-platform | + +## Environment Variables + +| Variable | Value | +|----------|-------| +| `EVENTGRID_TOPIC_ENDPOINT` | Topic endpoint URL | +| `EVENTGRID_TOPIC_KEY` | Topic access key (Key Vault) | + +## References + +- [Bicep Patterns](bicep.md) +- [Subscriptions](subscriptions.md) diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/event-grid/bicep.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/event-grid/bicep.md index 4e558179..09b9c800 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/event-grid/bicep.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/event-grid/bicep.md @@ -1,79 +1,79 @@ -# Event Grid - Bicep Patterns - -## Custom Topic - -```bicep -resource eventGridTopic 'Microsoft.EventGrid/topics@2023-12-15-preview' = { - name: '${resourcePrefix}-egt-${uniqueHash}' - location: location - properties: { - inputSchema: 'EventGridSchema' - publicNetworkAccess: 'Enabled' - } -} -``` - -## System Topic (Azure Resource Events) - -```bicep -resource storageSystemTopic 'Microsoft.EventGrid/systemTopics@2023-12-15-preview' = { - name: '${resourcePrefix}-storage-topic' - location: location - properties: { - source: storageAccount.id - topicType: 'Microsoft.Storage.StorageAccounts' - } -} -``` - -## Event Domain - -```bicep -resource eventDomain 'Microsoft.EventGrid/domains@2023-12-15-preview' = { - name: '${resourcePrefix}-domain' - location: location - properties: { - inputSchema: 'EventGridSchema' - } -} -``` - -## Publishing Events - -### Node.js - -```javascript -const { EventGridPublisherClient, AzureKeyCredential } = require("@azure/eventgrid"); - -const client = new EventGridPublisherClient( - process.env.EVENTGRID_TOPIC_ENDPOINT, - "EventGrid", - new AzureKeyCredential(process.env.EVENTGRID_TOPIC_KEY) -); - -await client.send([{ - eventType: "Order.Created", - subject: "/orders/12345", - dataVersion: "1.0", - data: { orderId: "12345" } -}]); -``` - -### Python - -```python -from azure.eventgrid import EventGridPublisherClient, EventGridEvent -from azure.core.credentials import AzureKeyCredential - -client = EventGridPublisherClient( - os.environ["EVENTGRID_TOPIC_ENDPOINT"], - AzureKeyCredential(os.environ["EVENTGRID_TOPIC_KEY"]) -) - -client.send([EventGridEvent( - event_type="Order.Created", - subject="/orders/12345", - data={"orderId": "12345"}, - data_version="1.0" -)]) -``` +# Event Grid - Bicep Patterns + +## Custom Topic + +```bicep +resource eventGridTopic 'Microsoft.EventGrid/topics@2023-12-15-preview' = { + name: '${resourcePrefix}-egt-${uniqueHash}' + location: location + properties: { + inputSchema: 'EventGridSchema' + publicNetworkAccess: 'Enabled' + } +} +``` + +## System Topic (Azure Resource Events) + +```bicep +resource storageSystemTopic 'Microsoft.EventGrid/systemTopics@2023-12-15-preview' = { + name: '${resourcePrefix}-storage-topic' + location: location + properties: { + source: storageAccount.id + topicType: 'Microsoft.Storage.StorageAccounts' + } +} +``` + +## Event Domain + +```bicep +resource eventDomain 'Microsoft.EventGrid/domains@2023-12-15-preview' = { + name: '${resourcePrefix}-domain' + location: location + properties: { + inputSchema: 'EventGridSchema' + } +} +``` + +## Publishing Events + +### Node.js + +```javascript +const { EventGridPublisherClient, AzureKeyCredential } = require("@azure/eventgrid"); + +const client = new EventGridPublisherClient( + process.env.EVENTGRID_TOPIC_ENDPOINT, + "EventGrid", + new AzureKeyCredential(process.env.EVENTGRID_TOPIC_KEY) +); + +await client.send([{ + eventType: "Order.Created", + subject: "/orders/12345", + dataVersion: "1.0", + data: { orderId: "12345" } +}]); +``` + +### Python + +```python +from azure.eventgrid import EventGridPublisherClient, EventGridEvent +from azure.core.credentials import AzureKeyCredential + +client = EventGridPublisherClient( + os.environ["EVENTGRID_TOPIC_ENDPOINT"], + AzureKeyCredential(os.environ["EVENTGRID_TOPIC_KEY"]) +) + +client.send([EventGridEvent( + event_type="Order.Created", + subject="/orders/12345", + data={"orderId": "12345"}, + data_version="1.0" +)]) +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/event-grid/subscriptions.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/event-grid/subscriptions.md index 195a29fe..4139c175 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/event-grid/subscriptions.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/event-grid/subscriptions.md @@ -1,110 +1,110 @@ -# Event Grid - Subscriptions - -## Event Subscription - -```bicep -resource eventGridSubscription 'Microsoft.EventGrid/topics/eventSubscriptions@2023-12-15-preview' = { - parent: eventGridTopic - name: 'order-processor-subscription' - properties: { - destination: { - endpointType: 'WebHook' - properties: { - endpointUrl: 'https://my-api.azurecontainerapps.io/webhooks/orders' - } - } - filter: { - includedEventTypes: [ - 'Order.Created' - 'Order.Updated' - ] - } - retryPolicy: { - maxDeliveryAttempts: 30 - eventTimeToLiveInMinutes: 1440 - } - } -} -``` - -## Destination Types - -### Webhook - -```bicep -destination: { - endpointType: 'WebHook' - properties: { - endpointUrl: 'https://my-api.example.com/events' - } -} -``` - -### Azure Function - -```bicep -destination: { - endpointType: 'AzureFunction' - properties: { - resourceId: functionApp.id - } -} -``` - -### Service Bus Queue - -```bicep -destination: { - endpointType: 'ServiceBusQueue' - properties: { - resourceId: '${serviceBus.id}/queues/events' - } -} -``` - -### Event Hub - -```bicep -destination: { - endpointType: 'EventHub' - properties: { - resourceId: eventHub.id - } -} -``` - -## Filtering - -### Event Type Filter - -```bicep -filter: { - includedEventTypes: [ - 'Order.Created' - 'Order.Shipped' - ] -} -``` - -### Subject Filter - -```bicep -filter: { - subjectBeginsWith: '/orders/priority' - subjectEndsWith: '.json' -} -``` - -### Advanced Filter - -```bicep -filter: { - advancedFilters: [ - { - operatorType: 'NumberGreaterThan' - key: 'data.amount' - value: 100 - } - ] -} -``` +# Event Grid - Subscriptions + +## Event Subscription + +```bicep +resource eventGridSubscription 'Microsoft.EventGrid/topics/eventSubscriptions@2023-12-15-preview' = { + parent: eventGridTopic + name: 'order-processor-subscription' + properties: { + destination: { + endpointType: 'WebHook' + properties: { + endpointUrl: 'https://my-api.azurecontainerapps.io/webhooks/orders' + } + } + filter: { + includedEventTypes: [ + 'Order.Created' + 'Order.Updated' + ] + } + retryPolicy: { + maxDeliveryAttempts: 30 + eventTimeToLiveInMinutes: 1440 + } + } +} +``` + +## Destination Types + +### Webhook + +```bicep +destination: { + endpointType: 'WebHook' + properties: { + endpointUrl: 'https://my-api.example.com/events' + } +} +``` + +### Azure Function + +```bicep +destination: { + endpointType: 'AzureFunction' + properties: { + resourceId: functionApp.id + } +} +``` + +### Service Bus Queue + +```bicep +destination: { + endpointType: 'ServiceBusQueue' + properties: { + resourceId: '${serviceBus.id}/queues/events' + } +} +``` + +### Event Hub + +```bicep +destination: { + endpointType: 'EventHub' + properties: { + resourceId: eventHub.id + } +} +``` + +## Filtering + +### Event Type Filter + +```bicep +filter: { + includedEventTypes: [ + 'Order.Created' + 'Order.Shipped' + ] +} +``` + +### Subject Filter + +```bicep +filter: { + subjectBeginsWith: '/orders/priority' + subjectEndsWith: '.json' +} +``` + +### Advanced Filter + +```bicep +filter: { + advancedFilters: [ + { + operatorType: 'NumberGreaterThan' + key: 'data.amount' + value: 100 + } + ] +} +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/foundry/README.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/foundry/README.md index af2631c8..12b0d0d5 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/foundry/README.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/foundry/README.md @@ -1,45 +1,45 @@ -# Azure AI Foundry - -Azure AI Foundry (formerly Azure OpenAI) for building AI-powered applications with models like GPT-4o, GPT-4, and embeddings. - -> **💡 For detailed AI guidance**, invoke the **`microsoft-foundry`** skill. It provides model catalog access, RAG patterns, agent creation, and evaluation workflows. - -## When to Use - -- Chat and conversational AI applications -- Text generation and completion -- Code generation assistants -- Document analysis and summarization -- Embeddings for search and RAG -- Multi-modal applications (vision + text) - -## Service Type in azure.yaml - -```yaml -services: - my-ai-service: - host: containerapp # AI services typically deployed via Container Apps - project: ./src/ai-service -``` - -## Required Supporting Resources - -| Resource | Purpose | -|----------|---------| -| Azure AI Foundry account | Model hosting | -| Model deployment | Specific model (GPT-4o, GPT-4, etc.) | -| Key Vault | Store API keys securely | -| Application Insights | Monitor usage and costs | - -## Model Selection - -| Model | Best For | Context Window | -|-------|----------|----------------| -| GPT-4o | General purpose, vision, latest | 128K | -| GPT-4 | Complex reasoning | 32K | -| GPT-3.5-Turbo | Cost-effective, simple tasks | 16K | -| text-embedding-ada-002 | Embeddings for RAG/search | 8K | - -## References - -- [Region Availability](region-availability.md) +# Azure AI Foundry + +Azure AI Foundry (formerly Azure OpenAI) for building AI-powered applications with models like GPT-4o, GPT-4, and embeddings. + +> **💡 For detailed AI guidance**, invoke the **`microsoft-foundry`** skill. It provides model catalog access, RAG patterns, agent creation, and evaluation workflows. + +## When to Use + +- Chat and conversational AI applications +- Text generation and completion +- Code generation assistants +- Document analysis and summarization +- Embeddings for search and RAG +- Multi-modal applications (vision + text) + +## Service Type in azure.yaml + +```yaml +services: + my-ai-service: + host: containerapp # AI services typically deployed via Container Apps + project: ./src/ai-service +``` + +## Required Supporting Resources + +| Resource | Purpose | +|----------|---------| +| Azure AI Foundry account | Model hosting | +| Model deployment | Specific model (GPT-4o, GPT-4, etc.) | +| Key Vault | Store API keys securely | +| Application Insights | Monitor usage and costs | + +## Model Selection + +| Model | Best For | Context Window | +|-------|----------|----------------| +| GPT-4o | General purpose, vision, latest | 128K | +| GPT-4 | Complex reasoning | 32K | +| GPT-3.5-Turbo | Cost-effective, simple tasks | 16K | +| text-embedding-ada-002 | Embeddings for RAG/search | 8K | + +## References + +- [Region Availability](region-availability.md) diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/foundry/region-availability.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/foundry/region-availability.md index f7532313..86c3ae96 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/foundry/region-availability.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/foundry/region-availability.md @@ -1,23 +1,23 @@ -# Foundry Region Availability - -⚠️ **Very limited — varies by model** - -| Region | GPT-4o | GPT-4 | GPT-3.5 | Embeddings | -|--------|:------:|:-----:|:-------:|:----------:| -| `eastus` | ✅ | ✅ | ✅ | ✅ | -| `eastus2` | ✅ | ✅ | ✅ | ✅ | -| `westus` | ⚠️ | ⚠️ | ✅ | ✅ | -| `westus3` | ✅ | ⚠️ | ✅ | ✅ | -| `southcentralus` | ✅ | ✅ | ✅ | ✅ | -| `swedencentral` | ✅ | ✅ | ✅ | ✅ | -| `westeurope` | ⚠️ | ✅ | ✅ | ✅ | - -> Check https://learn.microsoft.com/azure/ai-services/openai/concepts/models for current model availability. - -## Recommended Regions - -| Need | Recommended Region | -|------|--------------------| -| Full model availability | `eastus`, `eastus2`, `swedencentral` | -| Europe compliance | `swedencentral`, `westeurope` | -| With SWA | `eastus2` (only overlap) | +# Foundry Region Availability + +⚠️ **Very limited — varies by model** + +| Region | GPT-4o | GPT-4 | GPT-3.5 | Embeddings | +|--------|:------:|:-----:|:-------:|:----------:| +| `eastus` | ✅ | ✅ | ✅ | ✅ | +| `eastus2` | ✅ | ✅ | ✅ | ✅ | +| `westus` | ⚠️ | ⚠️ | ✅ | ✅ | +| `westus3` | ✅ | ⚠️ | ✅ | ✅ | +| `southcentralus` | ✅ | ✅ | ✅ | ✅ | +| `swedencentral` | ✅ | ✅ | ✅ | ✅ | +| `westeurope` | ⚠️ | ✅ | ✅ | ✅ | + +> Check https://learn.microsoft.com/azure/ai-services/openai/concepts/models for current model availability. + +## Recommended Regions + +| Need | Recommended Region | +|------|--------------------| +| Full model availability | `eastus`, `eastus2`, `swedencentral` | +| Europe compliance | `swedencentral`, `westeurope` | +| With SWA | `eastus2` (only overlap) | diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/key-vault/README.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/key-vault/README.md index 9e7744b9..cd04c287 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/key-vault/README.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/key-vault/README.md @@ -1,55 +1,55 @@ -# Azure Key Vault - -Centralized secrets, keys, and certificate management. - -## When to Use - -- Storing application secrets -- Managing certificates -- Storing encryption keys -- Centralizing secret management -- Enabling secret rotation - -## Required Supporting Resources - -| Resource | Purpose | -|----------|---------| -| None required | Key Vault is self-contained | -| Private Endpoint | Secure access (optional) | - -## SKU Selection - -| SKU | Features | -|-----|----------| -| Standard | Software-protected keys | -| Premium | HSM-protected keys | - -## RBAC Roles - -| Role | Permissions | -|------|-------------| -| Key Vault Administrator | Full access | -| Key Vault Secrets Officer | Manage secrets | -| Key Vault Secrets User | Read secrets | -| Key Vault Certificates Officer | Manage certificates | -| Key Vault Crypto Officer | Manage keys | - -## Environment Variables - -| Variable | Value | -|----------|-------| -| `KEY_VAULT_URL` | `https://{vault-name}.vault.azure.net/` | -| `KEY_VAULT_NAME` | Vault name | - -## Best Practices - -1. **Always use RBAC** over access policies -2. **Enable soft delete and purge protection** for production -3. **Use managed identities** instead of storing keys in apps -4. **Set expiration dates** on secrets -5. **Use separate vaults** for different environments - -## References - -- [Bicep Patterns](bicep.md) -- [SDK Access](sdk.md) +# Azure Key Vault + +Centralized secrets, keys, and certificate management. + +## When to Use + +- Storing application secrets +- Managing certificates +- Storing encryption keys +- Centralizing secret management +- Enabling secret rotation + +## Required Supporting Resources + +| Resource | Purpose | +|----------|---------| +| None required | Key Vault is self-contained | +| Private Endpoint | Secure access (optional) | + +## SKU Selection + +| SKU | Features | +|-----|----------| +| Standard | Software-protected keys | +| Premium | HSM-protected keys | + +## RBAC Roles + +| Role | Permissions | +|------|-------------| +| Key Vault Administrator | Full access | +| Key Vault Secrets Officer | Manage secrets | +| Key Vault Secrets User | Read secrets | +| Key Vault Certificates Officer | Manage certificates | +| Key Vault Crypto Officer | Manage keys | + +## Environment Variables + +| Variable | Value | +|----------|-------| +| `KEY_VAULT_URL` | `https://{vault-name}.vault.azure.net/` | +| `KEY_VAULT_NAME` | Vault name | + +## Best Practices + +1. **Always use RBAC** over access policies +2. **Enable soft delete and purge protection** for production +3. **Use managed identities** instead of storing keys in apps +4. **Set expiration dates** on secrets +5. **Use separate vaults** for different environments + +## References + +- [Bicep Patterns](bicep.md) +- [SDK Access](sdk.md) diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/key-vault/bicep.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/key-vault/bicep.md index 370768d0..15de9082 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/key-vault/bicep.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/key-vault/bicep.md @@ -1,85 +1,85 @@ -# Key Vault - Bicep Patterns - -## Basic Vault - -```bicep -resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' = { - name: '${resourcePrefix}-kv-${uniqueHash}' - location: location - properties: { - tenantId: subscription().tenantId - sku: { - family: 'A' - name: 'standard' - } - enableRbacAuthorization: true - enableSoftDelete: true - softDeleteRetentionInDays: 90 - enablePurgeProtection: true - } -} -``` - -## Storing Secrets - -```bicep -resource secret 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = { - parent: keyVault - name: 'database-connection-string' - properties: { - value: databaseConnectionString - } -} -``` - -## Role Assignment (Managed Identity) - -```bicep -resource keyVaultRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(keyVault.id, principalId, 'Key Vault Secrets User') - scope: keyVault - properties: { - roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4633458b-17de-408a-b874-0445c86b69e6') - principalId: principalId - principalType: 'ServicePrincipal' - } -} -``` - -## Referencing in App Service / Functions - -```bicep -appSettings: [ - { - name: 'DATABASE_URL' - value: '@Microsoft.KeyVault(VaultName=${keyVault.name};SecretName=database-connection-string)' - } -] -``` - -## Referencing in Container Apps - -```bicep -secrets: [ - { - name: 'db-connection' - keyVaultUrl: '${keyVault.properties.vaultUri}secrets/database-connection-string' - identity: containerApp.identity.principalId - } -] -``` - -## Secret with Expiration - -```bicep -resource secret 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = { - parent: keyVault - name: 'api-key' - properties: { - value: apiKey - attributes: { - exp: dateTimeToEpoch(dateTimeAdd(utcNow(), 'P90D')) - } - } -} -``` +# Key Vault - Bicep Patterns + +## Basic Vault + +```bicep +resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' = { + name: '${resourcePrefix}-kv-${uniqueHash}' + location: location + properties: { + tenantId: subscription().tenantId + sku: { + family: 'A' + name: 'standard' + } + enableRbacAuthorization: true + enableSoftDelete: true + softDeleteRetentionInDays: 90 + enablePurgeProtection: true + } +} +``` + +## Storing Secrets + +```bicep +resource secret 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = { + parent: keyVault + name: 'database-connection-string' + properties: { + value: databaseConnectionString + } +} +``` + +## Role Assignment (Managed Identity) + +```bicep +resource keyVaultRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(keyVault.id, principalId, 'Key Vault Secrets User') + scope: keyVault + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4633458b-17de-408a-b874-0445c86b69e6') + principalId: principalId + principalType: 'ServicePrincipal' + } +} +``` + +## Referencing in App Service / Functions + +```bicep +appSettings: [ + { + name: 'DATABASE_URL' + value: '@Microsoft.KeyVault(VaultName=${keyVault.name};SecretName=database-connection-string)' + } +] +``` + +## Referencing in Container Apps + +```bicep +secrets: [ + { + name: 'db-connection' + keyVaultUrl: '${keyVault.properties.vaultUri}secrets/database-connection-string' + identity: containerApp.identity.principalId + } +] +``` + +## Secret with Expiration + +```bicep +resource secret 'Microsoft.KeyVault/vaults/secrets@2023-07-01' = { + parent: keyVault + name: 'api-key' + properties: { + value: apiKey + attributes: { + exp: dateTimeToEpoch(dateTimeAdd(utcNow(), 'P90D')) + } + } +} +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/key-vault/sdk.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/key-vault/sdk.md index baa7518c..3dbca912 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/key-vault/sdk.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/key-vault/sdk.md @@ -1,72 +1,72 @@ -# Key Vault - SDK Patterns - -## Node.js - -> **Auth:** `DefaultAzureCredential` is for local development. See [auth-best-practices.md](../../auth-best-practices.md) for production patterns. - -```javascript -const { SecretClient } = require("@azure/keyvault-secrets"); -const { DefaultAzureCredential } = require("@azure/identity"); - -const client = new SecretClient( - process.env.KEY_VAULT_URL, - new DefaultAzureCredential() -); - -const secret = await client.getSecret("database-connection-string"); -console.log(secret.value); -``` - -## Python - -> **Auth:** `DefaultAzureCredential` is for local development. See [auth-best-practices.md](../../auth-best-practices.md) for production patterns. - -```python -from azure.keyvault.secrets import SecretClient -from azure.identity import DefaultAzureCredential - -client = SecretClient( - vault_url=os.environ["KEY_VAULT_URL"], - credential=DefaultAzureCredential() -) - -secret = client.get_secret("database-connection-string") -print(secret.value) -``` - -## .NET - -> **Auth:** `DefaultAzureCredential` is for local development. See [auth-best-practices.md](../../auth-best-practices.md) for production patterns. - -```csharp -var client = new SecretClient( - new Uri(Environment.GetEnvironmentVariable("KEY_VAULT_URL")), - new DefaultAzureCredential() -); - -KeyVaultSecret secret = await client.GetSecretAsync("database-connection-string"); -Console.WriteLine(secret.Value); -``` - -## Event Grid Integration (Expiry Notifications) - -```bicep -resource kvEventSubscription 'Microsoft.EventGrid/eventSubscriptions@2023-12-15-preview' = { - name: 'secret-expiry-notification' - scope: keyVault - properties: { - destination: { - endpointType: 'WebHook' - properties: { - endpointUrl: 'https://my-api.example.com/secret-rotation' - } - } - filter: { - includedEventTypes: [ - 'Microsoft.KeyVault.SecretNearExpiry' - 'Microsoft.KeyVault.SecretExpired' - ] - } - } -} -``` +# Key Vault - SDK Patterns + +## Node.js + +> **Auth:** `DefaultAzureCredential` is for local development. See [auth-best-practices.md](../../auth-best-practices.md) for production patterns. + +```javascript +const { SecretClient } = require("@azure/keyvault-secrets"); +const { DefaultAzureCredential } = require("@azure/identity"); + +const client = new SecretClient( + process.env.KEY_VAULT_URL, + new DefaultAzureCredential() +); + +const secret = await client.getSecret("database-connection-string"); +console.log(secret.value); +``` + +## Python + +> **Auth:** `DefaultAzureCredential` is for local development. See [auth-best-practices.md](../../auth-best-practices.md) for production patterns. + +```python +from azure.keyvault.secrets import SecretClient +from azure.identity import DefaultAzureCredential + +client = SecretClient( + vault_url=os.environ["KEY_VAULT_URL"], + credential=DefaultAzureCredential() +) + +secret = client.get_secret("database-connection-string") +print(secret.value) +``` + +## .NET + +> **Auth:** `DefaultAzureCredential` is for local development. See [auth-best-practices.md](../../auth-best-practices.md) for production patterns. + +```csharp +var client = new SecretClient( + new Uri(Environment.GetEnvironmentVariable("KEY_VAULT_URL")), + new DefaultAzureCredential() +); + +KeyVaultSecret secret = await client.GetSecretAsync("database-connection-string"); +Console.WriteLine(secret.Value); +``` + +## Event Grid Integration (Expiry Notifications) + +```bicep +resource kvEventSubscription 'Microsoft.EventGrid/eventSubscriptions@2023-12-15-preview' = { + name: 'secret-expiry-notification' + scope: keyVault + properties: { + destination: { + endpointType: 'WebHook' + properties: { + endpointUrl: 'https://my-api.example.com/secret-rotation' + } + } + filter: { + includedEventTypes: [ + 'Microsoft.KeyVault.SecretNearExpiry' + 'Microsoft.KeyVault.SecretExpired' + ] + } + } +} +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/logic-apps/README.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/logic-apps/README.md index 386aaa78..9c98a372 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/logic-apps/README.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/logic-apps/README.md @@ -1,43 +1,43 @@ -# Azure Logic Apps - -Low-code workflow automation and integration platform. - -## When to Use - -- Integration-heavy workloads -- Business process automation -- Connecting multiple SaaS services -- Approval and human workflow processes -- Low-code/visual workflow design -- Event-driven orchestration - -## Deployment Note - -Logic Apps are typically deployed as infrastructure, not application services: - -```yaml -# Logic Apps are defined in Bicep infrastructure -``` - -## Required Supporting Resources - -| Resource | Purpose | -|----------|---------| -| Storage Account | Workflow state (Standard only) | -| Log Analytics | Monitoring | -| API Connections | External service connections | - -## Consumption vs Standard - -| Feature | Consumption | Standard | -|---------|-------------|----------| -| Pricing | Per execution | App Service Plan | -| VNET | Limited | Full support | -| State | Azure-managed | Custom storage | -| Deployment | ARM/Bicep | VS Code deployment | -| Multi-workflow | One per resource | Multiple per app | - -## References - -- [Bicep Patterns](bicep.md) -- [Triggers](triggers.md) +# Azure Logic Apps + +Low-code workflow automation and integration platform. + +## When to Use + +- Integration-heavy workloads +- Business process automation +- Connecting multiple SaaS services +- Approval and human workflow processes +- Low-code/visual workflow design +- Event-driven orchestration + +## Deployment Note + +Logic Apps are typically deployed as infrastructure, not application services: + +```yaml +# Logic Apps are defined in Bicep infrastructure +``` + +## Required Supporting Resources + +| Resource | Purpose | +|----------|---------| +| Storage Account | Workflow state (Standard only) | +| Log Analytics | Monitoring | +| API Connections | External service connections | + +## Consumption vs Standard + +| Feature | Consumption | Standard | +|---------|-------------|----------| +| Pricing | Per execution | App Service Plan | +| VNET | Limited | Full support | +| State | Azure-managed | Custom storage | +| Deployment | ARM/Bicep | VS Code deployment | +| Multi-workflow | One per resource | Multiple per app | + +## References + +- [Bicep Patterns](bicep.md) +- [Triggers](triggers.md) diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/logic-apps/bicep.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/logic-apps/bicep.md index 008eb199..db33c563 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/logic-apps/bicep.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/logic-apps/bicep.md @@ -1,87 +1,87 @@ -# Logic Apps - Bicep Patterns - -## Consumption (Multi-tenant) - -```bicep -resource logicApp 'Microsoft.Logic/workflows@2019-05-01' = { - name: '${resourcePrefix}-logic-${uniqueHash}' - location: location - properties: { - state: 'Enabled' - definition: { - '$schema': 'https://schema.management.azure.com/providers/Microsoft.Logic/schemas/2016-06-01/workflowdefinition.json#' - contentVersion: '1.0.0.0' - triggers: { - manual: { - type: 'Request' - kind: 'Http' - inputs: { - schema: {} - } - } - } - actions: {} - } - parameters: {} - } -} -``` - -## Standard (Single-tenant) - -```bicep -resource logicAppPlan 'Microsoft.Web/serverfarms@2022-09-01' = { - name: '${resourcePrefix}-logicplan-${uniqueHash}' - location: location - sku: { - name: 'WS1' - tier: 'WorkflowStandard' - } - properties: { - reserved: true - } -} - -resource logicAppStandard 'Microsoft.Web/sites@2022-09-01' = { - name: '${resourcePrefix}-logic-${uniqueHash}' - location: location - kind: 'functionapp,workflowapp' - properties: { - serverFarmId: logicAppPlan.id - siteConfig: { - appSettings: [ - { - name: 'FUNCTIONS_EXTENSION_VERSION' - value: '~4' - } - { - name: 'FUNCTIONS_WORKER_RUNTIME' - value: 'node' - } - { - name: 'AzureWebJobsStorage' - value: storageConnectionString - } - ] - } - } -} -``` - -## API Connection - -```bicep -resource serviceBusConnection 'Microsoft.Web/connections@2016-06-01' = { - name: 'servicebus-connection' - location: location - properties: { - displayName: 'Service Bus Connection' - api: { - id: subscriptionResourceId('Microsoft.Web/locations/managedApis', location, 'servicebus') - } - parameterValues: { - connectionString: serviceBus.listKeys().primaryConnectionString - } - } -} -``` +# Logic Apps - Bicep Patterns + +## Consumption (Multi-tenant) + +```bicep +resource logicApp 'Microsoft.Logic/workflows@2019-05-01' = { + name: '${resourcePrefix}-logic-${uniqueHash}' + location: location + properties: { + state: 'Enabled' + definition: { + '$schema': 'https://schema.management.azure.com/providers/Microsoft.Logic/schemas/2016-06-01/workflowdefinition.json#' + contentVersion: '1.0.0.0' + triggers: { + manual: { + type: 'Request' + kind: 'Http' + inputs: { + schema: {} + } + } + } + actions: {} + } + parameters: {} + } +} +``` + +## Standard (Single-tenant) + +```bicep +resource logicAppPlan 'Microsoft.Web/serverfarms@2022-09-01' = { + name: '${resourcePrefix}-logicplan-${uniqueHash}' + location: location + sku: { + name: 'WS1' + tier: 'WorkflowStandard' + } + properties: { + reserved: true + } +} + +resource logicAppStandard 'Microsoft.Web/sites@2022-09-01' = { + name: '${resourcePrefix}-logic-${uniqueHash}' + location: location + kind: 'functionapp,workflowapp' + properties: { + serverFarmId: logicAppPlan.id + siteConfig: { + appSettings: [ + { + name: 'FUNCTIONS_EXTENSION_VERSION' + value: '~4' + } + { + name: 'FUNCTIONS_WORKER_RUNTIME' + value: 'node' + } + { + name: 'AzureWebJobsStorage' + value: storageConnectionString + } + ] + } + } +} +``` + +## API Connection + +```bicep +resource serviceBusConnection 'Microsoft.Web/connections@2016-06-01' = { + name: 'servicebus-connection' + location: location + properties: { + displayName: 'Service Bus Connection' + api: { + id: subscriptionResourceId('Microsoft.Web/locations/managedApis', location, 'servicebus') + } + parameterValues: { + connectionString: serviceBus.listKeys().primaryConnectionString + } + } +} +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/logic-apps/triggers.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/logic-apps/triggers.md index 3633f81d..301d1d8a 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/logic-apps/triggers.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/logic-apps/triggers.md @@ -1,105 +1,105 @@ -# Logic Apps - Triggers - -## HTTP Request - -```json -{ - "triggers": { - "manual": { - "type": "Request", - "kind": "Http", - "inputs": { - "schema": { - "type": "object", - "properties": { - "orderId": { "type": "string" } - } - } - } - } - } -} -``` - -## Recurrence (Schedule) - -```json -{ - "triggers": { - "Recurrence": { - "type": "Recurrence", - "recurrence": { - "frequency": "Hour", - "interval": 1 - } - } - } -} -``` - -## Service Bus Queue - -```json -{ - "triggers": { - "When_a_message_is_received": { - "type": "ApiConnection", - "inputs": { - "host": { - "connection": { - "name": "@parameters('$connections')['servicebus']['connectionId']" - } - }, - "method": "get", - "path": "/@{encodeURIComponent('orders')}/messages/head" - } - } - } -} -``` - -## Common Actions - -### HTTP Action - -```json -{ - "HTTP": { - "type": "Http", - "inputs": { - "method": "POST", - "uri": "https://api.example.com/orders", - "headers": { - "Content-Type": "application/json" - }, - "body": "@triggerBody()" - } - } -} -``` - -### Approval Email - -```json -{ - "Send_approval_email": { - "type": "ApiConnectionWebhook", - "inputs": { - "host": { - "connection": { - "name": "@parameters('$connections')['office365']['connectionId']" - } - }, - "body": { - "NotificationUrl": "@{listCallbackUrl()}", - "Message": { - "To": "approver@example.com", - "Subject": "Approval Required", - "Options": "Approve, Reject" - } - }, - "path": "/approvalmail/$subscriptions" - } - } -} -``` +# Logic Apps - Triggers + +## HTTP Request + +```json +{ + "triggers": { + "manual": { + "type": "Request", + "kind": "Http", + "inputs": { + "schema": { + "type": "object", + "properties": { + "orderId": { "type": "string" } + } + } + } + } + } +} +``` + +## Recurrence (Schedule) + +```json +{ + "triggers": { + "Recurrence": { + "type": "Recurrence", + "recurrence": { + "frequency": "Hour", + "interval": 1 + } + } + } +} +``` + +## Service Bus Queue + +```json +{ + "triggers": { + "When_a_message_is_received": { + "type": "ApiConnection", + "inputs": { + "host": { + "connection": { + "name": "@parameters('$connections')['servicebus']['connectionId']" + } + }, + "method": "get", + "path": "/@{encodeURIComponent('orders')}/messages/head" + } + } + } +} +``` + +## Common Actions + +### HTTP Action + +```json +{ + "HTTP": { + "type": "Http", + "inputs": { + "method": "POST", + "uri": "https://api.example.com/orders", + "headers": { + "Content-Type": "application/json" + }, + "body": "@triggerBody()" + } + } +} +``` + +### Approval Email + +```json +{ + "Send_approval_email": { + "type": "ApiConnectionWebhook", + "inputs": { + "host": { + "connection": { + "name": "@parameters('$connections')['office365']['connectionId']" + } + }, + "body": { + "NotificationUrl": "@{listCallbackUrl()}", + "Message": { + "To": "approver@example.com", + "Subject": "Approval Required", + "Options": "Approve, Reject" + } + }, + "path": "/approvalmail/$subscriptions" + } + } +} +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/service-bus/README.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/service-bus/README.md index ed11dfe8..50f49ea4 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/service-bus/README.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/service-bus/README.md @@ -1,54 +1,54 @@ -# Azure Service Bus - -Enterprise messaging with queues and pub/sub topics. - -## When to Use - -- Reliable message delivery -- Pub/sub messaging patterns -- Message ordering requirements -- Dead-letter handling -- Transaction support -- Enterprise integration - -## Required Supporting Resources - -| Resource | Purpose | -|----------|---------| -| None required | Service Bus is self-contained | -| Key Vault | Store connection strings (legacy) | - -## SKU Selection - -| SKU | Features | Use Case | -|-----|----------|----------| -| Basic | Queues only, 256KB messages | Simple messaging | -| Standard | Topics, 256KB messages | Pub/sub patterns | -| Premium | 100MB messages, VNET, zones | Enterprise, high throughput | - -## Environment Variables - -### Managed Identity (Recommended) - -| Variable | Value | -|----------|-------| -| `SERVICEBUS__fullyQualifiedNamespace` | `.servicebus.windows.net` | -| `SERVICEBUS_NAMESPACE` | Namespace name (for SDK) | -| `SERVICEBUS_QUEUE` | Queue name | - -**Required RBAC roles:** -- `Azure Service Bus Data Sender` (69a216fc-b8fb-44d8-bc22-1f3c2cd27a39) - for sending -- `Azure Service Bus Data Receiver` (4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0) - for receiving - -### Connection String (Legacy) - -| Variable | Value | -|----------|-------| -| `SERVICEBUS_CONNECTION_STRING` | Connection string (Key Vault) | -| `SERVICEBUS_NAMESPACE` | Namespace name | -| `SERVICEBUS_QUEUE` | Queue name | - -## References - -- [Bicep Patterns](bicep.md) -- [Messaging Patterns](patterns.md) +# Azure Service Bus + +Enterprise messaging with queues and pub/sub topics. + +## When to Use + +- Reliable message delivery +- Pub/sub messaging patterns +- Message ordering requirements +- Dead-letter handling +- Transaction support +- Enterprise integration + +## Required Supporting Resources + +| Resource | Purpose | +|----------|---------| +| None required | Service Bus is self-contained | +| Key Vault | Store connection strings (legacy) | + +## SKU Selection + +| SKU | Features | Use Case | +|-----|----------|----------| +| Basic | Queues only, 256KB messages | Simple messaging | +| Standard | Topics, 256KB messages | Pub/sub patterns | +| Premium | 100MB messages, VNET, zones | Enterprise, high throughput | + +## Environment Variables + +### Managed Identity (Recommended) + +| Variable | Value | +|----------|-------| +| `SERVICEBUS__fullyQualifiedNamespace` | `.servicebus.windows.net` | +| `SERVICEBUS_NAMESPACE` | Namespace name (for SDK) | +| `SERVICEBUS_QUEUE` | Queue name | + +**Required RBAC roles:** +- `Azure Service Bus Data Sender` (69a216fc-b8fb-44d8-bc22-1f3c2cd27a39) - for sending +- `Azure Service Bus Data Receiver` (4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0) - for receiving + +### Connection String (Legacy) + +| Variable | Value | +|----------|-------| +| `SERVICEBUS_CONNECTION_STRING` | Connection string (Key Vault) | +| `SERVICEBUS_NAMESPACE` | Namespace name | +| `SERVICEBUS_QUEUE` | Queue name | + +## References + +- [Bicep Patterns](bicep.md) +- [Messaging Patterns](patterns.md) diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/service-bus/bicep.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/service-bus/bicep.md index 36b20dd9..485246a5 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/service-bus/bicep.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/service-bus/bicep.md @@ -1,144 +1,144 @@ -# Service Bus - Bicep Patterns - -## Namespace - -```bicep -resource serviceBus 'Microsoft.ServiceBus/namespaces@2022-10-01-preview' = { - name: '${resourcePrefix}-sb-${uniqueHash}' - location: location - sku: { - name: 'Standard' - tier: 'Standard' - } -} -``` - -## Queue - -```bicep -resource queue 'Microsoft.ServiceBus/namespaces/queues@2022-10-01-preview' = { - parent: serviceBus - name: 'orders' - properties: { - maxDeliveryCount: 10 - deadLetteringOnMessageExpiration: true - defaultMessageTimeToLive: 'P14D' - lockDuration: 'PT5M' - } -} -``` - -## Topic and Subscription - -```bicep -resource topic 'Microsoft.ServiceBus/namespaces/topics@2022-10-01-preview' = { - parent: serviceBus - name: 'events' - properties: { - defaultMessageTimeToLive: 'P14D' - } -} - -resource subscription 'Microsoft.ServiceBus/namespaces/topics/subscriptions@2022-10-01-preview' = { - parent: topic - name: 'order-processor' - properties: { - maxDeliveryCount: 10 - deadLetteringOnMessageExpiration: true - lockDuration: 'PT5M' - } -} -``` - -## Subscription Filters - -### SQL Filter - -```bicep -resource filterRule 'Microsoft.ServiceBus/namespaces/topics/subscriptions/rules@2022-10-01-preview' = { - parent: subscription - name: 'high-priority' - properties: { - filterType: 'SqlFilter' - sqlFilter: { - sqlExpression: 'priority = \'high\'' - } - } -} -``` - -### Correlation Filter - -```bicep -resource correlationRule 'Microsoft.ServiceBus/namespaces/topics/subscriptions/rules@2022-10-01-preview' = { - parent: subscription - name: 'orders-only' - properties: { - filterType: 'CorrelationFilter' - correlationFilter: { - label: 'order' - } - } -} -``` - -## Managed Identity Access - -### Service Bus Data Receiver (for triggers/consumers) - -```bicep -resource serviceBusReceiverRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(serviceBus.id, principalId, 'Azure Service Bus Data Receiver') - scope: serviceBus - properties: { - roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0') - principalId: principalId - principalType: 'ServicePrincipal' - } -} -``` - -### Service Bus Data Sender (for producers) - -```bicep -resource serviceBusSenderRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(serviceBus.id, principalId, 'Azure Service Bus Data Sender') - scope: serviceBus - properties: { - roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '69a216fc-b8fb-44d8-bc22-1f3c2cd27a39') - principalId: principalId - principalType: 'ServicePrincipal' - } -} -``` - -### Both Sender and Receiver - -```bicep -// Grant both sender and receiver roles for bidirectional messaging -resource serviceBusReceiverRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(serviceBus.id, principalId, 'receiver') - scope: serviceBus - properties: { - roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0') - principalId: principalId - principalType: 'ServicePrincipal' - } -} - -resource serviceBusSenderRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(serviceBus.id, principalId, 'sender') - scope: serviceBus - properties: { - roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '69a216fc-b8fb-44d8-bc22-1f3c2cd27a39') - principalId: principalId - principalType: 'ServicePrincipal' - } -} -``` - -> 💡 **Role Selection:** -> - Use **Data Receiver** for Function triggers or message consumers -> - Use **Data Sender** for applications that send messages -> - Use **both roles** for bidirectional communication -> - Roles can be scoped to namespace (all queues/topics) or specific queue/topic +# Service Bus - Bicep Patterns + +## Namespace + +```bicep +resource serviceBus 'Microsoft.ServiceBus/namespaces@2022-10-01-preview' = { + name: '${resourcePrefix}-sb-${uniqueHash}' + location: location + sku: { + name: 'Standard' + tier: 'Standard' + } +} +``` + +## Queue + +```bicep +resource queue 'Microsoft.ServiceBus/namespaces/queues@2022-10-01-preview' = { + parent: serviceBus + name: 'orders' + properties: { + maxDeliveryCount: 10 + deadLetteringOnMessageExpiration: true + defaultMessageTimeToLive: 'P14D' + lockDuration: 'PT5M' + } +} +``` + +## Topic and Subscription + +```bicep +resource topic 'Microsoft.ServiceBus/namespaces/topics@2022-10-01-preview' = { + parent: serviceBus + name: 'events' + properties: { + defaultMessageTimeToLive: 'P14D' + } +} + +resource subscription 'Microsoft.ServiceBus/namespaces/topics/subscriptions@2022-10-01-preview' = { + parent: topic + name: 'order-processor' + properties: { + maxDeliveryCount: 10 + deadLetteringOnMessageExpiration: true + lockDuration: 'PT5M' + } +} +``` + +## Subscription Filters + +### SQL Filter + +```bicep +resource filterRule 'Microsoft.ServiceBus/namespaces/topics/subscriptions/rules@2022-10-01-preview' = { + parent: subscription + name: 'high-priority' + properties: { + filterType: 'SqlFilter' + sqlFilter: { + sqlExpression: 'priority = \'high\'' + } + } +} +``` + +### Correlation Filter + +```bicep +resource correlationRule 'Microsoft.ServiceBus/namespaces/topics/subscriptions/rules@2022-10-01-preview' = { + parent: subscription + name: 'orders-only' + properties: { + filterType: 'CorrelationFilter' + correlationFilter: { + label: 'order' + } + } +} +``` + +## Managed Identity Access + +### Service Bus Data Receiver (for triggers/consumers) + +```bicep +resource serviceBusReceiverRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(serviceBus.id, principalId, 'Azure Service Bus Data Receiver') + scope: serviceBus + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0') + principalId: principalId + principalType: 'ServicePrincipal' + } +} +``` + +### Service Bus Data Sender (for producers) + +```bicep +resource serviceBusSenderRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(serviceBus.id, principalId, 'Azure Service Bus Data Sender') + scope: serviceBus + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '69a216fc-b8fb-44d8-bc22-1f3c2cd27a39') + principalId: principalId + principalType: 'ServicePrincipal' + } +} +``` + +### Both Sender and Receiver + +```bicep +// Grant both sender and receiver roles for bidirectional messaging +resource serviceBusReceiverRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(serviceBus.id, principalId, 'receiver') + scope: serviceBus + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0') + principalId: principalId + principalType: 'ServicePrincipal' + } +} + +resource serviceBusSenderRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(serviceBus.id, principalId, 'sender') + scope: serviceBus + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '69a216fc-b8fb-44d8-bc22-1f3c2cd27a39') + principalId: principalId + principalType: 'ServicePrincipal' + } +} +``` + +> 💡 **Role Selection:** +> - Use **Data Receiver** for Function triggers or message consumers +> - Use **Data Sender** for applications that send messages +> - Use **both roles** for bidirectional communication +> - Roles can be scoped to namespace (all queues/topics) or specific queue/topic diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/service-bus/patterns.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/service-bus/patterns.md index 0fc635a0..d675dd08 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/service-bus/patterns.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/service-bus/patterns.md @@ -1,155 +1,155 @@ -# Service Bus - Messaging Patterns - -## Point-to-Point (Queue) - -``` -Producer → Queue → Consumer -``` - -Use for: Work distribution, command processing - -## Pub/Sub (Topic + Subscriptions) - -``` -Publisher → Topic → Subscription A → Consumer A - → Subscription B → Consumer B -``` - -Use for: Event broadcasting, multiple consumers - -## SDK Patterns - -### Managed Identity (Recommended) - -#### Node.js - -> **Auth:** `DefaultAzureCredential` is for local development. See [auth-best-practices.md](../../auth-best-practices.md) for production patterns. - -```javascript -const { ServiceBusClient } = require("@azure/service-bus"); -const { DefaultAzureCredential } = require("@azure/identity"); - -const credential = new DefaultAzureCredential(); -const fullyQualifiedNamespace = process.env.SERVICEBUS_NAMESPACE + ".servicebus.windows.net"; -const client = new ServiceBusClient(fullyQualifiedNamespace, credential); - -// Send -const sender = client.createSender("orders"); -await sender.sendMessages({ body: { orderId: "123" } }); - -// Receive -const receiver = client.createReceiver("orders"); -const messages = await receiver.receiveMessages(10); -for (const message of messages) { - await receiver.completeMessage(message); -} -``` - -#### Python - -> **Auth:** `DefaultAzureCredential` is for local development. See [auth-best-practices.md](../../auth-best-practices.md) for production patterns. - -```python -from azure.servicebus import ServiceBusClient, ServiceBusMessage -from azure.identity import DefaultAzureCredential - -credential = DefaultAzureCredential() -fully_qualified_namespace = f"{os.environ['SERVICEBUS_NAMESPACE']}.servicebus.windows.net" -client = ServiceBusClient(fully_qualified_namespace, credential) - -# Send -sender = client.get_queue_sender("orders") -with sender: - sender.send_messages(ServiceBusMessage('{"orderId": "123"}')) - -# Receive -receiver = client.get_queue_receiver("orders") -with receiver: - messages = receiver.receive_messages(max_message_count=10, max_wait_time=5) - for message in messages: - print(message) - receiver.complete_message(message) -``` - -#### .NET - -> **Auth:** `DefaultAzureCredential` is for local development. See [auth-best-practices.md](../../auth-best-practices.md) for production patterns. - -```csharp -using Azure.Identity; -using Azure.Messaging.ServiceBus; - -var credential = new DefaultAzureCredential(); -var fullyQualifiedNamespace = $"{Environment.GetEnvironmentVariable("SERVICEBUS_NAMESPACE")}.servicebus.windows.net"; -var client = new ServiceBusClient(fullyQualifiedNamespace, credential); - -// Send -var sender = client.CreateSender("orders"); -await sender.SendMessageAsync(new ServiceBusMessage("{\"orderId\": \"123\"}")); - -// Receive -var receiver = client.CreateReceiver("orders"); -var messages = await receiver.ReceiveMessagesAsync(maxMessages: 10); -foreach (var message in messages) -{ - await receiver.CompleteMessageAsync(message); -} -``` - -> 💡 **Required Permissions:** -> - `Azure Service Bus Data Sender` (69a216fc-b8fb-44d8-bc22-1f3c2cd27a39) - for sending -> - `Azure Service Bus Data Receiver` (4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0) - for receiving - -### Connection String (Legacy) - -#### Node.js - -```javascript -const { ServiceBusClient } = require("@azure/service-bus"); - -const client = new ServiceBusClient(process.env.SERVICEBUS_CONNECTION_STRING); - -// Send -const sender = client.createSender("orders"); -await sender.sendMessages({ body: { orderId: "123" } }); - -// Receive -const receiver = client.createReceiver("orders"); -const messages = await receiver.receiveMessages(10); -for (const message of messages) { - await receiver.completeMessage(message); -} -``` - -#### Python - -```python -from azure.servicebus import ServiceBusClient, ServiceBusMessage - -client = ServiceBusClient.from_connection_string( - os.environ["SERVICEBUS_CONNECTION_STRING"] -) -sender = client.get_queue_sender("orders") - -with sender: - sender.send_messages(ServiceBusMessage('{"orderId": "123"}')) -``` - -#### .NET - -```csharp -var client = new ServiceBusClient( - Environment.GetEnvironmentVariable("SERVICEBUS_CONNECTION_STRING") -); -var sender = client.CreateSender("orders"); - -await sender.SendMessageAsync(new ServiceBusMessage("{\"orderId\": \"123\"}")); -``` - -## Dead Letter Handling - -```javascript -const dlqReceiver = client.createReceiver("orders", { - subQueueType: "deadLetter" -}); -``` +# Service Bus - Messaging Patterns + +## Point-to-Point (Queue) + +``` +Producer → Queue → Consumer +``` + +Use for: Work distribution, command processing + +## Pub/Sub (Topic + Subscriptions) + +``` +Publisher → Topic → Subscription A → Consumer A + → Subscription B → Consumer B +``` + +Use for: Event broadcasting, multiple consumers + +## SDK Patterns + +### Managed Identity (Recommended) + +#### Node.js + +> **Auth:** `DefaultAzureCredential` is for local development. See [auth-best-practices.md](../../auth-best-practices.md) for production patterns. + +```javascript +const { ServiceBusClient } = require("@azure/service-bus"); +const { DefaultAzureCredential } = require("@azure/identity"); + +const credential = new DefaultAzureCredential(); +const fullyQualifiedNamespace = process.env.SERVICEBUS_NAMESPACE + ".servicebus.windows.net"; +const client = new ServiceBusClient(fullyQualifiedNamespace, credential); + +// Send +const sender = client.createSender("orders"); +await sender.sendMessages({ body: { orderId: "123" } }); + +// Receive +const receiver = client.createReceiver("orders"); +const messages = await receiver.receiveMessages(10); +for (const message of messages) { + await receiver.completeMessage(message); +} +``` + +#### Python + +> **Auth:** `DefaultAzureCredential` is for local development. See [auth-best-practices.md](../../auth-best-practices.md) for production patterns. + +```python +from azure.servicebus import ServiceBusClient, ServiceBusMessage +from azure.identity import DefaultAzureCredential + +credential = DefaultAzureCredential() +fully_qualified_namespace = f"{os.environ['SERVICEBUS_NAMESPACE']}.servicebus.windows.net" +client = ServiceBusClient(fully_qualified_namespace, credential) + +# Send +sender = client.get_queue_sender("orders") +with sender: + sender.send_messages(ServiceBusMessage('{"orderId": "123"}')) + +# Receive +receiver = client.get_queue_receiver("orders") +with receiver: + messages = receiver.receive_messages(max_message_count=10, max_wait_time=5) + for message in messages: + print(message) + receiver.complete_message(message) +``` + +#### .NET + +> **Auth:** `DefaultAzureCredential` is for local development. See [auth-best-practices.md](../../auth-best-practices.md) for production patterns. + +```csharp +using Azure.Identity; +using Azure.Messaging.ServiceBus; + +var credential = new DefaultAzureCredential(); +var fullyQualifiedNamespace = $"{Environment.GetEnvironmentVariable("SERVICEBUS_NAMESPACE")}.servicebus.windows.net"; +var client = new ServiceBusClient(fullyQualifiedNamespace, credential); + +// Send +var sender = client.CreateSender("orders"); +await sender.SendMessageAsync(new ServiceBusMessage("{\"orderId\": \"123\"}")); + +// Receive +var receiver = client.CreateReceiver("orders"); +var messages = await receiver.ReceiveMessagesAsync(maxMessages: 10); +foreach (var message in messages) +{ + await receiver.CompleteMessageAsync(message); +} +``` + +> 💡 **Required Permissions:** +> - `Azure Service Bus Data Sender` (69a216fc-b8fb-44d8-bc22-1f3c2cd27a39) - for sending +> - `Azure Service Bus Data Receiver` (4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0) - for receiving + +### Connection String (Legacy) + +#### Node.js + +```javascript +const { ServiceBusClient } = require("@azure/service-bus"); + +const client = new ServiceBusClient(process.env.SERVICEBUS_CONNECTION_STRING); + +// Send +const sender = client.createSender("orders"); +await sender.sendMessages({ body: { orderId: "123" } }); + +// Receive +const receiver = client.createReceiver("orders"); +const messages = await receiver.receiveMessages(10); +for (const message of messages) { + await receiver.completeMessage(message); +} +``` + +#### Python + +```python +from azure.servicebus import ServiceBusClient, ServiceBusMessage + +client = ServiceBusClient.from_connection_string( + os.environ["SERVICEBUS_CONNECTION_STRING"] +) +sender = client.get_queue_sender("orders") + +with sender: + sender.send_messages(ServiceBusMessage('{"orderId": "123"}')) +``` + +#### .NET + +```csharp +var client = new ServiceBusClient( + Environment.GetEnvironmentVariable("SERVICEBUS_CONNECTION_STRING") +); +var sender = client.CreateSender("orders"); + +await sender.SendMessageAsync(new ServiceBusMessage("{\"orderId\": \"123\"}")); +``` + +## Dead Letter Handling + +```javascript +const dlqReceiver = client.createReceiver("orders", { + subQueueType: "deadLetter" +}); +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/sql-database/README.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/sql-database/README.md index 37fa5bec..1465611a 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/sql-database/README.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/sql-database/README.md @@ -1,58 +1,58 @@ -# Azure SQL Database - -Managed relational database with ACID compliance and full SQL Server compatibility. - -## When to Use - -- Relational data with ACID requirements -- Complex queries and joins -- Existing SQL Server workloads -- Reporting and analytics -- Strong schema enforcement - -## Authentication - -**Default:** Entra-only authentication (recommended) -- Required for subscriptions with Entra-only policies -- More secure than SQL authentication -- Eliminates password management - -## Required Supporting Resources - -| Resource | Purpose | -|----------|---------| -| Key Vault | Store connection strings | -| Private Endpoint | Secure access (optional) | - -## SKU Selection - -| Tier | Use Case | Features | -|------|----------|----------| -| **Basic** | Dev/test, light workloads | 5 DTUs, 2GB | -| **Standard** | Production workloads | 10-3000 DTUs | -| **Premium** | High-performance | In-memory OLTP | -| **Serverless** | Variable workloads | Auto-pause, auto-scale | -| **Hyperscale** | Large databases | 100TB+, instant backup | - -## Environment Variables - -| Variable | Value | When to Set | -|----------|-------|-------------| -| `AZURE_PRINCIPAL_ID` | Current user's object ID | After `azd init`, before `azd provision` | -| `AZURE_PRINCIPAL_NAME` | Current user's display name | After `azd init`, before `azd provision` | -| `SQL_SERVER` | `{server}.database.windows.net` | Runtime (from Bicep outputs) | -| `SQL_DATABASE` | Database name | Runtime (from Bicep outputs) | -| `SQL_CONNECTION_STRING` | Full connection string (Key Vault) | Runtime (from Bicep outputs) | - -**Set principal variables:** -```bash -PRINCIPAL_INFO=$(az ad signed-in-user show --query "{id:id, name:displayName}" -o json) -azd env set AZURE_PRINCIPAL_ID $(echo $PRINCIPAL_INFO | jq -r '.id') -azd env set AZURE_PRINCIPAL_NAME $(echo $PRINCIPAL_INFO | jq -r '.name') -``` - -## References - -- [Bicep Patterns](bicep.md) -- [Entra ID Auth](auth.md) -- [SDK Patterns](sdk.md) +# Azure SQL Database + +Managed relational database with ACID compliance and full SQL Server compatibility. + +## When to Use + +- Relational data with ACID requirements +- Complex queries and joins +- Existing SQL Server workloads +- Reporting and analytics +- Strong schema enforcement + +## Authentication + +**Default:** Entra-only authentication (recommended) +- Required for subscriptions with Entra-only policies +- More secure than SQL authentication +- Eliminates password management + +## Required Supporting Resources + +| Resource | Purpose | +|----------|---------| +| Key Vault | Store connection strings | +| Private Endpoint | Secure access (optional) | + +## SKU Selection + +| Tier | Use Case | Features | +|------|----------|----------| +| **Basic** | Dev/test, light workloads | 5 DTUs, 2GB | +| **Standard** | Production workloads | 10-3000 DTUs | +| **Premium** | High-performance | In-memory OLTP | +| **Serverless** | Variable workloads | Auto-pause, auto-scale | +| **Hyperscale** | Large databases | 100TB+, instant backup | + +## Environment Variables + +| Variable | Value | When to Set | +|----------|-------|-------------| +| `AZURE_PRINCIPAL_ID` | Current user's object ID | After `azd init`, before `azd provision` | +| `AZURE_PRINCIPAL_NAME` | Current user's display name | After `azd init`, before `azd provision` | +| `SQL_SERVER` | `{server}.database.windows.net` | Runtime (from Bicep outputs) | +| `SQL_DATABASE` | Database name | Runtime (from Bicep outputs) | +| `SQL_CONNECTION_STRING` | Full connection string (Key Vault) | Runtime (from Bicep outputs) | + +**Set principal variables:** +```bash +PRINCIPAL_INFO=$(az ad signed-in-user show --query "{id:id, name:displayName}" -o json) +azd env set AZURE_PRINCIPAL_ID $(echo $PRINCIPAL_INFO | jq -r '.id') +azd env set AZURE_PRINCIPAL_NAME $(echo $PRINCIPAL_INFO | jq -r '.name') +``` + +## References + +- [Bicep Patterns](bicep.md) +- [Entra ID Auth](auth.md) +- [SDK Patterns](sdk.md) diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/sql-database/sdk.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/sql-database/sdk.md index 19bfb3d1..e51c1e73 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/sql-database/sdk.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/sql-database/sdk.md @@ -1,77 +1,77 @@ -# SQL Database - SDK Patterns - -## Node.js (mssql) - -```javascript -const sql = require('mssql'); - -const config = { - server: process.env.SQL_SERVER, - database: process.env.SQL_DATABASE, - authentication: { - type: 'azure-active-directory-default' - }, - options: { - encrypt: true - } -}; - -const pool = await sql.connect(config); -``` - -## Python (pyodbc) - -> **Auth:** `DefaultAzureCredential` is for local development. See [auth-best-practices.md](../../auth-best-practices.md) for production patterns. - -```python -import pyodbc -from azure.identity import DefaultAzureCredential - -credential = DefaultAzureCredential() -token = credential.get_token("https://database.windows.net/.default") - -conn = pyodbc.connect( - f"Driver={{ODBC Driver 18 for SQL Server}};" - f"Server={os.environ['SQL_SERVER']};" - f"Database={os.environ['SQL_DATABASE']};" - f"Authentication=ActiveDirectoryMsi" -) -``` - -## .NET (Entity Framework Core) - -**Required NuGet Packages:** -```bash -dotnet add package Microsoft.EntityFrameworkCore.SqlServer -dotnet add package Microsoft.Data.SqlClient --version 5.1.0 -dotnet add package Azure.Identity -``` - -**Connection string (Entra ID):** -``` -Server=tcp:{server}.database.windows.net,1433;Database={database};Authentication=Active Directory Default;Encrypt=True; -``` - -**Configuration:** -```csharp -services.AddDbContext(options => - options.UseSqlServer( - Configuration.GetConnectionString("DefaultConnection"), - sqlOptions => sqlOptions.EnableRetryOnFailure() - )); -``` - -**appsettings.json:** -```json -{ - "ConnectionStrings": { - "DefaultConnection": "Server=tcp:myserver.database.windows.net,1433;Database=mydb;Authentication=Active Directory Default;Encrypt=True;" - } -} -``` - -## Connection String Format - -``` -Server=tcp:{server}.database.windows.net,1433;Database={database};Authentication=Active Directory Default;Encrypt=True; -``` +# SQL Database - SDK Patterns + +## Node.js (mssql) + +```javascript +const sql = require('mssql'); + +const config = { + server: process.env.SQL_SERVER, + database: process.env.SQL_DATABASE, + authentication: { + type: 'azure-active-directory-default' + }, + options: { + encrypt: true + } +}; + +const pool = await sql.connect(config); +``` + +## Python (pyodbc) + +> **Auth:** `DefaultAzureCredential` is for local development. See [auth-best-practices.md](../../auth-best-practices.md) for production patterns. + +```python +import pyodbc +from azure.identity import DefaultAzureCredential + +credential = DefaultAzureCredential() +token = credential.get_token("https://database.windows.net/.default") + +conn = pyodbc.connect( + f"Driver={{ODBC Driver 18 for SQL Server}};" + f"Server={os.environ['SQL_SERVER']};" + f"Database={os.environ['SQL_DATABASE']};" + f"Authentication=ActiveDirectoryMsi" +) +``` + +## .NET (Entity Framework Core) + +**Required NuGet Packages:** +```bash +dotnet add package Microsoft.EntityFrameworkCore.SqlServer +dotnet add package Microsoft.Data.SqlClient --version 5.1.0 +dotnet add package Azure.Identity +``` + +**Connection string (Entra ID):** +``` +Server=tcp:{server}.database.windows.net,1433;Database={database};Authentication=Active Directory Default;Encrypt=True; +``` + +**Configuration:** +```csharp +services.AddDbContext(options => + options.UseSqlServer( + Configuration.GetConnectionString("DefaultConnection"), + sqlOptions => sqlOptions.EnableRetryOnFailure() + )); +``` + +**appsettings.json:** +```json +{ + "ConnectionStrings": { + "DefaultConnection": "Server=tcp:myserver.database.windows.net,1433;Database=mydb;Authentication=Active Directory Default;Encrypt=True;" + } +} +``` + +## Connection String Format + +``` +Server=tcp:{server}.database.windows.net,1433;Database={database};Authentication=Active Directory Default;Encrypt=True; +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/README.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/README.md index cc4518e7..20aa03c5 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/README.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/README.md @@ -1,64 +1,64 @@ -# Azure Static Web Apps - -Serverless hosting for static sites and SPAs with integrated APIs. - -## When to Use - -- Single Page Applications (React, Vue, Angular) -- Static sites (HTML/CSS/JS) -- JAMstack applications -- Sites with serverless API backends -- Documentation sites - -## Service Type in azure.yaml - -```yaml -services: - my-web: - host: staticwebapp - project: ./src/web -``` - -## Required Supporting Resources - -| Resource | Purpose | -|----------|---------| -| None required | Static Web Apps is fully managed | -| Application Insights | Monitoring (optional) | - -## SKU Selection - -| SKU | Features | -|-----|----------| -| Free | 2 custom domains, 0.5GB storage, shared bandwidth | -| Standard | 5 custom domains, 2GB storage, SLA, auth customization | - -## Build Configuration - -| Framework | outputLocation | -|-----------|----------------| -| React | `build` | -| Vue | `dist` | -| Angular | `dist/my-app` | -| Next.js (Static) | `out` | - -## API Integration - -Integrated Functions API structure: - -``` -project/ -├── src/ # Frontend -└── api/ # Azure Functions API - ├── hello/ - │ └── index.js - └── host.json -``` - -## References - -- [Region Availability](region-availability.md) -- [Bicep Patterns](bicep.md) -- [Terraform Patterns](terraform.md) -- [Routing and Auth](routing.md) -- [Deployment](deployment.md) +# Azure Static Web Apps + +Serverless hosting for static sites and SPAs with integrated APIs. + +## When to Use + +- Single Page Applications (React, Vue, Angular) +- Static sites (HTML/CSS/JS) +- JAMstack applications +- Sites with serverless API backends +- Documentation sites + +## Service Type in azure.yaml + +```yaml +services: + my-web: + host: staticwebapp + project: ./src/web +``` + +## Required Supporting Resources + +| Resource | Purpose | +|----------|---------| +| None required | Static Web Apps is fully managed | +| Application Insights | Monitoring (optional) | + +## SKU Selection + +| SKU | Features | +|-----|----------| +| Free | 2 custom domains, 0.5GB storage, shared bandwidth | +| Standard | 5 custom domains, 2GB storage, SLA, auth customization | + +## Build Configuration + +| Framework | outputLocation | +|-----------|----------------| +| React | `build` | +| Vue | `dist` | +| Angular | `dist/my-app` | +| Next.js (Static) | `out` | + +## API Integration + +Integrated Functions API structure: + +``` +project/ +├── src/ # Frontend +└── api/ # Azure Functions API + ├── hello/ + │ └── index.js + └── host.json +``` + +## References + +- [Region Availability](region-availability.md) +- [Bicep Patterns](bicep.md) +- [Terraform Patterns](terraform.md) +- [Routing and Auth](routing.md) +- [Deployment](deployment.md) diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/bicep.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/bicep.md index 171153d7..7593179f 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/bicep.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/bicep.md @@ -1,51 +1,51 @@ -# Static Web Apps - Bicep Patterns - -## Basic Resource - -```bicep -resource staticWebApp 'Microsoft.Web/staticSites@2022-09-01' = { - name: '${resourcePrefix}-${serviceName}-${uniqueHash}' - location: location - sku: { - name: 'Standard' - tier: 'Standard' - } - properties: { - buildProperties: { - appLocation: '/' - apiLocation: 'api' - outputLocation: 'dist' - } - } -} -``` - -## Custom Domain - -```bicep -resource customDomain 'Microsoft.Web/staticSites/customDomains@2022-09-01' = { - parent: staticWebApp - name: 'www.example.com' - properties: {} -} -``` - -## Application Settings - -For the integrated API: - -```bicep -resource staticWebAppSettings 'Microsoft.Web/staticSites/config@2022-09-01' = { - parent: staticWebApp - name: 'appsettings' - properties: { - DATABASE_URL: '@Microsoft.KeyVault(VaultName=${keyVault.name};SecretName=db-url)' - } -} -``` - -## Deployment Token - -> ⚠️ **Security Warning:** Do NOT expose deployment tokens in Bicep outputs. - -See [deployment.md](deployment.md) for secure token handling. +# Static Web Apps - Bicep Patterns + +## Basic Resource + +```bicep +resource staticWebApp 'Microsoft.Web/staticSites@2022-09-01' = { + name: '${resourcePrefix}-${serviceName}-${uniqueHash}' + location: location + sku: { + name: 'Standard' + tier: 'Standard' + } + properties: { + buildProperties: { + appLocation: '/' + apiLocation: 'api' + outputLocation: 'dist' + } + } +} +``` + +## Custom Domain + +```bicep +resource customDomain 'Microsoft.Web/staticSites/customDomains@2022-09-01' = { + parent: staticWebApp + name: 'www.example.com' + properties: {} +} +``` + +## Application Settings + +For the integrated API: + +```bicep +resource staticWebAppSettings 'Microsoft.Web/staticSites/config@2022-09-01' = { + parent: staticWebApp + name: 'appsettings' + properties: { + DATABASE_URL: '@Microsoft.KeyVault(VaultName=${keyVault.name};SecretName=db-url)' + } +} +``` + +## Deployment Token + +> ⚠️ **Security Warning:** Do NOT expose deployment tokens in Bicep outputs. + +See [deployment.md](deployment.md) for secure token handling. diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/deployment.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/deployment.md index 9f4dae02..0db32fa9 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/deployment.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/deployment.md @@ -1,68 +1,68 @@ -# Static Web Apps - Deployment - -## azd Deploy (Default) - -Standard deployment via Azure Developer CLI: - -```bash -azd deploy -``` - -## GitHub-Linked Deployments - -For CI/CD builds on Azure (instead of azd deploy): - -```bicep -properties: { - repositoryUrl: 'https://github.com/owner/repo' - branch: 'main' - buildProperties: { - appLocation: 'src' - apiLocation: 'api' - outputLocation: 'dist' - } -} -``` - -## Deployment Token - -> ⚠️ **Security Warning:** Do NOT expose deployment tokens in ARM/Bicep outputs. Deployment outputs are visible in Azure portal deployment history and logs. - -**Recommended approach** - retrieve token via Azure CLI and store directly in secret store: - -```bash -# Capture token to variable (never echo or log) -DEPLOYMENT_TOKEN=$(az staticwebapp secrets list --name --query "properties.apiKey" -o tsv) - -# Store directly in Key Vault -az keyvault secret set --vault-name --name swa-deployment-token --value "$DEPLOYMENT_TOKEN" --output none -``` - -**Do NOT do this** (exposes token in deployment history): -```bicep -// ❌ INSECURE - token visible in deployment history -// output deploymentToken string = staticWebApp.listSecrets().properties.apiKey -``` - -## Terraform Deployment - -> ⚠️ **Use `azurerm_static_web_app`** — NOT Storage Account `static_website`. -> Storage Account static websites require anonymous blob access, which is blocked -> by enterprise Azure Policies. See [terraform.md](terraform.md) for correct patterns. - -Terraform-based deployments use `azd deploy` the same way as Bicep: - -```bash -azd deploy -``` - -The `azd-service-name` tag on the `azurerm_static_web_app` resource tells azd where to deploy. - -**Do NOT do this** (exposes token in Terraform state): -```hcl -# ❌ INSECURE - token stored in Terraform state -# output "deployment_token" { -# value = azurerm_static_web_app.web.api_key -# sensitive = true -# } -``` +# Static Web Apps - Deployment + +## azd Deploy (Default) + +Standard deployment via Azure Developer CLI: + +```bash +azd deploy +``` + +## GitHub-Linked Deployments + +For CI/CD builds on Azure (instead of azd deploy): + +```bicep +properties: { + repositoryUrl: 'https://github.com/owner/repo' + branch: 'main' + buildProperties: { + appLocation: 'src' + apiLocation: 'api' + outputLocation: 'dist' + } +} +``` + +## Deployment Token + +> ⚠️ **Security Warning:** Do NOT expose deployment tokens in ARM/Bicep outputs. Deployment outputs are visible in Azure portal deployment history and logs. + +**Recommended approach** - retrieve token via Azure CLI and store directly in secret store: + +```bash +# Capture token to variable (never echo or log) +DEPLOYMENT_TOKEN=$(az staticwebapp secrets list --name --query "properties.apiKey" -o tsv) + +# Store directly in Key Vault +az keyvault secret set --vault-name --name swa-deployment-token --value "$DEPLOYMENT_TOKEN" --output none +``` + +**Do NOT do this** (exposes token in deployment history): +```bicep +// ❌ INSECURE - token visible in deployment history +// output deploymentToken string = staticWebApp.listSecrets().properties.apiKey +``` + +## Terraform Deployment + +> ⚠️ **Use `azurerm_static_web_app`** — NOT Storage Account `static_website`. +> Storage Account static websites require anonymous blob access, which is blocked +> by enterprise Azure Policies. See [terraform.md](terraform.md) for correct patterns. + +Terraform-based deployments use `azd deploy` the same way as Bicep: + +```bash +azd deploy +``` + +The `azd-service-name` tag on the `azurerm_static_web_app` resource tells azd where to deploy. + +**Do NOT do this** (exposes token in Terraform state): +```hcl +# ❌ INSECURE - token stored in Terraform state +# output "deployment_token" { +# value = azurerm_static_web_app.web.api_key +# sensitive = true +# } +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/region-availability.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/region-availability.md index 4c8aead4..d9d1905e 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/region-availability.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/region-availability.md @@ -1,21 +1,21 @@ -# SWA Region Availability - -⚠️ **NOT available in many common regions** — Check before deployment. - -| ✅ Available | ❌ NOT Available (will FAIL) | -|-------------|------------------------------| -| `westus2` | `eastus` | -| `centralus` | `northeurope` | -| `eastus2` | `southeastasia` | -| `westeurope` | `uksouth` | -| `eastasia` | `canadacentral` | -| | `australiaeast` | -| | `westus3` | - -## Recommended Regions - -| Pattern | Use | -|---------|-----| -| SWA only | `westus2`, `centralus`, `eastus2`, `westeurope`, `eastasia` | -| SWA + backend | `westus2`, `centralus`, `eastus2`, `westeurope`, `eastasia` | -| SWA + Azure OpenAI | `eastus2` (only region with full overlap) | +# SWA Region Availability + +⚠️ **NOT available in many common regions** — Check before deployment. + +| ✅ Available | ❌ NOT Available (will FAIL) | +|-------------|------------------------------| +| `westus2` | `eastus` | +| `centralus` | `northeurope` | +| `eastus2` | `southeastasia` | +| `westeurope` | `uksouth` | +| `eastasia` | `canadacentral` | +| | `australiaeast` | +| | `westus3` | + +## Recommended Regions + +| Pattern | Use | +|---------|-----| +| SWA only | `westus2`, `centralus`, `eastus2`, `westeurope`, `eastasia` | +| SWA + backend | `westus2`, `centralus`, `eastus2`, `westeurope`, `eastasia` | +| SWA + Azure OpenAI | `eastus2` (only region with full overlap) | diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/routing.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/routing.md index 475c8fcd..1c2aa1a4 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/routing.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/static-web-apps/routing.md @@ -1,70 +1,70 @@ -# Static Web Apps - Routing & Authentication - -## Route Configuration - -Create `staticwebapp.config.json` in the app root: - -```json -{ - "routes": [ - { - "route": "/api/*", - "allowedRoles": ["authenticated"] - } - ], - "navigationFallback": { - "rewrite": "/index.html", - "exclude": ["/api/*", "/*.{png,jpg,gif}"] - }, - "responseOverrides": { - "404": { - "rewrite": "/404.html" - } - } -} -``` - -## Authentication - -### Built-in Providers - -```json -{ - "routes": [ - { - "route": "/admin/*", - "allowedRoles": ["admin"] - } - ], - "auth": { - "identityProviders": { - "azureActiveDirectory": { - "registration": { - "openIdIssuer": "https://login.microsoftonline.com/{tenant-id}", - "clientIdSettingName": "AAD_CLIENT_ID", - "clientSecretSettingName": "AAD_CLIENT_SECRET" - } - } - } - } -} -``` - -### Supported Providers - -- Azure Active Directory / Entra ID -- GitHub -- Twitter -- Custom OpenID Connect - -## Role-Based Access - -```json -{ - "routes": [ - { "route": "/admin/*", "allowedRoles": ["admin"] }, - { "route": "/account/*", "allowedRoles": ["authenticated"] }, - { "route": "/*", "allowedRoles": ["anonymous"] } - ] -} -``` +# Static Web Apps - Routing & Authentication + +## Route Configuration + +Create `staticwebapp.config.json` in the app root: + +```json +{ + "routes": [ + { + "route": "/api/*", + "allowedRoles": ["authenticated"] + } + ], + "navigationFallback": { + "rewrite": "/index.html", + "exclude": ["/api/*", "/*.{png,jpg,gif}"] + }, + "responseOverrides": { + "404": { + "rewrite": "/404.html" + } + } +} +``` + +## Authentication + +### Built-in Providers + +```json +{ + "routes": [ + { + "route": "/admin/*", + "allowedRoles": ["admin"] + } + ], + "auth": { + "identityProviders": { + "azureActiveDirectory": { + "registration": { + "openIdIssuer": "https://login.microsoftonline.com/{tenant-id}", + "clientIdSettingName": "AAD_CLIENT_ID", + "clientSecretSettingName": "AAD_CLIENT_SECRET" + } + } + } + } +} +``` + +### Supported Providers + +- Azure Active Directory / Entra ID +- GitHub +- Twitter +- Custom OpenID Connect + +## Role-Based Access + +```json +{ + "routes": [ + { "route": "/admin/*", "allowedRoles": ["admin"] }, + { "route": "/account/*", "allowedRoles": ["authenticated"] }, + { "route": "/*", "allowedRoles": ["anonymous"] } + ] +} +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/storage/README.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/storage/README.md index 78a91edc..0193eea5 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/storage/README.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/storage/README.md @@ -1,58 +1,58 @@ -# Azure Storage - -Scalable cloud storage for blobs, files, queues, and tables. - -## When to Use - -- Blob storage (files, images, videos) -- File shares (SMB/NFS) -- Queue storage (simple messaging) -- Table storage (NoSQL key-value) -- Static website hosting - -## Required Supporting Resources - -| Resource | Purpose | -|----------|---------| -| None required | Storage is self-contained | -| Key Vault | Store connection strings | -| Private Endpoint | Secure access (optional) | - -## SKU Selection - -| SKU | Replication | Use Case | -|-----|-------------|----------| -| Standard_LRS | Local (3 copies) | Dev/test, non-critical | -| Standard_ZRS | Zone-redundant | Production, regional HA | -| Standard_GRS | Geo-redundant | DR requirements | -| Premium_LRS | Premium SSD | High performance | - -## Storage Types - -| Type | Best For | -|------|----------| -| Blob | Files, images, videos, backups, logs | -| Queue | Simple message queuing, decoupling | -| Table | NoSQL key-value data | -| File Share | Lift-and-shift, SMB/NFS access | - -## Access Tiers - -| Tier | Use Case | -|------|----------| -| Hot | Frequent access | -| Cool | Infrequent access (30+ days) | -| Archive | Rare access (180+ days) | - -## Environment Variables - -| Variable | Value | -|----------|-------| -| `AZURE_STORAGE_CONNECTION_STRING` | Connection string (Key Vault) | -| `AZURE_STORAGE_ACCOUNT` | Account name | -| `AZURE_STORAGE_CONTAINER` | Container name | - -## References - -- [Bicep Patterns](bicep.md) -- [Access Patterns](access.md) +# Azure Storage + +Scalable cloud storage for blobs, files, queues, and tables. + +## When to Use + +- Blob storage (files, images, videos) +- File shares (SMB/NFS) +- Queue storage (simple messaging) +- Table storage (NoSQL key-value) +- Static website hosting + +## Required Supporting Resources + +| Resource | Purpose | +|----------|---------| +| None required | Storage is self-contained | +| Key Vault | Store connection strings | +| Private Endpoint | Secure access (optional) | + +## SKU Selection + +| SKU | Replication | Use Case | +|-----|-------------|----------| +| Standard_LRS | Local (3 copies) | Dev/test, non-critical | +| Standard_ZRS | Zone-redundant | Production, regional HA | +| Standard_GRS | Geo-redundant | DR requirements | +| Premium_LRS | Premium SSD | High performance | + +## Storage Types + +| Type | Best For | +|------|----------| +| Blob | Files, images, videos, backups, logs | +| Queue | Simple message queuing, decoupling | +| Table | NoSQL key-value data | +| File Share | Lift-and-shift, SMB/NFS access | + +## Access Tiers + +| Tier | Use Case | +|------|----------| +| Hot | Frequent access | +| Cool | Infrequent access (30+ days) | +| Archive | Rare access (180+ days) | + +## Environment Variables + +| Variable | Value | +|----------|-------| +| `AZURE_STORAGE_CONNECTION_STRING` | Connection string (Key Vault) | +| `AZURE_STORAGE_ACCOUNT` | Account name | +| `AZURE_STORAGE_CONTAINER` | Container name | + +## References + +- [Bicep Patterns](bicep.md) +- [Access Patterns](access.md) diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/storage/access.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/storage/access.md index 3a133248..fdce4f12 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/storage/access.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/storage/access.md @@ -1,96 +1,96 @@ -# Storage - Access Patterns - -## Prerequisites for Granting Storage Access - -> ⚠️ **Important**: To assign storage roles to managed identities, you need: -> - **User Access Administrator** or **Owner** role on the Storage Account (or parent resource group/subscription) -> - The role must include the `Microsoft.Authorization/roleAssignments/write` permission - -**Common scenarios**: -- Granting Storage Blob Data Owner to a Web App or Function App's managed identity (System Assigned or User Assigned) -- Adding read/write access to blobs, queues, and tables for application workloads -- Allowing user identities (developers, data admins) access in dev/test environments -- Allowing applications to access storage using managed identity instead of connection strings - -**Scope best practices**: -- Grant roles at the **smallest scope possible** (e.g., specific storage account, not resource group or subscription) -- Avoid broad scopes (Resource Group, Subscription, Tenant) unless absolutely necessary -- Prefer resource-level assignments for production workloads - -**Managed identity types**: -- **System Assigned**: Automatically created with the resource (Web App, Function). Default when using `DefaultAzureCredential`. -- **User Assigned**: Standalone identity that can be shared across resources. Requires additional configuration: - - Set `AZURE_CLIENT_ID` app setting to the User Assigned Managed Identity's client ID - - Configure identity in Bicep with both `type: 'SystemAssigned, UserAssigned'` and `userAssignedIdentities` - -If you encounter `AuthorizationFailed` errors when assigning roles, ensure you have User Access Administrator or Owner permissions at the target scope. - -## Managed Identity Role Assignment - -```bicep -resource storageRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(storageAccount.id, principalId, 'Storage Blob Data Contributor') - scope: storageAccount - properties: { - roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe') - principalId: principalId - principalType: 'ServicePrincipal' - } -} -``` - -## Storage Roles - -| Role | Permissions | -|------|-------------| -| Storage Blob Data Reader | Read blobs | -| Storage Blob Data Contributor | Read/write blobs | -| Storage Queue Data Contributor | Read/write queues | -| Storage Table Data Contributor | Read/write tables | - -## SDK Connection Patterns - -### Node.js - -```javascript -const { BlobServiceClient } = require("@azure/storage-blob"); - -const blobServiceClient = BlobServiceClient.fromConnectionString( - process.env.AZURE_STORAGE_CONNECTION_STRING -); -const containerClient = blobServiceClient.getContainerClient("uploads"); -``` - -### Python - -```python -from azure.storage.blob import BlobServiceClient - -blob_service_client = BlobServiceClient.from_connection_string( - os.environ["AZURE_STORAGE_CONNECTION_STRING"] -) -container_client = blob_service_client.get_container_client("uploads") -``` - -### .NET - -```csharp -var blobServiceClient = new BlobServiceClient( - Environment.GetEnvironmentVariable("AZURE_STORAGE_CONNECTION_STRING") -); -var containerClient = blobServiceClient.GetBlobContainerClient("uploads"); -``` - -## Managed Identity Access - -Use `DefaultAzureCredential` for local development (in production, use `ManagedIdentityCredential` — see [auth-best-practices.md](../../auth-best-practices.md)): - -```javascript -const { DefaultAzureCredential } = require("@azure/identity"); -const { BlobServiceClient } = require("@azure/storage-blob"); - -const client = new BlobServiceClient( - `https://${accountName}.blob.core.windows.net`, - new DefaultAzureCredential() -); -``` +# Storage - Access Patterns + +## Prerequisites for Granting Storage Access + +> ⚠️ **Important**: To assign storage roles to managed identities, you need: +> - **User Access Administrator** or **Owner** role on the Storage Account (or parent resource group/subscription) +> - The role must include the `Microsoft.Authorization/roleAssignments/write` permission + +**Common scenarios**: +- Granting Storage Blob Data Owner to a Web App or Function App's managed identity (System Assigned or User Assigned) +- Adding read/write access to blobs, queues, and tables for application workloads +- Allowing user identities (developers, data admins) access in dev/test environments +- Allowing applications to access storage using managed identity instead of connection strings + +**Scope best practices**: +- Grant roles at the **smallest scope possible** (e.g., specific storage account, not resource group or subscription) +- Avoid broad scopes (Resource Group, Subscription, Tenant) unless absolutely necessary +- Prefer resource-level assignments for production workloads + +**Managed identity types**: +- **System Assigned**: Automatically created with the resource (Web App, Function). Default when using `DefaultAzureCredential`. +- **User Assigned**: Standalone identity that can be shared across resources. Requires additional configuration: + - Set `AZURE_CLIENT_ID` app setting to the User Assigned Managed Identity's client ID + - Configure identity in Bicep with both `type: 'SystemAssigned, UserAssigned'` and `userAssignedIdentities` + +If you encounter `AuthorizationFailed` errors when assigning roles, ensure you have User Access Administrator or Owner permissions at the target scope. + +## Managed Identity Role Assignment + +```bicep +resource storageRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(storageAccount.id, principalId, 'Storage Blob Data Contributor') + scope: storageAccount + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe') + principalId: principalId + principalType: 'ServicePrincipal' + } +} +``` + +## Storage Roles + +| Role | Permissions | +|------|-------------| +| Storage Blob Data Reader | Read blobs | +| Storage Blob Data Contributor | Read/write blobs | +| Storage Queue Data Contributor | Read/write queues | +| Storage Table Data Contributor | Read/write tables | + +## SDK Connection Patterns + +### Node.js + +```javascript +const { BlobServiceClient } = require("@azure/storage-blob"); + +const blobServiceClient = BlobServiceClient.fromConnectionString( + process.env.AZURE_STORAGE_CONNECTION_STRING +); +const containerClient = blobServiceClient.getContainerClient("uploads"); +``` + +### Python + +```python +from azure.storage.blob import BlobServiceClient + +blob_service_client = BlobServiceClient.from_connection_string( + os.environ["AZURE_STORAGE_CONNECTION_STRING"] +) +container_client = blob_service_client.get_container_client("uploads") +``` + +### .NET + +```csharp +var blobServiceClient = new BlobServiceClient( + Environment.GetEnvironmentVariable("AZURE_STORAGE_CONNECTION_STRING") +); +var containerClient = blobServiceClient.GetBlobContainerClient("uploads"); +``` + +## Managed Identity Access + +Use `DefaultAzureCredential` for local development (in production, use `ManagedIdentityCredential` — see [auth-best-practices.md](../../auth-best-practices.md)): + +```javascript +const { DefaultAzureCredential } = require("@azure/identity"); +const { BlobServiceClient } = require("@azure/storage-blob"); + +const client = new BlobServiceClient( + `https://${accountName}.blob.core.windows.net`, + new DefaultAzureCredential() +); +``` diff --git a/.github/plugins/azure-skills/skills/azure-prepare/references/services/storage/bicep.md b/.github/plugins/azure-skills/skills/azure-prepare/references/services/storage/bicep.md index 5372c6be..767bc6de 100644 --- a/.github/plugins/azure-skills/skills/azure-prepare/references/services/storage/bicep.md +++ b/.github/plugins/azure-skills/skills/azure-prepare/references/services/storage/bicep.md @@ -1,88 +1,88 @@ -# Storage - Bicep Patterns - -## Storage Account - -```bicep -resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { - name: '${resourcePrefix}stor${uniqueHash}' - location: location - sku: { - name: 'Standard_LRS' - } - kind: 'StorageV2' - properties: { - accessTier: 'Hot' - allowBlobPublicAccess: false - minimumTlsVersion: 'TLS1_2' - supportsHttpsTrafficOnly: true - } -} -``` - -## Blob Container - -```bicep -resource blobService 'Microsoft.Storage/storageAccounts/blobServices@2023-01-01' = { - parent: storageAccount - name: 'default' - properties: { - deleteRetentionPolicy: { - enabled: true - days: 7 - } - } -} - -resource container 'Microsoft.Storage/storageAccounts/blobServices/containers@2023-01-01' = { - parent: blobService - name: 'uploads' - properties: { - publicAccess: 'None' - } -} -``` - -## Queue - -```bicep -resource queueService 'Microsoft.Storage/storageAccounts/queueServices@2023-01-01' = { - parent: storageAccount - name: 'default' -} - -resource queue 'Microsoft.Storage/storageAccounts/queueServices/queues@2023-01-01' = { - parent: queueService - name: 'orders' -} -``` - -## Table - -```bicep -resource tableService 'Microsoft.Storage/storageAccounts/tableServices@2023-01-01' = { - parent: storageAccount - name: 'default' -} - -resource table 'Microsoft.Storage/storageAccounts/tableServices/tables@2023-01-01' = { - parent: tableService - name: 'logs' -} -``` - -## File Share - -```bicep -resource fileService 'Microsoft.Storage/storageAccounts/fileServices@2023-01-01' = { - parent: storageAccount - name: 'default' -} - -resource fileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2023-01-01' = { - parent: fileService - name: 'shared' - properties: { - shareQuota: 100 // GB - } -} -``` +# Storage - Bicep Patterns + +## Storage Account + +```bicep +resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' = { + name: '${resourcePrefix}stor${uniqueHash}' + location: location + sku: { + name: 'Standard_LRS' + } + kind: 'StorageV2' + properties: { + accessTier: 'Hot' + allowBlobPublicAccess: false + minimumTlsVersion: 'TLS1_2' + supportsHttpsTrafficOnly: true + } +} +``` + +## Blob Container + +```bicep +resource blobService 'Microsoft.Storage/storageAccounts/blobServices@2023-01-01' = { + parent: storageAccount + name: 'default' + properties: { + deleteRetentionPolicy: { + enabled: true + days: 7 + } + } +} + +resource container 'Microsoft.Storage/storageAccounts/blobServices/containers@2023-01-01' = { + parent: blobService + name: 'uploads' + properties: { + publicAccess: 'None' + } +} +``` + +## Queue + +```bicep +resource queueService 'Microsoft.Storage/storageAccounts/queueServices@2023-01-01' = { + parent: storageAccount + name: 'default' +} + +resource queue 'Microsoft.Storage/storageAccounts/queueServices/queues@2023-01-01' = { + parent: queueService + name: 'orders' +} +``` + +## Table + +```bicep +resource tableService 'Microsoft.Storage/storageAccounts/tableServices@2023-01-01' = { + parent: storageAccount + name: 'default' +} + +resource table 'Microsoft.Storage/storageAccounts/tableServices/tables@2023-01-01' = { + parent: tableService + name: 'logs' +} +``` + +## File Share + +```bicep +resource fileService 'Microsoft.Storage/storageAccounts/fileServices@2023-01-01' = { + parent: storageAccount + name: 'default' +} + +resource fileShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2023-01-01' = { + parent: fileService + name: 'shared' + properties: { + shareQuota: 100 // GB + } +} +``` diff --git a/.github/plugins/azure-skills/skills/azure-resource-lookup/SKILL.md b/.github/plugins/azure-skills/skills/azure-resource-lookup/SKILL.md index ff5c2c98..896c1114 100644 --- a/.github/plugins/azure-skills/skills/azure-resource-lookup/SKILL.md +++ b/.github/plugins/azure-skills/skills/azure-resource-lookup/SKILL.md @@ -4,7 +4,7 @@ description: "List, find, and show Azure resources across subscriptions or resou license: MIT metadata: author: Microsoft - version: "1.2.2" + version: "1.2.3" --- # Azure Resource Lookup diff --git a/.github/plugins/azure-skills/skills/azure-resource-lookup/references/azure-resource-graph.md b/.github/plugins/azure-skills/skills/azure-resource-lookup/references/azure-resource-graph.md index 459b0775..e861de3c 100644 --- a/.github/plugins/azure-skills/skills/azure-resource-lookup/references/azure-resource-graph.md +++ b/.github/plugins/azure-skills/skills/azure-resource-lookup/references/azure-resource-graph.md @@ -1,184 +1,184 @@ -# Azure Resource Graph Query Patterns - -Azure Resource Graph (ARG) queries use a KQL subset against indexed Azure resource metadata. Results are near real-time across all subscriptions. - -## Command Format - -```bash -az graph query -q "" --query "data[].{col1:field1, col2:field2}" -o table -``` - -| Flag | Purpose | -|------|---------| -| `-q` | KQL query string | -| `--query` | JMESPath to shape output columns | -| `--first N` | Limit to N results | -| `--subscriptions` | Scope to specific subscription IDs | -| `-o table` | Table output (also: json, tsv) | - -## Key Tables - -| Table | Contents | -|-------|----------| -| `Resources` | All ARM resources — name, type, location, properties, tags, sku | -| `ResourceContainers` | Subscriptions, resource groups, management groups | -| `HealthResources` | Resource health availability status | -| `ServiceHealthResources` | Azure service health events/incidents | -| `AuthorizationResources` | Role assignments and definitions | -| `AdvisorResources` | Azure Advisor recommendations | - -## KQL Essentials - -- `=~` case-insensitive equals (use for `type` field — types are lowercase) -- `properties.fieldName` navigates the properties JSON bag -- `mv-expand` flattens arrays (subnets, IP configs) -- `isempty()` / `isnotnull()` checks for null/empty fields -- `tostring()` converts dynamic fields for display - ---- - -## Resource Inventory Patterns - -**Count all resources by type:** -```kql -Resources | summarize count() by type | order by count_ desc -``` - -**Inventory by type and location:** -```kql -Resources | summarize count() by type, location | order by type asc -``` - -**Cross-subscription inventory with subscription names:** -```kql -Resources -| join kind=leftouter ( - ResourceContainers - | where type == 'microsoft.resources/subscriptions' - | project subscriptionId, subscriptionName=name -) on subscriptionId -| summarize count() by subscriptionName, type -| order by subscriptionName asc, count_ desc -``` - -**All resources in a resource group:** -```kql -Resources -| where resourceGroup =~ '' -| project name, type, location, sku.name, kind -``` - -## Orphaned Resource Patterns - -**Unattached managed disks:** -```kql -Resources -| where type =~ 'microsoft.compute/disks' -| where isempty(managedBy) -| project name, resourceGroup, location, diskSizeGb=properties.diskSizeGB, sku=sku.name -``` - -**Unused public IP addresses:** -```kql -Resources -| where type =~ 'microsoft.network/publicipaddresses' -| where isempty(properties.ipConfiguration) -| project name, resourceGroup, location, sku=sku.name -``` - -**Orphaned network interfaces:** -```kql -Resources -| where type =~ 'microsoft.network/networkinterfaces' -| where isempty(properties.virtualMachine) -| project name, resourceGroup, location -``` - -**Idle load balancers (no backends):** -```kql -Resources -| where type =~ 'microsoft.network/loadbalancers' -| where array_length(properties.backendAddressPools) == 0 -| project name, resourceGroup, location -``` - -## Tag & Compliance Patterns - -**Resources missing a required tag:** -```kql -Resources -| where isnull(tags['Environment']) or isnull(tags['CostCenter']) -| project name, type, resourceGroup, tags -``` - -**Tag coverage analysis by type:** -```kql -Resources -| extend hasTag = isnotnull(tags['Environment']) -| summarize total=count(), tagged=countif(hasTag) by type -| extend coverage=round(100.0 * tagged / total, 1) -| order by coverage asc -``` - -**Resources with public network access:** -```kql -Resources -| where properties.publicNetworkAccess =~ 'Enabled' -| project name, type, resourceGroup, location -``` - -## Health & Diagnostics Patterns - -**Resource health status:** -```kql -HealthResources -| where type =~ 'microsoft.resourcehealth/availabilitystatuses' -| where properties.availabilityState != 'Available' -| project name, state=properties.availabilityState, reason=properties.reasonType -``` - -**Active service health incidents:** -```kql -ServiceHealthResources -| where type =~ 'microsoft.resourcehealth/events' -| where properties.Status == 'Active' -| project name, title=properties.Title, status=properties.Status -``` - -**Failed provisioning states:** -```kql -Resources -| where properties.provisioningState != 'Succeeded' -| project name, type, resourceGroup, state=properties.provisioningState -``` - -## Service-Specific Patterns - -**App Services and their plans:** -```kql -Resources -| where type =~ 'microsoft.web/sites' -| project name, kind, location, plan=properties.serverFarmId, state=properties.state, resourceGroup -``` - -**Container Apps:** -```kql -Resources -| where type =~ 'microsoft.app/containerapps' -| project name, location, provisioningState=properties.provisioningState, resourceGroup -``` - -**VNet and subnet discovery:** -```kql -Resources -| where type =~ 'microsoft.network/virtualnetworks' -| mv-expand subnet=properties.subnets -| project vnetName=name, subnetName=subnet.name, prefix=subnet.properties.addressPrefix -``` - -**Advisor cost recommendations:** -```kql -AdvisorResources -| where properties.category == 'Cost' -| project name, impact=properties.impact, solution=properties.shortDescription.solution -``` +# Azure Resource Graph Query Patterns + +Azure Resource Graph (ARG) queries use a KQL subset against indexed Azure resource metadata. Results are near real-time across all subscriptions. + +## Command Format + +```bash +az graph query -q "" --query "data[].{col1:field1, col2:field2}" -o table +``` + +| Flag | Purpose | +|------|---------| +| `-q` | KQL query string | +| `--query` | JMESPath to shape output columns | +| `--first N` | Limit to N results | +| `--subscriptions` | Scope to specific subscription IDs | +| `-o table` | Table output (also: json, tsv) | + +## Key Tables + +| Table | Contents | +|-------|----------| +| `Resources` | All ARM resources — name, type, location, properties, tags, sku | +| `ResourceContainers` | Subscriptions, resource groups, management groups | +| `HealthResources` | Resource health availability status | +| `ServiceHealthResources` | Azure service health events/incidents | +| `AuthorizationResources` | Role assignments and definitions | +| `AdvisorResources` | Azure Advisor recommendations | + +## KQL Essentials + +- `=~` case-insensitive equals (use for `type` field — types are lowercase) +- `properties.fieldName` navigates the properties JSON bag +- `mv-expand` flattens arrays (subnets, IP configs) +- `isempty()` / `isnotnull()` checks for null/empty fields +- `tostring()` converts dynamic fields for display + +--- + +## Resource Inventory Patterns + +**Count all resources by type:** +```kql +Resources | summarize count() by type | order by count_ desc +``` + +**Inventory by type and location:** +```kql +Resources | summarize count() by type, location | order by type asc +``` + +**Cross-subscription inventory with subscription names:** +```kql +Resources +| join kind=leftouter ( + ResourceContainers + | where type == 'microsoft.resources/subscriptions' + | project subscriptionId, subscriptionName=name +) on subscriptionId +| summarize count() by subscriptionName, type +| order by subscriptionName asc, count_ desc +``` + +**All resources in a resource group:** +```kql +Resources +| where resourceGroup =~ '' +| project name, type, location, sku.name, kind +``` + +## Orphaned Resource Patterns + +**Unattached managed disks:** +```kql +Resources +| where type =~ 'microsoft.compute/disks' +| where isempty(managedBy) +| project name, resourceGroup, location, diskSizeGb=properties.diskSizeGB, sku=sku.name +``` + +**Unused public IP addresses:** +```kql +Resources +| where type =~ 'microsoft.network/publicipaddresses' +| where isempty(properties.ipConfiguration) +| project name, resourceGroup, location, sku=sku.name +``` + +**Orphaned network interfaces:** +```kql +Resources +| where type =~ 'microsoft.network/networkinterfaces' +| where isempty(properties.virtualMachine) +| project name, resourceGroup, location +``` + +**Idle load balancers (no backends):** +```kql +Resources +| where type =~ 'microsoft.network/loadbalancers' +| where array_length(properties.backendAddressPools) == 0 +| project name, resourceGroup, location +``` + +## Tag & Compliance Patterns + +**Resources missing a required tag:** +```kql +Resources +| where isnull(tags['Environment']) or isnull(tags['CostCenter']) +| project name, type, resourceGroup, tags +``` + +**Tag coverage analysis by type:** +```kql +Resources +| extend hasTag = isnotnull(tags['Environment']) +| summarize total=count(), tagged=countif(hasTag) by type +| extend coverage=round(100.0 * tagged / total, 1) +| order by coverage asc +``` + +**Resources with public network access:** +```kql +Resources +| where properties.publicNetworkAccess =~ 'Enabled' +| project name, type, resourceGroup, location +``` + +## Health & Diagnostics Patterns + +**Resource health status:** +```kql +HealthResources +| where type =~ 'microsoft.resourcehealth/availabilitystatuses' +| where properties.availabilityState != 'Available' +| project name, state=properties.availabilityState, reason=properties.reasonType +``` + +**Active service health incidents:** +```kql +ServiceHealthResources +| where type =~ 'microsoft.resourcehealth/events' +| where properties.Status == 'Active' +| project name, title=properties.Title, status=properties.Status +``` + +**Failed provisioning states:** +```kql +Resources +| where properties.provisioningState != 'Succeeded' +| project name, type, resourceGroup, state=properties.provisioningState +``` + +## Service-Specific Patterns + +**App Services and their plans:** +```kql +Resources +| where type =~ 'microsoft.web/sites' +| project name, kind, location, plan=properties.serverFarmId, state=properties.state, resourceGroup +``` + +**Container Apps:** +```kql +Resources +| where type =~ 'microsoft.app/containerapps' +| project name, location, provisioningState=properties.provisioningState, resourceGroup +``` + +**VNet and subnet discovery:** +```kql +Resources +| where type =~ 'microsoft.network/virtualnetworks' +| mv-expand subnet=properties.subnets +| project vnetName=name, subnetName=subnet.name, prefix=subnet.properties.addressPrefix +``` + +**Advisor cost recommendations:** +```kql +AdvisorResources +| where properties.category == 'Cost' +| project name, impact=properties.impact, solution=properties.shortDescription.solution +``` diff --git a/.github/plugins/azure-skills/skills/azure-resource-visualizer/SKILL.md b/.github/plugins/azure-skills/skills/azure-resource-visualizer/SKILL.md index 7f9e2226..24aa2ca9 100644 --- a/.github/plugins/azure-skills/skills/azure-resource-visualizer/SKILL.md +++ b/.github/plugins/azure-skills/skills/azure-resource-visualizer/SKILL.md @@ -4,7 +4,7 @@ description: "Analyze Azure resource groups and generate detailed Mermaid archit license: MIT metadata: author: Microsoft - version: "1.2.2" + version: "1.2.3" --- # Azure Resource Visualizer - Architecture Diagram Generator diff --git a/.github/plugins/azure-skills/skills/azure-resource-visualizer/references/azure-resource-graph.md b/.github/plugins/azure-skills/skills/azure-resource-visualizer/references/azure-resource-graph.md index 5128fd89..4ab7a3d9 100644 --- a/.github/plugins/azure-skills/skills/azure-resource-visualizer/references/azure-resource-graph.md +++ b/.github/plugins/azure-skills/skills/azure-resource-visualizer/references/azure-resource-graph.md @@ -1,87 +1,87 @@ -# Azure Resource Graph Queries for Resource Discovery - -Azure Resource Graph (ARG) enables fast, cross-subscription resource querying using KQL via `az graph query`. Use it for bulk resource discovery and relationship mapping. - -## How to Query - -Use the `extension_cli_generate` MCP tool to generate `az graph query` commands: - -```yaml -mcp_azure_mcp_extension_cli_generate - intent: "query Azure Resource Graph to " - cli-type: "az" -``` - -Or construct directly: - -```bash -az graph query -q "" --query "data[].{name:name, type:type}" -o table -``` - -> ⚠️ **Prerequisite:** `az extension add --name resource-graph` - -## Key Tables - -| Table | Contains | -|-------|----------| -| `Resources` | All ARM resource instances (name, type, location, properties, tags) | -| `ResourceContainers` | Subscriptions, resource groups, management groups | - -## Resource Discovery Patterns - -**List all resources by type:** - -```kql -Resources | summarize count() by type | order by count_ desc -``` - -**Inventory by location and type:** - -```kql -Resources | summarize count() by type, location | order by type asc -``` - -**List all resources in a resource group with details:** - -```kql -Resources -| where resourceGroup =~ '' -| project name, type, location, sku.name, kind -``` - -**Cross-subscription resource inventory with subscription names:** - -```kql -Resources -| join kind=leftouter ( - ResourceContainers - | where type == 'microsoft.resources/subscriptions' - | project subscriptionId, subscriptionName=name -) on subscriptionId -| project name, type, location, resourceGroup, subscriptionName -``` - -**Discover network relationships (VNets, subnets, NICs):** - -```kql -Resources -| where type =~ 'microsoft.network/virtualnetworks' -| mv-expand subnet=properties.subnets -| project vnetName=name, subnetName=subnet.name, addressPrefix=subnet.properties.addressPrefix, resourceGroup -``` - -**Find App Services and their plans:** - -```kql -Resources -| where type =~ 'microsoft.web/sites' -| project name, kind, location, serverFarmId=properties.serverFarmId, resourceGroup -``` - -## Tips - -- Use `=~` for case-insensitive type matching (resource types are lowercase) -- Navigate properties with `properties.fieldName` -- Use `--first N` to limit result count -- Use `--subscriptions` to scope to specific subscriptions -- Use `mv-expand` to flatten arrays (e.g., subnets, IP configurations) +# Azure Resource Graph Queries for Resource Discovery + +Azure Resource Graph (ARG) enables fast, cross-subscription resource querying using KQL via `az graph query`. Use it for bulk resource discovery and relationship mapping. + +## How to Query + +Use the `extension_cli_generate` MCP tool to generate `az graph query` commands: + +```yaml +mcp_azure_mcp_extension_cli_generate + intent: "query Azure Resource Graph to " + cli-type: "az" +``` + +Or construct directly: + +```bash +az graph query -q "" --query "data[].{name:name, type:type}" -o table +``` + +> ⚠️ **Prerequisite:** `az extension add --name resource-graph` + +## Key Tables + +| Table | Contains | +|-------|----------| +| `Resources` | All ARM resource instances (name, type, location, properties, tags) | +| `ResourceContainers` | Subscriptions, resource groups, management groups | + +## Resource Discovery Patterns + +**List all resources by type:** + +```kql +Resources | summarize count() by type | order by count_ desc +``` + +**Inventory by location and type:** + +```kql +Resources | summarize count() by type, location | order by type asc +``` + +**List all resources in a resource group with details:** + +```kql +Resources +| where resourceGroup =~ '' +| project name, type, location, sku.name, kind +``` + +**Cross-subscription resource inventory with subscription names:** + +```kql +Resources +| join kind=leftouter ( + ResourceContainers + | where type == 'microsoft.resources/subscriptions' + | project subscriptionId, subscriptionName=name +) on subscriptionId +| project name, type, location, resourceGroup, subscriptionName +``` + +**Discover network relationships (VNets, subnets, NICs):** + +```kql +Resources +| where type =~ 'microsoft.network/virtualnetworks' +| mv-expand subnet=properties.subnets +| project vnetName=name, subnetName=subnet.name, addressPrefix=subnet.properties.addressPrefix, resourceGroup +``` + +**Find App Services and their plans:** + +```kql +Resources +| where type =~ 'microsoft.web/sites' +| project name, kind, location, serverFarmId=properties.serverFarmId, resourceGroup +``` + +## Tips + +- Use `=~` for case-insensitive type matching (resource types are lowercase) +- Navigate properties with `properties.fieldName` +- Use `--first N` to limit result count +- Use `--subscriptions` to scope to specific subscriptions +- Use `mv-expand` to flatten arrays (e.g., subnets, IP configurations) diff --git a/.github/plugins/azure-skills/skills/azure-validate/SKILL.md b/.github/plugins/azure-skills/skills/azure-validate/SKILL.md index 92f74dcc..a90886aa 100644 --- a/.github/plugins/azure-skills/skills/azure-validate/SKILL.md +++ b/.github/plugins/azure-skills/skills/azure-validate/SKILL.md @@ -1,67 +1,67 @@ ---- -name: azure-validate -description: "Pre-deployment validation for Azure readiness. Run deep checks on configuration, infrastructure (Bicep or Terraform), RBAC role assignments, managed identity permissions, and prerequisites before deploying. WHEN: validate my app, check deployment readiness, run preflight checks, verify configuration, check if ready to deploy, validate azure.yaml, validate Bicep, test before deploying, troubleshoot deployment errors, validate Azure Functions, validate function app, validate serverless deployment, verify RBAC roles, check role assignments, review managed identity permissions, what-if analysis, validate Container Apps deployment." -license: MIT -metadata: - author: Microsoft - version: "1.2.2" ---- - -# Azure Validate - -> **AUTHORITATIVE GUIDANCE** — Follow these instructions exactly unless they contradict security policies given to you. - -> **⛔ STOP — PREREQUISITE CHECK REQUIRED** -> -> Before proceeding, verify this prerequisite is met: -> -> **azure-prepare** was invoked and completed → `.azure/deployment-plan.md` exists with status `Approved` or later -> -> If the plan is missing, **STOP IMMEDIATELY** and invoke **azure-prepare** first. -> -> The complete workflow ensures success: -> -> `azure-prepare` → `azure-validate` → `azure-deploy` - -## Triggers - -- Check if app is ready to deploy -- Validate azure.yaml or Bicep -- Run preflight checks -- Troubleshoot deployment errors - -## Rules - -1. Run after azure-prepare, before azure-deploy -2. All checks must pass—do not deploy with failures -3. ⛔ **Destructive actions require `ask_user`** — [global-rules](references/global-rules.md) - -## Steps - -Run the workflow script and follow its instructions. It walks you through each validation step one at a time, recording progress in `.azure/validate-status.json`. Use [references/scripts/workflow.ps1](references/scripts/workflow.ps1) on Windows or [references/scripts/workflow.sh](references/scripts/workflow.sh) on macOS/Linux. - -Start by calling the script **without** the completed-step argument: - -```bash -pwsh references/scripts/workflow.ps1 -WorkspacePath -# macOS/Linux: bash references/scripts/workflow.sh --workspace-path -``` - -Each run prints the next action and the value to pass next. Perform the action, then re-run with that value (`-CompletedStep ` for pwsh, `--completed-step ` for bash). Repeat until it reports the azure-validate workflow is complete. - -The steps reference recipe details in [references/recipes/README.md](references/recipes/README.md) and role checks in [references/role-verification.md](references/role-verification.md). - -> **⛔ VALIDATION AUTHORITY** -> -> This skill is the officially verified way to set plan status to `Validated`. You MUST follow the script's instructions to completion before setting status to `Validated`. -> Do NOT set status to `Validated` without doing so. - ---- - -> **⚠️ NEXT STEP — DEPENDS ON USER INTENT** -> -> After ALL validations pass, check whether the user asked to deploy: -> - **If the user explicitly requested deployment**, you **MUST** invoke **azure-deploy** to execute it. Do NOT run `azd up`, `azd deploy`, or any deployment commands directly — let azure-deploy handle execution. -> - **If the user only asked to validate or prepare** (not deploy), STOP after recording proof and setting status to `Validated`. Report the validation results and do NOT invoke azure-deploy. -> -> If any validation failed, fix the issues and re-run azure-validate before proceeding. +--- +name: azure-validate +description: "Pre-deployment validation for Azure readiness. Run deep checks on configuration, infrastructure (Bicep or Terraform), RBAC role assignments, managed identity permissions, and prerequisites before deploying. WHEN: validate my app, check deployment readiness, run preflight checks, verify configuration, check if ready to deploy, validate azure.yaml, validate Bicep, test before deploying, troubleshoot deployment errors, validate Azure Functions, validate function app, validate serverless deployment, verify RBAC roles, check role assignments, review managed identity permissions, what-if analysis, validate Container Apps deployment." +license: MIT +metadata: + author: Microsoft + version: "1.2.3" +--- + +# Azure Validate + +> **AUTHORITATIVE GUIDANCE** — Follow these instructions exactly unless they contradict security policies given to you. + +> **⛔ STOP — PREREQUISITE CHECK REQUIRED** +> +> Before proceeding, verify this prerequisite is met: +> +> **azure-prepare** was invoked and completed → `.azure/deployment-plan.md` exists with status `Approved` or later +> +> If the plan is missing, **STOP IMMEDIATELY** and invoke **azure-prepare** first. +> +> The complete workflow ensures success: +> +> `azure-prepare` → `azure-validate` → `azure-deploy` + +## Triggers + +- Check if app is ready to deploy +- Validate azure.yaml or Bicep +- Run preflight checks +- Troubleshoot deployment errors + +## Rules + +1. Run after azure-prepare, before azure-deploy +2. All checks must pass—do not deploy with failures +3. ⛔ **Destructive actions require `ask_user`** — [global-rules](references/global-rules.md) + +## Steps + +Run the workflow script and follow its instructions. It walks you through each validation step one at a time, recording progress in `.azure/validate-status.json`. Use [references/scripts/workflow.ps1](references/scripts/workflow.ps1) on Windows or [references/scripts/workflow.sh](references/scripts/workflow.sh) on macOS/Linux. + +Start by calling the script **without** the completed-step argument: + +```bash +pwsh references/scripts/workflow.ps1 -WorkspacePath +# macOS/Linux: bash references/scripts/workflow.sh --workspace-path +``` + +Each run prints the next action and the value to pass next. Perform the action, then re-run with that value (`-CompletedStep ` for pwsh, `--completed-step ` for bash). Repeat until it reports the azure-validate workflow is complete. + +The steps reference recipe details in [references/recipes/README.md](references/recipes/README.md) and role checks in [references/role-verification.md](references/role-verification.md). + +> **⛔ VALIDATION AUTHORITY** +> +> This skill is the officially verified way to set plan status to `Validated`. You MUST follow the script's instructions to completion before setting status to `Validated`. +> Do NOT set status to `Validated` without doing so. + +--- + +> **⚠️ NEXT STEP — DEPENDS ON USER INTENT** +> +> After ALL validations pass, check whether the user asked to deploy: +> - **If the user explicitly requested deployment**, you **MUST** invoke **azure-deploy** to execute it. Do NOT run `azd up`, `azd deploy`, or any deployment commands directly — let azure-deploy handle execution. +> - **If the user only asked to validate or prepare** (not deploy), STOP after recording proof and setting status to `Validated`. Report the validation results and do NOT invoke azure-deploy. +> +> If any validation failed, fix the issues and re-run azure-validate before proceeding. diff --git a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/README.md b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/README.md index d1d855cb..b7653515 100644 --- a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/README.md +++ b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/README.md @@ -1,10 +1,10 @@ -# Recipes - -Validation recipes for different infrastructure approaches. - -| Recipe | When to Use | -|--------|-------------| -| [AZD](azd/README.md) | Projects using Azure Developer CLI | -| [AZCLI](azcli/README.md) | Projects using Azure CLI scripts | -| [Bicep](bicep/README.md) | Projects using Bicep templates | -| [Terraform](terraform/README.md) | Projects using Terraform | +# Recipes + +Validation recipes for different infrastructure approaches. + +| Recipe | When to Use | +|--------|-------------| +| [AZD](azd/README.md) | Projects using Azure Developer CLI | +| [AZCLI](azcli/README.md) | Projects using Azure CLI scripts | +| [Bicep](bicep/README.md) | Projects using Bicep templates | +| [Terraform](terraform/README.md) | Projects using Terraform | diff --git a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azcli/README.md b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azcli/README.md index e6f198f5..eece00c1 100644 --- a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azcli/README.md +++ b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azcli/README.md @@ -1,88 +1,88 @@ -# AZCLI Validation - -Validation steps for Azure CLI deployments. - -## Prerequisites - -- `./infra/main.bicep` exists -- Docker available (if containerized) - -## Validation Steps - -- [ ] 1. Core Validation (CLI, auth, build, validate, what-if) — run [`validate-deployment` script](../scripts/validate-deployment.sh) -- [ ] 2. Docker Build (if containerized) -- [ ] 3. Azure Policy Validation - -## Validation Details - -### 1. Core Validation Script - -The core validation checks are a fixed, deterministic sequence. Run the shared -**validate-deployment** helper instead of executing and parsing each command by hand. It -confirms the Azure CLI is installed and authenticated, compiles the Bicep template -(`az bicep build`), validates it against the target scope (`az deployment ... validate`), -and runs a what-if preview — printing a compact PASS/FAIL summary plus a what-if change -count (Create/Modify/Delete). - -- Bash: [`../scripts/validate-deployment.sh`](../scripts/validate-deployment.sh) -- PowerShell: [`../scripts/validate-deployment.ps1`](../scripts/validate-deployment.ps1) - -**Subscription scope:** - -```bash -../scripts/validate-deployment.sh --scope sub --location -``` -```powershell -../scripts/validate-deployment.ps1 -Scope sub -Location -``` - -**Resource group scope:** - -```bash -../scripts/validate-deployment.sh --scope group --resource-group -``` -```powershell -../scripts/validate-deployment.ps1 -Scope group -ResourceGroup -``` - -Defaults: `--template ./infra/main.bicep`, `--parameters ./infra/main.parameters.json` -(skipped if absent). Pass `--subscription ` to target a specific subscription. - -**Interpreting results:** - -- `OVERALL: PASS` — all five checks passed; record the summary in Section 7 (Validation Proof). -- Any step `FAIL` — the script prints the failing command's error. Remediate: - - **Authenticated** fails → `az login`, then `az account set --subscription `. - - **Azure CLI installed** fails → install via `mcp_azure_mcp_extension_cli_install(cli-type: "az")`. - - Otherwise see [Error handling](./errors.md). - -### 2. Docker Build (if containerized) - -**Before building**, validate the Docker build context: - -1. Read the `Dockerfile` in `./src/` -2. If the Dockerfile contains `npm ci`, verify `package-lock.json` exists in the same directory -3. If `package-lock.json` is missing, generate it: - -```bash -cd ./src/ -npm install --package-lock-only -``` - -**Then build:** - -```bash -docker build -t :test ./src/ -``` - -### 3. Azure Policy Validation - -See [Policy Validation Guide](../../policy-validation.md) for instructions on retrieving and validating Azure policies for your subscription. - -## References - -- [Error handling](./errors.md) - -## Next - -All checks pass → **azure-deploy** +# AZCLI Validation + +Validation steps for Azure CLI deployments. + +## Prerequisites + +- `./infra/main.bicep` exists +- Docker available (if containerized) + +## Validation Steps + +- [ ] 1. Core Validation (CLI, auth, build, validate, what-if) — run [`validate-deployment` script](../scripts/validate-deployment.sh) +- [ ] 2. Docker Build (if containerized) +- [ ] 3. Azure Policy Validation + +## Validation Details + +### 1. Core Validation Script + +The core validation checks are a fixed, deterministic sequence. Run the shared +**validate-deployment** helper instead of executing and parsing each command by hand. It +confirms the Azure CLI is installed and authenticated, compiles the Bicep template +(`az bicep build`), validates it against the target scope (`az deployment ... validate`), +and runs a what-if preview — printing a compact PASS/FAIL summary plus a what-if change +count (Create/Modify/Delete). + +- Bash: [`../scripts/validate-deployment.sh`](../scripts/validate-deployment.sh) +- PowerShell: [`../scripts/validate-deployment.ps1`](../scripts/validate-deployment.ps1) + +**Subscription scope:** + +```bash +../scripts/validate-deployment.sh --scope sub --location +``` +```powershell +../scripts/validate-deployment.ps1 -Scope sub -Location +``` + +**Resource group scope:** + +```bash +../scripts/validate-deployment.sh --scope group --resource-group +``` +```powershell +../scripts/validate-deployment.ps1 -Scope group -ResourceGroup +``` + +Defaults: `--template ./infra/main.bicep`, `--parameters ./infra/main.parameters.json` +(skipped if absent). Pass `--subscription ` to target a specific subscription. + +**Interpreting results:** + +- `OVERALL: PASS` — all five checks passed; record the summary in Section 7 (Validation Proof). +- Any step `FAIL` — the script prints the failing command's error. Remediate: + - **Authenticated** fails → `az login`, then `az account set --subscription `. + - **Azure CLI installed** fails → install via `mcp_azure_mcp_extension_cli_install(cli-type: "az")`. + - Otherwise see [Error handling](./errors.md). + +### 2. Docker Build (if containerized) + +**Before building**, validate the Docker build context: + +1. Read the `Dockerfile` in `./src/` +2. If the Dockerfile contains `npm ci`, verify `package-lock.json` exists in the same directory +3. If `package-lock.json` is missing, generate it: + +```bash +cd ./src/ +npm install --package-lock-only +``` + +**Then build:** + +```bash +docker build -t :test ./src/ +``` + +### 3. Azure Policy Validation + +See [Policy Validation Guide](../../policy-validation.md) for instructions on retrieving and validating Azure policies for your subscription. + +## References + +- [Error handling](./errors.md) + +## Next + +All checks pass → **azure-deploy** diff --git a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azcli/errors.md b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azcli/errors.md index bd85db7c..b1f283f2 100644 --- a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azcli/errors.md +++ b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azcli/errors.md @@ -1,16 +1,16 @@ -# AZCLI Validation Errors - -| Error | Fix | -|-------|-----| -| `AADSTS700082: Token expired` | `az login` | -| `Please run 'az login'` | `az login` | -| `AADSTS50076: MFA required` | `az login --use-device-code` | -| `AuthorizationFailed` | Request Contributor role | -| `npm ci` fails with `missing: package-lock.json` | Run `npm install --package-lock-only` in the service directory before building | -| `Template validation failed` | Check Bicep syntax | - -## Debug - -```bash -az --verbose --debug -``` +# AZCLI Validation Errors + +| Error | Fix | +|-------|-----| +| `AADSTS700082: Token expired` | `az login` | +| `Please run 'az login'` | `az login` | +| `AADSTS50076: MFA required` | `az login --use-device-code` | +| `AuthorizationFailed` | Request Contributor role | +| `npm ci` fails with `missing: package-lock.json` | Run `npm install --package-lock-only` in the service directory before building | +| `Template validation failed` | Check Bicep syntax | + +## Debug + +```bash +az --verbose --debug +``` diff --git a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azd/README.md b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azd/README.md index d6c4547c..c5edf5c1 100644 --- a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azd/README.md +++ b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azd/README.md @@ -1,166 +1,166 @@ -# AZD Validation - -Validation steps for Azure Developer CLI projects. - -## Prerequisites - -- `azure.yaml` exists in project root -- Infrastructure files exist: - - For Bicep: `./infra/` contains Bicep files - - For Terraform: `./infra/` contains `.tf` files and `azure.yaml` has `infra.provider: terraform` - -## Validation Steps - -- [ ] 1. AZD Installation -- [ ] 2. Schema Validation -- [ ] 3. Environment Setup -- [ ] 4. Authentication Check -- [ ] 5. Subscription/Location Check -- [ ] 6. Aspire Pre-Provisioning Checks -- [ ] 7. Provision Preview -- [ ] 8. Build Verification -- [ ] 9. Docker Build Context Validation -- [ ] 10. Package Validation -- [ ] 11. Azure Policy Validation -- [ ] 12. Aspire Post-Provisioning Checks - -## Validation Details - -### 1. AZD Installation - -Verify AZD is installed: - -```bash -azd version -``` - -**If not installed:** -``` -mcp_azure_mcp_extension_cli_install(cli-type: "azd") -``` - -### 2. Schema Validation - -Validate azure.yaml against official schema: - -``` -mcp_azure_mcp_azd(command: "validate_azure_yaml", parameters: { path: "./azure.yaml" }) -``` - -### 3. Environment Setup - -Verify AZD environment exists and is configured. See [Environment Setup](environment.md) for detailed steps. - -### 4. Authentication Check - -```bash -azd auth login --check-status -``` - -**If not logged in:** -```bash -azd auth login -``` - -### 5. Subscription/Location Check - -Check environment values: -```bash -azd env get-values -``` - -**If AZURE_SUBSCRIPTION_ID or AZURE_LOCATION not set:** - -Use Azure MCP tools to list subscriptions: -``` -mcp_azure_mcp_subscription_list -``` - -Use Azure MCP tools to list resource groups (check for conflicts): -``` -mcp_azure_mcp_group_list - subscription: -``` - -Prompt user to confirm subscription and location before continuing. - -Refer to the region availability reference to select a region supported by all services in this template: -- [Region availability](../../region-availability.md) - -```bash -azd env set AZURE_SUBSCRIPTION_ID -azd env set AZURE_LOCATION -``` - -### 6. Aspire Pre-Provisioning Checks - -**If this is a .NET Aspire project** (detected by `*.AppHost.csproj` or `Aspire.Hosting` package reference), run the **Pre-Provisioning** checks in [Aspire Validation](aspire.md) before continuing. **If not Aspire, skip this step.** - -### 7. Provision Preview - -Validate IaC is ready (must complete without error): - -```bash -azd provision --preview --no-prompt -``` - -> 💡 **Note:** This works for both Bicep and Terraform. azd will automatically detect the provider from `azure.yaml` and run the appropriate validation (`bicep build` or `terraform plan`). - -**If `azd provision --preview` fails with `unsupported resource type`:** - -⛔ **Stop deployment immediately.** Do NOT attempt to fix or work around the error. - -This error means the Aspire AppHost contains custom resource types that have no Azure deployment target (e.g., `HealthChecksUI`, custom child resources, or local-only integrations). These resources are intentionally designed for local development tooling and cannot be meaningfully deployed to Azure. - -**Required actions:** -1. ⛔ **Do NOT modify source code** to suppress the error (e.g., do not add `.ExcludeFromManifest()`). -2. ⛔ **Do NOT proceed with `azd provision` or `azd deploy`.** -3. ✅ Inform the user: "This application contains custom Aspire resource types (`unsupported resource type: `) that cannot be deployed to Azure. The application is designed for local development only." -4. ✅ Record a blocker: "Deployment blocked — AppHost contains unsupported resource types with no Azure deployment target." - -> ⚠️ Adding `.ExcludeFromManifest()` to suppress the error may allow provisioning to proceed, but the resulting deployment will not represent the application's actual functionality and is not a valid deployment. - -### 8. Build Verification - -Build the project and verify there are no errors. If the build fails, fix the issues and re-build until it succeeds. Do NOT proceed to packaging or deployment with build errors. - -### 9. Docker Build Context Validation - -**If any service in `azure.yaml` uses a Dockerfile** (check the service's `project` path from `azure.yaml` for a `Dockerfile`), validate the build context before packaging: - -1. Read each service's `Dockerfile` -2. If the Dockerfile contains `npm ci`, verify `package-lock.json` exists in the same directory -3. If `package-lock.json` is missing, generate it in the service's `project` path directory before proceeding: - -```bash -cd -npm install --package-lock-only -``` - -> ⚠️ **Warning:** `npm ci` will fail during Docker build if `package-lock.json` is missing. This check prevents Docker build failures during `azd package` and `azd up`. - -### 10. Package Validation - -Confirm all services package successfully: - -```bash -azd package --no-prompt -``` - -### 11. Azure Policy Validation - -See [Policy Validation Guide](../../policy-validation.md) for instructions on retrieving and validating Azure policies for your subscription. - -### 12. Aspire Post-Provisioning Checks - -**If this is a .NET Aspire project**, run the **Post-Provisioning** checks in [Aspire Validation](aspire.md) before proceeding to deployment. **If not Aspire, skip this step.** - -## References - -- [Environment Setup](environment.md) -- [Aspire Validation](aspire.md) -- [Error Handling](./errors.md) - -## Next - -All checks pass → **azure-deploy** +# AZD Validation + +Validation steps for Azure Developer CLI projects. + +## Prerequisites + +- `azure.yaml` exists in project root +- Infrastructure files exist: + - For Bicep: `./infra/` contains Bicep files + - For Terraform: `./infra/` contains `.tf` files and `azure.yaml` has `infra.provider: terraform` + +## Validation Steps + +- [ ] 1. AZD Installation +- [ ] 2. Schema Validation +- [ ] 3. Environment Setup +- [ ] 4. Authentication Check +- [ ] 5. Subscription/Location Check +- [ ] 6. Aspire Pre-Provisioning Checks +- [ ] 7. Provision Preview +- [ ] 8. Build Verification +- [ ] 9. Docker Build Context Validation +- [ ] 10. Package Validation +- [ ] 11. Azure Policy Validation +- [ ] 12. Aspire Post-Provisioning Checks + +## Validation Details + +### 1. AZD Installation + +Verify AZD is installed: + +```bash +azd version +``` + +**If not installed:** +``` +mcp_azure_mcp_extension_cli_install(cli-type: "azd") +``` + +### 2. Schema Validation + +Validate azure.yaml against official schema: + +``` +mcp_azure_mcp_azd(command: "validate_azure_yaml", parameters: { path: "./azure.yaml" }) +``` + +### 3. Environment Setup + +Verify AZD environment exists and is configured. See [Environment Setup](environment.md) for detailed steps. + +### 4. Authentication Check + +```bash +azd auth login --check-status +``` + +**If not logged in:** +```bash +azd auth login +``` + +### 5. Subscription/Location Check + +Check environment values: +```bash +azd env get-values +``` + +**If AZURE_SUBSCRIPTION_ID or AZURE_LOCATION not set:** + +Use Azure MCP tools to list subscriptions: +``` +mcp_azure_mcp_subscription_list +``` + +Use Azure MCP tools to list resource groups (check for conflicts): +``` +mcp_azure_mcp_group_list + subscription: +``` + +Prompt user to confirm subscription and location before continuing. + +Refer to the region availability reference to select a region supported by all services in this template: +- [Region availability](../../region-availability.md) + +```bash +azd env set AZURE_SUBSCRIPTION_ID +azd env set AZURE_LOCATION +``` + +### 6. Aspire Pre-Provisioning Checks + +**If this is a .NET Aspire project** (detected by `*.AppHost.csproj` or `Aspire.Hosting` package reference), run the **Pre-Provisioning** checks in [Aspire Validation](aspire.md) before continuing. **If not Aspire, skip this step.** + +### 7. Provision Preview + +Validate IaC is ready (must complete without error): + +```bash +azd provision --preview --no-prompt +``` + +> 💡 **Note:** This works for both Bicep and Terraform. azd will automatically detect the provider from `azure.yaml` and run the appropriate validation (`bicep build` or `terraform plan`). + +**If `azd provision --preview` fails with `unsupported resource type`:** + +⛔ **Stop deployment immediately.** Do NOT attempt to fix or work around the error. + +This error means the Aspire AppHost contains custom resource types that have no Azure deployment target (e.g., `HealthChecksUI`, custom child resources, or local-only integrations). These resources are intentionally designed for local development tooling and cannot be meaningfully deployed to Azure. + +**Required actions:** +1. ⛔ **Do NOT modify source code** to suppress the error (e.g., do not add `.ExcludeFromManifest()`). +2. ⛔ **Do NOT proceed with `azd provision` or `azd deploy`.** +3. ✅ Inform the user: "This application contains custom Aspire resource types (`unsupported resource type: `) that cannot be deployed to Azure. The application is designed for local development only." +4. ✅ Record a blocker: "Deployment blocked — AppHost contains unsupported resource types with no Azure deployment target." + +> ⚠️ Adding `.ExcludeFromManifest()` to suppress the error may allow provisioning to proceed, but the resulting deployment will not represent the application's actual functionality and is not a valid deployment. + +### 8. Build Verification + +Build the project and verify there are no errors. If the build fails, fix the issues and re-build until it succeeds. Do NOT proceed to packaging or deployment with build errors. + +### 9. Docker Build Context Validation + +**If any service in `azure.yaml` uses a Dockerfile** (check the service's `project` path from `azure.yaml` for a `Dockerfile`), validate the build context before packaging: + +1. Read each service's `Dockerfile` +2. If the Dockerfile contains `npm ci`, verify `package-lock.json` exists in the same directory +3. If `package-lock.json` is missing, generate it in the service's `project` path directory before proceeding: + +```bash +cd +npm install --package-lock-only +``` + +> ⚠️ **Warning:** `npm ci` will fail during Docker build if `package-lock.json` is missing. This check prevents Docker build failures during `azd package` and `azd up`. + +### 10. Package Validation + +Confirm all services package successfully: + +```bash +azd package --no-prompt +``` + +### 11. Azure Policy Validation + +See [Policy Validation Guide](../../policy-validation.md) for instructions on retrieving and validating Azure policies for your subscription. + +### 12. Aspire Post-Provisioning Checks + +**If this is a .NET Aspire project**, run the **Post-Provisioning** checks in [Aspire Validation](aspire.md) before proceeding to deployment. **If not Aspire, skip this step.** + +## References + +- [Environment Setup](environment.md) +- [Aspire Validation](aspire.md) +- [Error Handling](./errors.md) + +## Next + +All checks pass → **azure-deploy** diff --git a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azd/environment.md b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azd/environment.md index 4ffc7d99..3bf8caea 100644 --- a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azd/environment.md +++ b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azd/environment.md @@ -1,104 +1,104 @@ -# AZD Environment Setup - -> **⛔ MANDATORY**: You MUST set up an AZD environment before running any deployment commands. - -## Step 1: Check Existing Environments - -```bash -azd env list -``` - -**If an environment is already selected** (marked with `*`), check its current values: - -```bash -azd env get-values -``` - -If `AZURE_ENV_NAME`, `AZURE_SUBSCRIPTION_ID`, and `AZURE_LOCATION` are already set, confirm with user: - -``` -Question: "An AZD environment is already configured. Would you like to use it?" - - Environment: {env-name} - Subscription: {subscription-id} - Location: {location} - -Choices: [ - "Yes, use this environment (Recommended)", - "No, create a new environment" -] -``` - -If user confirms → skip to **Verify Configuration**. Otherwise → continue to Step 2. - -**If NO environment exists or none is selected:** -- Continue to Step 2 - ---- - -## Step 2: Create New Environment - -> **⛔ DO NOT use generic names like "dev", "prod", or "test"** -> -> These cause naming conflicts in Azure resource groups and resources. Always generate a unique name. - -### Generate Suggested Name - -Use this pattern: -``` -{project-name}-{random-4-chars} -``` - -**Examples:** -- `dadjokes-x7k2` -- `todoapp-m3p9` -- `myapi-q5w8` - -### Prompt User - -**You MUST use `ask_user` to confirm the environment name:** - -``` -I need to create an AZD environment for this deployment. - -Suggested name: {project-name}-{random-4-chars} - -Would you like to use this name or enter a custom one? -``` - -### Create Environment - -After user confirms: -```bash -azd env new --no-prompt -``` - ---- - -## Step 3: Configure Environment - -Set subscription and location: - -```bash -azd env set AZURE_SUBSCRIPTION_ID -azd env set AZURE_LOCATION -``` - ---- - -## Verify Configuration - -```bash -azd env get-values -``` - -Confirm these values are set: -- `AZURE_ENV_NAME` -- `AZURE_SUBSCRIPTION_ID` -- `AZURE_LOCATION` - ---- - -## Only Then Proceed - -After environment is configured, proceed with `azd up --no-prompt`. +# AZD Environment Setup + +> **⛔ MANDATORY**: You MUST set up an AZD environment before running any deployment commands. + +## Step 1: Check Existing Environments + +```bash +azd env list +``` + +**If an environment is already selected** (marked with `*`), check its current values: + +```bash +azd env get-values +``` + +If `AZURE_ENV_NAME`, `AZURE_SUBSCRIPTION_ID`, and `AZURE_LOCATION` are already set, confirm with user: + +``` +Question: "An AZD environment is already configured. Would you like to use it?" + + Environment: {env-name} + Subscription: {subscription-id} + Location: {location} + +Choices: [ + "Yes, use this environment (Recommended)", + "No, create a new environment" +] +``` + +If user confirms → skip to **Verify Configuration**. Otherwise → continue to Step 2. + +**If NO environment exists or none is selected:** +- Continue to Step 2 + +--- + +## Step 2: Create New Environment + +> **⛔ DO NOT use generic names like "dev", "prod", or "test"** +> +> These cause naming conflicts in Azure resource groups and resources. Always generate a unique name. + +### Generate Suggested Name + +Use this pattern: +``` +{project-name}-{random-4-chars} +``` + +**Examples:** +- `dadjokes-x7k2` +- `todoapp-m3p9` +- `myapi-q5w8` + +### Prompt User + +**You MUST use `ask_user` to confirm the environment name:** + +``` +I need to create an AZD environment for this deployment. + +Suggested name: {project-name}-{random-4-chars} + +Would you like to use this name or enter a custom one? +``` + +### Create Environment + +After user confirms: +```bash +azd env new --no-prompt +``` + +--- + +## Step 3: Configure Environment + +Set subscription and location: + +```bash +azd env set AZURE_SUBSCRIPTION_ID +azd env set AZURE_LOCATION +``` + +--- + +## Verify Configuration + +```bash +azd env get-values +``` + +Confirm these values are set: +- `AZURE_ENV_NAME` +- `AZURE_SUBSCRIPTION_ID` +- `AZURE_LOCATION` + +--- + +## Only Then Proceed + +After environment is configured, proceed with `azd up --no-prompt`. diff --git a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azd/errors.md b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azd/errors.md index 9a5f396e..af6cab59 100644 --- a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azd/errors.md +++ b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/azd/errors.md @@ -1,43 +1,43 @@ -# AZD Validation Errors - -## Pre-Deployment Errors - -These errors can be caught **before** running `azd up`: - -| Error | Cause | Resolution | -|-------|-------|------------| -| `Please run 'az login'` | Not authenticated | `az login` or `azd auth login` | -| `No environment selected` | Missing azd environment | `azd env select ` or `azd env new --no-prompt` | -| `no default response for prompt 'Enter a unique environment name'` | No azd environment created, or missing `-e` flag | Run `azd env new --no-prompt` OR use `azd init --from-code -e --no-prompt` with the `-e` flag | -| `no default response for prompt 'Enter a value for the 'environmentName'` | Environment variables not set | Run `azd env set AZURE_ENV_NAME ` | -| `Service not found` | Service name mismatch | Check service name in azure.yaml | -| `Invalid azure.yaml` | YAML syntax error | Fix YAML syntax | -| `Project path does not exist` | Wrong service project path | Fix service project path in azure.yaml | -| `Cannot connect to Docker daemon` | Docker not running | Start Docker Desktop | -| `npm ci` fails with `missing: package-lock.json` | Dockerfile uses `npm ci` but `package-lock.json` not in build context | Run `npm install --package-lock-only` in the service directory before building | -| `Could not find a part of the path 'infra\main.bicep'` | Missing infrastructure files | Generate infra/ folder before `azd up` | -| `Invalid resource group location ''. The Resource group already exists in location ''` | RG exists in different region | Check RG location first with `az group show`, use that region or new env name | -| `expecting only '1' resource tagged with 'azd-service-name: web', but found '2'` | Multiple resources with same tag **in the same RG** | Delete duplicate or rename service | - -## Static Web App Errors - -| Error | Cause | Fix | -|-------|-------|-----| -| `language 'html' is not supported` | Invalid language value | Omit `language` for pure static sites | -| `language 'static' is not supported` | Invalid language value | Omit `language` for pure static sites | -| `dist folder not found` | Wrong dist path or missing build | Check `dist` is relative to `project`; add `language: js` if build needed | -| `LocationNotAvailableForResourceType` | SWA not in region | See [Region Availability](../../region-availability.md) for valid regions | - -## SWA Path Validation - -Before deployment, verify: -1. `project` path exists and contains source files -2. For framework apps: `language: js` is set -3. `dist` is relative to `project` (not project root) -4. Bicep has `azd-service-name` tag matching service name - -## Debug - -```bash -azd --debug -``` +# AZD Validation Errors + +## Pre-Deployment Errors + +These errors can be caught **before** running `azd up`: + +| Error | Cause | Resolution | +|-------|-------|------------| +| `Please run 'az login'` | Not authenticated | `az login` or `azd auth login` | +| `No environment selected` | Missing azd environment | `azd env select ` or `azd env new --no-prompt` | +| `no default response for prompt 'Enter a unique environment name'` | No azd environment created, or missing `-e` flag | Run `azd env new --no-prompt` OR use `azd init --from-code -e --no-prompt` with the `-e` flag | +| `no default response for prompt 'Enter a value for the 'environmentName'` | Environment variables not set | Run `azd env set AZURE_ENV_NAME ` | +| `Service not found` | Service name mismatch | Check service name in azure.yaml | +| `Invalid azure.yaml` | YAML syntax error | Fix YAML syntax | +| `Project path does not exist` | Wrong service project path | Fix service project path in azure.yaml | +| `Cannot connect to Docker daemon` | Docker not running | Start Docker Desktop | +| `npm ci` fails with `missing: package-lock.json` | Dockerfile uses `npm ci` but `package-lock.json` not in build context | Run `npm install --package-lock-only` in the service directory before building | +| `Could not find a part of the path 'infra\main.bicep'` | Missing infrastructure files | Generate infra/ folder before `azd up` | +| `Invalid resource group location ''. The Resource group already exists in location ''` | RG exists in different region | Check RG location first with `az group show`, use that region or new env name | +| `expecting only '1' resource tagged with 'azd-service-name: web', but found '2'` | Multiple resources with same tag **in the same RG** | Delete duplicate or rename service | + +## Static Web App Errors + +| Error | Cause | Fix | +|-------|-------|-----| +| `language 'html' is not supported` | Invalid language value | Omit `language` for pure static sites | +| `language 'static' is not supported` | Invalid language value | Omit `language` for pure static sites | +| `dist folder not found` | Wrong dist path or missing build | Check `dist` is relative to `project`; add `language: js` if build needed | +| `LocationNotAvailableForResourceType` | SWA not in region | See [Region Availability](../../region-availability.md) for valid regions | + +## SWA Path Validation + +Before deployment, verify: +1. `project` path exists and contains source files +2. For framework apps: `language: js` is set +3. `dist` is relative to `project` (not project root) +4. Bicep has `azd-service-name` tag matching service name + +## Debug + +```bash +azd --debug +``` diff --git a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/bicep/README.md b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/bicep/README.md index 7d28edbb..29eb484b 100644 --- a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/bicep/README.md +++ b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/bicep/README.md @@ -1,83 +1,83 @@ -# Bicep Validation - -Validation steps for standalone Bicep deployments. - -## Prerequisites - -- `./infra/main.bicep` exists -- `./infra/main.parameters.json` exists -- Azure CLI authenticated - -## Validation Steps - -- [ ] 1. Core Validation (CLI, auth, build, validate, what-if) — run [`validate-deployment` script](../scripts/validate-deployment.sh) -- [ ] 2. Linting (optional) -- [ ] 3. Azure Policy Validation - -## Validation Details - -### 1. Core Validation Script - -The core validation checks are a fixed, deterministic sequence — identical to the AZCLI -recipe. Run the shared **validate-deployment** helper instead of executing and parsing each -command by hand. It confirms the Azure CLI is installed and authenticated, compiles the Bicep -template (`az bicep build`), validates it against the target scope (`az deployment ... -validate`), and runs a what-if preview — printing a compact PASS/FAIL summary plus a what-if -change count (Create/Modify/Delete). - -- Bash: [`../scripts/validate-deployment.sh`](../scripts/validate-deployment.sh) -- PowerShell: [`../scripts/validate-deployment.ps1`](../scripts/validate-deployment.ps1) - -**Subscription scope:** - -```bash -../scripts/validate-deployment.sh --scope sub --location -``` -```powershell -../scripts/validate-deployment.ps1 -Scope sub -Location -``` - -**Resource group scope:** - -```bash -../scripts/validate-deployment.sh --scope group --resource-group -``` -```powershell -../scripts/validate-deployment.ps1 -Scope group -ResourceGroup -``` - -Defaults: `--template ./infra/main.bicep`, `--parameters ./infra/main.parameters.json` -(skipped if absent). Pass `--subscription ` to target a specific subscription. - -**Interpreting results:** - -- `OVERALL: PASS` — all checks passed; record the summary in Section 7 (Validation Proof). -- Any step `FAIL` — the script prints the failing command's error. If **Authenticated** fails, - run `az login`. Otherwise see [Error handling](./errors.md). - -### 2. Linting (optional) - -Use Bicep linter rules: - -```bash -az bicep lint --file ./infra/main.bicep -``` - -### 3. Azure Policy Validation - -See [Policy Validation Guide](../../policy-validation.md) for instructions on retrieving and validating Azure policies for your subscription. - -## Checklist - -| Check | Command | Pass | -|-------|---------|------| -| Core validation (CLI, auth, build, validate, what-if) | `validate-deployment` script | ☐ | -| Policies validated | MCP Policy tool | ☐ | - -## References - -- [Error handling](./errors.md) - -## Next - -All checks pass → **azure-deploy** +# Bicep Validation + +Validation steps for standalone Bicep deployments. + +## Prerequisites + +- `./infra/main.bicep` exists +- `./infra/main.parameters.json` exists +- Azure CLI authenticated + +## Validation Steps + +- [ ] 1. Core Validation (CLI, auth, build, validate, what-if) — run [`validate-deployment` script](../scripts/validate-deployment.sh) +- [ ] 2. Linting (optional) +- [ ] 3. Azure Policy Validation + +## Validation Details + +### 1. Core Validation Script + +The core validation checks are a fixed, deterministic sequence — identical to the AZCLI +recipe. Run the shared **validate-deployment** helper instead of executing and parsing each +command by hand. It confirms the Azure CLI is installed and authenticated, compiles the Bicep +template (`az bicep build`), validates it against the target scope (`az deployment ... +validate`), and runs a what-if preview — printing a compact PASS/FAIL summary plus a what-if +change count (Create/Modify/Delete). + +- Bash: [`../scripts/validate-deployment.sh`](../scripts/validate-deployment.sh) +- PowerShell: [`../scripts/validate-deployment.ps1`](../scripts/validate-deployment.ps1) + +**Subscription scope:** + +```bash +../scripts/validate-deployment.sh --scope sub --location +``` +```powershell +../scripts/validate-deployment.ps1 -Scope sub -Location +``` + +**Resource group scope:** + +```bash +../scripts/validate-deployment.sh --scope group --resource-group +``` +```powershell +../scripts/validate-deployment.ps1 -Scope group -ResourceGroup +``` + +Defaults: `--template ./infra/main.bicep`, `--parameters ./infra/main.parameters.json` +(skipped if absent). Pass `--subscription ` to target a specific subscription. + +**Interpreting results:** + +- `OVERALL: PASS` — all checks passed; record the summary in Section 7 (Validation Proof). +- Any step `FAIL` — the script prints the failing command's error. If **Authenticated** fails, + run `az login`. Otherwise see [Error handling](./errors.md). + +### 2. Linting (optional) + +Use Bicep linter rules: + +```bash +az bicep lint --file ./infra/main.bicep +``` + +### 3. Azure Policy Validation + +See [Policy Validation Guide](../../policy-validation.md) for instructions on retrieving and validating Azure policies for your subscription. + +## Checklist + +| Check | Command | Pass | +|-------|---------|------| +| Core validation (CLI, auth, build, validate, what-if) | `validate-deployment` script | ☐ | +| Policies validated | MCP Policy tool | ☐ | + +## References + +- [Error handling](./errors.md) + +## Next + +All checks pass → **azure-deploy** diff --git a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/bicep/errors.md b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/bicep/errors.md index 8399a9f0..4ff34489 100644 --- a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/bicep/errors.md +++ b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/bicep/errors.md @@ -1,15 +1,15 @@ -# Bicep Validation Errors - -| Error | Fix | -|-------|-----| -| `BCP035: Invalid type` | Check API version | -| `BCP037: Not a member` | Check resource schema | -| `BCP018: Expected character` | Fix syntax | -| `Module not found` | Check relative paths | -| `Template validation failed` | Review error details | - -## Debug - -```bash -az bicep build --file ./infra/main.bicep 2>&1 -``` +# Bicep Validation Errors + +| Error | Fix | +|-------|-----| +| `BCP035: Invalid type` | Check API version | +| `BCP037: Not a member` | Check resource schema | +| `BCP018: Expected character` | Fix syntax | +| `Module not found` | Check relative paths | +| `Template validation failed` | Review error details | + +## Debug + +```bash +az bicep build --file ./infra/main.bicep 2>&1 +``` diff --git a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/terraform/README.md b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/terraform/README.md index b2321974..86288cad 100644 --- a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/terraform/README.md +++ b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/terraform/README.md @@ -1,116 +1,116 @@ -# Terraform Validation - -Validation steps for Terraform deployments. - -## Prerequisites - -- `./infra/main.tf` exists -- State backend accessible - -## Run the preflight script - -Run the pre-built validation script instead of executing each check by hand. It runs the -full deterministic preflight sequence in one call and prints a compact **PASS / FAIL / SKIP** -summary plus captured error text for any failed step — jump straight to remediation without -re-parsing raw command output. - -| Script | Purpose | -|--------|---------| -| [`scripts/validate-terraform.sh`](scripts/validate-terraform.sh) | Bash preflight runner | -| [`scripts/validate-terraform.ps1`](scripts/validate-terraform.ps1) | PowerShell preflight runner | - -The script runs, in order: Terraform installed → Azure CLI installed → authenticated -(`az account show`) → `terraform init` → `fmt -check` → `validate` → `plan` → -`state list` → Go-style `{{ .Env.* }}` template-variable scan → `main.tfvars.json` -JSON-syntax check. A subscription-selection step is added when a subscription id is -supplied. It runs **every** check even if an earlier one fails, and exits non-zero when -any step fails. - -**Usage:** - -```bash -./scripts/validate-terraform.sh [infra-dir] [subscription-id] # infra-dir defaults to ./infra -``` -```powershell -.\scripts\validate-terraform.ps1 [-InfraDir ] [-SubscriptionId ] -``` - -**Examples:** - -```bash -./scripts/validate-terraform.sh # validate ./infra -./scripts/validate-terraform.sh ./infra 00000000-0000-0000-0000-000000000000 -``` -```powershell -.\scripts\validate-terraform.ps1 -InfraDir ./infra -``` - -**Reading the output:** the summary table lists every step as `PASS`, `FAIL`, or `SKIP` -(skipped when a prerequisite such as Terraform or the infra directory is missing). Each -`FAIL` is expanded in a **FAILURE DETAILS** section with the captured error text. Fix -failed steps using the guidance below, then re-run the script. - -## Remediation - -The script only **runs and reports** — fixing failures is manual. Guidance per step: - -### Terraform / Azure CLI not installed - -- Terraform: see https://developer.hashicorp.com/terraform/install -- Azure CLI: `mcp_azure_mcp_extension_cli_install(cli-type: "az")` - -### Not authenticated / wrong subscription - -```bash -az login -az account set --subscription -``` - -### Format check failed - -```bash -terraform fmt -recursive -``` - -### Init / validate / plan / state failures - -Read the captured error text in the script output, then consult -[Error handling](./errors.md). - -### Azure Policy Validation - -The script does not cover policy checks. See -[Policy Validation Guide](../../policy-validation.md) for retrieving and validating Azure -policies for your subscription. - -### Template Variable Resolution (AZD+Terraform) - -> ⚠️ **CRITICAL for azd+Terraform projects.** azd substitutes `${VAR}` references in -> `main.tfvars.json` via envsubst, but does NOT interpolate Go-style template variables -> (`{{ .Env.* }}`). Unresolved Go-style template strings passed to Terraform cause -> cascading deployment failures, state conflicts, and timeouts. - -When the template-variable scan reports `FAIL`: - -1. **Fix the syntax** in `main.tfvars.json` — replace `{{ .Env.VAR }}` with `${VAR}`: - ```json - { "environment_name": "${AZURE_ENV_NAME}", "location": "${AZURE_LOCATION}" } - ``` -2. For additional variables, use **`TF_VAR_*` environment variables**: - ```bash - azd env set TF_VAR_environment_name "$(azd env get-value AZURE_ENV_NAME)" - ``` -3. **Verify** that `variables.tf` declares all required variables. -4. **Re-run** the script to confirm `terraform validate` / `plan` and the scan now pass. - -> Prefer putting static defaults in `variables.tf` `default` values. Using `terraform.tfvars` -> (HCL) for static defaults is acceptable if your team prefers it; this restriction is -> specifically about avoiding Go-style template expressions in `.tfvars.json` files. - -## References - -- [Error handling](./errors.md) - -## Next - -All checks pass → **azure-deploy** +# Terraform Validation + +Validation steps for Terraform deployments. + +## Prerequisites + +- `./infra/main.tf` exists +- State backend accessible + +## Run the preflight script + +Run the pre-built validation script instead of executing each check by hand. It runs the +full deterministic preflight sequence in one call and prints a compact **PASS / FAIL / SKIP** +summary plus captured error text for any failed step — jump straight to remediation without +re-parsing raw command output. + +| Script | Purpose | +|--------|---------| +| [`scripts/validate-terraform.sh`](scripts/validate-terraform.sh) | Bash preflight runner | +| [`scripts/validate-terraform.ps1`](scripts/validate-terraform.ps1) | PowerShell preflight runner | + +The script runs, in order: Terraform installed → Azure CLI installed → authenticated +(`az account show`) → `terraform init` → `fmt -check` → `validate` → `plan` → +`state list` → Go-style `{{ .Env.* }}` template-variable scan → `main.tfvars.json` +JSON-syntax check. A subscription-selection step is added when a subscription id is +supplied. It runs **every** check even if an earlier one fails, and exits non-zero when +any step fails. + +**Usage:** + +```bash +./scripts/validate-terraform.sh [infra-dir] [subscription-id] # infra-dir defaults to ./infra +``` +```powershell +.\scripts\validate-terraform.ps1 [-InfraDir ] [-SubscriptionId ] +``` + +**Examples:** + +```bash +./scripts/validate-terraform.sh # validate ./infra +./scripts/validate-terraform.sh ./infra 00000000-0000-0000-0000-000000000000 +``` +```powershell +.\scripts\validate-terraform.ps1 -InfraDir ./infra +``` + +**Reading the output:** the summary table lists every step as `PASS`, `FAIL`, or `SKIP` +(skipped when a prerequisite such as Terraform or the infra directory is missing). Each +`FAIL` is expanded in a **FAILURE DETAILS** section with the captured error text. Fix +failed steps using the guidance below, then re-run the script. + +## Remediation + +The script only **runs and reports** — fixing failures is manual. Guidance per step: + +### Terraform / Azure CLI not installed + +- Terraform: see https://developer.hashicorp.com/terraform/install +- Azure CLI: `mcp_azure_mcp_extension_cli_install(cli-type: "az")` + +### Not authenticated / wrong subscription + +```bash +az login +az account set --subscription +``` + +### Format check failed + +```bash +terraform fmt -recursive +``` + +### Init / validate / plan / state failures + +Read the captured error text in the script output, then consult +[Error handling](./errors.md). + +### Azure Policy Validation + +The script does not cover policy checks. See +[Policy Validation Guide](../../policy-validation.md) for retrieving and validating Azure +policies for your subscription. + +### Template Variable Resolution (AZD+Terraform) + +> ⚠️ **CRITICAL for azd+Terraform projects.** azd substitutes `${VAR}` references in +> `main.tfvars.json` via envsubst, but does NOT interpolate Go-style template variables +> (`{{ .Env.* }}`). Unresolved Go-style template strings passed to Terraform cause +> cascading deployment failures, state conflicts, and timeouts. + +When the template-variable scan reports `FAIL`: + +1. **Fix the syntax** in `main.tfvars.json` — replace `{{ .Env.VAR }}` with `${VAR}`: + ```json + { "environment_name": "${AZURE_ENV_NAME}", "location": "${AZURE_LOCATION}" } + ``` +2. For additional variables, use **`TF_VAR_*` environment variables**: + ```bash + azd env set TF_VAR_environment_name "$(azd env get-value AZURE_ENV_NAME)" + ``` +3. **Verify** that `variables.tf` declares all required variables. +4. **Re-run** the script to confirm `terraform validate` / `plan` and the scan now pass. + +> Prefer putting static defaults in `variables.tf` `default` values. Using `terraform.tfvars` +> (HCL) for static defaults is acceptable if your team prefers it; this restriction is +> specifically about avoiding Go-style template expressions in `.tfvars.json` files. + +## References + +- [Error handling](./errors.md) + +## Next + +All checks pass → **azure-deploy** diff --git a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/terraform/errors.md b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/terraform/errors.md index dc6b6cf1..b7caf02e 100644 --- a/.github/plugins/azure-skills/skills/azure-validate/references/recipes/terraform/errors.md +++ b/.github/plugins/azure-skills/skills/azure-validate/references/recipes/terraform/errors.md @@ -1,80 +1,80 @@ -# Terraform Validation Errors - -| Error | Fix | -|-------|-----| -| `Backend init failed` | Check storage account access | -| `Provider version conflict` | Update required_providers | -| `State lock failed` | Wait or force unlock | -| `Validation failed` | Check terraform validate output | -| `Error: Cycle:` | See [Cycle Errors](#cycle-errors) below | - -## Cycle Errors - -`terraform validate` reports a cycle when two or more resources reference each other's attributes, creating a circular dependency. - -### Common Pattern: CORS Cross-Reference - -Multi-service App Service deployments often introduce a cycle when the API's CORS configuration references the frontend hostname and the frontend's app settings reference the API hostname: - -``` -Error: Cycle: azurerm_linux_web_app.frontend, azurerm_linux_web_app.api -``` - -**Cause — circular attribute references:** - -```hcl -# API references frontend.default_hostname in CORS -resource "azurerm_linux_web_app" "api" { - site_config { - cors { - allowed_origins = ["https://${azurerm_linux_web_app.frontend.default_hostname}"] - } - } -} - -# Frontend references api.default_hostname in app_settings -resource "azurerm_linux_web_app" "frontend" { - app_settings = { - API_URL = "https://${azurerm_linux_web_app.api.default_hostname}" - } -} -``` - -### Fix Strategies - -**Option A (recommended):** Use a Terraform variable for the frontend origin so CORS is restrictive by default and the cycle is broken. Define the variable with a sensible default and pass the real frontend URL after the first deployment: - -```hcl -variable "frontend_origin" { - type = string - description = "Frontend origin for API CORS. Set after first deployment." - default = "" -} - -resource "azurerm_linux_web_app" "api" { - site_config { - cors { - allowed_origins = var.frontend_origin != "" ? [var.frontend_origin] : ["*"] - support_credentials = var.frontend_origin != "" ? true : false - } - } -} -``` - -> ⚠️ **Warning:** If using `["*"]` as a temporary bootstrap value, you **must** set `frontend_origin` to the actual URL (e.g., `https://app-web-*.azurewebsites.net`) and re-run `terraform apply` in the same deployment session before reporting success. Do not leave wildcard CORS in a completed deployment. - -**Option B:** Use `azurerm_app_service_custom_hostname_binding` or a `null_resource` with a `local-exec` provisioner to configure CORS after both resources are created, breaking the dependency chain. - -**Option C:** Use `lifecycle { ignore_changes = [site_config[0].cors] }` on the API resource and configure CORS via a separate `azurerm_web_app_active_slot` or post-deployment script. - -### After Fixing - -1. Run `terraform fmt -recursive` to fix formatting -2. Re-run `terraform validate` to confirm the cycle is resolved -3. Run `terraform plan` to verify the configuration is correct - -## Debug - -```bash -TF_LOG=DEBUG terraform plan -``` +# Terraform Validation Errors + +| Error | Fix | +|-------|-----| +| `Backend init failed` | Check storage account access | +| `Provider version conflict` | Update required_providers | +| `State lock failed` | Wait or force unlock | +| `Validation failed` | Check terraform validate output | +| `Error: Cycle:` | See [Cycle Errors](#cycle-errors) below | + +## Cycle Errors + +`terraform validate` reports a cycle when two or more resources reference each other's attributes, creating a circular dependency. + +### Common Pattern: CORS Cross-Reference + +Multi-service App Service deployments often introduce a cycle when the API's CORS configuration references the frontend hostname and the frontend's app settings reference the API hostname: + +``` +Error: Cycle: azurerm_linux_web_app.frontend, azurerm_linux_web_app.api +``` + +**Cause — circular attribute references:** + +```hcl +# API references frontend.default_hostname in CORS +resource "azurerm_linux_web_app" "api" { + site_config { + cors { + allowed_origins = ["https://${azurerm_linux_web_app.frontend.default_hostname}"] + } + } +} + +# Frontend references api.default_hostname in app_settings +resource "azurerm_linux_web_app" "frontend" { + app_settings = { + API_URL = "https://${azurerm_linux_web_app.api.default_hostname}" + } +} +``` + +### Fix Strategies + +**Option A (recommended):** Use a Terraform variable for the frontend origin so CORS is restrictive by default and the cycle is broken. Define the variable with a sensible default and pass the real frontend URL after the first deployment: + +```hcl +variable "frontend_origin" { + type = string + description = "Frontend origin for API CORS. Set after first deployment." + default = "" +} + +resource "azurerm_linux_web_app" "api" { + site_config { + cors { + allowed_origins = var.frontend_origin != "" ? [var.frontend_origin] : ["*"] + support_credentials = var.frontend_origin != "" ? true : false + } + } +} +``` + +> ⚠️ **Warning:** If using `["*"]` as a temporary bootstrap value, you **must** set `frontend_origin` to the actual URL (e.g., `https://app-web-*.azurewebsites.net`) and re-run `terraform apply` in the same deployment session before reporting success. Do not leave wildcard CORS in a completed deployment. + +**Option B:** Use `azurerm_app_service_custom_hostname_binding` or a `null_resource` with a `local-exec` provisioner to configure CORS after both resources are created, breaking the dependency chain. + +**Option C:** Use `lifecycle { ignore_changes = [site_config[0].cors] }` on the API resource and configure CORS via a separate `azurerm_web_app_active_slot` or post-deployment script. + +### After Fixing + +1. Run `terraform fmt -recursive` to fix formatting +2. Re-run `terraform validate` to confirm the cycle is resolved +3. Run `terraform plan` to verify the configuration is correct + +## Debug + +```bash +TF_LOG=DEBUG terraform plan +``` diff --git a/.github/plugins/azure-skills/skills/microsoft-foundry/SKILL.md b/.github/plugins/azure-skills/skills/microsoft-foundry/SKILL.md index a1e93234..59e37425 100644 --- a/.github/plugins/azure-skills/skills/microsoft-foundry/SKILL.md +++ b/.github/plugins/azure-skills/skills/microsoft-foundry/SKILL.md @@ -4,7 +4,7 @@ description: "Build, deploy, evaluate, optimize, fine-tune, and manage Microsoft license: MIT metadata: author: Microsoft - version: "1.2.26" + version: "1.2.27" --- # Microsoft Foundry Skill diff --git a/.github/plugins/azure-skills/skills/microsoft-foundry/foundry-agent/agent-optimizer/agent-optimizer.md b/.github/plugins/azure-skills/skills/microsoft-foundry/foundry-agent/agent-optimizer/agent-optimizer.md index 04c1a49f..1cd592ac 100644 --- a/.github/plugins/azure-skills/skills/microsoft-foundry/foundry-agent/agent-optimizer/agent-optimizer.md +++ b/.github/plugins/azure-skills/skills/microsoft-foundry/foundry-agent/agent-optimizer/agent-optimizer.md @@ -1,51 +1,51 @@ -# Agent Optimizer in Foundry — Scaffold Python Agent - -Prepare an existing Python hosted agent for Agent Optimizer in Foundry, then run optimization, apply the selected candidate locally, and deploy through azd after review. - -## When to Use This Skill - -USE FOR: make my Python agent optimizable with Agent Optimizer in Foundry, scaffold optimizer config, add `load_config`, prepare `.agent_configs`, configure eval.yaml, run azd ai agent optimize, apply optimizer candidate, deploy optimized agent. - -DO NOT USE FOR: non-Python agents, prompt agents, running standalone batch evaluations, prompt optimization of an already deployed agent, or general Foundry deployment. For normal deployment, use [deploy](../deploy/deploy.md). For eval analysis loops, use [observe](../observe/observe.md). - -## Quick Reference - -| Property | Value | -| -------- | ----- | -| Phase | Scaffold, optimize, apply locally, deploy | -| Supported language | Python | -| Required runtime | azd project with hosted agent | -| Required package | `azure-ai-agentserver-optimization` | -| Required import | `from azure.ai.agentserver.optimization import load_config` | -| Required baseline | `.agent_configs/baseline/` in the agent's service source directory | -| Supported targets | instruction, model, skill folder, function tool definitions | -| azd setup | [azd Setup](references/azd-setup.md) | -| Detailed scaffold steps | [Scaffold Workflow](references/scaffold.md) | -| Python/file patterns | [Python Patterns](references/python-patterns.md) | -| Eval config | [eval.yaml Guidance](references/eval-yaml.md) | -| Optimize flow | [Optimize Workflow](references/optimize-workflow.md) | - -## High-Level Lifecycle - -1. **Prepare azd:** Verify azd, login, and `azure.ai.agents` extension with [azd Setup](references/azd-setup.md). -2. **Scaffold:** Follow [Scaffold Workflow](references/scaffold.md) when SDK wiring or `.agent_configs/baseline/` is missing; stop for review if files changed. -3. **Configure eval:** Create or update `eval.yaml` using [eval.yaml Guidance](references/eval-yaml.md). -4. **Optimize:** Run and monitor `azd ai agent optimize` with [Optimize Workflow](references/optimize-workflow.md). -5. **Apply and deploy:** Apply the selected candidate locally, review the diff, then deploy with `azd deploy`. - -## Workflow - -1. Resolve the target agent root and confirm it is a Python hosted agent. -2. Read [azd Setup](references/azd-setup.md), then [Scaffold Workflow](references/scaffold.md) if scaffolding is needed. -3. Read [eval.yaml Guidance](references/eval-yaml.md) and configure optimization inputs from known dataset/evaluator context. -4. Read [Optimize Workflow](references/optimize-workflow.md), run optimization, and ask before applying a candidate. -5. After local review and approval, deploy with `azd deploy`, then invoke via [invoke](../invoke/invoke.md). - -## Guardrails - -- Target hosted Python agents only. -- Preserve existing frameworks, tools, hosting adapters, protocols, and entrypoints. -- Do not use one global scaffold across multi-agent roles unless the architecture already has one global prompt/model or the user approves. -- Keep edits scoped to the selected agent root. -- Do not apply candidates or deploy automatically; stop for review first. -- Prefer `azd ai agent optimize apply --candidate` plus `azd deploy` over direct optimize deploy so source changes are reviewable. +# Agent Optimizer in Foundry — Scaffold Python Agent + +Prepare an existing Python hosted agent for Agent Optimizer in Foundry, then run optimization, apply the selected candidate locally, and deploy through azd after review. + +## When to Use This Skill + +USE FOR: make my Python agent optimizable with Agent Optimizer in Foundry, scaffold optimizer config, add `load_config`, prepare `.agent_configs`, configure eval.yaml, run azd ai agent optimize, apply optimizer candidate, deploy optimized agent. + +DO NOT USE FOR: non-Python agents, prompt agents, running standalone batch evaluations, prompt optimization of an already deployed agent, or general Foundry deployment. For normal deployment, use [deploy](../deploy/deploy.md). For eval analysis loops, use [observe](../observe/observe.md). + +## Quick Reference + +| Property | Value | +| -------- | ----- | +| Phase | Scaffold, optimize, apply locally, deploy | +| Supported language | Python | +| Required runtime | azd project with hosted agent | +| Required package | `azure-ai-agentserver-optimization` | +| Required import | `from azure.ai.agentserver.optimization import load_config` | +| Required baseline | `.agent_configs/baseline/` in the agent's service source directory | +| Supported targets | instruction, model, skill folder, function tool definitions | +| azd setup | [azd Setup](references/azd-setup.md) | +| Detailed scaffold steps | [Scaffold Workflow](references/scaffold.md) | +| Python/file patterns | [Python Patterns](references/python-patterns.md) | +| Eval config | [eval.yaml Guidance](references/eval-yaml.md) | +| Optimize flow | [Optimize Workflow](references/optimize-workflow.md) | + +## High-Level Lifecycle + +1. **Prepare azd:** Verify azd, login, and `azure.ai.agents` extension with [azd Setup](references/azd-setup.md). +2. **Scaffold:** Follow [Scaffold Workflow](references/scaffold.md) when SDK wiring or `.agent_configs/baseline/` is missing; stop for review if files changed. +3. **Configure eval:** Create or update `eval.yaml` using [eval.yaml Guidance](references/eval-yaml.md). +4. **Optimize:** Run and monitor `azd ai agent optimize` with [Optimize Workflow](references/optimize-workflow.md). +5. **Apply and deploy:** Apply the selected candidate locally, review the diff, then deploy with `azd deploy`. + +## Workflow + +1. Resolve the target agent root and confirm it is a Python hosted agent. +2. Read [azd Setup](references/azd-setup.md), then [Scaffold Workflow](references/scaffold.md) if scaffolding is needed. +3. Read [eval.yaml Guidance](references/eval-yaml.md) and configure optimization inputs from known dataset/evaluator context. +4. Read [Optimize Workflow](references/optimize-workflow.md), run optimization, and ask before applying a candidate. +5. After local review and approval, deploy with `azd deploy`, then invoke via [invoke](../invoke/invoke.md). + +## Guardrails + +- Target hosted Python agents only. +- Preserve existing frameworks, tools, hosting adapters, protocols, and entrypoints. +- Do not use one global scaffold across multi-agent roles unless the architecture already has one global prompt/model or the user approves. +- Keep edits scoped to the selected agent root. +- Do not apply candidates or deploy automatically; stop for review first. +- Prefer `azd ai agent optimize apply --candidate` plus `azd deploy` over direct optimize deploy so source changes are reviewable. diff --git a/.github/plugins/foundry-iq-skills/.claude-plugin/plugin.json b/.github/plugins/foundry-iq-skills/.claude-plugin/plugin.json index 35959234..46337b88 100644 --- a/.github/plugins/foundry-iq-skills/.claude-plugin/plugin.json +++ b/.github/plugins/foundry-iq-skills/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "foundry-iq-skills", "description": "Create and retrieve from a Foundry IQ knowledge base using File, Azure Blob, or ADLS Gen2, then connect an existing Prompt or Hosted Agent.", - "version": "0.1.9", + "version": "0.1.10", "author": { "name": "Microsoft", "url": "https://www.microsoft.com" diff --git a/.github/plugins/foundry-iq-skills/.cursor-plugin/plugin.json b/.github/plugins/foundry-iq-skills/.cursor-plugin/plugin.json index dbe267af..8e90b06f 100644 --- a/.github/plugins/foundry-iq-skills/.cursor-plugin/plugin.json +++ b/.github/plugins/foundry-iq-skills/.cursor-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "foundry-iq-skills", "description": "Create and retrieve from a Foundry IQ knowledge base using File, Azure Blob, or ADLS Gen2, then connect an existing Prompt or Hosted Agent.", - "version": "0.1.9", + "version": "0.1.10", "author": { "name": "Microsoft", "url": "https://www.microsoft.com" diff --git a/.github/plugins/foundry-iq-skills/.plugin/plugin.json b/.github/plugins/foundry-iq-skills/.plugin/plugin.json index bd393480..c92f38f7 100644 --- a/.github/plugins/foundry-iq-skills/.plugin/plugin.json +++ b/.github/plugins/foundry-iq-skills/.plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "foundry-iq-skills", "description": "Create and retrieve from a Foundry IQ knowledge base using File, Azure Blob, or ADLS Gen2, then connect an existing Prompt or Hosted Agent.", - "version": "0.1.9", + "version": "0.1.10", "author": { "name": "Microsoft", "url": "https://www.microsoft.com" diff --git a/.github/plugins/foundry-iq-skills/CHANGELOG.md b/.github/plugins/foundry-iq-skills/CHANGELOG.md index b8959128..716436c9 100644 --- a/.github/plugins/foundry-iq-skills/CHANGELOG.md +++ b/.github/plugins/foundry-iq-skills/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.1.10 + +- use consistent line ending ([#3313](https://github.com/microsoft/GitHub-Copilot-for-Azure/pull/3313)) + ## 0.1.9 - misc: document Foundry IQ plugin telemetry ([#3306](https://github.com/microsoft/GitHub-Copilot-for-Azure/pull/3306)) diff --git a/.github/plugins/foundry-iq-skills/LICENSE b/.github/plugins/foundry-iq-skills/LICENSE index 0cd75c0a..356b112b 100644 --- a/.github/plugins/foundry-iq-skills/LICENSE +++ b/.github/plugins/foundry-iq-skills/LICENSE @@ -1,21 +1,21 @@ -MIT License - -Copyright 2025 (c) Microsoft Corporation. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +MIT License + +Copyright 2025 (c) Microsoft Corporation. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE \ No newline at end of file diff --git a/.github/plugins/foundry-iq-skills/README.md b/.github/plugins/foundry-iq-skills/README.md index 03a04019..29c88b91 100644 --- a/.github/plugins/foundry-iq-skills/README.md +++ b/.github/plugins/foundry-iq-skills/README.md @@ -1,95 +1,95 @@ -# Foundry IQ Skills - -Build grounded knowledge experiences with [Azure AI Search (Foundry IQ)](https://learn.microsoft.com/azure/search/agentic-retrieval-overview). - -This plugin helps GitHub Copilot CLI create and use Foundry IQ knowledge bases, using your existing Azure resources when possible. - -## Security - -> [!WARNING] -> The `foundry-iq-skills` plugin uses `npx` to download and run the Azure MCP Server, inheriting the local environment's `.npmrc` configuration. Install this plugin only on trusted devices. A compromised `.npmrc` configuration could cause `npx` to download and execute malicious code, potentially resulting in remote code execution. - -## Telemetry - -The `track-telemetry` hook script uses `npx` to download and run Azure MCP to -collect telemetry for usage of skills and MCP tools from this plugin. To opt -out of telemetry collection, set -`AZURE_MCP_COLLECT_TELEMETRY=false` in the environment of the process running -the agent. - -## Skill - -- **foundry-iq**: Build, connect, query, and troubleshoot Foundry IQ knowledge experiences using supported Azure resources. - -## What it helps with - -- Create or reuse an Azure AI Search (Foundry IQ) service -- Create a File knowledge source from local files -- Create a knowledge source from Azure Blob Storage or Azure Data Lake Storage -- Create a knowledge base and validate that its content is searchable -- Query an existing knowledge base and return citations -- Connect an existing knowledge base to an agent -- Diagnose knowledge base failures and unsupported requests -- Prepare a cleanup plan for resources created by the workflow - -The skill is intended for Foundry IQ knowledge-base workflows. It does not support classic Azure AI Search (Foundry IQ) application, index, or query development, generic agent creation, or repository-file search. - -## Prerequisites - -- [Git](https://git-scm.com/downloads), required to add the plugin marketplace -- [GitHub Copilot CLI](https://github.com/github/copilot-cli) -- [Python 3.12 or later](https://www.python.org/downloads/), required to run the plugin's helper scripts -- Access to the Azure subscriptions and resources involved in your request -- An authenticated Azure identity with the permissions required for the requested read or change - -If you use Azure CLI for authentication, install the [Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli) and sign in: - -```bash -az login -``` - -The plugin supports Copilot CLI on Windows and Linux. Depending on the task, it uses supported Azure SDK, REST, infrastructure-as-code, Azure MCP Server, or native knowledge-base MCP interfaces. - -## Installation - -Run these commands in Copilot CLI: - -```text -/plugin marketplace add microsoft/azure-skills -/plugin install foundry-iq-skills@azure-skills -``` - -To update the plugin: - -```text -/plugin update foundry-iq-skills@azure-skills -``` - -## Example prompts - -- "Create a knowledge base from `./docs`." -- "Create a knowledge base from this Blob container." -- "Query this knowledge base with citations." -- "Connect this knowledge base to my existing agent." -- "Why is retrieval from this knowledge base failing?" - -## Before changes are made - -The skill starts with read-only checks and can reuse Azure resources you identify. It asks whether to use an existing service, find compatible services, or create a new one before performing broader discovery. - -It asks for your approval before it: - -- Creates an Azure AI Search (Foundry IQ) service, knowledge source, or knowledge base -- Connects a knowledge base to an agent -- Makes another planned change to Azure resources - -Approval applies to the plan shown to you. If the plan changes, the skill asks again. Cleanup is handled as a separate planning workflow. The skill can also perform supported deletion of workflow-owned resources after separate approval. - -## Learn more - -- [Azure AI Search (Foundry IQ) overview](https://learn.microsoft.com/azure/search/search-what-is-azure-search) -- [Foundry IQ and agentic retrieval](https://learn.microsoft.com/azure/search/agentic-retrieval-overview) -- [Azure AI Search (Foundry IQ) API and SDK versions](https://learn.microsoft.com/azure/search/search-api-versions) -- [Azure AI Search (Foundry IQ) REST API](https://learn.microsoft.com/en-us/rest/api/searchservice/?source=recommendations) -- [Azure MCP Server tools for Azure AI Search (Foundry IQ)](https://learn.microsoft.com/azure/developer/azure-mcp-server/tools/azure-ai-search) -- [Foundry IQ skill details](skills/foundry-iq/SKILL.md) +# Foundry IQ Skills + +Build grounded knowledge experiences with [Azure AI Search (Foundry IQ)](https://learn.microsoft.com/azure/search/agentic-retrieval-overview). + +This plugin helps GitHub Copilot CLI create and use Foundry IQ knowledge bases, using your existing Azure resources when possible. + +## Security + +> [!WARNING] +> The `foundry-iq-skills` plugin uses `npx` to download and run the Azure MCP Server, inheriting the local environment's `.npmrc` configuration. Install this plugin only on trusted devices. A compromised `.npmrc` configuration could cause `npx` to download and execute malicious code, potentially resulting in remote code execution. + +## Telemetry + +The `track-telemetry` hook script uses `npx` to download and run Azure MCP to +collect telemetry for usage of skills and MCP tools from this plugin. To opt +out of telemetry collection, set +`AZURE_MCP_COLLECT_TELEMETRY=false` in the environment of the process running +the agent. + +## Skill + +- **foundry-iq**: Build, connect, query, and troubleshoot Foundry IQ knowledge experiences using supported Azure resources. + +## What it helps with + +- Create or reuse an Azure AI Search (Foundry IQ) service +- Create a File knowledge source from local files +- Create a knowledge source from Azure Blob Storage or Azure Data Lake Storage +- Create a knowledge base and validate that its content is searchable +- Query an existing knowledge base and return citations +- Connect an existing knowledge base to an agent +- Diagnose knowledge base failures and unsupported requests +- Prepare a cleanup plan for resources created by the workflow + +The skill is intended for Foundry IQ knowledge-base workflows. It does not support classic Azure AI Search (Foundry IQ) application, index, or query development, generic agent creation, or repository-file search. + +## Prerequisites + +- [Git](https://git-scm.com/downloads), required to add the plugin marketplace +- [GitHub Copilot CLI](https://github.com/github/copilot-cli) +- [Python 3.12 or later](https://www.python.org/downloads/), required to run the plugin's helper scripts +- Access to the Azure subscriptions and resources involved in your request +- An authenticated Azure identity with the permissions required for the requested read or change + +If you use Azure CLI for authentication, install the [Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli) and sign in: + +```bash +az login +``` + +The plugin supports Copilot CLI on Windows and Linux. Depending on the task, it uses supported Azure SDK, REST, infrastructure-as-code, Azure MCP Server, or native knowledge-base MCP interfaces. + +## Installation + +Run these commands in Copilot CLI: + +```text +/plugin marketplace add microsoft/azure-skills +/plugin install foundry-iq-skills@azure-skills +``` + +To update the plugin: + +```text +/plugin update foundry-iq-skills@azure-skills +``` + +## Example prompts + +- "Create a knowledge base from `./docs`." +- "Create a knowledge base from this Blob container." +- "Query this knowledge base with citations." +- "Connect this knowledge base to my existing agent." +- "Why is retrieval from this knowledge base failing?" + +## Before changes are made + +The skill starts with read-only checks and can reuse Azure resources you identify. It asks whether to use an existing service, find compatible services, or create a new one before performing broader discovery. + +It asks for your approval before it: + +- Creates an Azure AI Search (Foundry IQ) service, knowledge source, or knowledge base +- Connects a knowledge base to an agent +- Makes another planned change to Azure resources + +Approval applies to the plan shown to you. If the plan changes, the skill asks again. Cleanup is handled as a separate planning workflow. The skill can also perform supported deletion of workflow-owned resources after separate approval. + +## Learn more + +- [Azure AI Search (Foundry IQ) overview](https://learn.microsoft.com/azure/search/search-what-is-azure-search) +- [Foundry IQ and agentic retrieval](https://learn.microsoft.com/azure/search/agentic-retrieval-overview) +- [Azure AI Search (Foundry IQ) API and SDK versions](https://learn.microsoft.com/azure/search/search-api-versions) +- [Azure AI Search (Foundry IQ) REST API](https://learn.microsoft.com/en-us/rest/api/searchservice/?source=recommendations) +- [Azure MCP Server tools for Azure AI Search (Foundry IQ)](https://learn.microsoft.com/azure/developer/azure-mcp-server/tools/azure-ai-search) +- [Foundry IQ skill details](skills/foundry-iq/SKILL.md) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/SKILL.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/SKILL.md index 588aa164..5f2867d4 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/SKILL.md +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/SKILL.md @@ -1,59 +1,59 @@ ---- -name: foundry-iq -description: "Foundry IQ knowledge bases. WHEN: make local or Blob documents searchable; create/diagnose KBs; triage unsupported connectors or multi-source KB creation/reconfiguration; connect existing KB to agents (including multi-source KBs); create/reuse a Search service; retrieve from an existing knowledge base with citations; no brand words needed. Read its procedure before query/target questions. NOT: other KB providers, repository-file search, classic Azure AI Search index/query/app work, generic agent creation." -license: MIT -compatibility: Azure -metadata: - author: Microsoft - version: "0.1.3" ---- - -# Foundry IQ -Read one procedure before questions/actions; invocation is not a read. -Read the owning procedure before blocked/unsupported responses too. -Before mutation plans/approval, successfully read its required pre-action references, selected -branches only. Do not reload successful reads. On failure, try only permitted -bounded exact-path reads with any supported reader; never bypass restrictions -or search broadly. If still unavailable: `blocked: reference-unavailable`, name -missing references and inability to plan. Never invent requirements/plans. -Failures first. - -Cleanup and receipt-backed execution go directly to their lifecycle/producer owner; -do not reopen Search intake, provisioning or hardening. - -Before expensive service discovery, reuse supplied resource/intent; otherwise ask -one early **USE EXISTING / FIND CANDIDATES / CREATE NEW** choice. -Only FIND enumerates; supplied identity uses exact/minimum scoped resolution. -CREATE checks its proposed name, not existing-service inventories. Preserve these -answers across source/KB/CU/model handoffs; selection is not write approval. - -Searchable docs: KB + validated retrieval. Confirm intent once; KS-only must be explicit. -Child success is not KB completion. -Unclear/compound/mode/completion: [read](references/intent-routing.md). - -Route by requested operation, not existing KB source count. -Connecting an existing KB with two or more sources, without changing the KB, uses Connect. -Read-only operations use their owning procedure and actual helper constraints; -this does not add retrieval modes or supported source kinds. -Explicit unsupported provisioning or multi-source KB creation/reconfiguration uses Diagnose -and stops before discovery, even when connecting an agent is also requested. -Do not silently execute only the supported part of a compound request. -If existing-KB connection versus KB creation/reconfiguration is unclear, read -intent-routing and clarify that scope before Azure discovery. Never infer KB mutation. - -|Outcome|Read| -|---|---| -|Cleanup|[Plan cleanup](lifecycle/cleanup.md)| -| Failure/drift | [Diagnose](troubleshooting/diagnose.md) | -| Unsupported connector provisioning / multi-source KB creation or reconfiguration | [Diagnose](troubleshooting/diagnose.md) | -| Connect | [Connect](agents/connect.md) | -| Read KB | [Query](knowledge-bases/retrieve.md) | -| Search only | [Search](search-services/create.md) | -| File KS only | [File](knowledge-sources/create-file.md) | -| Blob/ADLS KS only | [Blob](knowledge-sources/create-azure-blob.md) | -| Searchable/KB | [KB](knowledge-bases/create.md) | - -Generic agents: `microsoft-foundry`. -Reads: no approval; approve unchanged plans before writes. -Hide hashes. No Search/Storage keys, scope widening, guessed identity/boundaries, -drift repair or joint cleanup/creation approval. Acceptance != success. +--- +name: foundry-iq +description: "Foundry IQ knowledge bases. WHEN: make local or Blob documents searchable; create/diagnose KBs; triage unsupported connectors or multi-source KB creation/reconfiguration; connect existing KB to agents (including multi-source KBs); create/reuse a Search service; retrieve from an existing knowledge base with citations; no brand words needed. Read its procedure before query/target questions. NOT: other KB providers, repository-file search, classic Azure AI Search index/query/app work, generic agent creation." +license: MIT +compatibility: Azure +metadata: + author: Microsoft + version: "0.1.4" +--- + +# Foundry IQ +Read one procedure before questions/actions; invocation is not a read. +Read the owning procedure before blocked/unsupported responses too. +Before mutation plans/approval, successfully read its required pre-action references, selected +branches only. Do not reload successful reads. On failure, try only permitted +bounded exact-path reads with any supported reader; never bypass restrictions +or search broadly. If still unavailable: `blocked: reference-unavailable`, name +missing references and inability to plan. Never invent requirements/plans. +Failures first. + +Cleanup and receipt-backed execution go directly to their lifecycle/producer owner; +do not reopen Search intake, provisioning or hardening. + +Before expensive service discovery, reuse supplied resource/intent; otherwise ask +one early **USE EXISTING / FIND CANDIDATES / CREATE NEW** choice. +Only FIND enumerates; supplied identity uses exact/minimum scoped resolution. +CREATE checks its proposed name, not existing-service inventories. Preserve these +answers across source/KB/CU/model handoffs; selection is not write approval. + +Searchable docs: KB + validated retrieval. Confirm intent once; KS-only must be explicit. +Child success is not KB completion. +Unclear/compound/mode/completion: [read](references/intent-routing.md). + +Route by requested operation, not existing KB source count. +Connecting an existing KB with two or more sources, without changing the KB, uses Connect. +Read-only operations use their owning procedure and actual helper constraints; +this does not add retrieval modes or supported source kinds. +Explicit unsupported provisioning or multi-source KB creation/reconfiguration uses Diagnose +and stops before discovery, even when connecting an agent is also requested. +Do not silently execute only the supported part of a compound request. +If existing-KB connection versus KB creation/reconfiguration is unclear, read +intent-routing and clarify that scope before Azure discovery. Never infer KB mutation. + +|Outcome|Read| +|---|---| +|Cleanup|[Plan cleanup](lifecycle/cleanup.md)| +| Failure/drift | [Diagnose](troubleshooting/diagnose.md) | +| Unsupported connector provisioning / multi-source KB creation or reconfiguration | [Diagnose](troubleshooting/diagnose.md) | +| Connect | [Connect](agents/connect.md) | +| Read KB | [Query](knowledge-bases/retrieve.md) | +| Search only | [Search](search-services/create.md) | +| File KS only | [File](knowledge-sources/create-file.md) | +| Blob/ADLS KS only | [Blob](knowledge-sources/create-azure-blob.md) | +| Searchable/KB | [KB](knowledge-bases/create.md) | + +Generic agents: `microsoft-foundry`. +Reads: no approval; approve unchanged plans before writes. +Hide hashes. No Search/Storage keys, scope widening, guessed identity/boundaries, +drift repair or joint cleanup/creation approval. Acceptance != success. diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-hosted-toolbox.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-hosted-toolbox.md index 72b36544..9e877b61 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-hosted-toolbox.md +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-hosted-toolbox.md @@ -1,140 +1,140 @@ -# Hosted toolbox-only connection - -For an existing unversioned toolbox consumer, changing the default needs no source, -`azure.yaml`, environment setter or deployment. Preserve the observed agent version/definition. - -**Execution limit:** service version creation/promotion exists; this helper is -GET-only. Conditional ETag/default-update enforcement is unverified. Changed bindings -return `toolbox-promotion-concurrency-unverified` before connection/version creation. -No arbitrary PATCH headers or unconditional `azd ai toolbox publish` bypass. -Obtain the service owner's conditional-update contract, not Hosted source. - -Disclose upfront: the owner is the **retained connection/toolbox-version owner**. -Hosted cleanup is unsupported. Retain prior immutable versions and legacy -connections; rollback needs its own explicit plan/approval, never automatic action. - -## Resolve and inspect - -Accept project/KB names, resource IDs or endpoints. Exact selections win. -Resolve missing parents only within selected subscription/RG/account. -Ambiguity returns `selection_candidates`; denied/malformed pages block, never widen scope. -Search index lists never prove KB identity or absence. - -Four layers: **project connection** authenticates KB MCP; **toolbox** holds the default; -**immutable toolbox version** references connections; **Hosted runtime binding** consumes -the endpoint. The KB stays unchanged in Search. - -This recipe requires `RemoteTool` + `AgenticIdentityToken` + Search audience -`https://search.azure.com/` + exact KB MCP endpoint. `CustomKeys` and -`ProjectManagedIdentity` are not interchangeable here; other authorized custom -Hosted runtimes may support those auth modes. Preserve legacy names. -Collide by connection NAME, not endpoint. New explicit names may share a KB endpoint; -never overwrite mismatches or retry random names. Opaque tool auth/headers block; -preserve tool approval/filter policies. - -The access principal is the active published Hosted agent's `instance_identity.principal_id`, -not the blueprint or project principal used by the Prompt `ProjectManagedIdentity` -recipe. Verify existing Search Index Data Reader at the Search service and Foundry -User at the project. The helper checks these exact built-in grants; equivalent -custom grants need separate authoritative verification, not broader assignments. - -The consumer endpoint is `/toolboxes//mcp?api-version=v1`. -`/toolboxes//versions//mcp` is a pinned developer endpoint, not -equivalent. A mismatched endpoint/environment requires explaining the actual -runtime/config/deploy change and then handing off/requesting source if needed. -FoundryToolbox prefers present `TOOLBOX_ENDPOINT` (empty is invalid); only when absent does it combine -`FOUNDRY_PROJECT_ENDPOINT` (trailing slash removed) and `TOOLBOX_NAME`. -Explicit endpoint wins over name/project settings; pinned or mismatched results block. -These are supported configuration conventions, not proof arbitrary code uses them: -a constructor URL overrides the resolver. Runtime usage stays unverified until actual acceptance. - -## GET-only assessment - -Save semantic choices as `intent.json`: - -```json -{ - "schema_version": "1.0", - "scope": {"subscription_id": "", "resource_group": "", "account_name": ""}, - "project": "", - "search_service": "", - "knowledge_base": "", - "agent_name": "", - "agent_version": "", - "toolbox_name": "", - "tool_label": "", - "connection_name": "", - "reader_assignment_id": "/providers/Microsoft.Authorization/roleAssignments/", - "project_assignment_id": "/providers/Microsoft.Authorization/roleAssignments/", - "retention_owner": "" -} -``` - -```text -python helpers/hosted_connect.py --plan intent.json -``` - -Omit `account_name` only when scoped account/project discovery is needed. -Optional `known_agents: [{"name":"...","version":"..."}]` adds up to 20 known -binding readbacks, not an account-wide consumer scan. Limits: 1 MiB input/REST -responses; 20 pages/100 items per scoped inventory; at most 20 accounts and 200 tools. - -Output: `approval_summary`, private fingerprint, request IDs, zero writes. -Exact configured reuse is `planned`, without mutation approval. -Changed state is `blocked`; `execution_input` is null and `execution_available` is false; -no consent/apply mode. Fresh agent/default/version/connection or CLI-context drift blocks. - -Present this compact delta, not integrity hashes: - -| Item | Before | Proposed after | -|---|---|---| -| KB tool binding | Observed endpoint/connection | Selected endpoint/new compatible name | -| Toolbox default | Observed immutable version | New immutable version; ID not yet created | -| Agent | Exact observed version/runtime | **UNCHANGED** | - -## Shared-default approval and future write gate - -Promotion affects **every consumer** following the default, including external -clients. Show known bindings and unknown scope; never infer exclusive ownership -from a selected-agent read or partial inventory. Require explicit approval of this -shared-default effect plus the precise binding/auth/tool delta and retention. - -SDK 2.4 `project.toolboxes` exposes `get(name)`, `get_version(name, version)`, -`create_version(name, tools=...)`, `update(name, default_version=...)`. -REST `v1` uses GET `/toolboxes/` and `/toolboxes//versions/`, -POST `/toolboxes//versions`, then PATCH `/toolboxes/` with -`{"default_version":""}`. These are service capabilities, -**not executable approval through this helper**. - -Future writes require conditional promotion semantics, fresh default/ETag/protected state, -conflict blocking and preservation of unrelated tools/skills/metadata/policies. -The first toolbox version becomes default automatically; never create it accidentally. -Verify connection/version before promotion, then default and unchanged agent binding. -Partial failures retain original errors/ownership; no automatic rollback or cleanup. - -## Acceptance and portal help - -If no known-answer question is supplied, first reuse authorized retrieval evidence for this KB. -Otherwise the workflow owner may perform bounded KB retrieval only within an -approved data/model-cost boundary, then propose an evidence-backed question for -customer approval. Read-only retrieval may invoke embedding/chat and incur cost. -Do not fabricate payloads, add a probe when evidence is already known, or query -during this GET-only assessment. Optional `supported_question` and -`unrelated_question` are candidates, not invocation consent. Require a separate -unrelated abstention question and approval. - -Acceptance requires the actual selected agent's `knowledge_base_retrieve` activity, -original-source citations and unsupported-question abstention, not merely KB REST. -Pin its version, keep requested sessions/conversations and usage distinct, and -leave unavailable runtime/tool payload evidence unverified. - -UI help is optional, never a setup gate. Current first-party guidance: -**Manage > Project details > Connected resources**; older layouts/user screenshots -may say **Management center > Connected resources**. For the reported portal layout, -look under **Build > Tools > Toolboxes**; labels vary and this path is not API evidence. -Authorities: failure/conflict/uncertainty only. - -- [FoundryToolbox resolver](https://github.com/microsoft/agent-framework/blob/main/python/packages/foundry_hosting/agent_framework_foundry_hosting/_toolbox.py) -- [Toolbox operations and endpoints](https://learn.microsoft.com/azure/foundry/agents/how-to/tools/toolbox) -- [Connection UI](https://learn.microsoft.com/azure/foundry/how-to/connections-add) -- [Hosted code ownership](https://learn.microsoft.com/azure/foundry/agents/concepts/hosted-agents) +# Hosted toolbox-only connection + +For an existing unversioned toolbox consumer, changing the default needs no source, +`azure.yaml`, environment setter or deployment. Preserve the observed agent version/definition. + +**Execution limit:** service version creation/promotion exists; this helper is +GET-only. Conditional ETag/default-update enforcement is unverified. Changed bindings +return `toolbox-promotion-concurrency-unverified` before connection/version creation. +No arbitrary PATCH headers or unconditional `azd ai toolbox publish` bypass. +Obtain the service owner's conditional-update contract, not Hosted source. + +Disclose upfront: the owner is the **retained connection/toolbox-version owner**. +Hosted cleanup is unsupported. Retain prior immutable versions and legacy +connections; rollback needs its own explicit plan/approval, never automatic action. + +## Resolve and inspect + +Accept project/KB names, resource IDs or endpoints. Exact selections win. +Resolve missing parents only within selected subscription/RG/account. +Ambiguity returns `selection_candidates`; denied/malformed pages block, never widen scope. +Search index lists never prove KB identity or absence. + +Four layers: **project connection** authenticates KB MCP; **toolbox** holds the default; +**immutable toolbox version** references connections; **Hosted runtime binding** consumes +the endpoint. The KB stays unchanged in Search. + +This recipe requires `RemoteTool` + `AgenticIdentityToken` + Search audience +`https://search.azure.com/` + exact KB MCP endpoint. `CustomKeys` and +`ProjectManagedIdentity` are not interchangeable here; other authorized custom +Hosted runtimes may support those auth modes. Preserve legacy names. +Collide by connection NAME, not endpoint. New explicit names may share a KB endpoint; +never overwrite mismatches or retry random names. Opaque tool auth/headers block; +preserve tool approval/filter policies. + +The access principal is the active published Hosted agent's `instance_identity.principal_id`, +not the blueprint or project principal used by the Prompt `ProjectManagedIdentity` +recipe. Verify existing Search Index Data Reader at the Search service and Foundry +User at the project. The helper checks these exact built-in grants; equivalent +custom grants need separate authoritative verification, not broader assignments. + +The consumer endpoint is `/toolboxes//mcp?api-version=v1`. +`/toolboxes//versions//mcp` is a pinned developer endpoint, not +equivalent. A mismatched endpoint/environment requires explaining the actual +runtime/config/deploy change and then handing off/requesting source if needed. +FoundryToolbox prefers present `TOOLBOX_ENDPOINT` (empty is invalid); only when absent does it combine +`FOUNDRY_PROJECT_ENDPOINT` (trailing slash removed) and `TOOLBOX_NAME`. +Explicit endpoint wins over name/project settings; pinned or mismatched results block. +These are supported configuration conventions, not proof arbitrary code uses them: +a constructor URL overrides the resolver. Runtime usage stays unverified until actual acceptance. + +## GET-only assessment + +Save semantic choices as `intent.json`: + +```json +{ + "schema_version": "1.0", + "scope": {"subscription_id": "", "resource_group": "", "account_name": ""}, + "project": "", + "search_service": "", + "knowledge_base": "", + "agent_name": "", + "agent_version": "", + "toolbox_name": "", + "tool_label": "", + "connection_name": "", + "reader_assignment_id": "/providers/Microsoft.Authorization/roleAssignments/", + "project_assignment_id": "/providers/Microsoft.Authorization/roleAssignments/", + "retention_owner": "" +} +``` + +```text +python helpers/hosted_connect.py --plan intent.json +``` + +Omit `account_name` only when scoped account/project discovery is needed. +Optional `known_agents: [{"name":"...","version":"..."}]` adds up to 20 known +binding readbacks, not an account-wide consumer scan. Limits: 1 MiB input/REST +responses; 20 pages/100 items per scoped inventory; at most 20 accounts and 200 tools. + +Output: `approval_summary`, private fingerprint, request IDs, zero writes. +Exact configured reuse is `planned`, without mutation approval. +Changed state is `blocked`; `execution_input` is null and `execution_available` is false; +no consent/apply mode. Fresh agent/default/version/connection or CLI-context drift blocks. + +Present this compact delta, not integrity hashes: + +| Item | Before | Proposed after | +|---|---|---| +| KB tool binding | Observed endpoint/connection | Selected endpoint/new compatible name | +| Toolbox default | Observed immutable version | New immutable version; ID not yet created | +| Agent | Exact observed version/runtime | **UNCHANGED** | + +## Shared-default approval and future write gate + +Promotion affects **every consumer** following the default, including external +clients. Show known bindings and unknown scope; never infer exclusive ownership +from a selected-agent read or partial inventory. Require explicit approval of this +shared-default effect plus the precise binding/auth/tool delta and retention. + +SDK 2.4 `project.toolboxes` exposes `get(name)`, `get_version(name, version)`, +`create_version(name, tools=...)`, `update(name, default_version=...)`. +REST `v1` uses GET `/toolboxes/` and `/toolboxes//versions/`, +POST `/toolboxes//versions`, then PATCH `/toolboxes/` with +`{"default_version":""}`. These are service capabilities, +**not executable approval through this helper**. + +Future writes require conditional promotion semantics, fresh default/ETag/protected state, +conflict blocking and preservation of unrelated tools/skills/metadata/policies. +The first toolbox version becomes default automatically; never create it accidentally. +Verify connection/version before promotion, then default and unchanged agent binding. +Partial failures retain original errors/ownership; no automatic rollback or cleanup. + +## Acceptance and portal help + +If no known-answer question is supplied, first reuse authorized retrieval evidence for this KB. +Otherwise the workflow owner may perform bounded KB retrieval only within an +approved data/model-cost boundary, then propose an evidence-backed question for +customer approval. Read-only retrieval may invoke embedding/chat and incur cost. +Do not fabricate payloads, add a probe when evidence is already known, or query +during this GET-only assessment. Optional `supported_question` and +`unrelated_question` are candidates, not invocation consent. Require a separate +unrelated abstention question and approval. + +Acceptance requires the actual selected agent's `knowledge_base_retrieve` activity, +original-source citations and unsupported-question abstention, not merely KB REST. +Pin its version, keep requested sessions/conversations and usage distinct, and +leave unavailable runtime/tool payload evidence unverified. + +UI help is optional, never a setup gate. Current first-party guidance: +**Manage > Project details > Connected resources**; older layouts/user screenshots +may say **Management center > Connected resources**. For the reported portal layout, +look under **Build > Tools > Toolboxes**; labels vary and this path is not API evidence. +Authorities: failure/conflict/uncertainty only. + +- [FoundryToolbox resolver](https://github.com/microsoft/agent-framework/blob/main/python/packages/foundry_hosting/agent_framework_foundry_hosting/_toolbox.py) +- [Toolbox operations and endpoints](https://learn.microsoft.com/azure/foundry/agents/how-to/tools/toolbox) +- [Connection UI](https://learn.microsoft.com/azure/foundry/how-to/connections-add) +- [Hosted code ownership](https://learn.microsoft.com/azure/foundry/agents/concepts/hosted-agents) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-hosted.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-hosted.md index 28f5c0c7..7e273c31 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-hosted.md +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-hosted.md @@ -1,168 +1,168 @@ -# Existing Hosted Agent connection - -Branch of [Connect](connect.md), not a creation workflow. -Inspect remotely first; no source path for inspection or exact reuse. -Missing `azure.yaml` in cwd is not a discovery blocker. -Do not reinstall, reinitialize or inspect policy upfront. -Retain resources; Hosted cleanup is unsupported. - -## Inspect - -Use known project endpoint, exact agent name/version, KB MCP identity, runtime -principal and actual connection/toolbox/binding readbacks through authenticated, -observed read operations. Missing read tooling is a tooling blocker, not a source -request. Run these azd reads only with already available project/environment: - -```text -azd ai agent show --output json --no-prompt -azd ai connection show --output json --no-prompt -azd ai toolbox show --output json --no-prompt -``` - -Never use `azd ai project set` as a read; context changes require approval. -Inspect compatible connections/toolboxes before choosing a new name. Never -overwrite a Prompt connection using `ProjectManagedIdentity` to make it Hosted. - -Compare agent identity/version, model, telemetry, ACR, -network, environment, connection target/auth/audience and toolbox/default tools. -For omitted audience or `UnknownConnectionPropertiesV2`, use the exact -[nonsecret connection readback](../references/platform-interfaces.md#hosted-connection-readback). -Projection gaps are warnings only after authoritative equality is proven; -denial, conflicting properties or still-missing required evidence blocks. - -Resolve the actual published agent principal, not its blueprint or project MI. -Check effective runtime access and plan only missing grants: - -| Permission | Grant scope | -|---|---| -| Search Index Data Reader | `` | -| Foundry User or documented equivalent for toolbox access | `` | - -Reuse compatible effective grants. Disclose that Search service scope covers -its indexes, while Foundry User grants broader project data operations, not just -toolbox reads. No automatic grant, scope widening or caller-role change. A newly -observed principal or access requirement needs a separate approved access plan. - -## Select binding before approval - -Unchanged consumer: [toolbox-only branch](connect-hosted-toolbox.md); no deploy. - -Source-free changes require observed runtime support and an authoritative surface. -Otherwise explain the exact code/config/deploy delta and handoff/request source -only then. Hosted code orchestrates tools: existing agents may need source too; -no arbitrary Hosted tool changes through Prompt APIs. - -For the source-backed recipe, inspect its supported mode and emitted environment, -not just local variables. For `FoundryToolbox`, an explicitly -empty `TOOLBOX_ENDPOINT` raises instead of falling back to `TOOLBOX_NAME`. - -| Mode | `TOOLBOX_ENDPOINT` | `TOOLBOX_NAME` | -|---|---|---| -| Endpoint | `` | `""` or absent | -| Name | absent | `` | - -The consumer endpoint is -`/toolboxes//mcp?api-version=v1`. -Do not substitute the version-specific developer endpoint returned by some -toolbox readers. Preserve an already compatible exact binding without -standardizing its mode. For new/repair bindings, select one supported mode: -if the manifest injects an empty endpoint, prefer endpoint mode when supported. -Unsetting a local variable does not remove a manifest's runtime environment key. -If no valid binding is possible without source edits, stop for separate review. - -If installed, validate resolution offline with the SDK; do not install it for -this check. No token/model call or standalone -agentic toolbox `tools/list`: runtime identity requires published-agent context. - -## Approved changes only - -This source-backed recipe requires the approved existing `azd ai agent` source project; -unsupported/image-only source blocks this recipe, not remote inspection. -Retain source/config hashes for source/config changes or source-backed redeployment. -Plan grants/resource/binding deltas, source digest, costs and verification. -Exact state skips every mutation below. -After approved access changes, create only absent exact resources: - -```text -azd ai connection create --kind remote-tool --target --auth-type agentic-identity --audience https://search.azure.com/ --output json --no-prompt -azd ai toolbox create --from-file --output json --no-prompt -``` - -Verify stored `AgenticIdentityToken`, audience, target and project before -toolbox creation. YAML contains only description and the exact connection name; -no credentials or extra tools. Creation auto-publishes the first toolbox version -and writes `TOOLBOX__MCP_ENDPOINT` locally; disclose that write. - -Apply only the selected approved binding delta. Endpoint mode: - -```text -azd env set TOOLBOX_ENDPOINT --no-prompt -``` - -If the source exposes an unused name variable, clear it only as an approved delta: - -```text -azd env set TOOLBOX_NAME "" --no-prompt -``` - -Name mode, only with the endpoint absent from the emitted runtime environment: - -```text -azd env set TOOLBOX_NAME --no-prompt -``` - -Deploy only when remote configuration differs, under the same agent name: - -```text -azd deploy --no-prompt -``` - -Independently read back the returned version, actual principal, binding and -protected state. `active` metadata is not runtime-health proof. Unknown/changed -identity, source drift or failure stops; no blind retry, `--force`, handcrafted -mutation API, replacement agent or automatic cleanup. - -## Verify and reconcile - -Pin the observed version at session creation. A new session does not guarantee a new Responses -conversation. Start both fresh: - -```text -azd ai agent invoke "" --protocol responses --version --new-session --new-conversation --no-prompt -``` - -Capture the first response's session ID as `` and its -conversation ID. Verify both are distinct from prior trial IDs. Missing or reused -first-response IDs block the second call. Do not rely on automatic session selection; -explicitly pin the captured session while starting a fresh conversation: - -```text -azd ai agent invoke "" --protocol responses --session-id --new-conversation --no-prompt -``` - -Sessions bind their version at creation; `--version` and `--session-id` are mutually exclusive. -If the installed CLI lacks `--session-id`, block verification rather than falling -back to a saved session. Require the second response's session ID to equal the -captured ID and its conversation ID to differ from the first and all prior trials. -Missing or mismatched IDs make isolation unverified: stop before monitoring or -further invocations. Do not silently add invocations to repair a bounded test. - -Correlate actual `knowledge_base_retrieve` success with each request and response, -using returned events or supported logs: - -```text -azd ai agent monitor --session-id --tail 300 --no-prompt -``` - -Require supported answers with original citations and unsupported answers exactly -`I don't know.` without citations. Errors are not abstention. Tool success proves -execution, not the contents of an unavailable payload: record missing arguments, -results or usage and leave payload-level faithfulness unverified. Never infer -empty retrieval or change source/telemetry to manufacture proof. - -Compare identities, versions, tools, roles, protected source and binding. -Repeat reconciliation through reads only: no setters, create, publish -or deploy calls on exact state. Zero configuration writes excludes authorized -invocation/conversation/session activity and local test metadata; report both. -Retain resources. Cleanup remains separately approved and -`hosted-cleanup-unsupported` in this skill. +# Existing Hosted Agent connection + +Branch of [Connect](connect.md), not a creation workflow. +Inspect remotely first; no source path for inspection or exact reuse. +Missing `azure.yaml` in cwd is not a discovery blocker. +Do not reinstall, reinitialize or inspect policy upfront. +Retain resources; Hosted cleanup is unsupported. + +## Inspect + +Use known project endpoint, exact agent name/version, KB MCP identity, runtime +principal and actual connection/toolbox/binding readbacks through authenticated, +observed read operations. Missing read tooling is a tooling blocker, not a source +request. Run these azd reads only with already available project/environment: + +```text +azd ai agent show --output json --no-prompt +azd ai connection show --output json --no-prompt +azd ai toolbox show --output json --no-prompt +``` + +Never use `azd ai project set` as a read; context changes require approval. +Inspect compatible connections/toolboxes before choosing a new name. Never +overwrite a Prompt connection using `ProjectManagedIdentity` to make it Hosted. + +Compare agent identity/version, model, telemetry, ACR, +network, environment, connection target/auth/audience and toolbox/default tools. +For omitted audience or `UnknownConnectionPropertiesV2`, use the exact +[nonsecret connection readback](../references/platform-interfaces.md#hosted-connection-readback). +Projection gaps are warnings only after authoritative equality is proven; +denial, conflicting properties or still-missing required evidence blocks. + +Resolve the actual published agent principal, not its blueprint or project MI. +Check effective runtime access and plan only missing grants: + +| Permission | Grant scope | +|---|---| +| Search Index Data Reader | `` | +| Foundry User or documented equivalent for toolbox access | `` | + +Reuse compatible effective grants. Disclose that Search service scope covers +its indexes, while Foundry User grants broader project data operations, not just +toolbox reads. No automatic grant, scope widening or caller-role change. A newly +observed principal or access requirement needs a separate approved access plan. + +## Select binding before approval + +Unchanged consumer: [toolbox-only branch](connect-hosted-toolbox.md); no deploy. + +Source-free changes require observed runtime support and an authoritative surface. +Otherwise explain the exact code/config/deploy delta and handoff/request source +only then. Hosted code orchestrates tools: existing agents may need source too; +no arbitrary Hosted tool changes through Prompt APIs. + +For the source-backed recipe, inspect its supported mode and emitted environment, +not just local variables. For `FoundryToolbox`, an explicitly +empty `TOOLBOX_ENDPOINT` raises instead of falling back to `TOOLBOX_NAME`. + +| Mode | `TOOLBOX_ENDPOINT` | `TOOLBOX_NAME` | +|---|---|---| +| Endpoint | `` | `""` or absent | +| Name | absent | `` | + +The consumer endpoint is +`/toolboxes//mcp?api-version=v1`. +Do not substitute the version-specific developer endpoint returned by some +toolbox readers. Preserve an already compatible exact binding without +standardizing its mode. For new/repair bindings, select one supported mode: +if the manifest injects an empty endpoint, prefer endpoint mode when supported. +Unsetting a local variable does not remove a manifest's runtime environment key. +If no valid binding is possible without source edits, stop for separate review. + +If installed, validate resolution offline with the SDK; do not install it for +this check. No token/model call or standalone +agentic toolbox `tools/list`: runtime identity requires published-agent context. + +## Approved changes only + +This source-backed recipe requires the approved existing `azd ai agent` source project; +unsupported/image-only source blocks this recipe, not remote inspection. +Retain source/config hashes for source/config changes or source-backed redeployment. +Plan grants/resource/binding deltas, source digest, costs and verification. +Exact state skips every mutation below. +After approved access changes, create only absent exact resources: + +```text +azd ai connection create --kind remote-tool --target --auth-type agentic-identity --audience https://search.azure.com/ --output json --no-prompt +azd ai toolbox create --from-file --output json --no-prompt +``` + +Verify stored `AgenticIdentityToken`, audience, target and project before +toolbox creation. YAML contains only description and the exact connection name; +no credentials or extra tools. Creation auto-publishes the first toolbox version +and writes `TOOLBOX__MCP_ENDPOINT` locally; disclose that write. + +Apply only the selected approved binding delta. Endpoint mode: + +```text +azd env set TOOLBOX_ENDPOINT --no-prompt +``` + +If the source exposes an unused name variable, clear it only as an approved delta: + +```text +azd env set TOOLBOX_NAME "" --no-prompt +``` + +Name mode, only with the endpoint absent from the emitted runtime environment: + +```text +azd env set TOOLBOX_NAME --no-prompt +``` + +Deploy only when remote configuration differs, under the same agent name: + +```text +azd deploy --no-prompt +``` + +Independently read back the returned version, actual principal, binding and +protected state. `active` metadata is not runtime-health proof. Unknown/changed +identity, source drift or failure stops; no blind retry, `--force`, handcrafted +mutation API, replacement agent or automatic cleanup. + +## Verify and reconcile + +Pin the observed version at session creation. A new session does not guarantee a new Responses +conversation. Start both fresh: + +```text +azd ai agent invoke "" --protocol responses --version --new-session --new-conversation --no-prompt +``` + +Capture the first response's session ID as `` and its +conversation ID. Verify both are distinct from prior trial IDs. Missing or reused +first-response IDs block the second call. Do not rely on automatic session selection; +explicitly pin the captured session while starting a fresh conversation: + +```text +azd ai agent invoke "" --protocol responses --session-id --new-conversation --no-prompt +``` + +Sessions bind their version at creation; `--version` and `--session-id` are mutually exclusive. +If the installed CLI lacks `--session-id`, block verification rather than falling +back to a saved session. Require the second response's session ID to equal the +captured ID and its conversation ID to differ from the first and all prior trials. +Missing or mismatched IDs make isolation unverified: stop before monitoring or +further invocations. Do not silently add invocations to repair a bounded test. + +Correlate actual `knowledge_base_retrieve` success with each request and response, +using returned events or supported logs: + +```text +azd ai agent monitor --session-id --tail 300 --no-prompt +``` + +Require supported answers with original citations and unsupported answers exactly +`I don't know.` without citations. Errors are not abstention. Tool success proves +execution, not the contents of an unavailable payload: record missing arguments, +results or usage and leave payload-level faithfulness unverified. Never infer +empty retrieval or change source/telemetry to manufacture proof. + +Compare identities, versions, tools, roles, protected source and binding. +Repeat reconciliation through reads only: no setters, create, publish +or deploy calls on exact state. Zero configuration writes excludes authorized +invocation/conversation/session activity and local test metadata; report both. +Retain resources. Cleanup remains separately approved and +`hosted-cleanup-unsupported` in this skill. diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-prompt-sdk-fallback.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-prompt-sdk-fallback.md index 0931c8d1..a6626085 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-prompt-sdk-fallback.md +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-prompt-sdk-fallback.md @@ -1,139 +1,139 @@ -# Existing Prompt Agent SDK fallback - -Use this fallback only when authenticated Foundry MCP is unavailable -or lacks connection/version operations, never denial/conflict. -Select before planning; require the parent's approved child fingerprint before ARM creation; -do not read policy unless a creation failure implicates it. Unsupported settings block. -Read `helpers/contracts.md`; verify SDK/CLI identity. Install only missing/incompatible dependencies after approval: - -```text -python -m pip install --pre "azure-ai-projects>=2.4.0,<3" "azure-identity>=1.25.0,<2" -``` - -Run `az login` only if unauthenticated. ARM uses signed-in CLI identity; -SDK and shared cleanup loader use `AzureCliCredential`, not environment/MI fallback. -Missing CLI authentication blocks noninteractively, never falls back to another identity. -Keep CLI context unchanged; never silently switch tenant/identity. -SDK errors: safe code/HTTP status/server ID (`x-ms-request-id`, then `request-id`). -Client request IDs are not server provenance; raw exception text is withheld. -Ambiguous creation stays partial with original status/ID even if recovery fails. -Observed matching versions do not prove creation ownership. CLI output separates -`resources_remaining.unverified` from run-owned resources; unknown versions remain unnamed. -`request.json` (resolved intent, not hand-built bodies/hashes): - -```json -{ - "schema_version": "1.0", - "project_resource_id": "", - "project_endpoint": "https://account.services.ai.azure.com/api/projects/project", - "search_resource_id": "", - "knowledge_base_name": "kb", - "agent_name": "agent", - "agent_version": "1", - "connection_name": "kb-project-mi", - "is_shared_to_all": false, - "binding_action": "ensure", - "role_assignment_id": "/providers/Microsoft.Authorization/roleAssignments/", - "permission_forwarding": {"mode": "not-applicable"}, - "network": {"posture": "public", "evidence": ""}, - "owner": "" -} -``` - -```text -python helpers/prompt_connect.py --plan request.json -``` - -Use observed choices, never latest. Output: `status: planned`, `approval_summary`, -`execution_input`, `approval.confirmed: false`; no writes/installation/inference. -Read CLI context, exact project/KB/role/connection, selected version and all scoped -version pages including drafts; no account/index/connection enumeration. -Bounds: 1 MiB input/readback/definition, 200 KB sources, 200 agent versions/pages. -At 200 versions, new-version apply/plan blocks before writes; exact reuse remains valid. -Unresolved KB profiles block, never transition the KB. Verify the PROJECT principal's -exact Search Index Data Reader grant; missing roles block, never assign. -Search provisioning/degraded warns only with healthy KB GET; -failed/deleting/disabled or unavailable reads block. - -Review names, binding/grounding delta, sharing, principal/scope and preserved state. -Save `execution_input` privately; after consent set `approval.confirmed` true, -without changing plan or fingerprint. Fresh exact reuse: -`execution_required: false`, `mutation_approval_required: false`; stop without approval/apply. -Otherwise: - -```text -python helpers/prompt_connect.py --input -``` - -Bind project ID/endpoint, agent/version/model/digest, absent/exact ARM connection, -KB MCP endpoint, `allowed_tools: ["knowledge_base_retrieve"]`, `require_approval: "never"`, -role ID/principal/scope, permission forwarding, owner, `cleanup_approved: false`. -`grounding_instructions` binds the exact appended evidence-only text; -old envelopes cannot authorize grounding upgrades. - -```text -For every user question, call knowledge_base_retrieve before answering, including questions that seem unrelated to the knowledge base. Answer only from evidence returned for that question and cite the original sources. Do not answer from general knowledge or assume an answer without retrieval. If the retrieved evidence does not support an answer, reply exactly: I don't know. Do not add citations to an unsupported answer. If retrieval fails, report the failure instead of treating it as no evidence or answering from general knowledge. -``` - -Model/type/connection conflicts and duplicate same-label MCP tools block. Same-label MCP tool drift -requires `binding_action: replace-selected`; inline auth/connectors/conflicting headers -still block. Preserve definition/metadata/description/draft/blueprint reference. -Reuse the sole exact version without `create_version`, or create at most one ARM -connection and one same-agent version via `project.agents.create_version`; read both back. -Normalize only selected MCP `allowed_tools` list versus `tool_names` object encoding -(including `read_only: null`); other fields stay strict, including unrelated tools. -Use that comparison for reuse/recovery/verification, but approval/ownership hashes -bind original readback. Same-name non-equivalent connections conflict even with legacy -`update`; choose a new name, retain legacy/key connections on the same KB endpoint. -Legacy schema/hashes unchanged. Refresh version inventory/prerequisites before writes; -portal changes invalidate plans. Replan after execution; snapshots are not concurrency locks. -Permission forwarding: not applicable or named `search_auth_token` structured input, -value per request only. Exit `2`: no-write blocked; `3`: partial/ambiguous, never success. - -Use `Microsoft.CognitiveServices/accounts/projects/connections`. -Disclose `connection.is_shared_to_all` (default `true`) and actual sharing. -Server type metadata or boolean true→false restriction differences warn, not rewrite. -Missing/malformed sharing, broadened access, identity/target/auth/audience/category/ -required-metadata mismatches block; explicit false rejects nonempty `sharedUserList`. -Preserve warnings after agent failure; compare creation, ambiguous-write recovery and reuse -consistently. `completed` verifies configuration only; actual -invocation/citations/unsupported-question checks remain required. - -Connection approval excludes cleanup; use the separate cleanup planner. -After separate approval, invoke: - -```text -python helpers/prompt_cleanup.py --input -``` - -Cleanup binds run-owned identities, digests and connection ETag; deletes version before -connection; verifies absence including drafts. Preserve prior versions/project/model/KB/roles. - -## Creation receipts - -Before create: -`--cleanup-receipt-dir ""` to approved -`--input`; keep `cleanup_receipts` and original input. -Only `verified` receipts seed [separate cleanup](../lifecycle/cleanup.md); -`acknowledged`/reuse/update/recovery are not ownership proof. -Store native IDs/hashes/versions, never credentials/bodies. - -### Initial Prompt creation only - -Verify project/model/identity/network prerequisites before local initial creation. -`python helpers/prompt_connect.py --plan-initial `. -Closed request: `schema_version: "1.0"`, `owner`, `project_resource_id`, `project_endpoint`, -`agent: {name, definition}`, `prerequisites` evidence strings keyed by -`project`, `model`, `identity`, `network`; optional `inventory_limits`: -integer `max_pages`/`max_resources` (default 20/500, max 100/5000). -Closed definition: `kind: "prompt"`, verified deployment `model`, `instructions`, `tools: []`. -Evidence descriptions are not platform verification. - -Review `execution_input`; change only `approval.confirmed`. -Apply `--input ` with receipt flag: -Foundry v1 `POST /agents?api-version=v1`, never create-version on replay. -Complete project inventory/exact GET prove absence; existing names, changes, -denied/partial reads or ambiguous writes block. No POST retry. -HTTP 200 `AgentObject.versions.latest` plus independent version GET bind the receipt. -Post-ACK failure is partial, not ownership recovery. No invocation/provisioning/grants/ -Hosted creation. Cleanup retains container and other versions. +# Existing Prompt Agent SDK fallback + +Use this fallback only when authenticated Foundry MCP is unavailable +or lacks connection/version operations, never denial/conflict. +Select before planning; require the parent's approved child fingerprint before ARM creation; +do not read policy unless a creation failure implicates it. Unsupported settings block. +Read `helpers/contracts.md`; verify SDK/CLI identity. Install only missing/incompatible dependencies after approval: + +```text +python -m pip install --pre "azure-ai-projects>=2.4.0,<3" "azure-identity>=1.25.0,<2" +``` + +Run `az login` only if unauthenticated. ARM uses signed-in CLI identity; +SDK and shared cleanup loader use `AzureCliCredential`, not environment/MI fallback. +Missing CLI authentication blocks noninteractively, never falls back to another identity. +Keep CLI context unchanged; never silently switch tenant/identity. +SDK errors: safe code/HTTP status/server ID (`x-ms-request-id`, then `request-id`). +Client request IDs are not server provenance; raw exception text is withheld. +Ambiguous creation stays partial with original status/ID even if recovery fails. +Observed matching versions do not prove creation ownership. CLI output separates +`resources_remaining.unverified` from run-owned resources; unknown versions remain unnamed. +`request.json` (resolved intent, not hand-built bodies/hashes): + +```json +{ + "schema_version": "1.0", + "project_resource_id": "", + "project_endpoint": "https://account.services.ai.azure.com/api/projects/project", + "search_resource_id": "", + "knowledge_base_name": "kb", + "agent_name": "agent", + "agent_version": "1", + "connection_name": "kb-project-mi", + "is_shared_to_all": false, + "binding_action": "ensure", + "role_assignment_id": "/providers/Microsoft.Authorization/roleAssignments/", + "permission_forwarding": {"mode": "not-applicable"}, + "network": {"posture": "public", "evidence": ""}, + "owner": "" +} +``` + +```text +python helpers/prompt_connect.py --plan request.json +``` + +Use observed choices, never latest. Output: `status: planned`, `approval_summary`, +`execution_input`, `approval.confirmed: false`; no writes/installation/inference. +Read CLI context, exact project/KB/role/connection, selected version and all scoped +version pages including drafts; no account/index/connection enumeration. +Bounds: 1 MiB input/readback/definition, 200 KB sources, 200 agent versions/pages. +At 200 versions, new-version apply/plan blocks before writes; exact reuse remains valid. +Unresolved KB profiles block, never transition the KB. Verify the PROJECT principal's +exact Search Index Data Reader grant; missing roles block, never assign. +Search provisioning/degraded warns only with healthy KB GET; +failed/deleting/disabled or unavailable reads block. + +Review names, binding/grounding delta, sharing, principal/scope and preserved state. +Save `execution_input` privately; after consent set `approval.confirmed` true, +without changing plan or fingerprint. Fresh exact reuse: +`execution_required: false`, `mutation_approval_required: false`; stop without approval/apply. +Otherwise: + +```text +python helpers/prompt_connect.py --input +``` + +Bind project ID/endpoint, agent/version/model/digest, absent/exact ARM connection, +KB MCP endpoint, `allowed_tools: ["knowledge_base_retrieve"]`, `require_approval: "never"`, +role ID/principal/scope, permission forwarding, owner, `cleanup_approved: false`. +`grounding_instructions` binds the exact appended evidence-only text; +old envelopes cannot authorize grounding upgrades. + +```text +For every user question, call knowledge_base_retrieve before answering, including questions that seem unrelated to the knowledge base. Answer only from evidence returned for that question and cite the original sources. Do not answer from general knowledge or assume an answer without retrieval. If the retrieved evidence does not support an answer, reply exactly: I don't know. Do not add citations to an unsupported answer. If retrieval fails, report the failure instead of treating it as no evidence or answering from general knowledge. +``` + +Model/type/connection conflicts and duplicate same-label MCP tools block. Same-label MCP tool drift +requires `binding_action: replace-selected`; inline auth/connectors/conflicting headers +still block. Preserve definition/metadata/description/draft/blueprint reference. +Reuse the sole exact version without `create_version`, or create at most one ARM +connection and one same-agent version via `project.agents.create_version`; read both back. +Normalize only selected MCP `allowed_tools` list versus `tool_names` object encoding +(including `read_only: null`); other fields stay strict, including unrelated tools. +Use that comparison for reuse/recovery/verification, but approval/ownership hashes +bind original readback. Same-name non-equivalent connections conflict even with legacy +`update`; choose a new name, retain legacy/key connections on the same KB endpoint. +Legacy schema/hashes unchanged. Refresh version inventory/prerequisites before writes; +portal changes invalidate plans. Replan after execution; snapshots are not concurrency locks. +Permission forwarding: not applicable or named `search_auth_token` structured input, +value per request only. Exit `2`: no-write blocked; `3`: partial/ambiguous, never success. + +Use `Microsoft.CognitiveServices/accounts/projects/connections`. +Disclose `connection.is_shared_to_all` (default `true`) and actual sharing. +Server type metadata or boolean true→false restriction differences warn, not rewrite. +Missing/malformed sharing, broadened access, identity/target/auth/audience/category/ +required-metadata mismatches block; explicit false rejects nonempty `sharedUserList`. +Preserve warnings after agent failure; compare creation, ambiguous-write recovery and reuse +consistently. `completed` verifies configuration only; actual +invocation/citations/unsupported-question checks remain required. + +Connection approval excludes cleanup; use the separate cleanup planner. +After separate approval, invoke: + +```text +python helpers/prompt_cleanup.py --input +``` + +Cleanup binds run-owned identities, digests and connection ETag; deletes version before +connection; verifies absence including drafts. Preserve prior versions/project/model/KB/roles. + +## Creation receipts + +Before create: +`--cleanup-receipt-dir ""` to approved +`--input`; keep `cleanup_receipts` and original input. +Only `verified` receipts seed [separate cleanup](../lifecycle/cleanup.md); +`acknowledged`/reuse/update/recovery are not ownership proof. +Store native IDs/hashes/versions, never credentials/bodies. + +### Initial Prompt creation only + +Verify project/model/identity/network prerequisites before local initial creation. +`python helpers/prompt_connect.py --plan-initial `. +Closed request: `schema_version: "1.0"`, `owner`, `project_resource_id`, `project_endpoint`, +`agent: {name, definition}`, `prerequisites` evidence strings keyed by +`project`, `model`, `identity`, `network`; optional `inventory_limits`: +integer `max_pages`/`max_resources` (default 20/500, max 100/5000). +Closed definition: `kind: "prompt"`, verified deployment `model`, `instructions`, `tools: []`. +Evidence descriptions are not platform verification. + +Review `execution_input`; change only `approval.confirmed`. +Apply `--input ` with receipt flag: +Foundry v1 `POST /agents?api-version=v1`, never create-version on replay. +Complete project inventory/exact GET prove absence; existing names, changes, +denied/partial reads or ambiguous writes block. No POST retry. +HTTP 200 `AgentObject.versions.latest` plus independent version GET bind the receipt. +Post-ACK failure is partial, not ownership recovery. No invocation/provisioning/grants/ +Hosted creation. Cleanup retains container and other versions. diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect.md index fe95f0a4..32eef060 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect.md +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect.md @@ -1,146 +1,146 @@ -# Connect a knowledge base to an agent - -## When to use - -Connect one Prompt/Hosted Agent; optionally create it if missing. -KB: content; project connection: auth; agent MCP tool: versioned use. - -## Do not use - -Do not replace agents, attach multiple base tools, or handle base-free lifecycle. -Permission error is not absence; creation approval is separate. - -## Inputs and discovery order - -Resolve prompt, session, workspace, exact Azure readback, default, then one focused question. -Label sources; missing input blocks. Reads need no approval; silence approves nothing. - -| Input | Why needed | Required? | Discovery order | Safe default | If missing or unanswered | Reconfirmation trigger | -|---|---|---|---|---|---|---| -| Base return/MCP endpoint | Grounding | Required | Session, exact proof | None | Block | Base/API/evidence change | -| Project ID/endpoint | Target | Required | Prompt, workspace, readback | None | Block | ID/tenant change | -| Agent name/inventory/type | Identity | Required | Prompt, readback | Never latest/type | Block ambiguity | Inventory/type | -| Definition/model/baseline; Hosted telemetry | Preservation | By branch | Exact readback | Preserve unrelated fields | Block | Protected state | -| Connection/tool; Hosted toolbox/environment | Binding | By branch | Full readback | One retrieve binding | Block conflict | Binding/endpoint | -| Principal/reader role | Retrieval | Before assignment | Readback | Exact Search scope | Block | Assignment | -| Grounding delta | Evidence | Required | Instructions | Base/citations/`I don't know` | Block loss | Delta | -| Acceptance/unrelated questions | Verification | Before invocation | Authorized evidence | None | Propose candidate | Questions | -| Cleanup/retention owner | Ownership | Required | Prompt/session | None | Block | Owner | -| Provider/install or CLI choice | Missing agent | Conditional | Runtime, then choice | No automatic fallback | Block | Availability/choice | - -## Decisions - -Use [policy diagnostics](../references/search-substrate.md#azure-policy-diagnostics-after-creation-failure) -only after a creation failure implicates policy, never as a gate. - -Read exact IDs/all scoped pages; inaccessible is not absent: - -- **Prompt:** refresh the selected version and all its scoped version pages after - portal add/remove; never assume latest. Exact is zero-write. Collision is the - connection NAME, not KB endpoint: an approved new name can share that endpoint, - preserving legacy/key connections. Same-name mismatch blocks. An explicit - selected-tool switch creates a version, preserving all unrelated fields/tools. -- **Hosted:** inspect remote project, exact agent/version, KB MCP, runtime principal - and connection/toolbox/binding first: no source path for inspection or exact reuse. - Bind source/config digest for agent edits/redeployment. - Conflicts block. Read [Hosted connection](connect-hosted.md) - before plans or verification. - -Only for a missing agent, load [missing-agent creation](create-missing-agent.md) -before questions/plans. Prefer `microsoft-foundry`: delegate approved -identity/configuration and known requirements. Otherwise offer installation -or Prompt CLI creation. Installation refusal is not creation approval. -Require a typed return with project, agent, version/deployment, model/status/identity -and definition/source/config/environment. Independently read back; -use fresh discovery and a new fingerprinted connection plan; -never carry creation approval. Incomplete returns block. - -Read the authenticated `knowledgebases` API, never generated indexes -or `foundryextensions_knowledge_index_list`. Existing multi-source KBs are valid. -Preserve valid KB effort/output/models; preview MCP supports reasoning/synthesis. -Use the exact preview API in SDK; native GA minimal/extractive MCP exists. -KB model: low or medium with supported output; preview minimal/answerSynthesis is valid. -Use the approved [KB transition](../knowledge-bases/create.md#agent-compatible-minimal-transition) -only when an explicit model-free normalization is needed. -Never change the KB implicitly or add a model as a workaround. -Only when authenticated Foundry MCP is unavailable or lacks a required operation, -load [typed Prompt SDK fallback](connect-prompt-sdk-fallback.md). -Denial/conflict blocks. Bind the surface; switching requires fresh review, never bypass. - -## Proposed plan - -List delta, API, identity/scope, network/data, cost, verification, owner -and retained IDs. - -## Confirmation - -Approve concrete changes once; `plan_fingerprint` stays internal, with -`cleanup_approved: false`. Delegated creation is separate. A Hosted principal -known only after deploy requires a separate RBAC plan. Material/protected-state -drift invalidates approval; disclose bounded Prompt warnings below. - -## Mutation - -After confirmation, use exact identities; never retry another name. - -**Prompt:** Use the planned MCP or SDK surface; never invent operations. -Reconcile one `2025-10-01-preview` -`RemoteTool` connection with `ProjectManagedIdentity`, Search audience, and the -`2026-08-01-preview` KB MCP endpoint. Require the exact Search-service-scoped -`Search Index Data Reader` assignment to observed PROJECT `identity.principalId`, -not agent identity. Preserve all fields; add/switch at most one -same-label `MCPTool` with exact endpoint/connection, -`allowed_tools: ["knowledge_base_retrieve"]`, and `require_approval: "never"`. -Under the same agent name, require retrieval for every question, including -unrelated questions. Answer from retrieved evidence, not general knowledge; -unsupported answers are exactly `I don't know` without citations; -retrieval errors must remain errors. Approve exact appended instructions; -preserve previous instructions/versions. Duplicate labels block. -Search provisioning alone is not a connection-write blocker when the KB GET is -healthy; warn, never claim retrieval readiness. Failed/auth/deletion states block. - -After exact identity/binding checks, `type` metadata differences and boolean -`isSharedToAll: true`-to-`false` restriction warn without rewriting. Missing/malformed -sharing, broader access, wrong target/auth/audience/category/required metadata -block. Apply the same rules to reuse; require actual agent tests below. - -**Hosted:** Follow the selected [Hosted connection](connect-hosted.md) procedure. -Only approved missing resources, binding differences and required deployments -are writes; exact existing state skips them all. Never replace agents; -source edits need separate review/handoff. - -## Verification - -Verify protected fields, a `knowledge_base_retrieve` call with original -citations, and unrelated `I don't know` without citations. Preserve behavior; -rerun with stable IDs and zero configuration writes. Account separately for -requested invocations, conversations/sessions and their usage. Record actual -tool execution and returned evidence separately from answer behavior; unavailable -payloads leave payload-level faithfulness unverified, not an inferred empty result. - -## Failure and partial completion - -Preserve first status/message/request ID. Denial, conflict, incomplete -delegation, failed citations, unknown principal or drift blocks. -Record writes, evidence, ownership, recovery and warnings; never widen state. - -## Cleanup - -Separate fingerprinted cleanup deletes only run-owned artifacts in reverse. -Prompt uses `helpers/prompt_cleanup.py` with exact digests. Hosted returns -`hosted-cleanup-unsupported`, zero writes, retaining all resources. -Unclear ownership blocks. - -## Return contract - -Return `planned`/`completed`/`blocked`/`partial`: exact actions/resources, -invocation/readback evidence, ownership, failure, warnings and cleanup status. - -## References - -- [Shared audit schemas](../references/platform-contracts.md). -- [Interface versions](../references/platform-interfaces.md). - -Authorities: failure/conflict/uncertainty only. - -- [Connect Agents to Foundry IQ knowledge bases](https://learn.microsoft.com/azure/foundry/agents/how-to/foundry-iq-connect) +# Connect a knowledge base to an agent + +## When to use + +Connect one Prompt/Hosted Agent; optionally create it if missing. +KB: content; project connection: auth; agent MCP tool: versioned use. + +## Do not use + +Do not replace agents, attach multiple base tools, or handle base-free lifecycle. +Permission error is not absence; creation approval is separate. + +## Inputs and discovery order + +Resolve prompt, session, workspace, exact Azure readback, default, then one focused question. +Label sources; missing input blocks. Reads need no approval; silence approves nothing. + +| Input | Why needed | Required? | Discovery order | Safe default | If missing or unanswered | Reconfirmation trigger | +|---|---|---|---|---|---|---| +| Base return/MCP endpoint | Grounding | Required | Session, exact proof | None | Block | Base/API/evidence change | +| Project ID/endpoint | Target | Required | Prompt, workspace, readback | None | Block | ID/tenant change | +| Agent name/inventory/type | Identity | Required | Prompt, readback | Never latest/type | Block ambiguity | Inventory/type | +| Definition/model/baseline; Hosted telemetry | Preservation | By branch | Exact readback | Preserve unrelated fields | Block | Protected state | +| Connection/tool; Hosted toolbox/environment | Binding | By branch | Full readback | One retrieve binding | Block conflict | Binding/endpoint | +| Principal/reader role | Retrieval | Before assignment | Readback | Exact Search scope | Block | Assignment | +| Grounding delta | Evidence | Required | Instructions | Base/citations/`I don't know` | Block loss | Delta | +| Acceptance/unrelated questions | Verification | Before invocation | Authorized evidence | None | Propose candidate | Questions | +| Cleanup/retention owner | Ownership | Required | Prompt/session | None | Block | Owner | +| Provider/install or CLI choice | Missing agent | Conditional | Runtime, then choice | No automatic fallback | Block | Availability/choice | + +## Decisions + +Use [policy diagnostics](../references/search-substrate.md#azure-policy-diagnostics-after-creation-failure) +only after a creation failure implicates policy, never as a gate. + +Read exact IDs/all scoped pages; inaccessible is not absent: + +- **Prompt:** refresh the selected version and all its scoped version pages after + portal add/remove; never assume latest. Exact is zero-write. Collision is the + connection NAME, not KB endpoint: an approved new name can share that endpoint, + preserving legacy/key connections. Same-name mismatch blocks. An explicit + selected-tool switch creates a version, preserving all unrelated fields/tools. +- **Hosted:** inspect remote project, exact agent/version, KB MCP, runtime principal + and connection/toolbox/binding first: no source path for inspection or exact reuse. + Bind source/config digest for agent edits/redeployment. + Conflicts block. Read [Hosted connection](connect-hosted.md) + before plans or verification. + +Only for a missing agent, load [missing-agent creation](create-missing-agent.md) +before questions/plans. Prefer `microsoft-foundry`: delegate approved +identity/configuration and known requirements. Otherwise offer installation +or Prompt CLI creation. Installation refusal is not creation approval. +Require a typed return with project, agent, version/deployment, model/status/identity +and definition/source/config/environment. Independently read back; +use fresh discovery and a new fingerprinted connection plan; +never carry creation approval. Incomplete returns block. + +Read the authenticated `knowledgebases` API, never generated indexes +or `foundryextensions_knowledge_index_list`. Existing multi-source KBs are valid. +Preserve valid KB effort/output/models; preview MCP supports reasoning/synthesis. +Use the exact preview API in SDK; native GA minimal/extractive MCP exists. +KB model: low or medium with supported output; preview minimal/answerSynthesis is valid. +Use the approved [KB transition](../knowledge-bases/create.md#agent-compatible-minimal-transition) +only when an explicit model-free normalization is needed. +Never change the KB implicitly or add a model as a workaround. +Only when authenticated Foundry MCP is unavailable or lacks a required operation, +load [typed Prompt SDK fallback](connect-prompt-sdk-fallback.md). +Denial/conflict blocks. Bind the surface; switching requires fresh review, never bypass. + +## Proposed plan + +List delta, API, identity/scope, network/data, cost, verification, owner +and retained IDs. + +## Confirmation + +Approve concrete changes once; `plan_fingerprint` stays internal, with +`cleanup_approved: false`. Delegated creation is separate. A Hosted principal +known only after deploy requires a separate RBAC plan. Material/protected-state +drift invalidates approval; disclose bounded Prompt warnings below. + +## Mutation + +After confirmation, use exact identities; never retry another name. + +**Prompt:** Use the planned MCP or SDK surface; never invent operations. +Reconcile one `2025-10-01-preview` +`RemoteTool` connection with `ProjectManagedIdentity`, Search audience, and the +`2026-08-01-preview` KB MCP endpoint. Require the exact Search-service-scoped +`Search Index Data Reader` assignment to observed PROJECT `identity.principalId`, +not agent identity. Preserve all fields; add/switch at most one +same-label `MCPTool` with exact endpoint/connection, +`allowed_tools: ["knowledge_base_retrieve"]`, and `require_approval: "never"`. +Under the same agent name, require retrieval for every question, including +unrelated questions. Answer from retrieved evidence, not general knowledge; +unsupported answers are exactly `I don't know` without citations; +retrieval errors must remain errors. Approve exact appended instructions; +preserve previous instructions/versions. Duplicate labels block. +Search provisioning alone is not a connection-write blocker when the KB GET is +healthy; warn, never claim retrieval readiness. Failed/auth/deletion states block. + +After exact identity/binding checks, `type` metadata differences and boolean +`isSharedToAll: true`-to-`false` restriction warn without rewriting. Missing/malformed +sharing, broader access, wrong target/auth/audience/category/required metadata +block. Apply the same rules to reuse; require actual agent tests below. + +**Hosted:** Follow the selected [Hosted connection](connect-hosted.md) procedure. +Only approved missing resources, binding differences and required deployments +are writes; exact existing state skips them all. Never replace agents; +source edits need separate review/handoff. + +## Verification + +Verify protected fields, a `knowledge_base_retrieve` call with original +citations, and unrelated `I don't know` without citations. Preserve behavior; +rerun with stable IDs and zero configuration writes. Account separately for +requested invocations, conversations/sessions and their usage. Record actual +tool execution and returned evidence separately from answer behavior; unavailable +payloads leave payload-level faithfulness unverified, not an inferred empty result. + +## Failure and partial completion + +Preserve first status/message/request ID. Denial, conflict, incomplete +delegation, failed citations, unknown principal or drift blocks. +Record writes, evidence, ownership, recovery and warnings; never widen state. + +## Cleanup + +Separate fingerprinted cleanup deletes only run-owned artifacts in reverse. +Prompt uses `helpers/prompt_cleanup.py` with exact digests. Hosted returns +`hosted-cleanup-unsupported`, zero writes, retaining all resources. +Unclear ownership blocks. + +## Return contract + +Return `planned`/`completed`/`blocked`/`partial`: exact actions/resources, +invocation/readback evidence, ownership, failure, warnings and cleanup status. + +## References + +- [Shared audit schemas](../references/platform-contracts.md). +- [Interface versions](../references/platform-interfaces.md). + +Authorities: failure/conflict/uncertainty only. + +- [Connect Agents to Foundry IQ knowledge bases](https://learn.microsoft.com/azure/foundry/agents/how-to/foundry-iq-connect) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/create-missing-agent.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/create-missing-agent.md index c8ff31a8..f3126875 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/create-missing-agent.md +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/create-missing-agent.md @@ -1,163 +1,163 @@ -# Missing agent for a knowledge-base connection - -Load only from Connect for a missing agent, not generic creation, replacement -or the existing-agent SDK connection fallback. Preserve existing agents/versions. - -## Resolve and offer the creation capability - -Use prompt/session/workspace and exact Azure readback to recommend project, -type/name and existing model deployment. Ask only unresolved choices. -Inventory all pages; permission denial or incomplete discovery is not absence. -An existing incompatible agent does not authorize a new name: require explicit -new-agent intent and prove the selected new identity absent. - -Check the runtime registry or exact invocation for `microsoft-foundry`. Skill -availability and authenticated creation-tool availability are separate facts. -Unknown stays unknown; never claim unexecuted delegation. -If unavailable, ask one focused choice, unless already answered: - -- Install Microsoft's Azure Skills plugin (recommended). -- Decline installation and review local CLI creation for a Prompt Agent. -- Stop with setup guidance. - -Do not install automatically. Show the source, host/workspace scope and exact -commands. Installation consent is separate from Azure creation approval; -cancellation stops. In Copilot CLI these are interactive host commands, not -PowerShell commands: - -```text -/plugin marketplace add microsoft/azure-skills -/plugin install azure@azure-skills -/mcp show -``` - -With MCP already configured, the skill-only option is: - -```text -npx skills add https://github.com/microsoft/azure-skills --skill microsoft-foundry -``` - -Skill-only installation does not install or authenticate MCP tools. After -installation, reload if required and recheck runtime visibility, authentication -and exact creation operations. If installation/reload cannot run here, give -the commands to the user. If tools remain unavailable, report -`creation-provider-unavailable`, zero Azure writes, and offer the CLI choice; -do not silently switch. Never bypass an authorization -denial through another surface. - -## Creation plan and approval - -Prefer available delegation. Do not scan policy before either creation path. -Use [policy diagnostics](../references/search-substrate.md#azure-policy-diagnostics-after-creation-failure) -only after a creation failure implicates policy. Preserve known required -settings. Present one immutable creation plan with: - -- Exact tenant/subscription/project ID and matching endpoint, absent agent name, - Prompt/Hosted type, model deployment and full initial definition. -- Provider, API/version, exact expanded commands and request bodies or delegated - commands, local files/installations and digests. -- Identity/RBAC, network/data movement, known requirements, - existing and incremental costs, verification, retained resources and owner. -- `plan_fingerprint` and `cleanup_approved: false`. - -Let the user review delegated commands before execution. Changed commands, -model, scope, known requirements or definition require new approval. Approval to install -or select CLI does not approve creation. Creation does not approve the KB -connection, Search roles, invocation charges or cleanup. - -## Local CLI fallback: Prompt only - -Execute locally only after explicit CLI selection and creation-plan approval. -Reuse an accessible existing project and chat model deployment; read back the -deployment identity, model/version and status rather than copying a sample -model. Missing project/model/rights/network prerequisites block this fallback; -do not provision them, change permissions/networking or substitute a Hosted -Agent. Hosted creation remains delegated; if unavailable, return -`creation-provider-unavailable` with official guidance. - -Use signed-in Azure CLI identity with Foundry v1 REST. -Verify `az version` and `az account show`; authenticate only if needed. -`az rest --resource https://ai.azure.com/` obtains tokens internally. Never use -keys, ask for tokens, print tokens or enable verbose/debug credential logging. - -Before approval, review the complete body in a UTF-8 file: - -```json -{ - "name": "", - "definition": { - "kind": "prompt", - "model": "", - "instructions": "", - "tools": [] - } -} -``` - -No initial KB binding. These CLI templates use no shell variables (Bash, -PowerShell, cmd.exe). Replace placeholders before execution with the approved -endpoint (no trailing slash), name, returned version and OS-native absolute body -path. Keep URLs and the `@`-prefixed file argument quoted. - -```text -az rest --method get --url "/agents/?api-version=v1" --resource https://ai.azure.com/ -``` - -Inspect the actual HTTP result, not just a nonzero exit. A 404 is absence only -after successful project access and complete agent inventory; a 401/403, timeout -or malformed response blocks. For a present identity, list every version and -compare the full approved definition including model, instructions and tools. -A sole exact match is zero-write reuse; multiple matches or drift block. -Never choose latest automatically, overwrite, suffix-create or issue a version -POST on replay. Return connected versions to Connect; never reset their tools. - -Before writing, refresh absence/access and recompute the -approved plan/body digests. Changed evidence needs new review. Only after all -gates pass, issue one create-agent POST, not an update or create-version call: - -```text -az rest --method post --url "/agents?api-version=v1" --resource https://ai.azure.com/ --headers "Content-Type=application/json" --body "@" -``` - -Retain exit status and the first error/status/request ID. Never blindly -retry POST, including on timeout/409/5xx. Reconcile an ambiguous write by reading -the same identity and all versions. If the exact result or ownership is -unproven, report `partial` with potentially created resources and stop; never -switch to SDK/delegation or delete to make it pass. - -## Verify and return to Connect - -For local Prompt creation, independently GET the agent and returned version, -and list all versions; command acceptance is not proof: - -```text -az rest --method get --url "/agents/?api-version=v1" --resource https://ai.azure.com/ -az rest --method get --url "/agents//versions?api-version=v1" --resource https://ai.azure.com/ -az rest --method get --url "/agents//versions/?api-version=v1" --resource https://ai.azure.com/ -``` - -Follow pagination. Require matching project/name/version, Prompt kind, full -definition and usable status, with no duplicate versions or unexpected tools. -Record actual project/agent identities; never guess a principal. Unknown -required identity blocks connection, even if agent creation succeeded. - -Return `completed`, `blocked` or `partial`, IDs/version/model/status, definition -digest, identity evidence, first failure, API/provider/commands, approved -fingerprint, writes, ownership and retained resources. Return to Connect: -independently read back, use fresh discovery and a new fingerprinted connection -plan, and never carry creation approval. After connection approval, invoke the -exact version; require KB tool-call evidence, original citations and abstention. -Direct KB retrieval is not agent proof. Repeat Connect and compare -version/tool/connection IDs and counts with zero writes. - -Before local creation, select [receipt capture](connect-prompt-sdk-fallback.md#initial-local-prompt-creation-only). -No automatic cleanup: separate ownership/approval; retain prior versions, -shared resources and container. - -## Authorities - -Authorities: failure/conflict/uncertainty only. - -[Installation guide](https://learn.microsoft.com/azure/foundry/how-to/develop/use-microsoft-foundry-skill), -[Foundry v1 REST API](https://learn.microsoft.com/rest/api/microsoft-foundry/aiproject), -[Prompt quickstart](https://learn.microsoft.com/azure/foundry/agents/quickstarts/prompt-agent). +# Missing agent for a knowledge-base connection + +Load only from Connect for a missing agent, not generic creation, replacement +or the existing-agent SDK connection fallback. Preserve existing agents/versions. + +## Resolve and offer the creation capability + +Use prompt/session/workspace and exact Azure readback to recommend project, +type/name and existing model deployment. Ask only unresolved choices. +Inventory all pages; permission denial or incomplete discovery is not absence. +An existing incompatible agent does not authorize a new name: require explicit +new-agent intent and prove the selected new identity absent. + +Check the runtime registry or exact invocation for `microsoft-foundry`. Skill +availability and authenticated creation-tool availability are separate facts. +Unknown stays unknown; never claim unexecuted delegation. +If unavailable, ask one focused choice, unless already answered: + +- Install Microsoft's Azure Skills plugin (recommended). +- Decline installation and review local CLI creation for a Prompt Agent. +- Stop with setup guidance. + +Do not install automatically. Show the source, host/workspace scope and exact +commands. Installation consent is separate from Azure creation approval; +cancellation stops. In Copilot CLI these are interactive host commands, not +PowerShell commands: + +```text +/plugin marketplace add microsoft/azure-skills +/plugin install azure@azure-skills +/mcp show +``` + +With MCP already configured, the skill-only option is: + +```text +npx skills add https://github.com/microsoft/azure-skills --skill microsoft-foundry +``` + +Skill-only installation does not install or authenticate MCP tools. After +installation, reload if required and recheck runtime visibility, authentication +and exact creation operations. If installation/reload cannot run here, give +the commands to the user. If tools remain unavailable, report +`creation-provider-unavailable`, zero Azure writes, and offer the CLI choice; +do not silently switch. Never bypass an authorization +denial through another surface. + +## Creation plan and approval + +Prefer available delegation. Do not scan policy before either creation path. +Use [policy diagnostics](../references/search-substrate.md#azure-policy-diagnostics-after-creation-failure) +only after a creation failure implicates policy. Preserve known required +settings. Present one immutable creation plan with: + +- Exact tenant/subscription/project ID and matching endpoint, absent agent name, + Prompt/Hosted type, model deployment and full initial definition. +- Provider, API/version, exact expanded commands and request bodies or delegated + commands, local files/installations and digests. +- Identity/RBAC, network/data movement, known requirements, + existing and incremental costs, verification, retained resources and owner. +- `plan_fingerprint` and `cleanup_approved: false`. + +Let the user review delegated commands before execution. Changed commands, +model, scope, known requirements or definition require new approval. Approval to install +or select CLI does not approve creation. Creation does not approve the KB +connection, Search roles, invocation charges or cleanup. + +## Local CLI fallback: Prompt only + +Execute locally only after explicit CLI selection and creation-plan approval. +Reuse an accessible existing project and chat model deployment; read back the +deployment identity, model/version and status rather than copying a sample +model. Missing project/model/rights/network prerequisites block this fallback; +do not provision them, change permissions/networking or substitute a Hosted +Agent. Hosted creation remains delegated; if unavailable, return +`creation-provider-unavailable` with official guidance. + +Use signed-in Azure CLI identity with Foundry v1 REST. +Verify `az version` and `az account show`; authenticate only if needed. +`az rest --resource https://ai.azure.com/` obtains tokens internally. Never use +keys, ask for tokens, print tokens or enable verbose/debug credential logging. + +Before approval, review the complete body in a UTF-8 file: + +```json +{ + "name": "", + "definition": { + "kind": "prompt", + "model": "", + "instructions": "", + "tools": [] + } +} +``` + +No initial KB binding. These CLI templates use no shell variables (Bash, +PowerShell, cmd.exe). Replace placeholders before execution with the approved +endpoint (no trailing slash), name, returned version and OS-native absolute body +path. Keep URLs and the `@`-prefixed file argument quoted. + +```text +az rest --method get --url "/agents/?api-version=v1" --resource https://ai.azure.com/ +``` + +Inspect the actual HTTP result, not just a nonzero exit. A 404 is absence only +after successful project access and complete agent inventory; a 401/403, timeout +or malformed response blocks. For a present identity, list every version and +compare the full approved definition including model, instructions and tools. +A sole exact match is zero-write reuse; multiple matches or drift block. +Never choose latest automatically, overwrite, suffix-create or issue a version +POST on replay. Return connected versions to Connect; never reset their tools. + +Before writing, refresh absence/access and recompute the +approved plan/body digests. Changed evidence needs new review. Only after all +gates pass, issue one create-agent POST, not an update or create-version call: + +```text +az rest --method post --url "/agents?api-version=v1" --resource https://ai.azure.com/ --headers "Content-Type=application/json" --body "@" +``` + +Retain exit status and the first error/status/request ID. Never blindly +retry POST, including on timeout/409/5xx. Reconcile an ambiguous write by reading +the same identity and all versions. If the exact result or ownership is +unproven, report `partial` with potentially created resources and stop; never +switch to SDK/delegation or delete to make it pass. + +## Verify and return to Connect + +For local Prompt creation, independently GET the agent and returned version, +and list all versions; command acceptance is not proof: + +```text +az rest --method get --url "/agents/?api-version=v1" --resource https://ai.azure.com/ +az rest --method get --url "/agents//versions?api-version=v1" --resource https://ai.azure.com/ +az rest --method get --url "/agents//versions/?api-version=v1" --resource https://ai.azure.com/ +``` + +Follow pagination. Require matching project/name/version, Prompt kind, full +definition and usable status, with no duplicate versions or unexpected tools. +Record actual project/agent identities; never guess a principal. Unknown +required identity blocks connection, even if agent creation succeeded. + +Return `completed`, `blocked` or `partial`, IDs/version/model/status, definition +digest, identity evidence, first failure, API/provider/commands, approved +fingerprint, writes, ownership and retained resources. Return to Connect: +independently read back, use fresh discovery and a new fingerprinted connection +plan, and never carry creation approval. After connection approval, invoke the +exact version; require KB tool-call evidence, original citations and abstention. +Direct KB retrieval is not agent proof. Repeat Connect and compare +version/tool/connection IDs and counts with zero writes. + +Before local creation, select [receipt capture](connect-prompt-sdk-fallback.md#initial-local-prompt-creation-only). +No automatic cleanup: separate ownership/approval; retain prior versions, +shared resources and container. + +## Authorities + +Authorities: failure/conflict/uncertainty only. + +[Installation guide](https://learn.microsoft.com/azure/foundry/how-to/develop/use-microsoft-foundry-skill), +[Foundry v1 REST API](https://learn.microsoft.com/rest/api/microsoft-foundry/aiproject), +[Prompt quickstart](https://learn.microsoft.com/azure/foundry/agents/quickstarts/prompt-agent). diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_blob_observation.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_blob_observation.py index b31e1ca9..645e4a24 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_blob_observation.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_blob_observation.py @@ -1,83 +1,83 @@ -"""Credential-free semantic projections, separate from cleanup integrity.""" -from __future__ import annotations - -try: - from ._common import HelperFailure, digest - from . import _indexer_observation as indexer -except ImportError: - from _common import HelperFailure, digest - import _indexer_observation as indexer - - -FIELDS = frozenset(("name", "type", "container", "identity", "description", - "dataChangeDetectionPolicy", "dataDeletionDetectionPolicy", "encryptionKey", - "fields", "scoringProfiles", "defaultScoringProfile", "corsOptions", - "suggesters", "analyzers", "normalizers", "tokenizers", "tokenFilters", - "charFilters", "similarity", "semantic", "vectorSearch", - "skills", "cognitiveServices", "knowledgeStore", "indexProjections", - "dataSourceName", "targetIndexName", "skillsetName", "parameters", - "fieldMappings", "outputFieldMappings", "cache", "disabled")) -PROJECTION_FIELDS = {"core_digest", "field_digests"} - - -def observe(child, kind): - excluded = {"@odata.etag"} - if kind == "datasource": - excluded.add("credentials") - if kind == "indexer": - excluded.add("schedule") - core = {key: value for key, value in child.items() if key not in excluded} - fields = {key: digest(value) for key, value in core.items() if key in FIELDS} - fields["additionalProperties"] = digest({key: value for key, value in core.items() if key not in FIELDS}) - result = {"core_digest": digest(core), "field_digests": fields} - if kind == "datasource": - result["credential_digest"] = digest({"present": "credentials" in child, "value": child.get("credentials")}) - return result - - -def note(diagnostics, kind, severity, code, field, request_id): - indexer.note(diagnostics, severity, code, f"{kind}.{field}", - f"{kind}.{field}: {code}; values withheld.", request_id) - - -def compare(current, expected, kind, diagnostics, request_id): - if current["digest"] == expected["digest"] and current != expected: - raise HelperFailure(f"{kind}-evidence-inconsistent", "Equal full hashes have inconsistent projection evidence.", - blocked_at="verification", request_id=request_id) - if current["core_digest"] != expected["core_digest"]: - fields = current["field_digests"].keys() | expected["field_digests"].keys() - for field in sorted(fields): - if current["field_digests"].get(field) != expected["field_digests"].get(field): - note(diagnostics, kind, "error", "definition-drift", field, request_id) - raise HelperFailure("indexer-definition-drift" if kind == "indexer" else "definition-drift", - "Meaningful generated configuration changed; inspect child diagnostics.", - blocked_at="verification", request_id=request_id) - if current["etag"] != expected["etag"]: - note(diagnostics, kind, "info", "generated-version-changed", "@odata.etag", request_id) - if kind == "datasource" and current["credential_digest"] != expected["credential_digest"]: - note(diagnostics, kind, "warning", "datasource-credential-projection-changed", "credentials", request_id) - - -def indexer_only(diagnostics): - return [item for item in diagnostics - if not item["field"].startswith(("datasource.", "indexer.", "skillset.", "index."))] - - -def valid(value, kind, sha256): - fields = {"etag", "digest"} | PROJECTION_FIELDS - if kind == "indexer": - fields |= indexer.PROJECTION_FIELDS - if kind == "datasource": - fields |= {"credential_digest", "binding_proof"} - if not isinstance(value, dict) or set(value) != fields: - return False - hashes = fields - {"etag", "field_digests", "binding_proof"} - return (isinstance(value["etag"], str) and bool(value["etag"].strip()) - and all(isinstance(value[key], str) and sha256.fullmatch(value[key]) for key in hashes) - and isinstance(value["field_digests"], dict) - and set(value["field_digests"]) <= FIELDS | {"additionalProperties"} - and "additionalProperties" in value["field_digests"] - and all(isinstance(item, str) and sha256.fullmatch(item) - for item in value["field_digests"].values()) - and (kind != "datasource" or isinstance(value["binding_proof"], str) - and value["binding_proof"] in {"resource-id", "unverified"})) +"""Credential-free semantic projections, separate from cleanup integrity.""" +from __future__ import annotations + +try: + from ._common import HelperFailure, digest + from . import _indexer_observation as indexer +except ImportError: + from _common import HelperFailure, digest + import _indexer_observation as indexer + + +FIELDS = frozenset(("name", "type", "container", "identity", "description", + "dataChangeDetectionPolicy", "dataDeletionDetectionPolicy", "encryptionKey", + "fields", "scoringProfiles", "defaultScoringProfile", "corsOptions", + "suggesters", "analyzers", "normalizers", "tokenizers", "tokenFilters", + "charFilters", "similarity", "semantic", "vectorSearch", + "skills", "cognitiveServices", "knowledgeStore", "indexProjections", + "dataSourceName", "targetIndexName", "skillsetName", "parameters", + "fieldMappings", "outputFieldMappings", "cache", "disabled")) +PROJECTION_FIELDS = {"core_digest", "field_digests"} + + +def observe(child, kind): + excluded = {"@odata.etag"} + if kind == "datasource": + excluded.add("credentials") + if kind == "indexer": + excluded.add("schedule") + core = {key: value for key, value in child.items() if key not in excluded} + fields = {key: digest(value) for key, value in core.items() if key in FIELDS} + fields["additionalProperties"] = digest({key: value for key, value in core.items() if key not in FIELDS}) + result = {"core_digest": digest(core), "field_digests": fields} + if kind == "datasource": + result["credential_digest"] = digest({"present": "credentials" in child, "value": child.get("credentials")}) + return result + + +def note(diagnostics, kind, severity, code, field, request_id): + indexer.note(diagnostics, severity, code, f"{kind}.{field}", + f"{kind}.{field}: {code}; values withheld.", request_id) + + +def compare(current, expected, kind, diagnostics, request_id): + if current["digest"] == expected["digest"] and current != expected: + raise HelperFailure(f"{kind}-evidence-inconsistent", "Equal full hashes have inconsistent projection evidence.", + blocked_at="verification", request_id=request_id) + if current["core_digest"] != expected["core_digest"]: + fields = current["field_digests"].keys() | expected["field_digests"].keys() + for field in sorted(fields): + if current["field_digests"].get(field) != expected["field_digests"].get(field): + note(diagnostics, kind, "error", "definition-drift", field, request_id) + raise HelperFailure("indexer-definition-drift" if kind == "indexer" else "definition-drift", + "Meaningful generated configuration changed; inspect child diagnostics.", + blocked_at="verification", request_id=request_id) + if current["etag"] != expected["etag"]: + note(diagnostics, kind, "info", "generated-version-changed", "@odata.etag", request_id) + if kind == "datasource" and current["credential_digest"] != expected["credential_digest"]: + note(diagnostics, kind, "warning", "datasource-credential-projection-changed", "credentials", request_id) + + +def indexer_only(diagnostics): + return [item for item in diagnostics + if not item["field"].startswith(("datasource.", "indexer.", "skillset.", "index."))] + + +def valid(value, kind, sha256): + fields = {"etag", "digest"} | PROJECTION_FIELDS + if kind == "indexer": + fields |= indexer.PROJECTION_FIELDS + if kind == "datasource": + fields |= {"credential_digest", "binding_proof"} + if not isinstance(value, dict) or set(value) != fields: + return False + hashes = fields - {"etag", "field_digests", "binding_proof"} + return (isinstance(value["etag"], str) and bool(value["etag"].strip()) + and all(isinstance(value[key], str) and sha256.fullmatch(value[key]) for key in hashes) + and isinstance(value["field_digests"], dict) + and set(value["field_digests"]) <= FIELDS | {"additionalProperties"} + and "additionalProperties" in value["field_digests"] + and all(isinstance(item, str) and sha256.fullmatch(item) + for item in value["field_digests"].values()) + and (kind != "datasource" or isinstance(value["binding_proof"], str) + and value["binding_proof"] in {"resource-id", "unverified"})) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_bootstrap_io.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_bootstrap_io.py index 3dfbd89f..1e27f43f 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_bootstrap_io.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_bootstrap_io.py @@ -1,589 +1,589 @@ -"""Private storage and shell-free native CLI execution for the bootstrap owner.""" -from __future__ import annotations - -import ctypes -import json -import os -import re -import shutil -import stat -import subprocess -import uuid -from contextlib import contextmanager -from pathlib import Path - -try: - from ._common import HelperFailure, canonical_bytes -except ImportError: - from _common import HelperFailure, canonical_bytes - -MAX_BYTES = 1024 * 1024 - - -def failure(code, message): - return HelperFailure(code, message, blocked_at="verification") - - -def read_json(path): - try: - with Path(path).open("rb") as handle: - raw = handle.read(MAX_BYTES + 1) - if len(raw) > MAX_BYTES: - raise failure("bootstrap-input-invalid", "JSON exceeds the one MiB input limit.") - value = json.loads(raw.decode("utf-8")) - pending = [(value, 0)] - count = 0 - while pending: - item, depth = pending.pop() - count += 1 - if depth > 20 or count > 10000: - raise ValueError("JSON structure exceeds limits") - if isinstance(item, dict): - pending.extend((v, depth + 1) for v in item.values()) - elif isinstance(item, list): - pending.extend((v, depth + 1) for v in item) - json.dumps(value, ensure_ascii=False, allow_nan=False).encode("utf-8") - return value - except (OSError, UnicodeError, ValueError, RecursionError) as exc: - raise failure("bootstrap-input-invalid", "Select bounded, valid UTF-8 JSON.") from exc - - -def _windows_ancestor_acl(owner_text, entries, current): - # Windows volume roots are commonly owned by this fixed OS servicing principal. - installer = "S-1-5-80-956008885-3418522649-1831038044-1853292631-2271478464" - trusted = (current, "OW", "SY", "BA", installer) - if owner_text not in tuple("O:" + trustee for trustee in trusted if trustee != "OW"): - raise OSError("Ancestor owner can change the access boundary") - rights = { - "FA": 0x1F01FF, "FR": 0x120089, "FW": 0x120116, "FX": 0x1200A0, - "GA": 0x10000000, "GR": 0x80000000, "GW": 0x40000000, "GX": 0x20000000, - "SD": 0x10000, "RC": 0x20000, "WD": 0x40000, "WO": 0x80000, - "CC": 1, "DC": 2, "LC": 4, "SW": 8, "RP": 16, "WP": 32, "DT": 64, - "LO": 128, "CR": 256, - } - for entry in entries: - parts = entry.split(";") - if len(parts) != 6 or parts[0] not in ("A", "D") or parts[3] or parts[4]: - raise OSError("Unsupported ancestor ACL") - if parts[0] == "D" or "IO" in parts[1] or parts[5] in trusted: - continue - value = parts[2] - try: - mask = int(value, 16) if value.startswith("0x") else 0 - if not value.startswith("0x"): - if not value or len(value) % 2: - raise ValueError("Unsupported rights") - for start in range(0, len(value), 2): - mask |= rights[value[start:start + 2]] - except (KeyError, ValueError) as exc: - raise OSError("Unsupported ancestor rights") from exc - # Delete-child, delete, write-DACL, write-owner, or generic-all can replace retained paths. - if mask & 0x100D0040: - raise OSError("Another principal can substitute an ancestor or its children") - - -def _windows_private(path, *, ancestor=False): - # Inspect effective trustees, not chmod: Windows chmod does not establish privacy. - from ctypes import wintypes as w - adv = ctypes.WinDLL("advapi32", use_last_error=True) - kernel = ctypes.WinDLL("kernel32", use_last_error=True) - pointer = ctypes.c_void_p - adv.GetNamedSecurityInfoW.argtypes = [w.LPWSTR, w.DWORD, w.DWORD] + [ctypes.POINTER(pointer)] * 5 - adv.GetNamedSecurityInfoW.restype = w.DWORD - adv.ConvertSecurityDescriptorToStringSecurityDescriptorW.argtypes = [ - pointer, w.DWORD, w.DWORD, ctypes.POINTER(w.LPWSTR), ctypes.POINTER(w.DWORD), - ] - adv.ConvertSidToStringSidW.argtypes = [pointer, ctypes.POINTER(w.LPWSTR)] - adv.OpenProcessToken.argtypes = [w.HANDLE, w.DWORD, ctypes.POINTER(w.HANDLE)] - adv.GetTokenInformation.argtypes = [w.HANDLE, ctypes.c_int, pointer, w.DWORD, ctypes.POINTER(w.DWORD)] - kernel.GetCurrentProcess.restype = w.HANDLE - kernel.CloseHandle.argtypes = [w.HANDLE] - kernel.LocalFree.argtypes = [pointer] - descriptor, owner, group, dacl, sacl = (pointer() for _ in range(5)) - text, sid_text, token, size = w.LPWSTR(), w.LPWSTR(), w.HANDLE(), w.DWORD() - try: - if adv.GetNamedSecurityInfoW(str(path), 1, 5, ctypes.byref(owner), ctypes.byref(group), - ctypes.byref(dacl), ctypes.byref(sacl), ctypes.byref(descriptor)): - raise OSError("Cannot inspect private ACL") - if not adv.ConvertSecurityDescriptorToStringSecurityDescriptorW( - descriptor, 1, 5, ctypes.byref(text), None, - ): - raise OSError("Cannot inspect security descriptor") - if not adv.OpenProcessToken(kernel.GetCurrentProcess(), 8, ctypes.byref(token)): - raise OSError("Cannot inspect current owner") - adv.GetTokenInformation(token, 1, None, 0, ctypes.byref(size)) - buffer = ctypes.create_string_buffer(size.value) - if not adv.GetTokenInformation(token, 1, buffer, size, ctypes.byref(size)): - raise OSError("Cannot inspect current owner") - sid = ctypes.cast(buffer, ctypes.POINTER(pointer))[0] - if not adv.ConvertSidToStringSidW(sid, ctypes.byref(sid_text)): - raise OSError("Cannot inspect current owner") - sddl = text.value - current = sid_text.value - owner_text, separator, acl = sddl.partition("D:") - if not separator or not dacl: - raise OSError("Owner or DACL is not private") - entries = re.findall(r"\(([^()]*)\)", acl) - if not entries or re.sub(r"\([^()]*\)", "", acl) not in ("", "P", "AI", "PAI"): - raise OSError("Unsupported ACL") - if ancestor: - _windows_ancestor_acl(owner_text, entries, current) - return - if owner_text != "O:" + current: - raise OSError("Owner is not the operator") - for entry in entries: - parts = entry.split(";") - if len(parts) != 6 or parts[0] != "A" or parts[5] not in (current, "OW", "SY", "BA"): - raise OSError("ACL grants access to another principal") - finally: - if token: - kernel.CloseHandle(token) - for allocated in (descriptor, text, sid_text): - if allocated: - kernel.LocalFree(ctypes.cast(allocated, pointer)) - - -def _outside_plugin(path): - skill = Path(__file__).resolve().parents[1] - plugin = skill.parent.parent - protected = [skill] - if any((plugin / marker / "plugin.json").is_file() for marker in (".plugin", ".claude-plugin", ".cursor-plugin")): - protected.append(plugin) - if any(root in (path.resolve(), *path.resolve().parents) for root in protected): - raise OSError("Receipts must be outside the installed plugin") - - -def _safe_leaf(name): - return (isinstance(name, str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,199}", name) - and not name.endswith(".") - and not re.fullmatch(r"(?i)(?:con|prn|aux|nul|com[1-9]|lpt[1-9])", name.split(".")[0])) - - -def _validated_directory(value): - if not isinstance(value, str) or not Path(value).is_absolute(): - raise failure("bootstrap-receipt-private", "Select an absolute, existing private receipt directory.") - path = Path(value) - try: - for part in (path, *path.parents): - info = part.lstat() - if part == path: - selected = info - if stat.S_ISLNK(info.st_mode) or getattr(info, "st_file_attributes", 0) & 0x400: - raise OSError("Linked paths are unsupported") - _outside_plugin(path) - if not stat.S_ISDIR(selected.st_mode): - raise OSError("Receipt location must be a directory") - if os.name == "nt": - _windows_private(path) - else: - if selected.st_uid != os.getuid() or stat.S_IMODE(selected.st_mode) & 0o077: - raise OSError("Directory must be private to its owner") - except (OSError, ValueError) as exc: - raise failure("bootstrap-receipt-private", "Receipt location must be user-owned and private; no ACLs were changed.") from exc - return path, selected - - -def private_directory(value): - return _validated_directory(value)[0] - - -@contextmanager -def _windows_security(): - """An explicit protected, inheritable owner/SYSTEM/admin DACL, supplied at creation.""" - from ctypes import wintypes as w - adv = ctypes.WinDLL("advapi32", use_last_error=True) - kernel = ctypes.WinDLL("kernel32", use_last_error=True) - pointer = ctypes.c_void_p - class Attributes(ctypes.Structure): - _fields_ = [("length", w.DWORD), ("descriptor", pointer), ("inherit", w.BOOL)] - adv.OpenProcessToken.argtypes = [w.HANDLE, w.DWORD, ctypes.POINTER(w.HANDLE)] - adv.GetTokenInformation.argtypes = [w.HANDLE, ctypes.c_int, pointer, w.DWORD, ctypes.POINTER(w.DWORD)] - adv.ConvertSidToStringSidW.argtypes = [pointer, ctypes.POINTER(w.LPWSTR)] - adv.ConvertStringSecurityDescriptorToSecurityDescriptorW.argtypes = [ - w.LPCWSTR, w.DWORD, ctypes.POINTER(pointer), ctypes.POINTER(w.DWORD)] - kernel.GetCurrentProcess.restype = w.HANDLE - kernel.CloseHandle.argtypes = [w.HANDLE] - kernel.LocalFree.argtypes = [pointer] - token, size, sid_text, descriptor = w.HANDLE(), w.DWORD(), w.LPWSTR(), pointer() - try: - if not adv.OpenProcessToken(kernel.GetCurrentProcess(), 8, ctypes.byref(token)): - raise OSError("Owner unavailable") - adv.GetTokenInformation(token, 1, None, 0, ctypes.byref(size)) - buffer = ctypes.create_string_buffer(size.value) - if not adv.GetTokenInformation(token, 1, buffer, size, ctypes.byref(size)): - raise OSError("Owner unavailable") - if not adv.ConvertSidToStringSidW(ctypes.cast(buffer, ctypes.POINTER(pointer))[0], ctypes.byref(sid_text)): - raise OSError("Owner unavailable") - sddl = f"O:{sid_text.value}D:P(A;OICI;FA;;;{sid_text.value})(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)" - if not adv.ConvertStringSecurityDescriptorToSecurityDescriptorW(sddl, 1, ctypes.byref(descriptor), None): - raise OSError("Private descriptor unavailable") - yield Attributes(ctypes.sizeof(Attributes), descriptor, False) - finally: - if token: - kernel.CloseHandle(token) - for allocated in (sid_text, descriptor): - if allocated: - kernel.LocalFree(ctypes.cast(allocated, pointer)) - - -@contextmanager -def _pinned_directory(path, *, private=True): - """Pin every ancestor against substitution; POSIX writes remain handle-relative.""" - handles = [] - try: - if os.name == "nt": - from ctypes import wintypes as w - kernel = ctypes.WinDLL("kernel32", use_last_error=True) - kernel.CreateFileW.argtypes = [w.LPCWSTR, w.DWORD, w.DWORD, ctypes.c_void_p, - w.DWORD, w.DWORD, w.HANDLE] - kernel.CreateFileW.restype = w.HANDLE - kernel.CloseHandle.argtypes = [w.HANDLE] - for part in reversed((path, *path.parents)): - # No FILE_SHARE_DELETE: an opened ancestor cannot be renamed/replaced. - handle = kernel.CreateFileW(str(part), 0x81, 3, None, 3, 0x02200000, None) - if handle == ctypes.c_void_p(-1).value: - raise OSError("Directory cannot be pinned") - handles.append(handle) - info = part.lstat() - if not stat.S_ISDIR(info.st_mode) or getattr(info, "st_file_attributes", 0) & 0x400: - raise OSError("Linked directory") - _windows_private(part, ancestor=True) - if private: - _validated_directory(str(path)) - yield None - else: - if not hasattr(os, "O_DIRECTORY") or not hasattr(os, "O_NOFOLLOW"): - raise OSError("Handle-relative creation unavailable") - for part in reversed((path, *path.parents)): - fd = os.open(str(part) if not handles else part.name, - os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, - **({"dir_fd": handles[-1]} if handles else {})) - handles.append(fd) - info = os.fstat(fd) - # A foreign writable ancestor can substitute descendants, even if the leaf is 0700. - if info.st_uid not in (0, os.getuid()) or stat.S_IMODE(info.st_mode) & 0o022: - raise OSError("Unsafe ancestor ownership or write access") - if private: - _, selected = _validated_directory(str(path)) - opened = os.fstat(handles[-1]) - if (selected.st_dev, selected.st_ino) != (opened.st_dev, opened.st_ino): - raise OSError("Directory identity changed") - yield handles[-1] - finally: - cleanup_failed = False - for handle in reversed(handles): - try: - if os.name == "nt": - cleanup_failed = not kernel.CloseHandle(handle) or cleanup_failed - else: - os.close(handle) - except OSError: - cleanup_failed = True - if cleanup_failed: - raise OSError("Directory handle cleanup failed") - - -def create_private_directory(value): - """Create one new leaf, never modify an existing directory or its ancestors.""" - if not isinstance(value, str) or not Path(value).is_absolute(): - raise failure("bootstrap-receipt-private", "Select an absolute new private directory.") - path = Path(value) - try: - if not _safe_leaf(path.name): - raise OSError("Unsafe directory leaf") - _outside_plugin(path) - with _pinned_directory(path.parent, private=False) as parent_fd: - if os.name == "nt": - from ctypes import wintypes as w - kernel = ctypes.WinDLL("kernel32", use_last_error=True) - kernel.CreateDirectoryW.argtypes = [w.LPCWSTR, ctypes.c_void_p] - with _windows_security() as attributes: - if not kernel.CreateDirectoryW(str(path), ctypes.byref(attributes)): - raise OSError("Private directory creation failed") - else: - os.mkdir(path.name, 0o700, dir_fd=parent_fd) - with _pinned_directory(path): - return private_directory(str(path)) - except (OSError, ValueError, NotImplementedError) as exc: - raise failure("bootstrap-receipt-private", "Private leaf creation failed; no existing directory ACLs were changed.") from exc - - -def validate_private_artifact_directory(value): - path = private_directory(value) - try: - with _pinned_directory(path): - return private_directory(value) - except (OSError, ValueError, NotImplementedError) as exc: - raise failure("bootstrap-receipt-private", "Private directory or ancestor stability could not be verified; no ACLs were changed.") from exc - - -def _windows_private_open(path, *, create=True): - import msvcrt - from ctypes import wintypes as w - kernel = ctypes.WinDLL("kernel32", use_last_error=True) - kernel.CreateFileW.argtypes = [w.LPCWSTR, w.DWORD, w.DWORD, ctypes.c_void_p, - w.DWORD, w.DWORD, w.HANDLE] - kernel.CreateFileW.restype = w.HANDLE - kernel.CloseHandle.argtypes = [w.HANDLE] - if create: - with _windows_security() as attributes: - handle = kernel.CreateFileW(str(path), 0xC0010000, 1, ctypes.byref(attributes), 1, 0x80200000, None) - else: - handle = kernel.CreateFileW(str(path), 0x80000000, 1, None, 3, 0x00200000, None) - if handle == ctypes.c_void_p(-1).value: - raise OSError("Private file creation failed") - try: - return msvcrt.open_osfhandle(handle, (os.O_RDWR if create else os.O_RDONLY) | os.O_BINARY) - except BaseException: - kernel.CloseHandle(handle) - raise - - -def _windows_publish(fd, destination): - import msvcrt - from ctypes import wintypes as w - kernel = ctypes.WinDLL("kernel32", use_last_error=True) - target = str(destination) - size = len(target.encode("utf-16-le")) - class Rename(ctypes.Structure): - _fields_ = [("replace", w.BOOL), ("root", w.HANDLE), ("size", w.DWORD), - ("name", w.WCHAR * (size // 2 + 1))] - value = Rename(False, None, size, target) - kernel.SetFileInformationByHandle.argtypes = [w.HANDLE, ctypes.c_int, ctypes.c_void_p, w.DWORD] - # Rename the owned, write-through handle, never reopen a substitutable temporary pathname. - if not kernel.SetFileInformationByHandle(msvcrt.get_osfhandle(fd), 3, ctypes.byref(value), ctypes.sizeof(value)): - raise OSError("Private artifact publication failed") - - -def _cleanup_owned_temporary(directory, name, owned, *, directory_fd=None, primary=None): - path = directory / name if directory_fd is None else name - kwargs = {} if directory_fd is None else {"dir_fd": directory_fd} - try: - current = os.stat(path, follow_symlinks=False, **kwargs) - except FileNotFoundError: - outcome = "already-absent" - except (OSError, NotImplementedError): - outcome = "inspection-unavailable; entry left untouched" - else: - if (current.st_dev, current.st_ino) != (owned.st_dev, owned.st_ino): - outcome = "identity-changed; entry left untouched" - else: - try: - os.unlink(path, **kwargs) - outcome = "removed" - except FileNotFoundError: - outcome = "already-absent" - except (OSError, NotImplementedError): - outcome = "removal-unconfirmed; private temporary may remain" - warning = "Private temporary cleanup: " + outcome + "." - if primary is not None: - primary.warnings.append(warning) - elif outcome not in ("removed", "already-absent"): - error = failure("bootstrap-receipt-failed", "Private temporary cleanup could not be confirmed; no artifact success is reported.") - error.warnings.append(warning) - raise error - return outcome - - -def atomic_private_file(directory, name, value, *, max_bytes=16 * MAX_BYTES): - """Publish complete, verified JSON without replacing any existing filesystem entry.""" - if not _safe_leaf(name): - raise failure("bootstrap-receipt-failed", "Artifact name must be a safe ordinary leaf.") - try: - data = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, - allow_nan=False).encode("utf-8") + b"\n" - except (TypeError, ValueError, UnicodeError, RecursionError) as exc: - raise failure("bootstrap-receipt-failed", "Artifact must contain valid finite UTF-8 JSON.") from exc - if len(data) > max_bytes: - raise failure("bootstrap-receipt-failed", "Artifact exceeds its output byte bound.") - directory = Path(directory) - temporary = ".artifact-" + uuid.uuid4().hex - owned = None - primary = None - verified = False - try: - with _pinned_directory(directory) as directory_fd: - kwargs = {} if directory_fd is None else {"dir_fd": directory_fd} - leaf = lambda name: directory / name if directory_fd is None else name - try: - fd = (_windows_private_open(directory / temporary) if os.name == "nt" else - os.open(leaf(temporary), os.O_RDWR | os.O_CREAT | os.O_EXCL | - os.O_NOFOLLOW, 0o600, **kwargs)) - try: - handle = os.fdopen(fd, "w+b") - except BaseException: - os.close(fd) - raise - with handle: - owned = os.fstat(handle.fileno()) - if os.name == "nt": - _windows_private(directory / temporary) - handle.write(data) - handle.flush() - os.fsync(handle.fileno()) - handle.seek(0) - if handle.read() != data or os.fstat(handle.fileno()).st_nlink != 1: - raise OSError("Artifact integrity failed") - if os.name == "nt": - _validated_directory(str(directory)) - _windows_publish(handle.fileno(), directory / name) - published = owned - owned = None - os.fsync(handle.fileno()) - # Revalidate before publication; cleanup is restricted to our original inode. - if os.name != "nt": - current = os.stat(leaf(temporary), follow_symlinks=False, **kwargs) - if (current.st_dev, current.st_ino) != (owned.st_dev, owned.st_ino): - raise OSError("Artifact identity changed") - _validated_directory(str(directory)) - os.link(temporary, name, src_dir_fd=directory_fd, dst_dir_fd=directory_fd, - follow_symlinks=False) - for entry in (temporary, name): - current = os.stat(entry, dir_fd=directory_fd, follow_symlinks=False) - if (current.st_dev, current.st_ino) != (owned.st_dev, owned.st_ino): - raise OSError("Artifact changed during publication") - os.unlink(temporary, dir_fd=directory_fd) - published = owned - owned = None - os.fsync(directory_fd) - fd = (_windows_private_open(directory / name, create=False) if os.name == "nt" else - os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=directory_fd)) - try: - handle = os.fdopen(fd, "rb") - except BaseException: - os.close(fd) - raise - with handle: - observed = os.fstat(handle.fileno()) - if ((observed.st_dev, observed.st_ino) != (published.st_dev, published.st_ino) - or not stat.S_ISREG(observed.st_mode) or observed.st_nlink != 1 - or handle.read(len(data) + 1) != data): - raise OSError("Published artifact integrity failed") - if os.name == "nt": - _windows_private(directory / name) - _validated_directory(str(directory)) - verified = True - except HelperFailure as exc: - primary = exc - raise - except (OSError, ValueError, NotImplementedError) as exc: - primary = failure("bootstrap-receipt-failed", "Private artifact could not be verified and persisted; no artifact success is reported.") - raise primary from exc - finally: - if owned is not None: - try: - _cleanup_owned_temporary(directory, temporary, owned, - directory_fd=directory_fd, primary=primary) - except HelperFailure as exc: - primary = exc - raise - except (OSError, ValueError, NotImplementedError) as exc: - if primary is not None: - primary.warnings.append("Private directory handle cleanup could not be confirmed.") - raise primary from primary.__cause__ - error = failure("bootstrap-receipt-failed", "Private artifact could not be verified and persisted; no artifact success is reported.") - if verified: - error.warnings.append("Private directory handle cleanup could not be confirmed.") - raise error from exc - return directory / name - - -def private_file(directory, name, value): - return private_bytes(directory, name, canonical_bytes(value) + b"\n") - - -def private_bytes(directory, name, data): - if not _safe_leaf(name): - raise failure("bootstrap-receipt-failed", "Receipt name must be a safe, ordinary leaf name.") - if not isinstance(data, bytes): - raise failure("bootstrap-receipt-failed", "Private content must be bounded bytes.") - if len(data) > MAX_BYTES: - raise failure("bootstrap-receipt-failed", "Sanitized receipt exceeds its bound.") - directory_fd = file_fd = None - primary = None - try: - directory, selected = _validated_directory(str(directory)) - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) - if os.name == "nt": - file_fd = os.open(directory / name, flags, 0o600) - else: - if not hasattr(os, "O_DIRECTORY") or not hasattr(os, "O_NOFOLLOW"): - raise failure("bootstrap-receipt-private", "POSIX handle-relative private creation is unavailable.") - directory_fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) - opened = os.fstat(directory_fd) - if ((opened.st_dev, opened.st_ino) != (selected.st_dev, selected.st_ino) - or not stat.S_ISDIR(opened.st_mode) or opened.st_uid != os.getuid() - or stat.S_IMODE(opened.st_mode) & 0o077): - raise failure("bootstrap-receipt-private", "Opened receipt directory changed identity, ownership or permissions.") - file_fd = os.open(name, flags, 0o600, dir_fd=directory_fd) - handle = os.fdopen(file_fd, "wb") - file_fd = None - with handle: - handle.write(data) - handle.flush() - os.fsync(handle.fileno()) - except HelperFailure as exc: - primary = exc - raise - except (OSError, NotImplementedError) as exc: - primary = failure("bootstrap-receipt-failed", "Private receipt could not be persisted; retain the returned ownership handoff.") - raise primary from exc - finally: - cleanup_failed = False - for descriptor in (file_fd, directory_fd): - if descriptor is not None: - try: - os.close(descriptor) - except OSError: - cleanup_failed = True - if cleanup_failed: - if primary is not None: - primary.warnings.append("Receipt descriptor cleanup could not be confirmed.") - else: - raise failure("bootstrap-receipt-failed", "Receipt descriptor cleanup could not be confirmed.") - return directory / name - - -def cli_prefix(): - executable = shutil.which("az") - if executable is None: - raise failure("bootstrap-tool-unavailable", "A signed-in Azure CLI installation is required.") - if os.name == "nt": - if Path(executable).suffix.lower() not in (".cmd", ".bat"): - raise failure("bootstrap-tool-unavailable", "Windows requires the supported bundled CLI Python layout.") - # Use the installed CLI's own interpreter, never cmd.exe or caller commands. - # Azure/azure-cli: build_scripts/windows/scripts/az_msi.cmd (and az_zip.cmd). - python = Path(executable).parent.parent / "python.exe" - if not python.is_file(): - raise failure("bootstrap-tool-unavailable", "This Windows CLI layout has no supported bundled Python launcher.") - return [str(python), "-IBm", "azure.cli"] - return [executable] - - -def run_cli(arguments, timeout): - """Capture native CLI output; size validation is post-capture, not a memory bound.""" - env = dict(os.environ, AZURE_EXTENSION_USE_DYNAMIC_INSTALL="no", - AZURE_CORE_COLLECT_TELEMETRY="no", AZURE_CORE_ONLY_SHOW_ERRORS="true", - AZURE_CORE_NO_COLOR="true", AZURE_LOGGING_ENABLE_LOG_FILE="false", - AZURE_AUTO_UPGRADE_ENABLE="false") - try: - command = cli_prefix() + arguments + ["--output", "json", "--only-show-errors"] - result = subprocess.run(command, stdin=subprocess.DEVNULL, capture_output=True, - shell=False, env=env, timeout=timeout, check=False) - except subprocess.TimeoutExpired as exc: - raise failure("bootstrap-cli-timeout", "Native CLI timed out; remote completion and process-tree cleanup are not established.") from exc - except OSError as exc: - cause = exc - for _ in range(8): - if not isinstance(cause, OSError): - break - cause = cause.__context__ - if isinstance(cause, subprocess.TimeoutExpired): - error = failure("bootstrap-cli-timeout", "Native CLI timed out; remote completion is not established.") - error.warnings.append("Standard subprocess cleanup also failed; process state is unknown.") - else: - error = failure("bootstrap-cli-unavailable", "Native CLI execution failed; process and remote state may be unknown.") - raise error from exc - if len(result.stdout) > MAX_BYTES or len(result.stderr) > MAX_BYTES: - raise failure("bootstrap-cli-output-limit", "Captured CLI output exceeds one MiB per stream; this is not a capture-memory bound.") - return result.returncode, result.stdout, result.stderr +"""Private storage and shell-free native CLI execution for the bootstrap owner.""" +from __future__ import annotations + +import ctypes +import json +import os +import re +import shutil +import stat +import subprocess +import uuid +from contextlib import contextmanager +from pathlib import Path + +try: + from ._common import HelperFailure, canonical_bytes +except ImportError: + from _common import HelperFailure, canonical_bytes + +MAX_BYTES = 1024 * 1024 + + +def failure(code, message): + return HelperFailure(code, message, blocked_at="verification") + + +def read_json(path): + try: + with Path(path).open("rb") as handle: + raw = handle.read(MAX_BYTES + 1) + if len(raw) > MAX_BYTES: + raise failure("bootstrap-input-invalid", "JSON exceeds the one MiB input limit.") + value = json.loads(raw.decode("utf-8")) + pending = [(value, 0)] + count = 0 + while pending: + item, depth = pending.pop() + count += 1 + if depth > 20 or count > 10000: + raise ValueError("JSON structure exceeds limits") + if isinstance(item, dict): + pending.extend((v, depth + 1) for v in item.values()) + elif isinstance(item, list): + pending.extend((v, depth + 1) for v in item) + json.dumps(value, ensure_ascii=False, allow_nan=False).encode("utf-8") + return value + except (OSError, UnicodeError, ValueError, RecursionError) as exc: + raise failure("bootstrap-input-invalid", "Select bounded, valid UTF-8 JSON.") from exc + + +def _windows_ancestor_acl(owner_text, entries, current): + # Windows volume roots are commonly owned by this fixed OS servicing principal. + installer = "S-1-5-80-956008885-3418522649-1831038044-1853292631-2271478464" + trusted = (current, "OW", "SY", "BA", installer) + if owner_text not in tuple("O:" + trustee for trustee in trusted if trustee != "OW"): + raise OSError("Ancestor owner can change the access boundary") + rights = { + "FA": 0x1F01FF, "FR": 0x120089, "FW": 0x120116, "FX": 0x1200A0, + "GA": 0x10000000, "GR": 0x80000000, "GW": 0x40000000, "GX": 0x20000000, + "SD": 0x10000, "RC": 0x20000, "WD": 0x40000, "WO": 0x80000, + "CC": 1, "DC": 2, "LC": 4, "SW": 8, "RP": 16, "WP": 32, "DT": 64, + "LO": 128, "CR": 256, + } + for entry in entries: + parts = entry.split(";") + if len(parts) != 6 or parts[0] not in ("A", "D") or parts[3] or parts[4]: + raise OSError("Unsupported ancestor ACL") + if parts[0] == "D" or "IO" in parts[1] or parts[5] in trusted: + continue + value = parts[2] + try: + mask = int(value, 16) if value.startswith("0x") else 0 + if not value.startswith("0x"): + if not value or len(value) % 2: + raise ValueError("Unsupported rights") + for start in range(0, len(value), 2): + mask |= rights[value[start:start + 2]] + except (KeyError, ValueError) as exc: + raise OSError("Unsupported ancestor rights") from exc + # Delete-child, delete, write-DACL, write-owner, or generic-all can replace retained paths. + if mask & 0x100D0040: + raise OSError("Another principal can substitute an ancestor or its children") + + +def _windows_private(path, *, ancestor=False): + # Inspect effective trustees, not chmod: Windows chmod does not establish privacy. + from ctypes import wintypes as w + adv = ctypes.WinDLL("advapi32", use_last_error=True) + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + pointer = ctypes.c_void_p + adv.GetNamedSecurityInfoW.argtypes = [w.LPWSTR, w.DWORD, w.DWORD] + [ctypes.POINTER(pointer)] * 5 + adv.GetNamedSecurityInfoW.restype = w.DWORD + adv.ConvertSecurityDescriptorToStringSecurityDescriptorW.argtypes = [ + pointer, w.DWORD, w.DWORD, ctypes.POINTER(w.LPWSTR), ctypes.POINTER(w.DWORD), + ] + adv.ConvertSidToStringSidW.argtypes = [pointer, ctypes.POINTER(w.LPWSTR)] + adv.OpenProcessToken.argtypes = [w.HANDLE, w.DWORD, ctypes.POINTER(w.HANDLE)] + adv.GetTokenInformation.argtypes = [w.HANDLE, ctypes.c_int, pointer, w.DWORD, ctypes.POINTER(w.DWORD)] + kernel.GetCurrentProcess.restype = w.HANDLE + kernel.CloseHandle.argtypes = [w.HANDLE] + kernel.LocalFree.argtypes = [pointer] + descriptor, owner, group, dacl, sacl = (pointer() for _ in range(5)) + text, sid_text, token, size = w.LPWSTR(), w.LPWSTR(), w.HANDLE(), w.DWORD() + try: + if adv.GetNamedSecurityInfoW(str(path), 1, 5, ctypes.byref(owner), ctypes.byref(group), + ctypes.byref(dacl), ctypes.byref(sacl), ctypes.byref(descriptor)): + raise OSError("Cannot inspect private ACL") + if not adv.ConvertSecurityDescriptorToStringSecurityDescriptorW( + descriptor, 1, 5, ctypes.byref(text), None, + ): + raise OSError("Cannot inspect security descriptor") + if not adv.OpenProcessToken(kernel.GetCurrentProcess(), 8, ctypes.byref(token)): + raise OSError("Cannot inspect current owner") + adv.GetTokenInformation(token, 1, None, 0, ctypes.byref(size)) + buffer = ctypes.create_string_buffer(size.value) + if not adv.GetTokenInformation(token, 1, buffer, size, ctypes.byref(size)): + raise OSError("Cannot inspect current owner") + sid = ctypes.cast(buffer, ctypes.POINTER(pointer))[0] + if not adv.ConvertSidToStringSidW(sid, ctypes.byref(sid_text)): + raise OSError("Cannot inspect current owner") + sddl = text.value + current = sid_text.value + owner_text, separator, acl = sddl.partition("D:") + if not separator or not dacl: + raise OSError("Owner or DACL is not private") + entries = re.findall(r"\(([^()]*)\)", acl) + if not entries or re.sub(r"\([^()]*\)", "", acl) not in ("", "P", "AI", "PAI"): + raise OSError("Unsupported ACL") + if ancestor: + _windows_ancestor_acl(owner_text, entries, current) + return + if owner_text != "O:" + current: + raise OSError("Owner is not the operator") + for entry in entries: + parts = entry.split(";") + if len(parts) != 6 or parts[0] != "A" or parts[5] not in (current, "OW", "SY", "BA"): + raise OSError("ACL grants access to another principal") + finally: + if token: + kernel.CloseHandle(token) + for allocated in (descriptor, text, sid_text): + if allocated: + kernel.LocalFree(ctypes.cast(allocated, pointer)) + + +def _outside_plugin(path): + skill = Path(__file__).resolve().parents[1] + plugin = skill.parent.parent + protected = [skill] + if any((plugin / marker / "plugin.json").is_file() for marker in (".plugin", ".claude-plugin", ".cursor-plugin")): + protected.append(plugin) + if any(root in (path.resolve(), *path.resolve().parents) for root in protected): + raise OSError("Receipts must be outside the installed plugin") + + +def _safe_leaf(name): + return (isinstance(name, str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,199}", name) + and not name.endswith(".") + and not re.fullmatch(r"(?i)(?:con|prn|aux|nul|com[1-9]|lpt[1-9])", name.split(".")[0])) + + +def _validated_directory(value): + if not isinstance(value, str) or not Path(value).is_absolute(): + raise failure("bootstrap-receipt-private", "Select an absolute, existing private receipt directory.") + path = Path(value) + try: + for part in (path, *path.parents): + info = part.lstat() + if part == path: + selected = info + if stat.S_ISLNK(info.st_mode) or getattr(info, "st_file_attributes", 0) & 0x400: + raise OSError("Linked paths are unsupported") + _outside_plugin(path) + if not stat.S_ISDIR(selected.st_mode): + raise OSError("Receipt location must be a directory") + if os.name == "nt": + _windows_private(path) + else: + if selected.st_uid != os.getuid() or stat.S_IMODE(selected.st_mode) & 0o077: + raise OSError("Directory must be private to its owner") + except (OSError, ValueError) as exc: + raise failure("bootstrap-receipt-private", "Receipt location must be user-owned and private; no ACLs were changed.") from exc + return path, selected + + +def private_directory(value): + return _validated_directory(value)[0] + + +@contextmanager +def _windows_security(): + """An explicit protected, inheritable owner/SYSTEM/admin DACL, supplied at creation.""" + from ctypes import wintypes as w + adv = ctypes.WinDLL("advapi32", use_last_error=True) + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + pointer = ctypes.c_void_p + class Attributes(ctypes.Structure): + _fields_ = [("length", w.DWORD), ("descriptor", pointer), ("inherit", w.BOOL)] + adv.OpenProcessToken.argtypes = [w.HANDLE, w.DWORD, ctypes.POINTER(w.HANDLE)] + adv.GetTokenInformation.argtypes = [w.HANDLE, ctypes.c_int, pointer, w.DWORD, ctypes.POINTER(w.DWORD)] + adv.ConvertSidToStringSidW.argtypes = [pointer, ctypes.POINTER(w.LPWSTR)] + adv.ConvertStringSecurityDescriptorToSecurityDescriptorW.argtypes = [ + w.LPCWSTR, w.DWORD, ctypes.POINTER(pointer), ctypes.POINTER(w.DWORD)] + kernel.GetCurrentProcess.restype = w.HANDLE + kernel.CloseHandle.argtypes = [w.HANDLE] + kernel.LocalFree.argtypes = [pointer] + token, size, sid_text, descriptor = w.HANDLE(), w.DWORD(), w.LPWSTR(), pointer() + try: + if not adv.OpenProcessToken(kernel.GetCurrentProcess(), 8, ctypes.byref(token)): + raise OSError("Owner unavailable") + adv.GetTokenInformation(token, 1, None, 0, ctypes.byref(size)) + buffer = ctypes.create_string_buffer(size.value) + if not adv.GetTokenInformation(token, 1, buffer, size, ctypes.byref(size)): + raise OSError("Owner unavailable") + if not adv.ConvertSidToStringSidW(ctypes.cast(buffer, ctypes.POINTER(pointer))[0], ctypes.byref(sid_text)): + raise OSError("Owner unavailable") + sddl = f"O:{sid_text.value}D:P(A;OICI;FA;;;{sid_text.value})(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)" + if not adv.ConvertStringSecurityDescriptorToSecurityDescriptorW(sddl, 1, ctypes.byref(descriptor), None): + raise OSError("Private descriptor unavailable") + yield Attributes(ctypes.sizeof(Attributes), descriptor, False) + finally: + if token: + kernel.CloseHandle(token) + for allocated in (sid_text, descriptor): + if allocated: + kernel.LocalFree(ctypes.cast(allocated, pointer)) + + +@contextmanager +def _pinned_directory(path, *, private=True): + """Pin every ancestor against substitution; POSIX writes remain handle-relative.""" + handles = [] + try: + if os.name == "nt": + from ctypes import wintypes as w + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.CreateFileW.argtypes = [w.LPCWSTR, w.DWORD, w.DWORD, ctypes.c_void_p, + w.DWORD, w.DWORD, w.HANDLE] + kernel.CreateFileW.restype = w.HANDLE + kernel.CloseHandle.argtypes = [w.HANDLE] + for part in reversed((path, *path.parents)): + # No FILE_SHARE_DELETE: an opened ancestor cannot be renamed/replaced. + handle = kernel.CreateFileW(str(part), 0x81, 3, None, 3, 0x02200000, None) + if handle == ctypes.c_void_p(-1).value: + raise OSError("Directory cannot be pinned") + handles.append(handle) + info = part.lstat() + if not stat.S_ISDIR(info.st_mode) or getattr(info, "st_file_attributes", 0) & 0x400: + raise OSError("Linked directory") + _windows_private(part, ancestor=True) + if private: + _validated_directory(str(path)) + yield None + else: + if not hasattr(os, "O_DIRECTORY") or not hasattr(os, "O_NOFOLLOW"): + raise OSError("Handle-relative creation unavailable") + for part in reversed((path, *path.parents)): + fd = os.open(str(part) if not handles else part.name, + os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, + **({"dir_fd": handles[-1]} if handles else {})) + handles.append(fd) + info = os.fstat(fd) + # A foreign writable ancestor can substitute descendants, even if the leaf is 0700. + if info.st_uid not in (0, os.getuid()) or stat.S_IMODE(info.st_mode) & 0o022: + raise OSError("Unsafe ancestor ownership or write access") + if private: + _, selected = _validated_directory(str(path)) + opened = os.fstat(handles[-1]) + if (selected.st_dev, selected.st_ino) != (opened.st_dev, opened.st_ino): + raise OSError("Directory identity changed") + yield handles[-1] + finally: + cleanup_failed = False + for handle in reversed(handles): + try: + if os.name == "nt": + cleanup_failed = not kernel.CloseHandle(handle) or cleanup_failed + else: + os.close(handle) + except OSError: + cleanup_failed = True + if cleanup_failed: + raise OSError("Directory handle cleanup failed") + + +def create_private_directory(value): + """Create one new leaf, never modify an existing directory or its ancestors.""" + if not isinstance(value, str) or not Path(value).is_absolute(): + raise failure("bootstrap-receipt-private", "Select an absolute new private directory.") + path = Path(value) + try: + if not _safe_leaf(path.name): + raise OSError("Unsafe directory leaf") + _outside_plugin(path) + with _pinned_directory(path.parent, private=False) as parent_fd: + if os.name == "nt": + from ctypes import wintypes as w + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.CreateDirectoryW.argtypes = [w.LPCWSTR, ctypes.c_void_p] + with _windows_security() as attributes: + if not kernel.CreateDirectoryW(str(path), ctypes.byref(attributes)): + raise OSError("Private directory creation failed") + else: + os.mkdir(path.name, 0o700, dir_fd=parent_fd) + with _pinned_directory(path): + return private_directory(str(path)) + except (OSError, ValueError, NotImplementedError) as exc: + raise failure("bootstrap-receipt-private", "Private leaf creation failed; no existing directory ACLs were changed.") from exc + + +def validate_private_artifact_directory(value): + path = private_directory(value) + try: + with _pinned_directory(path): + return private_directory(value) + except (OSError, ValueError, NotImplementedError) as exc: + raise failure("bootstrap-receipt-private", "Private directory or ancestor stability could not be verified; no ACLs were changed.") from exc + + +def _windows_private_open(path, *, create=True): + import msvcrt + from ctypes import wintypes as w + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.CreateFileW.argtypes = [w.LPCWSTR, w.DWORD, w.DWORD, ctypes.c_void_p, + w.DWORD, w.DWORD, w.HANDLE] + kernel.CreateFileW.restype = w.HANDLE + kernel.CloseHandle.argtypes = [w.HANDLE] + if create: + with _windows_security() as attributes: + handle = kernel.CreateFileW(str(path), 0xC0010000, 1, ctypes.byref(attributes), 1, 0x80200000, None) + else: + handle = kernel.CreateFileW(str(path), 0x80000000, 1, None, 3, 0x00200000, None) + if handle == ctypes.c_void_p(-1).value: + raise OSError("Private file creation failed") + try: + return msvcrt.open_osfhandle(handle, (os.O_RDWR if create else os.O_RDONLY) | os.O_BINARY) + except BaseException: + kernel.CloseHandle(handle) + raise + + +def _windows_publish(fd, destination): + import msvcrt + from ctypes import wintypes as w + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + target = str(destination) + size = len(target.encode("utf-16-le")) + class Rename(ctypes.Structure): + _fields_ = [("replace", w.BOOL), ("root", w.HANDLE), ("size", w.DWORD), + ("name", w.WCHAR * (size // 2 + 1))] + value = Rename(False, None, size, target) + kernel.SetFileInformationByHandle.argtypes = [w.HANDLE, ctypes.c_int, ctypes.c_void_p, w.DWORD] + # Rename the owned, write-through handle, never reopen a substitutable temporary pathname. + if not kernel.SetFileInformationByHandle(msvcrt.get_osfhandle(fd), 3, ctypes.byref(value), ctypes.sizeof(value)): + raise OSError("Private artifact publication failed") + + +def _cleanup_owned_temporary(directory, name, owned, *, directory_fd=None, primary=None): + path = directory / name if directory_fd is None else name + kwargs = {} if directory_fd is None else {"dir_fd": directory_fd} + try: + current = os.stat(path, follow_symlinks=False, **kwargs) + except FileNotFoundError: + outcome = "already-absent" + except (OSError, NotImplementedError): + outcome = "inspection-unavailable; entry left untouched" + else: + if (current.st_dev, current.st_ino) != (owned.st_dev, owned.st_ino): + outcome = "identity-changed; entry left untouched" + else: + try: + os.unlink(path, **kwargs) + outcome = "removed" + except FileNotFoundError: + outcome = "already-absent" + except (OSError, NotImplementedError): + outcome = "removal-unconfirmed; private temporary may remain" + warning = "Private temporary cleanup: " + outcome + "." + if primary is not None: + primary.warnings.append(warning) + elif outcome not in ("removed", "already-absent"): + error = failure("bootstrap-receipt-failed", "Private temporary cleanup could not be confirmed; no artifact success is reported.") + error.warnings.append(warning) + raise error + return outcome + + +def atomic_private_file(directory, name, value, *, max_bytes=16 * MAX_BYTES): + """Publish complete, verified JSON without replacing any existing filesystem entry.""" + if not _safe_leaf(name): + raise failure("bootstrap-receipt-failed", "Artifact name must be a safe ordinary leaf.") + try: + data = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, + allow_nan=False).encode("utf-8") + b"\n" + except (TypeError, ValueError, UnicodeError, RecursionError) as exc: + raise failure("bootstrap-receipt-failed", "Artifact must contain valid finite UTF-8 JSON.") from exc + if len(data) > max_bytes: + raise failure("bootstrap-receipt-failed", "Artifact exceeds its output byte bound.") + directory = Path(directory) + temporary = ".artifact-" + uuid.uuid4().hex + owned = None + primary = None + verified = False + try: + with _pinned_directory(directory) as directory_fd: + kwargs = {} if directory_fd is None else {"dir_fd": directory_fd} + leaf = lambda name: directory / name if directory_fd is None else name + try: + fd = (_windows_private_open(directory / temporary) if os.name == "nt" else + os.open(leaf(temporary), os.O_RDWR | os.O_CREAT | os.O_EXCL | + os.O_NOFOLLOW, 0o600, **kwargs)) + try: + handle = os.fdopen(fd, "w+b") + except BaseException: + os.close(fd) + raise + with handle: + owned = os.fstat(handle.fileno()) + if os.name == "nt": + _windows_private(directory / temporary) + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + handle.seek(0) + if handle.read() != data or os.fstat(handle.fileno()).st_nlink != 1: + raise OSError("Artifact integrity failed") + if os.name == "nt": + _validated_directory(str(directory)) + _windows_publish(handle.fileno(), directory / name) + published = owned + owned = None + os.fsync(handle.fileno()) + # Revalidate before publication; cleanup is restricted to our original inode. + if os.name != "nt": + current = os.stat(leaf(temporary), follow_symlinks=False, **kwargs) + if (current.st_dev, current.st_ino) != (owned.st_dev, owned.st_ino): + raise OSError("Artifact identity changed") + _validated_directory(str(directory)) + os.link(temporary, name, src_dir_fd=directory_fd, dst_dir_fd=directory_fd, + follow_symlinks=False) + for entry in (temporary, name): + current = os.stat(entry, dir_fd=directory_fd, follow_symlinks=False) + if (current.st_dev, current.st_ino) != (owned.st_dev, owned.st_ino): + raise OSError("Artifact changed during publication") + os.unlink(temporary, dir_fd=directory_fd) + published = owned + owned = None + os.fsync(directory_fd) + fd = (_windows_private_open(directory / name, create=False) if os.name == "nt" else + os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=directory_fd)) + try: + handle = os.fdopen(fd, "rb") + except BaseException: + os.close(fd) + raise + with handle: + observed = os.fstat(handle.fileno()) + if ((observed.st_dev, observed.st_ino) != (published.st_dev, published.st_ino) + or not stat.S_ISREG(observed.st_mode) or observed.st_nlink != 1 + or handle.read(len(data) + 1) != data): + raise OSError("Published artifact integrity failed") + if os.name == "nt": + _windows_private(directory / name) + _validated_directory(str(directory)) + verified = True + except HelperFailure as exc: + primary = exc + raise + except (OSError, ValueError, NotImplementedError) as exc: + primary = failure("bootstrap-receipt-failed", "Private artifact could not be verified and persisted; no artifact success is reported.") + raise primary from exc + finally: + if owned is not None: + try: + _cleanup_owned_temporary(directory, temporary, owned, + directory_fd=directory_fd, primary=primary) + except HelperFailure as exc: + primary = exc + raise + except (OSError, ValueError, NotImplementedError) as exc: + if primary is not None: + primary.warnings.append("Private directory handle cleanup could not be confirmed.") + raise primary from primary.__cause__ + error = failure("bootstrap-receipt-failed", "Private artifact could not be verified and persisted; no artifact success is reported.") + if verified: + error.warnings.append("Private directory handle cleanup could not be confirmed.") + raise error from exc + return directory / name + + +def private_file(directory, name, value): + return private_bytes(directory, name, canonical_bytes(value) + b"\n") + + +def private_bytes(directory, name, data): + if not _safe_leaf(name): + raise failure("bootstrap-receipt-failed", "Receipt name must be a safe, ordinary leaf name.") + if not isinstance(data, bytes): + raise failure("bootstrap-receipt-failed", "Private content must be bounded bytes.") + if len(data) > MAX_BYTES: + raise failure("bootstrap-receipt-failed", "Sanitized receipt exceeds its bound.") + directory_fd = file_fd = None + primary = None + try: + directory, selected = _validated_directory(str(directory)) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + if os.name == "nt": + file_fd = os.open(directory / name, flags, 0o600) + else: + if not hasattr(os, "O_DIRECTORY") or not hasattr(os, "O_NOFOLLOW"): + raise failure("bootstrap-receipt-private", "POSIX handle-relative private creation is unavailable.") + directory_fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + opened = os.fstat(directory_fd) + if ((opened.st_dev, opened.st_ino) != (selected.st_dev, selected.st_ino) + or not stat.S_ISDIR(opened.st_mode) or opened.st_uid != os.getuid() + or stat.S_IMODE(opened.st_mode) & 0o077): + raise failure("bootstrap-receipt-private", "Opened receipt directory changed identity, ownership or permissions.") + file_fd = os.open(name, flags, 0o600, dir_fd=directory_fd) + handle = os.fdopen(file_fd, "wb") + file_fd = None + with handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + except HelperFailure as exc: + primary = exc + raise + except (OSError, NotImplementedError) as exc: + primary = failure("bootstrap-receipt-failed", "Private receipt could not be persisted; retain the returned ownership handoff.") + raise primary from exc + finally: + cleanup_failed = False + for descriptor in (file_fd, directory_fd): + if descriptor is not None: + try: + os.close(descriptor) + except OSError: + cleanup_failed = True + if cleanup_failed: + if primary is not None: + primary.warnings.append("Receipt descriptor cleanup could not be confirmed.") + else: + raise failure("bootstrap-receipt-failed", "Receipt descriptor cleanup could not be confirmed.") + return directory / name + + +def cli_prefix(): + executable = shutil.which("az") + if executable is None: + raise failure("bootstrap-tool-unavailable", "A signed-in Azure CLI installation is required.") + if os.name == "nt": + if Path(executable).suffix.lower() not in (".cmd", ".bat"): + raise failure("bootstrap-tool-unavailable", "Windows requires the supported bundled CLI Python layout.") + # Use the installed CLI's own interpreter, never cmd.exe or caller commands. + # Azure/azure-cli: build_scripts/windows/scripts/az_msi.cmd (and az_zip.cmd). + python = Path(executable).parent.parent / "python.exe" + if not python.is_file(): + raise failure("bootstrap-tool-unavailable", "This Windows CLI layout has no supported bundled Python launcher.") + return [str(python), "-IBm", "azure.cli"] + return [executable] + + +def run_cli(arguments, timeout): + """Capture native CLI output; size validation is post-capture, not a memory bound.""" + env = dict(os.environ, AZURE_EXTENSION_USE_DYNAMIC_INSTALL="no", + AZURE_CORE_COLLECT_TELEMETRY="no", AZURE_CORE_ONLY_SHOW_ERRORS="true", + AZURE_CORE_NO_COLOR="true", AZURE_LOGGING_ENABLE_LOG_FILE="false", + AZURE_AUTO_UPGRADE_ENABLE="false") + try: + command = cli_prefix() + arguments + ["--output", "json", "--only-show-errors"] + result = subprocess.run(command, stdin=subprocess.DEVNULL, capture_output=True, + shell=False, env=env, timeout=timeout, check=False) + except subprocess.TimeoutExpired as exc: + raise failure("bootstrap-cli-timeout", "Native CLI timed out; remote completion and process-tree cleanup are not established.") from exc + except OSError as exc: + cause = exc + for _ in range(8): + if not isinstance(cause, OSError): + break + cause = cause.__context__ + if isinstance(cause, subprocess.TimeoutExpired): + error = failure("bootstrap-cli-timeout", "Native CLI timed out; remote completion is not established.") + error.warnings.append("Standard subprocess cleanup also failed; process state is unknown.") + else: + error = failure("bootstrap-cli-unavailable", "Native CLI execution failed; process and remote state may be unknown.") + raise error from exc + if len(result.stdout) > MAX_BYTES or len(result.stderr) > MAX_BYTES: + raise failure("bootstrap-cli-output-limit", "Captured CLI output exceeds one MiB per stream; this is not a capture-memory bound.") + return result.returncode, result.stdout, result.stderr diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_cleanup_dependencies.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_cleanup_dependencies.py index 1bb6a766..7a21c841 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_cleanup_dependencies.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_cleanup_dependencies.py @@ -1,435 +1,435 @@ -from __future__ import annotations - -import inspect -import re -from typing import Any -from urllib.parse import parse_qs, urlencode, urljoin, urlsplit - -try: - from ._common import HelperFailure, digest, odata_name, require_allowed_fields, sdk_error_metadata -except ImportError: - from _common import HelperFailure, digest, odata_name, require_allowed_fields, sdk_error_metadata - - -COLLECTIONS = {"index": "indexes", "indexer": "indexers", "skillset": "skillsets", "datasource": "datasources"} -SHA = re.compile(r"sha256:[a-f0-9]{64}\Z") - - -def fail(code, message): - return HelperFailure(code, message, blocked_at="cleanup-dependencies") - - -def limits(value=None): - value = {"max_pages": 20, "max_resources": 500} if value is None else value - if not isinstance(value, dict) or set(value) != {"max_pages", "max_resources"}: - raise fail("input-schema-invalid", "Inventory limits require max_pages and max_resources.") - if any(type(value[key]) is not int or not 1 <= value[key] <= maximum - for key, maximum in (("max_pages", 100), ("max_resources", 5000))): - raise fail("input-schema-invalid", "Inventory limits exceed supported bounded discovery.") - return dict(value) - - -def generated(current): - kind = current.get("kind") - if kind not in ("file", "azureBlob"): - raise fail("source-cleanup-kind-unsupported", "Only File and Blob/ADLS generated ownership is supported.") - parameters = current.get("fileParameters" if kind == "file" else "azureBlobParameters") - raw = parameters.get("createdResources") if isinstance(parameters, dict) else None - required = {"index"} if kind == "file" else set(COLLECTIONS) - if not isinstance(raw, dict): - raise fail("generated-ownership-unproven", "The exact source must expose its generated resource identities.") - names = {("datasource" if key == "dataSourceConnection" else key): value for key, value in raw.items()} - if set(names) != required or len(names) != len(raw): - raise fail("generated-ownership-unproven", "Unexpected, missing or colliding generated child identities.") - for name in names.values(): - odata_name(name) - return names - - -def validate_guard(plan): - guard = plan.get("dependency_guard") - if guard is None: - return - if not isinstance(guard, dict) or plan.get("operation") != "delete": - raise fail("input-schema-invalid", "Dependency guards belong only to cleanup deletion plans.") - kind = guard.get("kind") - if kind == "search-source" and plan.get("resource_type") == "knowledge-source": - fields, records, record_fields = ( - {"kind", "limits", "generated", "inventory_digest"}, - guard.get("generated"), {"type", "name", "etag", "definition_digest"}, - ) - if not isinstance(guard.get("inventory_digest"), str) or not SHA.fullmatch(guard["inventory_digest"]): - raise fail("input-schema-invalid", "A complete scoped inventory digest is required.") - elif kind == "prompt-connection" and isinstance(plan.get("connection"), dict): - fields, records, record_fields = ( - {"kind", "limits", "versions", "toolboxes"}, guard.get("versions"), {"name", "version", "definition_digest"}, - ) - if not isinstance(guard.get("toolboxes"), list): - raise fail("input-schema-invalid", "Complete toolbox version snapshots are required.") - records = records + guard["toolboxes"] if isinstance(records, list) else records - else: - raise fail("input-schema-invalid", "Dependency guard and cleanup target must agree.") - require_allowed_fields(guard, fields, label="Cleanup dependency guard") - if set(guard) != fields or not isinstance(records, list): - raise fail("input-schema-invalid", "The dependency snapshot must be complete.") - bounds = limits(guard["limits"]) - if len(records) > bounds["max_resources"]: - raise fail("input-schema-invalid", "The dependency snapshot exceeds approved limits.") - identities = set() - for record_index, record in enumerate(records): - if not isinstance(record, dict) or set(record) != record_fields: - raise fail("input-schema-invalid", "Dependency snapshot records are closed objects.") - if any(not isinstance(value, str) or not value.strip() for value in record.values()): - raise fail("input-schema-invalid", "Dependency snapshots require exact nonempty identities and versions.") - if not SHA.fullmatch(record["definition_digest"]): - raise fail("input-schema-invalid", "Dependency snapshots require canonical definition digests.") - domain = "toolbox" if kind == "prompt-connection" and record_index >= len(guard["versions"]) else "agent" - identity = (domain, record.get("type", record.get("version")), record["name"]) - if identity in identities: - raise fail("input-schema-invalid", "Duplicate dependency snapshot identity.") - identities.add(identity) - if kind == "search-source": - kinds = {record["type"] for record in records} - if kinds not in ({"index"}, set(COLLECTIONS)) or len(kinds) != len(records): - raise fail("input-schema-invalid", "Generated snapshots must identify one complete supported cascade.") - - -def _list_search(plan, collection, token, transport, bounds): - base = f"{plan['endpoint'].rstrip('/')}/{collection}" - url = base + "?" + urlencode({"api-version": plan["api_version"]}) - seen, names, records = set(), set(), [] - for _ in range(bounds["max_pages"]): - if url in seen: - raise fail("dependency-inventory-partial", "A continuation cycle prevents complete dependency discovery.") - seen.add(url) - response = transport("GET", url, token) - body = response.body - if response.status != 200: - raise HelperFailure("dependency-inventory-failed", "Scoped collection GET failed; consumer absence is unproven.", - blocked_at="cleanup-dependencies", status=response.status, request_id=response.request_id) - if not isinstance(body, dict) or not isinstance(body.get("value"), list): - raise fail("dependency-inventory-invalid", "Scoped collection GET did not return a complete JSON page.") - if set(body) - {"value", "@odata.nextLink", "@odata.context"}: - raise fail("dependency-inventory-partial", "An unsupported pagination contract cannot prove absence.") - for item in body["value"]: - if not isinstance(item, dict): - raise fail("dependency-inventory-invalid", "A dependency inventory item is not an object.") - name = item.get("name") - odata_name(name) - if name in names: - raise fail("dependency-inventory-changed", "Duplicate identities across inventory pages require fresh planning.") - names.add(name) - records.append(item) - if len(records) > bounds["max_resources"]: - raise fail("dependency-inventory-limit", "Complete scoped discovery exceeds the approved object bound.") - next_link = body.get("@odata.nextLink") - if next_link is None: - return sorted(records, key=lambda item: item["name"]) - if len(records) >= bounds["max_resources"]: - raise fail("dependency-inventory-limit", "More inventory pages exceed the approved object bound.") - if not isinstance(next_link, str) or not next_link: - raise fail("dependency-inventory-partial", "Malformed continuation is not an empty final page.") - candidate = urljoin(base, next_link) - parsed, expected = urlsplit(candidate), urlsplit(base) - query = parse_qs(parsed.query, keep_blank_values=True) - if ( - parsed.scheme != expected.scheme or parsed.netloc != expected.netloc or parsed.path != expected.path - or parsed.fragment or parsed.username or parsed.password - or query.get("api-version") != [plan["api_version"]] - or set(query) - {"api-version", "$skiptoken", "service", "pageSize", "search", "searchType"} - or query.get("search", [""]) != [""] - or ("service" in query and query["service"][0].casefold() - not in {expected.hostname.casefold(), expected.hostname.split(".")[0].casefold()}) - or any(len(values) != 1 for values in query.values()) - ): - raise fail("dependency-inventory-scope", "Continuation changed the selected service, collection, API or filter.") - url = candidate - raise fail("dependency-inventory-limit", "Complete dependency discovery exceeds the approved page bound.") - - -def _projections(skillset): - projections = skillset.get("indexProjections") - if projections is None: - return [] - if not isinstance(projections, dict) or not isinstance(projections.get("selectors"), list): - raise fail("dependency-inventory-invalid", "Skillset projection consumers are incomplete.") - names = [] - for selector in projections["selectors"]: - if not isinstance(selector, dict): - raise fail("dependency-inventory-invalid", "A skillset projection selector is incomplete.") - name = selector.get("targetIndexName") - odata_name(name) - names.append(name) - return names - - -def search_snapshot(plan, current, token, *, transport, bounds): - try: - from . import search_reconcile - except ImportError: - import search_reconcile - names = generated(current) - inventories = {} - bases = _list_search(plan, "knowledgebases", token, transport, bounds) - inventories["knowledgebases"] = bases - for base in bases: - sources = base.get("knowledgeSources") - if not isinstance(sources, list) or any(not isinstance(item, dict) or not isinstance(item.get("name"), str) for item in sources): - raise fail("dependency-inventory-invalid", "Every KB definition must expose its complete source references.") - if any(item["name"] == plan["name"] for item in sources): - raise HelperFailure( - "source-in-use", "A retained KB still references this source; separately plan that KB first, never detach it.", - blocked_at="cleanup-dependencies", warnings=["referencing-kb:" + base["name"]], - ) - sources = _list_search(plan, "knowledgesources", token, transport, bounds) - inventories["knowledgesources"] = sources - if sum(item["name"] == plan["name"] for item in sources) != 1: - raise fail("dependency-inventory-changed", "The selected source disappeared from the complete service inventory.") - for source in sources: - if source["name"] == plan["name"]: - if generated(source) != names or source.get("@odata.etag") != current.get("@odata.etag"): - raise fail("definition-drift", "Source inventory and exact readback disagree.") - continue - if source.get("kind") == "searchIndex": - parameters = source.get("searchIndexParameters") - index = parameters.get("searchIndexName") if isinstance(parameters, dict) else None - if not isinstance(index, str): - raise fail("dependency-inventory-invalid", "An existing-index source has unknown target identity.") - shared = index == names["index"] - elif source.get("kind") in ("file", "azureBlob"): - other = generated(source) - shared = any(other.get(kind) == name for kind, name in names.items()) - else: - raise fail("dependency-consumer-opaque", "An unsupported source kind prevents complete generated-index consumer proof.") - if shared: - raise fail("generated-resource-shared", "Another knowledge source references a generated cleanup resource.") - indexers = _list_search(plan, "indexers", token, transport, bounds) - inventories["indexers"] = indexers - for indexer in indexers: - if not all(isinstance(indexer.get(key), str) for key in ("dataSourceName", "targetIndexName")): - raise fail("dependency-inventory-invalid", "Indexer dependency identities are incomplete.") - if indexer.get("skillsetName") is not None and not isinstance(indexer["skillsetName"], str): - raise fail("dependency-inventory-invalid", "Indexer skillset identity is malformed.") - if indexer["name"] == names.get("indexer"): - if any(indexer.get(field) != names[kind] for field, kind in ( - ("dataSourceName", "datasource"), ("targetIndexName", "index"), ("skillsetName", "skillset"), - )): - raise fail("definition-drift", "The generated indexer no longer binds the exact owned pipeline.") - elif any(indexer.get(field) == names[kind] for field, kind in ( - ("dataSourceName", "datasource"), ("targetIndexName", "index"), ("skillsetName", "skillset"), - ) if kind in names): - raise fail("generated-resource-shared", "Another indexer consumes a generated cleanup resource.") - skillsets = _list_search(plan, "skillsets", token, transport, bounds) - inventories["skillsets"] = skillsets - for skillset in skillsets: - if skillset["name"] == names.get("skillset"): - if any(index != names["index"] for index in _projections(skillset)): - raise fail("generated-resource-shared", "The generated skillset also serves an outside-plan index.") - continue - skills = skillset.get("skills") - if not isinstance(skills, list) or any(not isinstance(skill, dict) or not isinstance(skill.get("@odata.type"), str) for skill in skills): - raise fail("dependency-inventory-invalid", "Skillset consumers are incomplete.") - if any(".Custom." in skill["@odata.type"] for skill in skills): - raise fail("dependency-consumer-opaque", "Custom skill code has no complete native generated-resource consumer contract.") - if names["index"] in _projections(skillset): - raise fail("generated-resource-shared", "Another skillset projects into the generated index.") - snapshots = [] - for kind, name in sorted(names.items()): - url = f"{plan['endpoint'].rstrip('/')}/{COLLECTIONS[kind]}('{odata_name(name)}')?api-version={plan['api_version']}" - child, _ = search_reconcile.read_resource(url, token, transport=transport) - if child is None: - raise fail("generated-resource-missing", "A generated child is missing; source absence alone cannot prove the remaining cascade.") - etag = child.get("@odata.etag") - if child.get("name") != name or not isinstance(etag, str) or not etag.strip(): - raise fail("generated-version-unavailable", "Every generated object requires an exact name and fresh ETag.") - snapshots.append({"type": kind, "name": name, "etag": etag, "definition_digest": digest(child)}) - refreshed, _ = search_reconcile.read_resource(search_reconcile.resource_url(plan), token, transport=transport) - if (refreshed is None or search_reconcile._definition(refreshed) != search_reconcile._definition(current) - or refreshed.get("@odata.etag") != current.get("@odata.etag") or generated(refreshed) != names): - raise fail("definition-drift", "The source changed during dependency discovery.") - inventory_state = { - kind: [ - {"name": item["name"], "etag": item.get("@odata.etag"), - "definition": search_reconcile._definition(item), - "generated": generated(item) if kind == "knowledgesources" and item.get("kind") in ("file", "azureBlob") else None} - for item in items - ] - for kind, items in inventories.items() - } - return {"kind": "search-source", "limits": bounds, "generated": snapshots, "inventory_digest": digest(inventory_state)} - - -def verify_search(plan, current, token, *, transport): - guard = plan["dependency_guard"] - if search_snapshot(plan, current, token, transport=transport, bounds=guard["limits"]) != guard: - raise fail("dependency-drift", "A generated ETag/definition or scoped dependency inventory changed after approval.") - - -def _paged(items, bounds): - if not callable(getattr(items, "by_page", None)): - raise fail("connection-consumer-api-unavailable", "SDK ItemPaged.by_page is required to prove complete project inventory.") - pages = iter(items.by_page()) - if not hasattr(pages, "continuation_token"): - raise fail("connection-consumer-api-unavailable", "SDK page continuation state is required for bounded complete discovery.") - count = 0 - for _ in range(bounds["max_pages"]): - try: - page = next(pages) - except StopIteration: - return - for item in page: - count += 1 - if count > bounds["max_resources"]: - raise fail("dependency-inventory-limit", "Project inventory exceeds the approved object bound.") - yield item - if pages.continuation_token is None: - return - if count >= bounds["max_resources"]: - raise fail("dependency-inventory-limit", "More project pages exceed the approved object bound.") - raise fail("dependency-inventory-limit", "Project inventory exceeds the approved page bound.") - - -def _connection_use(definition, plan, connection): - if definition.get("kind") != "prompt": - raise fail("connection-consumer-opaque", "Project inventory contains Hosted/workflow/external consumers; their runtime connection use is not enumerable here.") - if set(definition) - {"rai_config", "kind", "model", "instructions", "temperature", "top_p", "reasoning", - "tools", "tool_choice", "text", "structured_inputs"}: - raise fail("connection-consumer-opaque", "Unknown Prompt definition fields prevent complete consumer interpretation.") - tools = definition.get("tools") - if not isinstance(tools, list): - raise fail("connection-consumer-opaque", "A Prompt version has no complete tool definition.") - expected = f"{plan['project_resource_id']}/connections/{connection['name']}".casefold() - uses = False - for tool in tools: - if not isinstance(tool, dict) or tool.get("type") != "mcp": - raise fail("connection-consumer-opaque", "A non-MCP tool needs a verified native connection-consumer contract.") - if (set(tool) - {"type", "name", "description", "tool_configs", "server_label", "server_url", "connector_id", - "authorization", "server_description", "headers", "allowed_tools", "require_approval", - "defer_loading", "project_connection_id"} or tool.get("connector_id") is not None): - raise fail("connection-consumer-opaque", "Unknown tool fields or connector resolution cannot prove project-connection absence.") - configurations = tool.get("tool_configs") - if configurations is not None and ( - not isinstance(configurations, dict) or any( - not isinstance(value, dict) or set(value) - {"pin", "additional_search_text"} - or (value.get("pin") is not None and type(value["pin"]) is not bool) - or (value.get("additional_search_text") is not None and not isinstance(value["additional_search_text"], str)) - for value in configurations.values() - ) - ): - raise fail("connection-consumer-opaque", "Tool configuration is not the documented visibility/search-only contract.") - reference = tool.get("project_connection_id") - if reference is None: - if not isinstance(tool.get("server_url"), str): - raise fail("connection-consumer-opaque", "An MCP tool lacks an explicit connection or server target.") - continue - if (not isinstance(reference, str) or not reference.strip() or reference != reference.strip() - or any(character in reference for character in ("%", "?", "#", "{", "}", "\\", "\r", "\n", "\t"))): - raise fail("connection-consumer-opaque", "An MCP connection reference is malformed.") - if "/" in reference: - try: - from .prompt_connect import PROJECT_ID - except ImportError: - from prompt_connect import PROJECT_ID - project, _, leaf = reference.rstrip("/").casefold().rpartition("/connections/") - if PROJECT_ID.fullmatch(project) is None or not leaf or "/" in leaf: - raise fail("connection-consumer-opaque", "An MCP reference is neither a project connection name nor an exact ARM ID.") - uses |= reference.rstrip("/").casefold() == expected - else: - uses |= reference.casefold() == connection["name"].casefold() - return uses - - -def prompt_snapshot(plan, current, *, sdk_loader, bounds, allow_selected=True): - properties = current.get("properties") - if not isinstance(properties, dict) or properties.get("isSharedToAll") is not False or properties.get("sharedUserList") not in (None, []): - raise fail("connection-sharing-unverified", "Shared/unknown connection scope requires account/external consumer enumeration, not project-only inventory.") - AIProjectClient, _, _, _, extras = sdk_loader() - AzureCliCredential, AzureError = extras - client = AIProjectClient(endpoint=plan["project_endpoint"], credential=AzureCliCredential()) - selected = plan.get("agent") - selected_id = (selected["name"], selected["version"]) if selected else None - versions, toolboxes, names, identities = [], [], set(), set() - try: - if "include_drafts" not in inspect.signature(client.agents.list_versions).parameters: - raise fail("connection-consumer-api-unavailable", "agents.list_versions(include_drafts=True) is required; this SDK cannot prove draft-consumer absence.") - for agent in _paged(client.agents.list(), bounds): - name = getattr(agent, "name", None) - if not isinstance(name, str) or not name or name in names: - raise fail("dependency-inventory-changed", "Project agent inventory contains missing or duplicate identities.") - names.add(name) - for item in _paged(client.agents.list_versions(agent_name=name, include_drafts=True), bounds): - version = str(getattr(item, "version", "")) - identity = (name, version) - if not re.fullmatch(r"[1-9][0-9]*", version) or identity in identities: - raise fail("dependency-inventory-changed", "Version inventory contains missing or duplicate identities.") - identities.add(identity) - if len(identities) > bounds["max_resources"]: - raise fail("dependency-inventory-limit", "Total project versions exceed the approved object bound.") - observed = client.agents.get_version(agent_name=name, agent_version=version) - definition = observed.definition.as_dict() - if observed.name != name or str(observed.version) != version or not isinstance(definition, dict): - raise fail("dependency-inventory-changed", "Exact version readback disagrees with project inventory.") - uses = _connection_use(definition, plan, plan["connection"]) - if identity == selected_id and digest(definition) != selected["owned_definition_digest"]: - raise fail("definition-drift", "The explicitly selected version changed during consumer discovery.") - if identity == selected_id and not allow_selected: - raise fail("agent-absence-unverified", "The selected version must be absent before connection deletion.") - if uses and identity != selected_id: - raise HelperFailure( - "connection-shared-consumer", "A retained agent version consumes this connection.", - blocked_at="cleanup-dependencies", warnings=[f"consumer:{name}:{version}"], - ) - versions.append({"name": name, "version": version, "definition_digest": digest(definition)}) - names, identities = set(), set() - for toolbox in _paged(client.toolboxes.list(), bounds): - name = getattr(toolbox, "name", None) - if not isinstance(name, str) or not name.strip() or name in names: - raise fail("dependency-inventory-changed", "Toolbox inventory contains missing or duplicate names.") - names.add(name) - for item in _paged(client.toolboxes.list_versions(name=name), bounds): - version = getattr(item, "version", None) - identity = (name, version) - if not isinstance(version, str) or not version.strip() or identity in identities: - raise fail("dependency-inventory-changed", "Toolbox version inventory is incomplete or duplicated.") - identities.add(identity) - if len(versions) + len(identities) > bounds["max_resources"]: - raise fail("dependency-inventory-limit", "Combined agent/toolbox versions exceed approved bounds.") - observed = client.toolboxes.get_version(name=name, version=version) - definition = observed.as_dict() - if (not isinstance(definition, dict) or definition.get("name") != name - or definition.get("version") != version): - raise fail("dependency-inventory-changed", "Exact toolbox version disagrees with inventory.") - if (set(definition) - {"metadata", "id", "name", "version", "description", "created_at", "tools", "skills", "policies"} - or definition.get("skills") not in (None, []) or definition.get("policies") not in (None, {})): - raise fail("connection-consumer-opaque", "Toolbox skills/policies have no complete connection-consumer contract.") - if _connection_use({"kind": "prompt", "tools": definition.get("tools")}, plan, plan["connection"]): - raise HelperFailure( - "connection-shared-consumer", "A retained toolbox version consumes this connection.", - blocked_at="cleanup-dependencies", warnings=[f"toolbox-consumer:{name}:{version}"], - ) - toolboxes.append({"name": name, "version": version, "definition_digest": digest(definition)}) - except AzureError as exc: - raise HelperFailure(message="Complete project agent/version GET inventory failed; absence is unproven.", - blocked_at="cleanup-dependencies", **sdk_error_metadata(exc, "connection-inventory-failed")) from exc - except (AttributeError, TypeError, ValueError) as exc: - raise fail("connection-consumer-api-unavailable", "The installed SDK lacks complete typed agent/version inventory readback.") from exc - finally: - client.close() - return {"kind": "prompt-connection", "limits": bounds, - "versions": sorted(versions, key=lambda item: (item["name"], item["version"])), - "toolboxes": sorted(toolboxes, key=lambda item: (item["name"], item["version"]))} - - -def verify_prompt(plan, current, *, sdk_loader, allow_selected=True): - connection = plan["connection"] - if (digest(current) != connection["owned_definition_digest"] - or (current.get("etag") or current.get("@odata.etag")) != connection["expected_etag"]): - raise fail("definition-drift", "The connection definition or ETag changed after cleanup approval.") - guard = plan["dependency_guard"] - actual = prompt_snapshot(plan, current, sdk_loader=sdk_loader, bounds=guard["limits"], allow_selected=allow_selected) - expected_versions = guard["versions"] - selected = plan.get("agent") - if selected and not any(item["name"] == selected["name"] and item["version"] == selected["version"] for item in actual["versions"]): - expected_versions = [item for item in expected_versions if (item["name"], item["version"]) != (selected["name"], selected["version"])] - if actual["versions"] != expected_versions or actual["toolboxes"] != guard["toolboxes"]: - raise fail("dependency-drift", "Protected project agent/version definitions changed after cleanup approval.") +from __future__ import annotations + +import inspect +import re +from typing import Any +from urllib.parse import parse_qs, urlencode, urljoin, urlsplit + +try: + from ._common import HelperFailure, digest, odata_name, require_allowed_fields, sdk_error_metadata +except ImportError: + from _common import HelperFailure, digest, odata_name, require_allowed_fields, sdk_error_metadata + + +COLLECTIONS = {"index": "indexes", "indexer": "indexers", "skillset": "skillsets", "datasource": "datasources"} +SHA = re.compile(r"sha256:[a-f0-9]{64}\Z") + + +def fail(code, message): + return HelperFailure(code, message, blocked_at="cleanup-dependencies") + + +def limits(value=None): + value = {"max_pages": 20, "max_resources": 500} if value is None else value + if not isinstance(value, dict) or set(value) != {"max_pages", "max_resources"}: + raise fail("input-schema-invalid", "Inventory limits require max_pages and max_resources.") + if any(type(value[key]) is not int or not 1 <= value[key] <= maximum + for key, maximum in (("max_pages", 100), ("max_resources", 5000))): + raise fail("input-schema-invalid", "Inventory limits exceed supported bounded discovery.") + return dict(value) + + +def generated(current): + kind = current.get("kind") + if kind not in ("file", "azureBlob"): + raise fail("source-cleanup-kind-unsupported", "Only File and Blob/ADLS generated ownership is supported.") + parameters = current.get("fileParameters" if kind == "file" else "azureBlobParameters") + raw = parameters.get("createdResources") if isinstance(parameters, dict) else None + required = {"index"} if kind == "file" else set(COLLECTIONS) + if not isinstance(raw, dict): + raise fail("generated-ownership-unproven", "The exact source must expose its generated resource identities.") + names = {("datasource" if key == "dataSourceConnection" else key): value for key, value in raw.items()} + if set(names) != required or len(names) != len(raw): + raise fail("generated-ownership-unproven", "Unexpected, missing or colliding generated child identities.") + for name in names.values(): + odata_name(name) + return names + + +def validate_guard(plan): + guard = plan.get("dependency_guard") + if guard is None: + return + if not isinstance(guard, dict) or plan.get("operation") != "delete": + raise fail("input-schema-invalid", "Dependency guards belong only to cleanup deletion plans.") + kind = guard.get("kind") + if kind == "search-source" and plan.get("resource_type") == "knowledge-source": + fields, records, record_fields = ( + {"kind", "limits", "generated", "inventory_digest"}, + guard.get("generated"), {"type", "name", "etag", "definition_digest"}, + ) + if not isinstance(guard.get("inventory_digest"), str) or not SHA.fullmatch(guard["inventory_digest"]): + raise fail("input-schema-invalid", "A complete scoped inventory digest is required.") + elif kind == "prompt-connection" and isinstance(plan.get("connection"), dict): + fields, records, record_fields = ( + {"kind", "limits", "versions", "toolboxes"}, guard.get("versions"), {"name", "version", "definition_digest"}, + ) + if not isinstance(guard.get("toolboxes"), list): + raise fail("input-schema-invalid", "Complete toolbox version snapshots are required.") + records = records + guard["toolboxes"] if isinstance(records, list) else records + else: + raise fail("input-schema-invalid", "Dependency guard and cleanup target must agree.") + require_allowed_fields(guard, fields, label="Cleanup dependency guard") + if set(guard) != fields or not isinstance(records, list): + raise fail("input-schema-invalid", "The dependency snapshot must be complete.") + bounds = limits(guard["limits"]) + if len(records) > bounds["max_resources"]: + raise fail("input-schema-invalid", "The dependency snapshot exceeds approved limits.") + identities = set() + for record_index, record in enumerate(records): + if not isinstance(record, dict) or set(record) != record_fields: + raise fail("input-schema-invalid", "Dependency snapshot records are closed objects.") + if any(not isinstance(value, str) or not value.strip() for value in record.values()): + raise fail("input-schema-invalid", "Dependency snapshots require exact nonempty identities and versions.") + if not SHA.fullmatch(record["definition_digest"]): + raise fail("input-schema-invalid", "Dependency snapshots require canonical definition digests.") + domain = "toolbox" if kind == "prompt-connection" and record_index >= len(guard["versions"]) else "agent" + identity = (domain, record.get("type", record.get("version")), record["name"]) + if identity in identities: + raise fail("input-schema-invalid", "Duplicate dependency snapshot identity.") + identities.add(identity) + if kind == "search-source": + kinds = {record["type"] for record in records} + if kinds not in ({"index"}, set(COLLECTIONS)) or len(kinds) != len(records): + raise fail("input-schema-invalid", "Generated snapshots must identify one complete supported cascade.") + + +def _list_search(plan, collection, token, transport, bounds): + base = f"{plan['endpoint'].rstrip('/')}/{collection}" + url = base + "?" + urlencode({"api-version": plan["api_version"]}) + seen, names, records = set(), set(), [] + for _ in range(bounds["max_pages"]): + if url in seen: + raise fail("dependency-inventory-partial", "A continuation cycle prevents complete dependency discovery.") + seen.add(url) + response = transport("GET", url, token) + body = response.body + if response.status != 200: + raise HelperFailure("dependency-inventory-failed", "Scoped collection GET failed; consumer absence is unproven.", + blocked_at="cleanup-dependencies", status=response.status, request_id=response.request_id) + if not isinstance(body, dict) or not isinstance(body.get("value"), list): + raise fail("dependency-inventory-invalid", "Scoped collection GET did not return a complete JSON page.") + if set(body) - {"value", "@odata.nextLink", "@odata.context"}: + raise fail("dependency-inventory-partial", "An unsupported pagination contract cannot prove absence.") + for item in body["value"]: + if not isinstance(item, dict): + raise fail("dependency-inventory-invalid", "A dependency inventory item is not an object.") + name = item.get("name") + odata_name(name) + if name in names: + raise fail("dependency-inventory-changed", "Duplicate identities across inventory pages require fresh planning.") + names.add(name) + records.append(item) + if len(records) > bounds["max_resources"]: + raise fail("dependency-inventory-limit", "Complete scoped discovery exceeds the approved object bound.") + next_link = body.get("@odata.nextLink") + if next_link is None: + return sorted(records, key=lambda item: item["name"]) + if len(records) >= bounds["max_resources"]: + raise fail("dependency-inventory-limit", "More inventory pages exceed the approved object bound.") + if not isinstance(next_link, str) or not next_link: + raise fail("dependency-inventory-partial", "Malformed continuation is not an empty final page.") + candidate = urljoin(base, next_link) + parsed, expected = urlsplit(candidate), urlsplit(base) + query = parse_qs(parsed.query, keep_blank_values=True) + if ( + parsed.scheme != expected.scheme or parsed.netloc != expected.netloc or parsed.path != expected.path + or parsed.fragment or parsed.username or parsed.password + or query.get("api-version") != [plan["api_version"]] + or set(query) - {"api-version", "$skiptoken", "service", "pageSize", "search", "searchType"} + or query.get("search", [""]) != [""] + or ("service" in query and query["service"][0].casefold() + not in {expected.hostname.casefold(), expected.hostname.split(".")[0].casefold()}) + or any(len(values) != 1 for values in query.values()) + ): + raise fail("dependency-inventory-scope", "Continuation changed the selected service, collection, API or filter.") + url = candidate + raise fail("dependency-inventory-limit", "Complete dependency discovery exceeds the approved page bound.") + + +def _projections(skillset): + projections = skillset.get("indexProjections") + if projections is None: + return [] + if not isinstance(projections, dict) or not isinstance(projections.get("selectors"), list): + raise fail("dependency-inventory-invalid", "Skillset projection consumers are incomplete.") + names = [] + for selector in projections["selectors"]: + if not isinstance(selector, dict): + raise fail("dependency-inventory-invalid", "A skillset projection selector is incomplete.") + name = selector.get("targetIndexName") + odata_name(name) + names.append(name) + return names + + +def search_snapshot(plan, current, token, *, transport, bounds): + try: + from . import search_reconcile + except ImportError: + import search_reconcile + names = generated(current) + inventories = {} + bases = _list_search(plan, "knowledgebases", token, transport, bounds) + inventories["knowledgebases"] = bases + for base in bases: + sources = base.get("knowledgeSources") + if not isinstance(sources, list) or any(not isinstance(item, dict) or not isinstance(item.get("name"), str) for item in sources): + raise fail("dependency-inventory-invalid", "Every KB definition must expose its complete source references.") + if any(item["name"] == plan["name"] for item in sources): + raise HelperFailure( + "source-in-use", "A retained KB still references this source; separately plan that KB first, never detach it.", + blocked_at="cleanup-dependencies", warnings=["referencing-kb:" + base["name"]], + ) + sources = _list_search(plan, "knowledgesources", token, transport, bounds) + inventories["knowledgesources"] = sources + if sum(item["name"] == plan["name"] for item in sources) != 1: + raise fail("dependency-inventory-changed", "The selected source disappeared from the complete service inventory.") + for source in sources: + if source["name"] == plan["name"]: + if generated(source) != names or source.get("@odata.etag") != current.get("@odata.etag"): + raise fail("definition-drift", "Source inventory and exact readback disagree.") + continue + if source.get("kind") == "searchIndex": + parameters = source.get("searchIndexParameters") + index = parameters.get("searchIndexName") if isinstance(parameters, dict) else None + if not isinstance(index, str): + raise fail("dependency-inventory-invalid", "An existing-index source has unknown target identity.") + shared = index == names["index"] + elif source.get("kind") in ("file", "azureBlob"): + other = generated(source) + shared = any(other.get(kind) == name for kind, name in names.items()) + else: + raise fail("dependency-consumer-opaque", "An unsupported source kind prevents complete generated-index consumer proof.") + if shared: + raise fail("generated-resource-shared", "Another knowledge source references a generated cleanup resource.") + indexers = _list_search(plan, "indexers", token, transport, bounds) + inventories["indexers"] = indexers + for indexer in indexers: + if not all(isinstance(indexer.get(key), str) for key in ("dataSourceName", "targetIndexName")): + raise fail("dependency-inventory-invalid", "Indexer dependency identities are incomplete.") + if indexer.get("skillsetName") is not None and not isinstance(indexer["skillsetName"], str): + raise fail("dependency-inventory-invalid", "Indexer skillset identity is malformed.") + if indexer["name"] == names.get("indexer"): + if any(indexer.get(field) != names[kind] for field, kind in ( + ("dataSourceName", "datasource"), ("targetIndexName", "index"), ("skillsetName", "skillset"), + )): + raise fail("definition-drift", "The generated indexer no longer binds the exact owned pipeline.") + elif any(indexer.get(field) == names[kind] for field, kind in ( + ("dataSourceName", "datasource"), ("targetIndexName", "index"), ("skillsetName", "skillset"), + ) if kind in names): + raise fail("generated-resource-shared", "Another indexer consumes a generated cleanup resource.") + skillsets = _list_search(plan, "skillsets", token, transport, bounds) + inventories["skillsets"] = skillsets + for skillset in skillsets: + if skillset["name"] == names.get("skillset"): + if any(index != names["index"] for index in _projections(skillset)): + raise fail("generated-resource-shared", "The generated skillset also serves an outside-plan index.") + continue + skills = skillset.get("skills") + if not isinstance(skills, list) or any(not isinstance(skill, dict) or not isinstance(skill.get("@odata.type"), str) for skill in skills): + raise fail("dependency-inventory-invalid", "Skillset consumers are incomplete.") + if any(".Custom." in skill["@odata.type"] for skill in skills): + raise fail("dependency-consumer-opaque", "Custom skill code has no complete native generated-resource consumer contract.") + if names["index"] in _projections(skillset): + raise fail("generated-resource-shared", "Another skillset projects into the generated index.") + snapshots = [] + for kind, name in sorted(names.items()): + url = f"{plan['endpoint'].rstrip('/')}/{COLLECTIONS[kind]}('{odata_name(name)}')?api-version={plan['api_version']}" + child, _ = search_reconcile.read_resource(url, token, transport=transport) + if child is None: + raise fail("generated-resource-missing", "A generated child is missing; source absence alone cannot prove the remaining cascade.") + etag = child.get("@odata.etag") + if child.get("name") != name or not isinstance(etag, str) or not etag.strip(): + raise fail("generated-version-unavailable", "Every generated object requires an exact name and fresh ETag.") + snapshots.append({"type": kind, "name": name, "etag": etag, "definition_digest": digest(child)}) + refreshed, _ = search_reconcile.read_resource(search_reconcile.resource_url(plan), token, transport=transport) + if (refreshed is None or search_reconcile._definition(refreshed) != search_reconcile._definition(current) + or refreshed.get("@odata.etag") != current.get("@odata.etag") or generated(refreshed) != names): + raise fail("definition-drift", "The source changed during dependency discovery.") + inventory_state = { + kind: [ + {"name": item["name"], "etag": item.get("@odata.etag"), + "definition": search_reconcile._definition(item), + "generated": generated(item) if kind == "knowledgesources" and item.get("kind") in ("file", "azureBlob") else None} + for item in items + ] + for kind, items in inventories.items() + } + return {"kind": "search-source", "limits": bounds, "generated": snapshots, "inventory_digest": digest(inventory_state)} + + +def verify_search(plan, current, token, *, transport): + guard = plan["dependency_guard"] + if search_snapshot(plan, current, token, transport=transport, bounds=guard["limits"]) != guard: + raise fail("dependency-drift", "A generated ETag/definition or scoped dependency inventory changed after approval.") + + +def _paged(items, bounds): + if not callable(getattr(items, "by_page", None)): + raise fail("connection-consumer-api-unavailable", "SDK ItemPaged.by_page is required to prove complete project inventory.") + pages = iter(items.by_page()) + if not hasattr(pages, "continuation_token"): + raise fail("connection-consumer-api-unavailable", "SDK page continuation state is required for bounded complete discovery.") + count = 0 + for _ in range(bounds["max_pages"]): + try: + page = next(pages) + except StopIteration: + return + for item in page: + count += 1 + if count > bounds["max_resources"]: + raise fail("dependency-inventory-limit", "Project inventory exceeds the approved object bound.") + yield item + if pages.continuation_token is None: + return + if count >= bounds["max_resources"]: + raise fail("dependency-inventory-limit", "More project pages exceed the approved object bound.") + raise fail("dependency-inventory-limit", "Project inventory exceeds the approved page bound.") + + +def _connection_use(definition, plan, connection): + if definition.get("kind") != "prompt": + raise fail("connection-consumer-opaque", "Project inventory contains Hosted/workflow/external consumers; their runtime connection use is not enumerable here.") + if set(definition) - {"rai_config", "kind", "model", "instructions", "temperature", "top_p", "reasoning", + "tools", "tool_choice", "text", "structured_inputs"}: + raise fail("connection-consumer-opaque", "Unknown Prompt definition fields prevent complete consumer interpretation.") + tools = definition.get("tools") + if not isinstance(tools, list): + raise fail("connection-consumer-opaque", "A Prompt version has no complete tool definition.") + expected = f"{plan['project_resource_id']}/connections/{connection['name']}".casefold() + uses = False + for tool in tools: + if not isinstance(tool, dict) or tool.get("type") != "mcp": + raise fail("connection-consumer-opaque", "A non-MCP tool needs a verified native connection-consumer contract.") + if (set(tool) - {"type", "name", "description", "tool_configs", "server_label", "server_url", "connector_id", + "authorization", "server_description", "headers", "allowed_tools", "require_approval", + "defer_loading", "project_connection_id"} or tool.get("connector_id") is not None): + raise fail("connection-consumer-opaque", "Unknown tool fields or connector resolution cannot prove project-connection absence.") + configurations = tool.get("tool_configs") + if configurations is not None and ( + not isinstance(configurations, dict) or any( + not isinstance(value, dict) or set(value) - {"pin", "additional_search_text"} + or (value.get("pin") is not None and type(value["pin"]) is not bool) + or (value.get("additional_search_text") is not None and not isinstance(value["additional_search_text"], str)) + for value in configurations.values() + ) + ): + raise fail("connection-consumer-opaque", "Tool configuration is not the documented visibility/search-only contract.") + reference = tool.get("project_connection_id") + if reference is None: + if not isinstance(tool.get("server_url"), str): + raise fail("connection-consumer-opaque", "An MCP tool lacks an explicit connection or server target.") + continue + if (not isinstance(reference, str) or not reference.strip() or reference != reference.strip() + or any(character in reference for character in ("%", "?", "#", "{", "}", "\\", "\r", "\n", "\t"))): + raise fail("connection-consumer-opaque", "An MCP connection reference is malformed.") + if "/" in reference: + try: + from .prompt_connect import PROJECT_ID + except ImportError: + from prompt_connect import PROJECT_ID + project, _, leaf = reference.rstrip("/").casefold().rpartition("/connections/") + if PROJECT_ID.fullmatch(project) is None or not leaf or "/" in leaf: + raise fail("connection-consumer-opaque", "An MCP reference is neither a project connection name nor an exact ARM ID.") + uses |= reference.rstrip("/").casefold() == expected + else: + uses |= reference.casefold() == connection["name"].casefold() + return uses + + +def prompt_snapshot(plan, current, *, sdk_loader, bounds, allow_selected=True): + properties = current.get("properties") + if not isinstance(properties, dict) or properties.get("isSharedToAll") is not False or properties.get("sharedUserList") not in (None, []): + raise fail("connection-sharing-unverified", "Shared/unknown connection scope requires account/external consumer enumeration, not project-only inventory.") + AIProjectClient, _, _, _, extras = sdk_loader() + AzureCliCredential, AzureError = extras + client = AIProjectClient(endpoint=plan["project_endpoint"], credential=AzureCliCredential()) + selected = plan.get("agent") + selected_id = (selected["name"], selected["version"]) if selected else None + versions, toolboxes, names, identities = [], [], set(), set() + try: + if "include_drafts" not in inspect.signature(client.agents.list_versions).parameters: + raise fail("connection-consumer-api-unavailable", "agents.list_versions(include_drafts=True) is required; this SDK cannot prove draft-consumer absence.") + for agent in _paged(client.agents.list(), bounds): + name = getattr(agent, "name", None) + if not isinstance(name, str) or not name or name in names: + raise fail("dependency-inventory-changed", "Project agent inventory contains missing or duplicate identities.") + names.add(name) + for item in _paged(client.agents.list_versions(agent_name=name, include_drafts=True), bounds): + version = str(getattr(item, "version", "")) + identity = (name, version) + if not re.fullmatch(r"[1-9][0-9]*", version) or identity in identities: + raise fail("dependency-inventory-changed", "Version inventory contains missing or duplicate identities.") + identities.add(identity) + if len(identities) > bounds["max_resources"]: + raise fail("dependency-inventory-limit", "Total project versions exceed the approved object bound.") + observed = client.agents.get_version(agent_name=name, agent_version=version) + definition = observed.definition.as_dict() + if observed.name != name or str(observed.version) != version or not isinstance(definition, dict): + raise fail("dependency-inventory-changed", "Exact version readback disagrees with project inventory.") + uses = _connection_use(definition, plan, plan["connection"]) + if identity == selected_id and digest(definition) != selected["owned_definition_digest"]: + raise fail("definition-drift", "The explicitly selected version changed during consumer discovery.") + if identity == selected_id and not allow_selected: + raise fail("agent-absence-unverified", "The selected version must be absent before connection deletion.") + if uses and identity != selected_id: + raise HelperFailure( + "connection-shared-consumer", "A retained agent version consumes this connection.", + blocked_at="cleanup-dependencies", warnings=[f"consumer:{name}:{version}"], + ) + versions.append({"name": name, "version": version, "definition_digest": digest(definition)}) + names, identities = set(), set() + for toolbox in _paged(client.toolboxes.list(), bounds): + name = getattr(toolbox, "name", None) + if not isinstance(name, str) or not name.strip() or name in names: + raise fail("dependency-inventory-changed", "Toolbox inventory contains missing or duplicate names.") + names.add(name) + for item in _paged(client.toolboxes.list_versions(name=name), bounds): + version = getattr(item, "version", None) + identity = (name, version) + if not isinstance(version, str) or not version.strip() or identity in identities: + raise fail("dependency-inventory-changed", "Toolbox version inventory is incomplete or duplicated.") + identities.add(identity) + if len(versions) + len(identities) > bounds["max_resources"]: + raise fail("dependency-inventory-limit", "Combined agent/toolbox versions exceed approved bounds.") + observed = client.toolboxes.get_version(name=name, version=version) + definition = observed.as_dict() + if (not isinstance(definition, dict) or definition.get("name") != name + or definition.get("version") != version): + raise fail("dependency-inventory-changed", "Exact toolbox version disagrees with inventory.") + if (set(definition) - {"metadata", "id", "name", "version", "description", "created_at", "tools", "skills", "policies"} + or definition.get("skills") not in (None, []) or definition.get("policies") not in (None, {})): + raise fail("connection-consumer-opaque", "Toolbox skills/policies have no complete connection-consumer contract.") + if _connection_use({"kind": "prompt", "tools": definition.get("tools")}, plan, plan["connection"]): + raise HelperFailure( + "connection-shared-consumer", "A retained toolbox version consumes this connection.", + blocked_at="cleanup-dependencies", warnings=[f"toolbox-consumer:{name}:{version}"], + ) + toolboxes.append({"name": name, "version": version, "definition_digest": digest(definition)}) + except AzureError as exc: + raise HelperFailure(message="Complete project agent/version GET inventory failed; absence is unproven.", + blocked_at="cleanup-dependencies", **sdk_error_metadata(exc, "connection-inventory-failed")) from exc + except (AttributeError, TypeError, ValueError) as exc: + raise fail("connection-consumer-api-unavailable", "The installed SDK lacks complete typed agent/version inventory readback.") from exc + finally: + client.close() + return {"kind": "prompt-connection", "limits": bounds, + "versions": sorted(versions, key=lambda item: (item["name"], item["version"])), + "toolboxes": sorted(toolboxes, key=lambda item: (item["name"], item["version"]))} + + +def verify_prompt(plan, current, *, sdk_loader, allow_selected=True): + connection = plan["connection"] + if (digest(current) != connection["owned_definition_digest"] + or (current.get("etag") or current.get("@odata.etag")) != connection["expected_etag"]): + raise fail("definition-drift", "The connection definition or ETag changed after cleanup approval.") + guard = plan["dependency_guard"] + actual = prompt_snapshot(plan, current, sdk_loader=sdk_loader, bounds=guard["limits"], allow_selected=allow_selected) + expected_versions = guard["versions"] + selected = plan.get("agent") + if selected and not any(item["name"] == selected["name"] and item["version"] == selected["version"] for item in actual["versions"]): + expected_versions = [item for item in expected_versions if (item["name"], item["version"]) != (selected["name"], selected["version"])] + if actual["versions"] != expected_versions or actual["toolboxes"] != guard["toolboxes"]: + raise fail("dependency-drift", "Protected project agent/version definitions changed after cleanup approval.") diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_cleanup_receipts.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_cleanup_receipts.py index 5533e5b1..761a70c7 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_cleanup_receipts.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_cleanup_receipts.py @@ -1,239 +1,239 @@ -"""Filtered, immutable creation evidence; never a credentialized callback surface.""" -from __future__ import annotations - -import copy -from datetime import datetime -from pathlib import Path -import uuid - -try: - from . import _bootstrap_io as private_io - from ._common import HelperFailure, digest, load_approved_input, reject_secrets, odata_name, validate_search_endpoint -except ImportError: - import _bootstrap_io as private_io - from _common import HelperFailure, digest, load_approved_input, reject_secrets, odata_name, validate_search_endpoint - - -def fail(code, message): - return HelperFailure(code, message, blocked_at="cleanup-provenance") - - -def version_identity(value): - get = value.get if isinstance(value, dict) else lambda key: getattr(value, key, None) - identifier, created = get("id"), get("created_at") - if isinstance(created, datetime): - created = int(created.timestamp()) if created.tzinfo is not None else None - if not isinstance(identifier, str) or not identifier.strip() or type(created) is not int or created <= 0: - return None - return {"id": identifier, "created_at": created} - - -def add_argument(parser): - parser.add_argument("--cleanup-receipt-dir", type=Path, - help="Existing absolute private directory for filtered original-create evidence.") - - -class Capture: - def __init__(self, directory, document): - self.directory = private_io.private_directory(str(directory)) - self.plan_digest = digest(document["plan"]) - self.owner = document["plan"].get("owner") - self.records = {} - self.summaries = [] - - def start(self, target, acknowledgement): - acknowledgement.setdefault("version_identity", None) - record = { - "schema_version": "1.0", "kind": "cleanup-creation-receipt", - "plan_digest": self.plan_digest, "owner": self.owner, "target": target, - "state": "acknowledged", "acknowledgement": acknowledgement, "snapshot": None, - } - key = digest(target) - self.records[key] = record - self._persist(record) - - def finish(self, target, snapshot): - record = copy.deepcopy(self.records[digest(target)]) - ack = record["acknowledgement"] - snapshot.setdefault("version_identity", None) - if (snapshot["version_identity"] != ack["version_identity"] - or (target["type"] == "prompt-agent-version" and ack["version_identity"] is None)): - raise fail("creation-version-unproven", "Native version identity/creation time must match the original acknowledgement.") - if not isinstance(ack["request_id"], str) or not ack["request_id"].strip(): - raise fail("creation-request-id-unavailable", "Original native acknowledgement lacks a request ID.") - if snapshot["definition_digest"] != ack["definition_digest"]: - raise fail("definition-drift", "Readback differs from the acknowledged original creation.") - record.update(state="verified", snapshot=snapshot) - self._persist(record) - - def _persist(self, record): - reject_secrets(record) - value = {**record, "integrity": digest(record)} - try: - path = private_io.private_file(self.directory, uuid.uuid4().hex + ".cleanup.json", value) - except OSError as exc: - raise fail("cleanup-receipt-persistence-failed", "Acknowledged creation evidence could not be retained privately.") from exc - self.summaries.append({"target": record["target"], "state": record["state"], - "receipt_file": str(path), "evidence_digest": value["integrity"]}) - - -def search_target(plan): - return {"type": plan["resource_type"], "endpoint": validate_search_endpoint(plan["endpoint"]), - **{key: plan[key] for key in ("name", "api_version")}} - - -def search_ack(capture, plan, response): - try: - from . import search_reconcile as search, _cleanup_dependencies as dependencies - except ImportError: - import search_reconcile as search, _cleanup_dependencies as dependencies - generated = None - if plan["resource_type"] == "knowledge-source": - try: - generated = dependencies.generated(response.body) if isinstance(response.body, dict) else None - except HelperFailure: - pass - capture.start(search_target(plan), { - "operation": "search-create", "status": response.status, "request_id": response.request_id, - "definition_digest": digest(search._definition(plan["desired"])), - "etag_evidence": search.response_etags(response), "generated": generated, "version": None, - }) - - -def search_finish(capture, plan, current, token, transport): - try: - from . import search_reconcile as search, _cleanup_dependencies as dependencies - except ImportError: - import search_reconcile as search, _cleanup_dependencies as dependencies - target = search_target(plan) - ack = capture.records[digest(target)]["acknowledgement"] - etag = search.resolve_etag(ack["etag_evidence"], ack["request_id"]) - if ack["status"] != 201 or not etag or current is None or current.get("@odata.etag") != etag: - raise fail("creation-version-unproven", "Require original HTTP 201 and the unchanged acknowledged ETag, not a later GET version.") - children = [] - if plan["resource_type"] == "knowledge-source": - if ack["generated"] is None or dependencies.generated(current) != ack["generated"]: - raise fail("generated-creation-evidence-unavailable", "The original create response did not identify this exact generated cascade.") - for kind, name in sorted(ack["generated"].items()): - url = f"{plan['endpoint'].rstrip('/')}/{dependencies.COLLECTIONS[kind]}('{odata_name(name)}')?api-version={plan['api_version']}" - child, _ = search.read_resource(url, token, transport=transport) - if child is None or child.get("name") != name or not child.get("@odata.etag"): - raise fail("generated-version-unavailable", "An original generated child lacks exact version readback.") - children.append({"type": kind, "name": name, "etag": child["@odata.etag"], "definition_digest": digest(child)}) - refreshed, _ = search.read_resource(search.resource_url(plan), token, transport=transport) - if (refreshed is None or refreshed.get("@odata.etag") != etag - or search._definition(refreshed) != search._definition(current) - or dependencies.generated(refreshed) != ack["generated"]): - raise fail("definition-drift", "Source changed during original generated-child capture.") - capture.finish(target, {"definition_digest": digest(search._definition(current)), "etag": etag, "generated": children}) - - -def project_target(plan, kind, *, name=None, version=None): - target = {"type": kind, "project_resource_id": plan["project_resource_id"].casefold(), - "project_endpoint": plan["project_endpoint"].rstrip("/"), - "name": name or plan["connection"]["name"]} - if version is not None: - target["version"] = version - return target - - -def connection_ack(capture, plan, response): - body = response.body if isinstance(response.body, dict) else {} - capture.start(project_target(plan, "project-connection"), { - "operation": "project-connection-create", "status": response.status, "request_id": response.request_id, - "definition_digest": digest(body), "version": None, "generated": None, - "etag_evidence": {"body": body.get("etag") or body.get("@odata.etag"), - "headers": [v for k, v in response.headers.items() if k.casefold() == "etag"]}, - }) - - -def connection_finish(capture, plan, body): - try: - from .search_reconcile import resolve_etag - except ImportError: - from search_reconcile import resolve_etag - target = project_target(plan, "project-connection") - ack = capture.records[digest(target)]["acknowledgement"] - etag = resolve_etag(ack["etag_evidence"], ack["request_id"]) - if ack["status"] != 201 or not etag or etag != (body.get("etag") or body.get("@odata.etag")): - raise fail("creation-version-unproven", "Connection readback must retain its original HTTP 201 ETag.") - capture.finish(target, {"definition_digest": digest(body), "etag": etag, "generated": []}) - - -def sdk_response_hook(metadata): - def capture(response): - native = response.http_response - metadata["request_id"] = next((v for k, v in native.headers.items() - if k.casefold() in ("request-id", "x-request-id", "x-ms-request-id", "apim-request-id")), None) - metadata["status"] = native.status_code - return capture - - -def agent_ack(capture, plan, created, metadata): - target = project_target(plan, "prompt-agent-version", name=created.name, version=str(created.version)) - capture.start(target, { - "operation": "agents.create_version", "status": metadata.get("status"), "request_id": metadata.get("request_id"), - "definition_digest": digest(created.definition.as_dict()), "version": str(created.version), - "etag_evidence": None, "generated": None, - "version_identity": version_identity(created), - }) - if (metadata.get("status") not in (200, 201) or created.name != plan["agent"]["name"] - or str(created.version) == plan["agent"]["version"] or version_identity(created) is None): - raise fail("ownership-unproven", "The original SDK return must identify a newly created version in the approved agent.") - return target - - -def load(input_path, receipt_path, target): - try: - from .blob_recheck import read_private - from . import search_reconcile as search - except ImportError: - from blob_recheck import read_private - import search_reconcile as search - _, plan, fingerprint = load_approved_input(input_path) - record = read_private(receipt_path) - reject_secrets(record) - fields = {"schema_version", "kind", "plan_digest", "owner", "target", "state", "acknowledgement", "snapshot", "integrity"} - if (not isinstance(record, dict) or set(record) != fields or record["schema_version"] != "1.0" - or record["kind"] != "cleanup-creation-receipt" or record["state"] != "verified" - or record["plan_digest"] != fingerprint or record["owner"] != plan.get("owner") - or record["target"] != target - or record["integrity"] != digest({k: v for k, v in record.items() if k != "integrity"})): - raise fail("ownership-unproven", "Require the exact original verified producer receipt and original approval; ACK/recovery/reuse cannot substitute.") - ack, snapshot = record["acknowledgement"], record["snapshot"] - if (not isinstance(ack, dict) or set(ack) != {"operation", "status", "request_id", "definition_digest", "etag_evidence", "generated", "version", "version_identity"} - or not isinstance(snapshot, dict) or set(snapshot) != {"definition_digest", "etag", "generated", "version_identity"} - or not isinstance(ack["request_id"], str) or not ack["request_id"].strip() - or snapshot["definition_digest"] != ack["definition_digest"] - or not isinstance(snapshot["definition_digest"], str) or not search.SHA256.fullmatch(snapshot["definition_digest"]) - or not isinstance(snapshot["generated"], list)): - raise fail("ownership-unproven", "Producer receipt lacks complete acknowledged version evidence.") - if (snapshot["version_identity"] != ack["version_identity"] - or (target["type"] == "prompt-agent-version" and ( - not isinstance(snapshot["version_identity"], dict) - or set(snapshot["version_identity"]) != {"id", "created_at"} - or version_identity(snapshot["version_identity"]) != snapshot["version_identity"]))): - raise fail("ownership-unproven", "Native version birth identity is missing or changed.") - if target["type"] != "prompt-agent-version" and ( - not isinstance(ack["etag_evidence"], dict) or set(ack["etag_evidence"]) != {"body", "headers"} - or not isinstance(ack["etag_evidence"]["headers"], list) - ): - raise fail("ownership-unproven", "Native ETag evidence must be complete.") - if target["type"] in ("knowledge-base", "knowledge-source"): - if (ack["operation"] != "search-create" or ack["status"] != 201 - or search.resolve_etag(ack["etag_evidence"], ack["request_id"]) != snapshot["etag"] - or not isinstance(snapshot["etag"], str) or not snapshot["etag"].strip()): - raise fail("creation-version-unproven", "Original Search create acknowledgement and snapshot versions disagree.") - elif target["type"] == "project-connection": - if (ack["operation"] != "project-connection-create" or ack["status"] != 201 - or search.resolve_etag(ack["etag_evidence"], ack["request_id"]) != snapshot["etag"] - or not snapshot["etag"] or snapshot["generated"]): - raise fail("creation-version-unproven", "Original connection acknowledgement and readback versions disagree.") - elif target["type"] == "prompt-agent-version": - if (ack["operation"] not in ("agents.create_version", "agents.create") or ack["status"] not in (200, 201) - or (ack["operation"] == "agents.create" and ack["status"] != 200) - or ack["version"] != target["version"] or snapshot["etag"] is not None or snapshot["generated"]): - raise fail("ownership-unproven", "Require original SDK version-create return and its exact readback.") - else: - raise fail("cleanup-kind-unsupported", "No producer receipt contract supports this target.") - return plan, record +"""Filtered, immutable creation evidence; never a credentialized callback surface.""" +from __future__ import annotations + +import copy +from datetime import datetime +from pathlib import Path +import uuid + +try: + from . import _bootstrap_io as private_io + from ._common import HelperFailure, digest, load_approved_input, reject_secrets, odata_name, validate_search_endpoint +except ImportError: + import _bootstrap_io as private_io + from _common import HelperFailure, digest, load_approved_input, reject_secrets, odata_name, validate_search_endpoint + + +def fail(code, message): + return HelperFailure(code, message, blocked_at="cleanup-provenance") + + +def version_identity(value): + get = value.get if isinstance(value, dict) else lambda key: getattr(value, key, None) + identifier, created = get("id"), get("created_at") + if isinstance(created, datetime): + created = int(created.timestamp()) if created.tzinfo is not None else None + if not isinstance(identifier, str) or not identifier.strip() or type(created) is not int or created <= 0: + return None + return {"id": identifier, "created_at": created} + + +def add_argument(parser): + parser.add_argument("--cleanup-receipt-dir", type=Path, + help="Existing absolute private directory for filtered original-create evidence.") + + +class Capture: + def __init__(self, directory, document): + self.directory = private_io.private_directory(str(directory)) + self.plan_digest = digest(document["plan"]) + self.owner = document["plan"].get("owner") + self.records = {} + self.summaries = [] + + def start(self, target, acknowledgement): + acknowledgement.setdefault("version_identity", None) + record = { + "schema_version": "1.0", "kind": "cleanup-creation-receipt", + "plan_digest": self.plan_digest, "owner": self.owner, "target": target, + "state": "acknowledged", "acknowledgement": acknowledgement, "snapshot": None, + } + key = digest(target) + self.records[key] = record + self._persist(record) + + def finish(self, target, snapshot): + record = copy.deepcopy(self.records[digest(target)]) + ack = record["acknowledgement"] + snapshot.setdefault("version_identity", None) + if (snapshot["version_identity"] != ack["version_identity"] + or (target["type"] == "prompt-agent-version" and ack["version_identity"] is None)): + raise fail("creation-version-unproven", "Native version identity/creation time must match the original acknowledgement.") + if not isinstance(ack["request_id"], str) or not ack["request_id"].strip(): + raise fail("creation-request-id-unavailable", "Original native acknowledgement lacks a request ID.") + if snapshot["definition_digest"] != ack["definition_digest"]: + raise fail("definition-drift", "Readback differs from the acknowledged original creation.") + record.update(state="verified", snapshot=snapshot) + self._persist(record) + + def _persist(self, record): + reject_secrets(record) + value = {**record, "integrity": digest(record)} + try: + path = private_io.private_file(self.directory, uuid.uuid4().hex + ".cleanup.json", value) + except OSError as exc: + raise fail("cleanup-receipt-persistence-failed", "Acknowledged creation evidence could not be retained privately.") from exc + self.summaries.append({"target": record["target"], "state": record["state"], + "receipt_file": str(path), "evidence_digest": value["integrity"]}) + + +def search_target(plan): + return {"type": plan["resource_type"], "endpoint": validate_search_endpoint(plan["endpoint"]), + **{key: plan[key] for key in ("name", "api_version")}} + + +def search_ack(capture, plan, response): + try: + from . import search_reconcile as search, _cleanup_dependencies as dependencies + except ImportError: + import search_reconcile as search, _cleanup_dependencies as dependencies + generated = None + if plan["resource_type"] == "knowledge-source": + try: + generated = dependencies.generated(response.body) if isinstance(response.body, dict) else None + except HelperFailure: + pass + capture.start(search_target(plan), { + "operation": "search-create", "status": response.status, "request_id": response.request_id, + "definition_digest": digest(search._definition(plan["desired"])), + "etag_evidence": search.response_etags(response), "generated": generated, "version": None, + }) + + +def search_finish(capture, plan, current, token, transport): + try: + from . import search_reconcile as search, _cleanup_dependencies as dependencies + except ImportError: + import search_reconcile as search, _cleanup_dependencies as dependencies + target = search_target(plan) + ack = capture.records[digest(target)]["acknowledgement"] + etag = search.resolve_etag(ack["etag_evidence"], ack["request_id"]) + if ack["status"] != 201 or not etag or current is None or current.get("@odata.etag") != etag: + raise fail("creation-version-unproven", "Require original HTTP 201 and the unchanged acknowledged ETag, not a later GET version.") + children = [] + if plan["resource_type"] == "knowledge-source": + if ack["generated"] is None or dependencies.generated(current) != ack["generated"]: + raise fail("generated-creation-evidence-unavailable", "The original create response did not identify this exact generated cascade.") + for kind, name in sorted(ack["generated"].items()): + url = f"{plan['endpoint'].rstrip('/')}/{dependencies.COLLECTIONS[kind]}('{odata_name(name)}')?api-version={plan['api_version']}" + child, _ = search.read_resource(url, token, transport=transport) + if child is None or child.get("name") != name or not child.get("@odata.etag"): + raise fail("generated-version-unavailable", "An original generated child lacks exact version readback.") + children.append({"type": kind, "name": name, "etag": child["@odata.etag"], "definition_digest": digest(child)}) + refreshed, _ = search.read_resource(search.resource_url(plan), token, transport=transport) + if (refreshed is None or refreshed.get("@odata.etag") != etag + or search._definition(refreshed) != search._definition(current) + or dependencies.generated(refreshed) != ack["generated"]): + raise fail("definition-drift", "Source changed during original generated-child capture.") + capture.finish(target, {"definition_digest": digest(search._definition(current)), "etag": etag, "generated": children}) + + +def project_target(plan, kind, *, name=None, version=None): + target = {"type": kind, "project_resource_id": plan["project_resource_id"].casefold(), + "project_endpoint": plan["project_endpoint"].rstrip("/"), + "name": name or plan["connection"]["name"]} + if version is not None: + target["version"] = version + return target + + +def connection_ack(capture, plan, response): + body = response.body if isinstance(response.body, dict) else {} + capture.start(project_target(plan, "project-connection"), { + "operation": "project-connection-create", "status": response.status, "request_id": response.request_id, + "definition_digest": digest(body), "version": None, "generated": None, + "etag_evidence": {"body": body.get("etag") or body.get("@odata.etag"), + "headers": [v for k, v in response.headers.items() if k.casefold() == "etag"]}, + }) + + +def connection_finish(capture, plan, body): + try: + from .search_reconcile import resolve_etag + except ImportError: + from search_reconcile import resolve_etag + target = project_target(plan, "project-connection") + ack = capture.records[digest(target)]["acknowledgement"] + etag = resolve_etag(ack["etag_evidence"], ack["request_id"]) + if ack["status"] != 201 or not etag or etag != (body.get("etag") or body.get("@odata.etag")): + raise fail("creation-version-unproven", "Connection readback must retain its original HTTP 201 ETag.") + capture.finish(target, {"definition_digest": digest(body), "etag": etag, "generated": []}) + + +def sdk_response_hook(metadata): + def capture(response): + native = response.http_response + metadata["request_id"] = next((v for k, v in native.headers.items() + if k.casefold() in ("request-id", "x-request-id", "x-ms-request-id", "apim-request-id")), None) + metadata["status"] = native.status_code + return capture + + +def agent_ack(capture, plan, created, metadata): + target = project_target(plan, "prompt-agent-version", name=created.name, version=str(created.version)) + capture.start(target, { + "operation": "agents.create_version", "status": metadata.get("status"), "request_id": metadata.get("request_id"), + "definition_digest": digest(created.definition.as_dict()), "version": str(created.version), + "etag_evidence": None, "generated": None, + "version_identity": version_identity(created), + }) + if (metadata.get("status") not in (200, 201) or created.name != plan["agent"]["name"] + or str(created.version) == plan["agent"]["version"] or version_identity(created) is None): + raise fail("ownership-unproven", "The original SDK return must identify a newly created version in the approved agent.") + return target + + +def load(input_path, receipt_path, target): + try: + from .blob_recheck import read_private + from . import search_reconcile as search + except ImportError: + from blob_recheck import read_private + import search_reconcile as search + _, plan, fingerprint = load_approved_input(input_path) + record = read_private(receipt_path) + reject_secrets(record) + fields = {"schema_version", "kind", "plan_digest", "owner", "target", "state", "acknowledgement", "snapshot", "integrity"} + if (not isinstance(record, dict) or set(record) != fields or record["schema_version"] != "1.0" + or record["kind"] != "cleanup-creation-receipt" or record["state"] != "verified" + or record["plan_digest"] != fingerprint or record["owner"] != plan.get("owner") + or record["target"] != target + or record["integrity"] != digest({k: v for k, v in record.items() if k != "integrity"})): + raise fail("ownership-unproven", "Require the exact original verified producer receipt and original approval; ACK/recovery/reuse cannot substitute.") + ack, snapshot = record["acknowledgement"], record["snapshot"] + if (not isinstance(ack, dict) or set(ack) != {"operation", "status", "request_id", "definition_digest", "etag_evidence", "generated", "version", "version_identity"} + or not isinstance(snapshot, dict) or set(snapshot) != {"definition_digest", "etag", "generated", "version_identity"} + or not isinstance(ack["request_id"], str) or not ack["request_id"].strip() + or snapshot["definition_digest"] != ack["definition_digest"] + or not isinstance(snapshot["definition_digest"], str) or not search.SHA256.fullmatch(snapshot["definition_digest"]) + or not isinstance(snapshot["generated"], list)): + raise fail("ownership-unproven", "Producer receipt lacks complete acknowledged version evidence.") + if (snapshot["version_identity"] != ack["version_identity"] + or (target["type"] == "prompt-agent-version" and ( + not isinstance(snapshot["version_identity"], dict) + or set(snapshot["version_identity"]) != {"id", "created_at"} + or version_identity(snapshot["version_identity"]) != snapshot["version_identity"]))): + raise fail("ownership-unproven", "Native version birth identity is missing or changed.") + if target["type"] != "prompt-agent-version" and ( + not isinstance(ack["etag_evidence"], dict) or set(ack["etag_evidence"]) != {"body", "headers"} + or not isinstance(ack["etag_evidence"]["headers"], list) + ): + raise fail("ownership-unproven", "Native ETag evidence must be complete.") + if target["type"] in ("knowledge-base", "knowledge-source"): + if (ack["operation"] != "search-create" or ack["status"] != 201 + or search.resolve_etag(ack["etag_evidence"], ack["request_id"]) != snapshot["etag"] + or not isinstance(snapshot["etag"], str) or not snapshot["etag"].strip()): + raise fail("creation-version-unproven", "Original Search create acknowledgement and snapshot versions disagree.") + elif target["type"] == "project-connection": + if (ack["operation"] != "project-connection-create" or ack["status"] != 201 + or search.resolve_etag(ack["etag_evidence"], ack["request_id"]) != snapshot["etag"] + or not snapshot["etag"] or snapshot["generated"]): + raise fail("creation-version-unproven", "Original connection acknowledgement and readback versions disagree.") + elif target["type"] == "prompt-agent-version": + if (ack["operation"] not in ("agents.create_version", "agents.create") or ack["status"] not in (200, 201) + or (ack["operation"] == "agents.create" and ack["status"] != 200) + or ack["version"] != target["version"] or snapshot["etag"] is not None or snapshot["generated"]): + raise fail("ownership-unproven", "Require original SDK version-create return and its exact readback.") + else: + raise fail("cleanup-kind-unsupported", "No producer receipt contract supports this target.") + return plan, record diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_common.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_common.py index 21021249..b8b2f49c 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_common.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_common.py @@ -1,949 +1,949 @@ -from __future__ import annotations - -import hashlib -import json -import math -import re -import shutil -import socket -import subprocess -import sys -import threading -import time -import uuid -from collections.abc import Mapping -from contextlib import suppress -from dataclasses import dataclass -from datetime import datetime, timedelta, timezone -from email.utils import parsedate_to_datetime -from http.client import HTTPException -from pathlib import Path -from typing import Any, Callable, NamedTuple -from urllib.error import HTTPError, URLError -from urllib.parse import quote, urlsplit -from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen - - -SEARCH_AUDIENCE = "https://search.azure.com" -MANAGEMENT_AUDIENCE = "https://management.azure.com" -SEARCH_HOST = re.compile( - r"^[a-z0-9](?:[a-z0-9-]{0,58}[a-z0-9])?\.search\.windows\.net$" -) -SECRET_FIELDS = { - "accesskey", - "accesstoken", - "accountkey", - "apikey", - "applicationsecret", - "authorization", - "clientsecret", - "credential", - "connectionsecret", - "key", - "password", - "privatekey", - "refreshtoken", - "sas", - "sastoken", - "secret", - "sharedaccesskey", - "storageaccountkey", - "token", -} -RESOURCE_ID_CONNECTION = re.compile(r"^ResourceId=/[^;\r\n]+;?$") - - -def normalize_azure_location(value: Any) -> str | None: - """Normalize ASCII case/whitespace only; this does not validate region availability.""" - if not isinstance(value, str) or len(value) > 128 or not value.isascii(): - return None - compact = re.sub(r"\s+", "", value, flags=re.ASCII).lower() - return compact if re.fullmatch(r"[a-z][a-z0-9]{1,40}", compact) else None - - -def _normalize_response_headers(headers: Mapping[str, str]) -> dict[str, str]: - normalized: dict[str, str] = {} - for name, value in headers.items(): - key = name.lower() - if key in {"x-ms-request-id", "request-id"} and normalized.get(key): - continue - if key == "retry-after" and key in normalized: - normalized[key] = "" - continue - normalized[key] = value - return normalized - - -def _request_id(headers: Mapping[str, str]) -> str | None: - normalized = _normalize_response_headers(headers) - return normalized.get("x-ms-request-id") or normalized.get("request-id") or None - - -@dataclass(frozen=True) -class HttpResult: - status: int - body: Any - headers: dict[str, str] - etag_values: tuple[str, ...] | None = None - recovery_deadline: float | None = None - ack_failure: HelperFailure | None = None - - @property - def request_id(self) -> str | None: - return _request_id(self.headers) - - @property - def retry_after(self) -> RetryAfter: - return retry_after_metadata(self.headers) - - -class RetryAfter(NamedTuple): - kind: str - value: float = 0 - - -class RetryAfterTiming(NamedTuple): - received_at_utc: float | None - not_before_utc: float | None - server_delay_seconds: int | None = None - - -def valid_utc_timestamp(value): - return type(value) in (int, float) and -62135596800 <= value < 253402300800 and math.isfinite(value) - - -def retry_after_not_before(metadata, received_at): - """Resolve typed metadata against UTC, without shortening a server interval.""" - if not valid_utc_timestamp(received_at) or not isinstance(metadata, RetryAfter): - return None - value = metadata.value - if not valid_utc_timestamp(value): - return None - if metadata.kind == "seconds": - value = received_at + value if value >= 0 else math.inf - elif metadata.kind == "date-rfc850": - try: - epoch = datetime(1970, 1, 1, tzinfo=timezone.utc) - parsed = epoch + timedelta(seconds=value) - current_year = (epoch + timedelta(seconds=received_at)).year - year = current_year // 100 * 100 + parsed.year % 100 - if year > current_year + 50: - year -= 100 - value = parsed.replace(year=year).timestamp() - except (ValueError, OverflowError): - return None - elif metadata.kind in {"missing", "invalid"}: - value = received_at + 1 - elif metadata.kind != "date": - return None - return value if valid_utc_timestamp(value) else None - - -def retry_after_timing(headers, *, received_at=None): - """Portable timing alongside the unchanged, capped RetryAfter protocol.""" - received_at = time.time() if received_at is None else received_at - if not valid_utc_timestamp(received_at): - return RetryAfterTiming(None, None) - metadata = retry_after_metadata(headers) - deadline = retry_after_not_before(metadata, received_at) - server_delay = metadata.value if metadata.kind == "seconds" else None - if metadata.kind == "overlong": - values = [value for name, value in headers.items() - if isinstance(name, str) and name.lower() == "retry-after"] - if (len(values) == 1 and isinstance(values[0], str) and len(values[0]) <= 128 - and re.fullmatch(r"[0-9]+", values[0].strip(" \t"))): - seconds = int(values[0].strip(" \t")) - if seconds < 253402300800 - received_at: - deadline = received_at + seconds - server_delay = seconds - return RetryAfterTiming(received_at, deadline, server_delay) - - -def retry_after_metadata(headers: Mapping[str, str]) -> RetryAfter: - """Retain only a bounded delay/date, never raw server header text.""" - values = [] - for name, value in headers.items(): - if isinstance(name, str) and name.lower() == "retry-after": - values.append(value) - if len(values) > 1: - return RetryAfter("invalid") - if not values: - return RetryAfter("missing") - value = values[0] - if not isinstance(value, str): - return RetryAfter("invalid") - # An unbounded field cannot safely authorize an early request. - if len(value) > 128: - return RetryAfter("overlong") - if not value.isascii(): - return RetryAfter("invalid") - value = value.strip(" \t") - if re.fullmatch(r"[0-9]+", value): - seconds = int(value) - return RetryAfter("seconds", seconds) if seconds <= 30 else RetryAfter("overlong") - # HTTP-date includes obsolete RFC850/asctime forms, but not arbitrary email dates. - if not re.fullmatch( - r"(?:[A-Z][a-z]{2}, [0-9]{2} [A-Z][a-z]{2} [0-9]{4} [0-9:]{8} GMT" - r"|[A-Z][a-z]+, [0-9]{2}-[A-Z][a-z]{2}-[0-9]{2} [0-9:]{8} GMT" - r"|[A-Z][a-z]{2} [A-Z][a-z]{2} [ 0-9][0-9] [0-9:]{8} [0-9]{4})", value - ): - return RetryAfter("invalid") - try: - parsed = parsedate_to_datetime(value) - stamp = parsed.replace(tzinfo=timezone.utc).timestamp() - return RetryAfter("date-rfc850" if "-" in value else "date", stamp) - except (TypeError, ValueError, OverflowError): - return RetryAfter("invalid") - - -class HelperFailure(RuntimeError): - def __init__( - self, - code: str, - message: str, - *, - blocked_at: str, - writes: list[dict[str, Any]] | None = None, - resources_remaining: list[dict[str, Any]] | None = None, - resources_reused: list[dict[str, Any]] | None = None, - resources_unverified: list[dict[str, Any]] | None = None, - request_id: str | None = None, - status: int | None = None, - partial: bool = False, - warnings: list[str] | None = None, - retry_after: RetryAfter | None = None, - recovery_deadline: float | None = None, - retry_after_timing: RetryAfterTiming | None = None, - ) -> None: - super().__init__(message) - self.code = code - self.message = message - self.blocked_at = blocked_at - self.writes = writes or [] - self.resources_remaining = resources_remaining or [] - self.resources_reused = resources_reused or [] - self.resources_unverified = resources_unverified or [] - self.request_id = request_id - self.http_status = status - self.partial = partial - self.warnings = warnings or [] - self.retry_after = retry_after or RetryAfter("missing") - self.recovery_deadline = recovery_deadline - self.retry_after_timing = retry_after_timing - self.response_close_failed = False - self.file_batch = None - - -class ReadRecovery: - """One 429 delay opportunity and one deadline across an explicit read sequence.""" - - def __init__(self, *, monotonic=None, wall_clock=None, sleeper=None, deadline=None, on_wait=None): - self.monotonic = monotonic or time.monotonic - self.wall_clock = wall_clock or time.time - self.sleeper = sleeper or time.sleep - self.deadline = min(self.monotonic() + 60, deadline if deadline is not None else math.inf) - self.delayed = False - self.request_ids: list[str] = [] - self.warnings: list[str] = [] - self.on_wait = on_wait - - @staticmethod - def safe_id(request_id): - return request_id if isinstance(request_id, str) and re.fullmatch( - r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}", request_id - ) else "[withheld]" - - def record(self, request_id): - if request_id and len(self.request_ids) < 202: - self.request_ids.append(self.safe_id(request_id)) - - def diagnostics(self): - return [*self.warnings, *( - ["Read-only request IDs: " + ", ".join(self.request_ids)] if self.request_ids else [] - )] - - def annotate(self, failure): - for warning in self.diagnostics(): - if warning not in failure.warnings: - failure.warnings.append(warning) - return failure - - def _stop(self, code, message): - self.warnings.append(message) - raise self.annotate(HelperFailure( - code, message + " Stop; resume only read-only verification, never replay the write.", - blocked_at="verification", - request_id=self.request_ids[-1] if self.request_ids else None, - )) - - def delay(self, failure): - if failure.blocked_at == "local-persistence": - self._stop("read-recovery-persistence-failed", "Required receipt persistence failed; recovery is terminal.") - if failure.response_close_failed: - self._stop("read-recovery-response-close-failed", "HTTP error response cleanup failed; delayed recovery is blocked.") - if failure.http_status != 429: - return - if failure.recovery_deadline is not None: - self.deadline = min(self.deadline, failure.recovery_deadline) - if self.delayed: - self._stop("read-recovery-exhausted", "The single HTTP 429 delay opportunity is exhausted.") - metadata = failure.retry_after - delay = metadata.value - if metadata.kind in {"date", "date-rfc850"}: - now = self.wall_clock() - deadline = retry_after_not_before(metadata, now) - if deadline is None: - self._stop("read-recovery-delay-exceeded", "Retry-After date cannot be represented safely.") - delay = max(0, deadline - now) - elif metadata.kind in {"missing", "invalid"}: - delay = 1 - self.warnings.append("Retry-After missing/invalid; using the fixed 1-second fallback.") - if metadata.kind == "overlong" or not math.isfinite(delay) or delay > 30: - self._stop("read-recovery-delay-exceeded", "Retry-After exceeds the 30-second wait allowance; it was not shortened.") - remaining = self.deadline - self.monotonic() - if remaining <= delay: - self._stop("read-recovery-budget-exhausted", "Insufficient recovery read budget for Retry-After.") - self.delayed = True - start = self.monotonic() - if self.on_wait is not None: - self.on_wait(delay) - self.sleeper(delay) - elapsed = self.monotonic() - start - if elapsed < delay: - self._stop("read-recovery-wait-incomplete", "The required Retry-After delay did not elapse.") - if self.monotonic() >= self.deadline: - self._stop("read-recovery-budget-exhausted", "Recovery read deadline elapsed during the wait.") - - def get(self, url, token, *, transport, max_requests=2, **kwargs): - self.deadline = min(self.deadline, kwargs.get("response_deadline", math.inf)) - first = None - for attempt in range(2): - try: - remaining = self.deadline - self.monotonic() - if remaining <= 0 or attempt >= max_requests: - self._stop("read-recovery-budget-exhausted", "The recovery read deadline elapsed.") - options = {**kwargs, "follow_redirects": False, "response_deadline": self.deadline, - "timeout": min(kwargs.get("timeout", 180), remaining), - "max_response_bytes": kwargs.get("max_response_bytes", 1024 * 1024)} - result = transport("GET", url, token, **options) - if result.recovery_deadline is not None: - self.deadline = min(self.deadline, result.recovery_deadline) - self.record(result.request_id) - if self.monotonic() >= self.deadline: - self._stop("read-recovery-budget-exhausted", "The recovery response exceeded its read deadline.") - if result.status != 429: - return result - raise HelperFailure( - "azure-http-error", "Azure request failed with HTTP 429.", - blocked_at="verification", status=429, request_id=result.request_id, - retry_after=result.retry_after, - recovery_deadline=result.recovery_deadline, - ) - except HelperFailure as failure: - if not self.request_ids or self.request_ids[-1] != failure.request_id: - self.record(failure.request_id) - if failure.http_status != 429 or attempt: - if first is not None: - self.warnings.append( - f"Recovery read stopped ({failure.code}); delay opportunity exhausted; " - "initial read failure retained. Resume read-only; never replay the write." - ) - raise self.annotate(first) from failure - raise self.annotate(failure) - first = failure - try: - if attempt + 1 >= max_requests: - self._stop("read-recovery-exhausted", "No requests remain in the recovery read allowance.") - self.delay(failure) - except HelperFailure as stopped: - self.warnings.append(stopped.message) - raise self.annotate(first) from stopped - raise AssertionError("unreachable") - - -def canonical_bytes(value: Any) -> bytes: - return json.dumps( - value, - ensure_ascii=True, - sort_keys=True, - separators=(",", ":"), - ).encode("utf-8") - - -def digest(value: Any) -> str: - return f"sha256:{hashlib.sha256(canonical_bytes(value)).hexdigest()}" - - -def file_digest(path: Path) -> str: - hasher = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - hasher.update(chunk) - return f"sha256:{hasher.hexdigest()}" - - -def load_approved_input(path: Path) -> tuple[dict[str, Any], dict[str, Any], str]: - try: - document = json.loads(path.read_text(encoding="utf-8")) - except OSError as exc: - raise HelperFailure( - "input-unreadable", - "Input file cannot be read.", - blocked_at="input-resolution", - ) from exc - except json.JSONDecodeError as exc: - raise HelperFailure( - "input-invalid-json", - f"Input file is not valid JSON: {exc}", - blocked_at="input-resolution", - ) from exc - if not isinstance(document, dict) or document.get("schema_version") != "1.0": - raise HelperFailure( - "input-schema-invalid", - "Input must be an object with schema_version 1.0.", - blocked_at="input-resolution", - ) - reject_secrets(document) - require_allowed_fields( - document, - {"schema_version", "plan", "approval"}, - label="input envelope", - ) - plan = document.get("plan") - approval = document.get("approval") - if not isinstance(plan, dict) or not isinstance(approval, dict): - raise HelperFailure( - "input-schema-invalid", - "Input must contain plan and approval objects.", - blocked_at="input-resolution", - ) - require_allowed_fields( - approval, - {"confirmed", "fingerprint"}, - label="approval", - ) - computed = digest(plan) - if approval.get("confirmed") is not True: - raise HelperFailure( - "approval-missing", - "The exact plan has not been explicitly approved.", - blocked_at="confirmation", - ) - if approval.get("fingerprint") != computed: - raise HelperFailure( - "approval-mismatch", - "The approved fingerprint does not match the canonical plan.", - blocked_at="confirmation", - ) - return document, plan, computed - - -def reject_secrets(value: Any, *, path: str = "") -> None: - if isinstance(value, dict): - for key, child in value.items(): - normalized = re.sub(r"[^a-z0-9]", "", str(key).casefold()) - child_path = f"{path}/{key}" - if normalized in SECRET_FIELDS and child not in (None, "", []): - raise HelperFailure( - "secret-input-forbidden", - f"Secret-bearing field is forbidden at {child_path}.", - blocked_at="input-resolution", - ) - if normalized == "connectionstring" and child not in (None, ""): - if ( - not isinstance(child, str) - or RESOURCE_ID_CONNECTION.fullmatch(child) is None - ): - raise HelperFailure( - "secret-input-forbidden", - "Only one ResourceId storage connectionString component is allowed.", - blocked_at="input-resolution", - ) - reject_secrets(child, path=child_path) - elif isinstance(value, list): - for index, child in enumerate(value): - reject_secrets(child, path=f"{path}/{index}") - - -def require_allowed_fields( - value: dict[str, Any], - allowed: set[str], - *, - label: str, -) -> None: - if set(value) - allowed: - raise HelperFailure( - "input-schema-invalid", - f"{label} contains unsupported fields.", - blocked_at="input-resolution", - ) - - -def is_ambiguous_mutation_failure(failure: HelperFailure) -> bool: - return ( - failure.partial - or failure.code == "azure-response-ambiguous" - or failure.http_status in {408, 429} - or ( - isinstance(failure.http_status, int) - and failure.http_status >= 500 - ) - ) - - -def is_ambiguous_status(status: int) -> bool: - return status in {408, 429} or status >= 500 - - -def sdk_error_status(error: Exception) -> int | None: - for source in (error, getattr(error, "response", None)): - status = getattr(source, "status_code", None) - if type(status) is int and 100 <= status <= 599: - return status - return None - - -def sdk_error_metadata(error: Exception, fallback_code: str | None = None) -> dict[str, Any]: - """Read only bounded identifier fields, never exception text or response bodies.""" - def identifier(value: Any) -> str | None: - if isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}", value): - return value - return None - - response = getattr(error, "response", None) - raw_headers = getattr(response, "headers", None) - headers = {} - if isinstance(raw_headers, Mapping): - headers = _normalize_response_headers({ - key: identifier(value) - for key, value in raw_headers.items() - if isinstance(key, str) and key.lower() in {"x-ms-request-id", "request-id", "x-ms-error-code"} - }) - result = {"status": sdk_error_status(error), "request_id": _request_id(headers)} - if fallback_code is not None: - detail = getattr(error, "error", None) - code = detail.get("code") if isinstance(detail, Mapping) else getattr(detail, "code", None) - result["code"] = ( - identifier(code) or identifier(getattr(error, "code", None)) - or headers.get("x-ms-error-code") or fallback_code - ) - return result - - -def is_ambiguous_sdk_error(error: Exception) -> bool: - status = sdk_error_status(error) - return status is None or is_ambiguous_status(status) - - -def redact_sensitive(value: Any) -> Any: - if isinstance(value, dict): - result: dict[str, Any] = {} - for key, child in value.items(): - normalized = re.sub(r"[^a-z0-9]", "", str(key).casefold()) - if normalized in SECRET_FIELDS or normalized == "connectionstring": - result[str(key)] = "[REDACTED]" - else: - result[str(key)] = redact_sensitive(child) - return result - if isinstance(value, list): - return [redact_sensitive(child) for child in value] - return value - - -def validate_search_endpoint(endpoint: Any) -> str: - if not isinstance(endpoint, str): - raise HelperFailure( - "endpoint-invalid", - "Search endpoint must be a string.", - blocked_at="input-resolution", - ) - try: - endpoint.encode("utf-8") - parsed = urlsplit(endpoint) - port = parsed.port - except (ValueError, UnicodeEncodeError) as exc: - raise HelperFailure( - "endpoint-invalid", - "Search endpoint must be a valid UTF-8 HTTPS service root.", - blocked_at="input-resolution", - ) from exc - if ( - parsed.scheme != "https" - or not parsed.hostname - or SEARCH_HOST.fullmatch(parsed.hostname) is None - or parsed.path not in {"", "/"} - or parsed.query - or parsed.fragment - or parsed.username - or parsed.password - or port not in {None, 443} - ): - raise HelperFailure( - "endpoint-invalid", - "Search endpoint must be an HTTPS search.windows.net service root.", - blocked_at="input-resolution", - ) - return endpoint.rstrip("/") - - -def odata_name(name: Any) -> str: - if not isinstance(name, str) or not name or len(name) > 128: - raise HelperFailure( - "name-invalid", - "Resource name must be a non-empty string no longer than 128 characters.", - blocked_at="input-resolution", - ) - try: - return quote(name.replace("'", "''"), safe="") - except UnicodeEncodeError as exc: - raise HelperFailure( - "name-invalid", - "Resource name must be valid UTF-8 text.", - blocked_at="input-resolution", - ) from exc - - -def azure_cli_token(resource: str) -> str: - try: - executable = shutil.which("az") - if executable is None: - raise FileNotFoundError("Azure CLI executable was not found on PATH.") - completed = subprocess.run( - [ - executable, - "account", - "get-access-token", - "--resource", - resource, - "--query", - "accessToken", - "--output", - "tsv", - ], - check=True, - capture_output=True, - text=True, - timeout=60, - ) - except FileNotFoundError as exc: - raise HelperFailure( - "azure-cli-unavailable", - "Azure CLI is required for keyless authentication.", - blocked_at="execution", - ) from exc - except subprocess.CalledProcessError as exc: - raise HelperFailure( - "azure-authentication-failed", - "Azure CLI could not acquire the required access token.", - blocked_at="execution", - ) from exc - except subprocess.TimeoutExpired as exc: - raise HelperFailure( - "azure-authentication-timeout", - "Azure CLI did not return an access token within 60 seconds.", - blocked_at="execution", - ) from exc - token = completed.stdout.strip() - if not token: - raise HelperFailure( - "azure-authentication-failed", - "Azure CLI returned an empty access token.", - blocked_at="execution", - ) - return token - - -class _NoRedirect(HTTPRedirectHandler): - def redirect_request(self, req: Any, fp: Any, code: int, msg: str, headers: Any, newurl: str) -> None: - return None - - -def _read_response_with_deadline( - response: Any, deadline: float, max_bytes: int, *, method: str -) -> bytes: - def timed_out() -> HelperFailure: - return HelperFailure( - "response-deadline-exceeded", - "HTTP response body exceeded its monotonic deadline.", - blocked_at="verification", - request_id=_request_id(response.headers), - status=response.status, - partial=method not in {"GET", "HEAD"}, - ) - - remaining = deadline - time.monotonic() - if remaining <= 0: - raise timed_out() - connection = getattr(getattr(getattr(response, "fp", None), "raw", None), "_sock", None) - if not isinstance(connection, socket.socket): - raise HelperFailure( - "response-deadline-unsupported", - "Deadline reads require an interruptible urllib HTTP socket.", - blocked_at="verification", - request_id=_request_id(response.headers), - status=response.status, - partial=method not in {"GET", "HEAD"}, - ) - expired = threading.Event() - - def interrupt() -> None: - expired.set() - # BufferedReader.close can wait on the active read lock; interrupt its socket instead. - with suppress(OSError): - connection.shutdown(socket.SHUT_RDWR) - with suppress(OSError): - connection.close() - - watchdog = threading.Timer(max(0, deadline - time.monotonic()), interrupt) - watchdog.start() - try: - if expired.is_set() or time.monotonic() >= deadline: - raise timed_out() - try: - payload = response.read(max_bytes + 1) - except (OSError, HTTPException, ValueError) as exc: - if expired.is_set() or time.monotonic() >= deadline: - raise timed_out() from exc - raise HelperFailure( - "response-read-failed", "HTTP response body could not be read.", - blocked_at="verification", - request_id=_request_id(response.headers), - status=response.status, - partial=method not in {"GET", "HEAD"}, - ) from exc - if expired.is_set() or time.monotonic() >= deadline: - raise timed_out() - return payload - finally: - watchdog.cancel() - watchdog.join() - - -def http_request( - method: str, - url: str, - token: str, - *, - body: bytes | None = None, - headers: dict[str, str] | None = None, - timeout: int = 180, - raw_response: bool = False, - max_response_bytes: int | None = None, - follow_redirects: bool = True, - response_deadline: float | None = None, -) -> HttpResult: - if response_deadline is not None: - if ( - not isinstance(response_deadline, (int, float)) - or not math.isfinite(response_deadline) - or response_deadline - time.monotonic() > threading.TIMEOUT_MAX - ): - raise HelperFailure( - "response-deadline-invalid", "Response deadline must be a supported finite monotonic time.", - blocked_at="input-resolution", - ) - if max_response_bytes is None: - max_response_bytes = 1024 * 1024 - if not isinstance(max_response_bytes, int) or max_response_bytes < 0: - raise HelperFailure( - "response-limit-invalid", "Response byte limit must be a nonnegative integer.", - blocked_at="input-resolution", - ) - if response_deadline <= time.monotonic(): - raise HelperFailure( - "response-deadline-exceeded", "HTTP deadline elapsed before the request.", - blocked_at="verification", - ) - request_headers = { - "Accept": "application/json;odata.metadata=minimal", - "Authorization": f"Bearer {token}", - "x-ms-client-request-id": str(uuid.uuid4()), - } - request_headers.update(headers or {}) - request = Request(url=url, data=body, headers=request_headers, method=method) - try: - open_request = urlopen if follow_redirects else build_opener(_NoRedirect).open - with open_request(request, timeout=timeout) as response: - if response_deadline is not None: - payload = _read_response_with_deadline( - response, response_deadline, max_response_bytes, method=method - ) - else: - payload = ( - response.read(max_response_bytes + 1) - if max_response_bytes is not None - else response.read() - ) - etag_values = tuple(value for name, value in response.headers.items() if name.lower() == "etag") - response_headers = _normalize_response_headers(response.headers) - if max_response_bytes is not None and len(payload) > max_response_bytes: - raise HelperFailure( - "response-too-large", - "Azure response exceeded the bounded read limit.", - blocked_at="verification", - request_id=_request_id(response_headers), - status=response.status, - partial=method not in {"GET", "HEAD"}, - ) - if raw_response: - parsed: Any = payload - elif not payload: - parsed = None - else: - try: - parsed = json.loads(payload.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise HelperFailure( - "response-invalid-json", - "Azure returned a non-JSON response where JSON was required.", - blocked_at="verification", - request_id=_request_id(response_headers), - status=response.status, - partial=method not in {"GET", "HEAD"}, - ) from exc - return HttpResult(response.status, parsed, response_headers, etag_values, response_deadline) - except HTTPError as exc: - retry_after = retry_after_metadata(exc.headers) - timing = retry_after_timing(exc.headers) - response_headers = _normalize_response_headers(exc.headers) - status = int(exc.code) - close_failed = False - if status == 429: - try: - exc.close() - except (OSError, HTTPException): - close_failed = True - elif response_deadline is not None: - exc.close() - ambiguous = method not in {"GET", "HEAD"} and is_ambiguous_status(status) - failure = HelperFailure( - "azure-http-error", - f"Azure request failed with HTTP {status}.", - blocked_at="execution", - request_id=_request_id(response_headers), - status=status, - partial=ambiguous, - retry_after=retry_after, - recovery_deadline=response_deadline, - retry_after_timing=timing, - warnings=(["response-close-failed: HTTP 429 response cleanup failed; " - "original HTTP failure retained and delayed recovery blocked."] if close_failed else []), - ) - failure.response_close_failed = close_failed - raise failure from exc - except (URLError, TimeoutError) as exc: - raise HelperFailure( - "azure-response-ambiguous", - "Azure request did not return an authoritative response.", - blocked_at="verification", - partial=method not in {"GET", "HEAD"}, - ) from exc - - -def blocked_result( - failure: HelperFailure, - *, - outcome: str, - fingerprint: str | None, - owner: Any = None, -) -> dict[str, Any]: - result = _blocked_result(failure, outcome=outcome, fingerprint=fingerprint, owner=owner) - if failure.file_batch is not None: - result["file_batch"] = failure.file_batch - result["safe_next_decision"] = ( - "Retain the source and original ACK/journal. Do not rerun creation, replay an uncertain upload, " - "or delete/reset resources. Plan upload-only continuation for never-attempted files with " - "file_upload.py --plan; missing original evidence blocks continuation, not retention." - ) - return result - - -def _blocked_result(failure, *, outcome, fingerprint, owner): - if failure.partial or failure.writes: - return { - "status": "partial", - "outcome": outcome, - "approved_plan": { - "fingerprint": fingerprint, - "confirmed": fingerprint is not None, - }, - "first_failure": { - "code": failure.code, - "operation": failure.blocked_at, - "status": failure.http_status, - "message": failure.message, - "request_id": failure.request_id, - }, - "completed_writes": failure.writes, - "failed_or_unverified_postconditions": [failure.code], - "resources_remaining": { - "run_owned": failure.resources_remaining, - "reused": failure.resources_reused, - **({"unverified": failure.resources_unverified} if failure.resources_unverified else {}), - }, - "rollback": {"possible": False, "exact_plan": []}, - "cleanup": {"status": "separate-plan-and-approval-required"}, - "owner": owner, - "warnings": failure.warnings, - } - result = { - "status": "blocked", - "outcome": outcome, - "blocked_at": failure.blocked_at, - "first_blocker": { - "code": failure.code, - "message": failure.message, - "status": failure.http_status, - "request_id": failure.request_id, - }, - "missing_or_conflicting_input": failure.code, - "read_only_evidence": [], - "writes_performed": [], - "safe_next_decision": "Resolve the first blocker, rebuild the plan, and obtain new approval.", - "ownership": {"run_owned": [], "reused_not_owned": []}, - "cleanup": "not applicable", - } - if fingerprint is not None: - result["approved_plan"] = { - "fingerprint": fingerprint, - "confirmed": True, - } - if failure.warnings: - result["warnings"] = failure.warnings - return result - - -def emit_result( - result: dict[str, Any], *, stream: Any = None, preserve_unapproved_input: bool = False -) -> None: - if stream is None: - stream = sys.stdout - safe = redact_sensitive(result) - if preserve_unapproved_input: - document = result.get("execution_input") - if ( - result.get("status") != "planned" or not isinstance(document, dict) - or document.get("schema_version") != "1.0" or not isinstance(document.get("plan"), dict) - or not isinstance(document.get("approval"), dict) - or document["approval"].get("confirmed") is not False - or document.get("approval") != {"confirmed": False, "fingerprint": digest(document["plan"])} - ): - raise HelperFailure( - "planning-output-invalid", "Only an exact unapproved execution input can retain ResourceId bindings.", - blocked_at="verification", - ) - reject_secrets(document) - require_allowed_fields(document, {"schema_version", "plan", "approval"}, label="planning envelope") - safe["execution_input"] = document - stream.write( - json.dumps( - safe, - sort_keys=True, - separators=(",", ":"), - ) - + "\n" - ) - - -TokenProvider = Callable[[str], str] -Transport = Callable[..., HttpResult] +from __future__ import annotations + +import hashlib +import json +import math +import re +import shutil +import socket +import subprocess +import sys +import threading +import time +import uuid +from collections.abc import Mapping +from contextlib import suppress +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from email.utils import parsedate_to_datetime +from http.client import HTTPException +from pathlib import Path +from typing import Any, Callable, NamedTuple +from urllib.error import HTTPError, URLError +from urllib.parse import quote, urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen + + +SEARCH_AUDIENCE = "https://search.azure.com" +MANAGEMENT_AUDIENCE = "https://management.azure.com" +SEARCH_HOST = re.compile( + r"^[a-z0-9](?:[a-z0-9-]{0,58}[a-z0-9])?\.search\.windows\.net$" +) +SECRET_FIELDS = { + "accesskey", + "accesstoken", + "accountkey", + "apikey", + "applicationsecret", + "authorization", + "clientsecret", + "credential", + "connectionsecret", + "key", + "password", + "privatekey", + "refreshtoken", + "sas", + "sastoken", + "secret", + "sharedaccesskey", + "storageaccountkey", + "token", +} +RESOURCE_ID_CONNECTION = re.compile(r"^ResourceId=/[^;\r\n]+;?$") + + +def normalize_azure_location(value: Any) -> str | None: + """Normalize ASCII case/whitespace only; this does not validate region availability.""" + if not isinstance(value, str) or len(value) > 128 or not value.isascii(): + return None + compact = re.sub(r"\s+", "", value, flags=re.ASCII).lower() + return compact if re.fullmatch(r"[a-z][a-z0-9]{1,40}", compact) else None + + +def _normalize_response_headers(headers: Mapping[str, str]) -> dict[str, str]: + normalized: dict[str, str] = {} + for name, value in headers.items(): + key = name.lower() + if key in {"x-ms-request-id", "request-id"} and normalized.get(key): + continue + if key == "retry-after" and key in normalized: + normalized[key] = "" + continue + normalized[key] = value + return normalized + + +def _request_id(headers: Mapping[str, str]) -> str | None: + normalized = _normalize_response_headers(headers) + return normalized.get("x-ms-request-id") or normalized.get("request-id") or None + + +@dataclass(frozen=True) +class HttpResult: + status: int + body: Any + headers: dict[str, str] + etag_values: tuple[str, ...] | None = None + recovery_deadline: float | None = None + ack_failure: HelperFailure | None = None + + @property + def request_id(self) -> str | None: + return _request_id(self.headers) + + @property + def retry_after(self) -> RetryAfter: + return retry_after_metadata(self.headers) + + +class RetryAfter(NamedTuple): + kind: str + value: float = 0 + + +class RetryAfterTiming(NamedTuple): + received_at_utc: float | None + not_before_utc: float | None + server_delay_seconds: int | None = None + + +def valid_utc_timestamp(value): + return type(value) in (int, float) and -62135596800 <= value < 253402300800 and math.isfinite(value) + + +def retry_after_not_before(metadata, received_at): + """Resolve typed metadata against UTC, without shortening a server interval.""" + if not valid_utc_timestamp(received_at) or not isinstance(metadata, RetryAfter): + return None + value = metadata.value + if not valid_utc_timestamp(value): + return None + if metadata.kind == "seconds": + value = received_at + value if value >= 0 else math.inf + elif metadata.kind == "date-rfc850": + try: + epoch = datetime(1970, 1, 1, tzinfo=timezone.utc) + parsed = epoch + timedelta(seconds=value) + current_year = (epoch + timedelta(seconds=received_at)).year + year = current_year // 100 * 100 + parsed.year % 100 + if year > current_year + 50: + year -= 100 + value = parsed.replace(year=year).timestamp() + except (ValueError, OverflowError): + return None + elif metadata.kind in {"missing", "invalid"}: + value = received_at + 1 + elif metadata.kind != "date": + return None + return value if valid_utc_timestamp(value) else None + + +def retry_after_timing(headers, *, received_at=None): + """Portable timing alongside the unchanged, capped RetryAfter protocol.""" + received_at = time.time() if received_at is None else received_at + if not valid_utc_timestamp(received_at): + return RetryAfterTiming(None, None) + metadata = retry_after_metadata(headers) + deadline = retry_after_not_before(metadata, received_at) + server_delay = metadata.value if metadata.kind == "seconds" else None + if metadata.kind == "overlong": + values = [value for name, value in headers.items() + if isinstance(name, str) and name.lower() == "retry-after"] + if (len(values) == 1 and isinstance(values[0], str) and len(values[0]) <= 128 + and re.fullmatch(r"[0-9]+", values[0].strip(" \t"))): + seconds = int(values[0].strip(" \t")) + if seconds < 253402300800 - received_at: + deadline = received_at + seconds + server_delay = seconds + return RetryAfterTiming(received_at, deadline, server_delay) + + +def retry_after_metadata(headers: Mapping[str, str]) -> RetryAfter: + """Retain only a bounded delay/date, never raw server header text.""" + values = [] + for name, value in headers.items(): + if isinstance(name, str) and name.lower() == "retry-after": + values.append(value) + if len(values) > 1: + return RetryAfter("invalid") + if not values: + return RetryAfter("missing") + value = values[0] + if not isinstance(value, str): + return RetryAfter("invalid") + # An unbounded field cannot safely authorize an early request. + if len(value) > 128: + return RetryAfter("overlong") + if not value.isascii(): + return RetryAfter("invalid") + value = value.strip(" \t") + if re.fullmatch(r"[0-9]+", value): + seconds = int(value) + return RetryAfter("seconds", seconds) if seconds <= 30 else RetryAfter("overlong") + # HTTP-date includes obsolete RFC850/asctime forms, but not arbitrary email dates. + if not re.fullmatch( + r"(?:[A-Z][a-z]{2}, [0-9]{2} [A-Z][a-z]{2} [0-9]{4} [0-9:]{8} GMT" + r"|[A-Z][a-z]+, [0-9]{2}-[A-Z][a-z]{2}-[0-9]{2} [0-9:]{8} GMT" + r"|[A-Z][a-z]{2} [A-Z][a-z]{2} [ 0-9][0-9] [0-9:]{8} [0-9]{4})", value + ): + return RetryAfter("invalid") + try: + parsed = parsedate_to_datetime(value) + stamp = parsed.replace(tzinfo=timezone.utc).timestamp() + return RetryAfter("date-rfc850" if "-" in value else "date", stamp) + except (TypeError, ValueError, OverflowError): + return RetryAfter("invalid") + + +class HelperFailure(RuntimeError): + def __init__( + self, + code: str, + message: str, + *, + blocked_at: str, + writes: list[dict[str, Any]] | None = None, + resources_remaining: list[dict[str, Any]] | None = None, + resources_reused: list[dict[str, Any]] | None = None, + resources_unverified: list[dict[str, Any]] | None = None, + request_id: str | None = None, + status: int | None = None, + partial: bool = False, + warnings: list[str] | None = None, + retry_after: RetryAfter | None = None, + recovery_deadline: float | None = None, + retry_after_timing: RetryAfterTiming | None = None, + ) -> None: + super().__init__(message) + self.code = code + self.message = message + self.blocked_at = blocked_at + self.writes = writes or [] + self.resources_remaining = resources_remaining or [] + self.resources_reused = resources_reused or [] + self.resources_unverified = resources_unverified or [] + self.request_id = request_id + self.http_status = status + self.partial = partial + self.warnings = warnings or [] + self.retry_after = retry_after or RetryAfter("missing") + self.recovery_deadline = recovery_deadline + self.retry_after_timing = retry_after_timing + self.response_close_failed = False + self.file_batch = None + + +class ReadRecovery: + """One 429 delay opportunity and one deadline across an explicit read sequence.""" + + def __init__(self, *, monotonic=None, wall_clock=None, sleeper=None, deadline=None, on_wait=None): + self.monotonic = monotonic or time.monotonic + self.wall_clock = wall_clock or time.time + self.sleeper = sleeper or time.sleep + self.deadline = min(self.monotonic() + 60, deadline if deadline is not None else math.inf) + self.delayed = False + self.request_ids: list[str] = [] + self.warnings: list[str] = [] + self.on_wait = on_wait + + @staticmethod + def safe_id(request_id): + return request_id if isinstance(request_id, str) and re.fullmatch( + r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}", request_id + ) else "[withheld]" + + def record(self, request_id): + if request_id and len(self.request_ids) < 202: + self.request_ids.append(self.safe_id(request_id)) + + def diagnostics(self): + return [*self.warnings, *( + ["Read-only request IDs: " + ", ".join(self.request_ids)] if self.request_ids else [] + )] + + def annotate(self, failure): + for warning in self.diagnostics(): + if warning not in failure.warnings: + failure.warnings.append(warning) + return failure + + def _stop(self, code, message): + self.warnings.append(message) + raise self.annotate(HelperFailure( + code, message + " Stop; resume only read-only verification, never replay the write.", + blocked_at="verification", + request_id=self.request_ids[-1] if self.request_ids else None, + )) + + def delay(self, failure): + if failure.blocked_at == "local-persistence": + self._stop("read-recovery-persistence-failed", "Required receipt persistence failed; recovery is terminal.") + if failure.response_close_failed: + self._stop("read-recovery-response-close-failed", "HTTP error response cleanup failed; delayed recovery is blocked.") + if failure.http_status != 429: + return + if failure.recovery_deadline is not None: + self.deadline = min(self.deadline, failure.recovery_deadline) + if self.delayed: + self._stop("read-recovery-exhausted", "The single HTTP 429 delay opportunity is exhausted.") + metadata = failure.retry_after + delay = metadata.value + if metadata.kind in {"date", "date-rfc850"}: + now = self.wall_clock() + deadline = retry_after_not_before(metadata, now) + if deadline is None: + self._stop("read-recovery-delay-exceeded", "Retry-After date cannot be represented safely.") + delay = max(0, deadline - now) + elif metadata.kind in {"missing", "invalid"}: + delay = 1 + self.warnings.append("Retry-After missing/invalid; using the fixed 1-second fallback.") + if metadata.kind == "overlong" or not math.isfinite(delay) or delay > 30: + self._stop("read-recovery-delay-exceeded", "Retry-After exceeds the 30-second wait allowance; it was not shortened.") + remaining = self.deadline - self.monotonic() + if remaining <= delay: + self._stop("read-recovery-budget-exhausted", "Insufficient recovery read budget for Retry-After.") + self.delayed = True + start = self.monotonic() + if self.on_wait is not None: + self.on_wait(delay) + self.sleeper(delay) + elapsed = self.monotonic() - start + if elapsed < delay: + self._stop("read-recovery-wait-incomplete", "The required Retry-After delay did not elapse.") + if self.monotonic() >= self.deadline: + self._stop("read-recovery-budget-exhausted", "Recovery read deadline elapsed during the wait.") + + def get(self, url, token, *, transport, max_requests=2, **kwargs): + self.deadline = min(self.deadline, kwargs.get("response_deadline", math.inf)) + first = None + for attempt in range(2): + try: + remaining = self.deadline - self.monotonic() + if remaining <= 0 or attempt >= max_requests: + self._stop("read-recovery-budget-exhausted", "The recovery read deadline elapsed.") + options = {**kwargs, "follow_redirects": False, "response_deadline": self.deadline, + "timeout": min(kwargs.get("timeout", 180), remaining), + "max_response_bytes": kwargs.get("max_response_bytes", 1024 * 1024)} + result = transport("GET", url, token, **options) + if result.recovery_deadline is not None: + self.deadline = min(self.deadline, result.recovery_deadline) + self.record(result.request_id) + if self.monotonic() >= self.deadline: + self._stop("read-recovery-budget-exhausted", "The recovery response exceeded its read deadline.") + if result.status != 429: + return result + raise HelperFailure( + "azure-http-error", "Azure request failed with HTTP 429.", + blocked_at="verification", status=429, request_id=result.request_id, + retry_after=result.retry_after, + recovery_deadline=result.recovery_deadline, + ) + except HelperFailure as failure: + if not self.request_ids or self.request_ids[-1] != failure.request_id: + self.record(failure.request_id) + if failure.http_status != 429 or attempt: + if first is not None: + self.warnings.append( + f"Recovery read stopped ({failure.code}); delay opportunity exhausted; " + "initial read failure retained. Resume read-only; never replay the write." + ) + raise self.annotate(first) from failure + raise self.annotate(failure) + first = failure + try: + if attempt + 1 >= max_requests: + self._stop("read-recovery-exhausted", "No requests remain in the recovery read allowance.") + self.delay(failure) + except HelperFailure as stopped: + self.warnings.append(stopped.message) + raise self.annotate(first) from stopped + raise AssertionError("unreachable") + + +def canonical_bytes(value: Any) -> bytes: + return json.dumps( + value, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def digest(value: Any) -> str: + return f"sha256:{hashlib.sha256(canonical_bytes(value)).hexdigest()}" + + +def file_digest(path: Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + hasher.update(chunk) + return f"sha256:{hasher.hexdigest()}" + + +def load_approved_input(path: Path) -> tuple[dict[str, Any], dict[str, Any], str]: + try: + document = json.loads(path.read_text(encoding="utf-8")) + except OSError as exc: + raise HelperFailure( + "input-unreadable", + "Input file cannot be read.", + blocked_at="input-resolution", + ) from exc + except json.JSONDecodeError as exc: + raise HelperFailure( + "input-invalid-json", + f"Input file is not valid JSON: {exc}", + blocked_at="input-resolution", + ) from exc + if not isinstance(document, dict) or document.get("schema_version") != "1.0": + raise HelperFailure( + "input-schema-invalid", + "Input must be an object with schema_version 1.0.", + blocked_at="input-resolution", + ) + reject_secrets(document) + require_allowed_fields( + document, + {"schema_version", "plan", "approval"}, + label="input envelope", + ) + plan = document.get("plan") + approval = document.get("approval") + if not isinstance(plan, dict) or not isinstance(approval, dict): + raise HelperFailure( + "input-schema-invalid", + "Input must contain plan and approval objects.", + blocked_at="input-resolution", + ) + require_allowed_fields( + approval, + {"confirmed", "fingerprint"}, + label="approval", + ) + computed = digest(plan) + if approval.get("confirmed") is not True: + raise HelperFailure( + "approval-missing", + "The exact plan has not been explicitly approved.", + blocked_at="confirmation", + ) + if approval.get("fingerprint") != computed: + raise HelperFailure( + "approval-mismatch", + "The approved fingerprint does not match the canonical plan.", + blocked_at="confirmation", + ) + return document, plan, computed + + +def reject_secrets(value: Any, *, path: str = "") -> None: + if isinstance(value, dict): + for key, child in value.items(): + normalized = re.sub(r"[^a-z0-9]", "", str(key).casefold()) + child_path = f"{path}/{key}" + if normalized in SECRET_FIELDS and child not in (None, "", []): + raise HelperFailure( + "secret-input-forbidden", + f"Secret-bearing field is forbidden at {child_path}.", + blocked_at="input-resolution", + ) + if normalized == "connectionstring" and child not in (None, ""): + if ( + not isinstance(child, str) + or RESOURCE_ID_CONNECTION.fullmatch(child) is None + ): + raise HelperFailure( + "secret-input-forbidden", + "Only one ResourceId storage connectionString component is allowed.", + blocked_at="input-resolution", + ) + reject_secrets(child, path=child_path) + elif isinstance(value, list): + for index, child in enumerate(value): + reject_secrets(child, path=f"{path}/{index}") + + +def require_allowed_fields( + value: dict[str, Any], + allowed: set[str], + *, + label: str, +) -> None: + if set(value) - allowed: + raise HelperFailure( + "input-schema-invalid", + f"{label} contains unsupported fields.", + blocked_at="input-resolution", + ) + + +def is_ambiguous_mutation_failure(failure: HelperFailure) -> bool: + return ( + failure.partial + or failure.code == "azure-response-ambiguous" + or failure.http_status in {408, 429} + or ( + isinstance(failure.http_status, int) + and failure.http_status >= 500 + ) + ) + + +def is_ambiguous_status(status: int) -> bool: + return status in {408, 429} or status >= 500 + + +def sdk_error_status(error: Exception) -> int | None: + for source in (error, getattr(error, "response", None)): + status = getattr(source, "status_code", None) + if type(status) is int and 100 <= status <= 599: + return status + return None + + +def sdk_error_metadata(error: Exception, fallback_code: str | None = None) -> dict[str, Any]: + """Read only bounded identifier fields, never exception text or response bodies.""" + def identifier(value: Any) -> str | None: + if isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}", value): + return value + return None + + response = getattr(error, "response", None) + raw_headers = getattr(response, "headers", None) + headers = {} + if isinstance(raw_headers, Mapping): + headers = _normalize_response_headers({ + key: identifier(value) + for key, value in raw_headers.items() + if isinstance(key, str) and key.lower() in {"x-ms-request-id", "request-id", "x-ms-error-code"} + }) + result = {"status": sdk_error_status(error), "request_id": _request_id(headers)} + if fallback_code is not None: + detail = getattr(error, "error", None) + code = detail.get("code") if isinstance(detail, Mapping) else getattr(detail, "code", None) + result["code"] = ( + identifier(code) or identifier(getattr(error, "code", None)) + or headers.get("x-ms-error-code") or fallback_code + ) + return result + + +def is_ambiguous_sdk_error(error: Exception) -> bool: + status = sdk_error_status(error) + return status is None or is_ambiguous_status(status) + + +def redact_sensitive(value: Any) -> Any: + if isinstance(value, dict): + result: dict[str, Any] = {} + for key, child in value.items(): + normalized = re.sub(r"[^a-z0-9]", "", str(key).casefold()) + if normalized in SECRET_FIELDS or normalized == "connectionstring": + result[str(key)] = "[REDACTED]" + else: + result[str(key)] = redact_sensitive(child) + return result + if isinstance(value, list): + return [redact_sensitive(child) for child in value] + return value + + +def validate_search_endpoint(endpoint: Any) -> str: + if not isinstance(endpoint, str): + raise HelperFailure( + "endpoint-invalid", + "Search endpoint must be a string.", + blocked_at="input-resolution", + ) + try: + endpoint.encode("utf-8") + parsed = urlsplit(endpoint) + port = parsed.port + except (ValueError, UnicodeEncodeError) as exc: + raise HelperFailure( + "endpoint-invalid", + "Search endpoint must be a valid UTF-8 HTTPS service root.", + blocked_at="input-resolution", + ) from exc + if ( + parsed.scheme != "https" + or not parsed.hostname + or SEARCH_HOST.fullmatch(parsed.hostname) is None + or parsed.path not in {"", "/"} + or parsed.query + or parsed.fragment + or parsed.username + or parsed.password + or port not in {None, 443} + ): + raise HelperFailure( + "endpoint-invalid", + "Search endpoint must be an HTTPS search.windows.net service root.", + blocked_at="input-resolution", + ) + return endpoint.rstrip("/") + + +def odata_name(name: Any) -> str: + if not isinstance(name, str) or not name or len(name) > 128: + raise HelperFailure( + "name-invalid", + "Resource name must be a non-empty string no longer than 128 characters.", + blocked_at="input-resolution", + ) + try: + return quote(name.replace("'", "''"), safe="") + except UnicodeEncodeError as exc: + raise HelperFailure( + "name-invalid", + "Resource name must be valid UTF-8 text.", + blocked_at="input-resolution", + ) from exc + + +def azure_cli_token(resource: str) -> str: + try: + executable = shutil.which("az") + if executable is None: + raise FileNotFoundError("Azure CLI executable was not found on PATH.") + completed = subprocess.run( + [ + executable, + "account", + "get-access-token", + "--resource", + resource, + "--query", + "accessToken", + "--output", + "tsv", + ], + check=True, + capture_output=True, + text=True, + timeout=60, + ) + except FileNotFoundError as exc: + raise HelperFailure( + "azure-cli-unavailable", + "Azure CLI is required for keyless authentication.", + blocked_at="execution", + ) from exc + except subprocess.CalledProcessError as exc: + raise HelperFailure( + "azure-authentication-failed", + "Azure CLI could not acquire the required access token.", + blocked_at="execution", + ) from exc + except subprocess.TimeoutExpired as exc: + raise HelperFailure( + "azure-authentication-timeout", + "Azure CLI did not return an access token within 60 seconds.", + blocked_at="execution", + ) from exc + token = completed.stdout.strip() + if not token: + raise HelperFailure( + "azure-authentication-failed", + "Azure CLI returned an empty access token.", + blocked_at="execution", + ) + return token + + +class _NoRedirect(HTTPRedirectHandler): + def redirect_request(self, req: Any, fp: Any, code: int, msg: str, headers: Any, newurl: str) -> None: + return None + + +def _read_response_with_deadline( + response: Any, deadline: float, max_bytes: int, *, method: str +) -> bytes: + def timed_out() -> HelperFailure: + return HelperFailure( + "response-deadline-exceeded", + "HTTP response body exceeded its monotonic deadline.", + blocked_at="verification", + request_id=_request_id(response.headers), + status=response.status, + partial=method not in {"GET", "HEAD"}, + ) + + remaining = deadline - time.monotonic() + if remaining <= 0: + raise timed_out() + connection = getattr(getattr(getattr(response, "fp", None), "raw", None), "_sock", None) + if not isinstance(connection, socket.socket): + raise HelperFailure( + "response-deadline-unsupported", + "Deadline reads require an interruptible urllib HTTP socket.", + blocked_at="verification", + request_id=_request_id(response.headers), + status=response.status, + partial=method not in {"GET", "HEAD"}, + ) + expired = threading.Event() + + def interrupt() -> None: + expired.set() + # BufferedReader.close can wait on the active read lock; interrupt its socket instead. + with suppress(OSError): + connection.shutdown(socket.SHUT_RDWR) + with suppress(OSError): + connection.close() + + watchdog = threading.Timer(max(0, deadline - time.monotonic()), interrupt) + watchdog.start() + try: + if expired.is_set() or time.monotonic() >= deadline: + raise timed_out() + try: + payload = response.read(max_bytes + 1) + except (OSError, HTTPException, ValueError) as exc: + if expired.is_set() or time.monotonic() >= deadline: + raise timed_out() from exc + raise HelperFailure( + "response-read-failed", "HTTP response body could not be read.", + blocked_at="verification", + request_id=_request_id(response.headers), + status=response.status, + partial=method not in {"GET", "HEAD"}, + ) from exc + if expired.is_set() or time.monotonic() >= deadline: + raise timed_out() + return payload + finally: + watchdog.cancel() + watchdog.join() + + +def http_request( + method: str, + url: str, + token: str, + *, + body: bytes | None = None, + headers: dict[str, str] | None = None, + timeout: int = 180, + raw_response: bool = False, + max_response_bytes: int | None = None, + follow_redirects: bool = True, + response_deadline: float | None = None, +) -> HttpResult: + if response_deadline is not None: + if ( + not isinstance(response_deadline, (int, float)) + or not math.isfinite(response_deadline) + or response_deadline - time.monotonic() > threading.TIMEOUT_MAX + ): + raise HelperFailure( + "response-deadline-invalid", "Response deadline must be a supported finite monotonic time.", + blocked_at="input-resolution", + ) + if max_response_bytes is None: + max_response_bytes = 1024 * 1024 + if not isinstance(max_response_bytes, int) or max_response_bytes < 0: + raise HelperFailure( + "response-limit-invalid", "Response byte limit must be a nonnegative integer.", + blocked_at="input-resolution", + ) + if response_deadline <= time.monotonic(): + raise HelperFailure( + "response-deadline-exceeded", "HTTP deadline elapsed before the request.", + blocked_at="verification", + ) + request_headers = { + "Accept": "application/json;odata.metadata=minimal", + "Authorization": f"Bearer {token}", + "x-ms-client-request-id": str(uuid.uuid4()), + } + request_headers.update(headers or {}) + request = Request(url=url, data=body, headers=request_headers, method=method) + try: + open_request = urlopen if follow_redirects else build_opener(_NoRedirect).open + with open_request(request, timeout=timeout) as response: + if response_deadline is not None: + payload = _read_response_with_deadline( + response, response_deadline, max_response_bytes, method=method + ) + else: + payload = ( + response.read(max_response_bytes + 1) + if max_response_bytes is not None + else response.read() + ) + etag_values = tuple(value for name, value in response.headers.items() if name.lower() == "etag") + response_headers = _normalize_response_headers(response.headers) + if max_response_bytes is not None and len(payload) > max_response_bytes: + raise HelperFailure( + "response-too-large", + "Azure response exceeded the bounded read limit.", + blocked_at="verification", + request_id=_request_id(response_headers), + status=response.status, + partial=method not in {"GET", "HEAD"}, + ) + if raw_response: + parsed: Any = payload + elif not payload: + parsed = None + else: + try: + parsed = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise HelperFailure( + "response-invalid-json", + "Azure returned a non-JSON response where JSON was required.", + blocked_at="verification", + request_id=_request_id(response_headers), + status=response.status, + partial=method not in {"GET", "HEAD"}, + ) from exc + return HttpResult(response.status, parsed, response_headers, etag_values, response_deadline) + except HTTPError as exc: + retry_after = retry_after_metadata(exc.headers) + timing = retry_after_timing(exc.headers) + response_headers = _normalize_response_headers(exc.headers) + status = int(exc.code) + close_failed = False + if status == 429: + try: + exc.close() + except (OSError, HTTPException): + close_failed = True + elif response_deadline is not None: + exc.close() + ambiguous = method not in {"GET", "HEAD"} and is_ambiguous_status(status) + failure = HelperFailure( + "azure-http-error", + f"Azure request failed with HTTP {status}.", + blocked_at="execution", + request_id=_request_id(response_headers), + status=status, + partial=ambiguous, + retry_after=retry_after, + recovery_deadline=response_deadline, + retry_after_timing=timing, + warnings=(["response-close-failed: HTTP 429 response cleanup failed; " + "original HTTP failure retained and delayed recovery blocked."] if close_failed else []), + ) + failure.response_close_failed = close_failed + raise failure from exc + except (URLError, TimeoutError) as exc: + raise HelperFailure( + "azure-response-ambiguous", + "Azure request did not return an authoritative response.", + blocked_at="verification", + partial=method not in {"GET", "HEAD"}, + ) from exc + + +def blocked_result( + failure: HelperFailure, + *, + outcome: str, + fingerprint: str | None, + owner: Any = None, +) -> dict[str, Any]: + result = _blocked_result(failure, outcome=outcome, fingerprint=fingerprint, owner=owner) + if failure.file_batch is not None: + result["file_batch"] = failure.file_batch + result["safe_next_decision"] = ( + "Retain the source and original ACK/journal. Do not rerun creation, replay an uncertain upload, " + "or delete/reset resources. Plan upload-only continuation for never-attempted files with " + "file_upload.py --plan; missing original evidence blocks continuation, not retention." + ) + return result + + +def _blocked_result(failure, *, outcome, fingerprint, owner): + if failure.partial or failure.writes: + return { + "status": "partial", + "outcome": outcome, + "approved_plan": { + "fingerprint": fingerprint, + "confirmed": fingerprint is not None, + }, + "first_failure": { + "code": failure.code, + "operation": failure.blocked_at, + "status": failure.http_status, + "message": failure.message, + "request_id": failure.request_id, + }, + "completed_writes": failure.writes, + "failed_or_unverified_postconditions": [failure.code], + "resources_remaining": { + "run_owned": failure.resources_remaining, + "reused": failure.resources_reused, + **({"unverified": failure.resources_unverified} if failure.resources_unverified else {}), + }, + "rollback": {"possible": False, "exact_plan": []}, + "cleanup": {"status": "separate-plan-and-approval-required"}, + "owner": owner, + "warnings": failure.warnings, + } + result = { + "status": "blocked", + "outcome": outcome, + "blocked_at": failure.blocked_at, + "first_blocker": { + "code": failure.code, + "message": failure.message, + "status": failure.http_status, + "request_id": failure.request_id, + }, + "missing_or_conflicting_input": failure.code, + "read_only_evidence": [], + "writes_performed": [], + "safe_next_decision": "Resolve the first blocker, rebuild the plan, and obtain new approval.", + "ownership": {"run_owned": [], "reused_not_owned": []}, + "cleanup": "not applicable", + } + if fingerprint is not None: + result["approved_plan"] = { + "fingerprint": fingerprint, + "confirmed": True, + } + if failure.warnings: + result["warnings"] = failure.warnings + return result + + +def emit_result( + result: dict[str, Any], *, stream: Any = None, preserve_unapproved_input: bool = False +) -> None: + if stream is None: + stream = sys.stdout + safe = redact_sensitive(result) + if preserve_unapproved_input: + document = result.get("execution_input") + if ( + result.get("status") != "planned" or not isinstance(document, dict) + or document.get("schema_version") != "1.0" or not isinstance(document.get("plan"), dict) + or not isinstance(document.get("approval"), dict) + or document["approval"].get("confirmed") is not False + or document.get("approval") != {"confirmed": False, "fingerprint": digest(document["plan"])} + ): + raise HelperFailure( + "planning-output-invalid", "Only an exact unapproved execution input can retain ResourceId bindings.", + blocked_at="verification", + ) + reject_secrets(document) + require_allowed_fields(document, {"schema_version", "plan", "approval"}, label="planning envelope") + safe["execution_input"] = document + stream.write( + json.dumps( + safe, + sort_keys=True, + separators=(",", ":"), + ) + + "\n" + ) + + +TokenProvider = Callable[[str], str] +Transport = Callable[..., HttpResult] diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_document_adapters.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_document_adapters.py index be1cd49d..6f71c20e 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_document_adapters.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_document_adapters.py @@ -1,336 +1,336 @@ -"""Bounded lexical, OOXML-part and image-header adapters; no conversion.""" -from __future__ import annotations - -import codecs -from html.parser import HTMLParser -import io -import json -from pathlib import PurePosixPath -import re -import struct -import sys -import zipfile -import zlib - -XML_VERSION = "0.7.1" -MAX_TEXT = 2 * 1024 * 1024 -MAX_ENTRIES = 256 -MAX_INFLATED = 16 * 1024 * 1024 -MAX_RATIO = 100 -MAX_NODES = 100_000 -MAX_DEPTH = 64 -MAX_IMAGE_PIXELS = 100_000_000 -CT = "http://schemas.openxmlformats.org/package/2006/content-types" -W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" -P = "http://schemas.openxmlformats.org/presentationml/2006/main" -A = "http://schemas.openxmlformats.org/drawingml/2006/main" -S = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" -OFFICE = { - "docx": ("/word/document.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"), - "pptx": ("/ppt/presentation.xml", "application/vnd.openxmlformats-officedocument.presentationml.presentation.main+xml"), - "xlsx": ("/xl/workbook.xml", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"), -} - - -class AdapterBlocked(ValueError): - """Fixed content-free reason; callers never forward parser exceptions.""" - - -def facts(**values): - return { - "page_count": "not-applicable", - "text_characters": "not-assessed", "text_available": "not-assessed", - "total_images": "not-assessed", "total_drawings": "not-assessed", - "table_structure": "not-assessed", "layout_relationships": "not-assessed", - "input_profile": "unknown", "answerability": "not-assessed", - "ocr_need": "not-assessed", **values, - } - - -def text_facts(count, basis, **values): - return facts(text_characters=count, text_available=count > 0, text_basis=basis, **values) - - -class HTMLCounts(HTMLParser): - def __init__(self): - super().__init__(convert_charrefs=True) - self.characters = 0 - self.tags = {"img": 0, "table": 0, "tr": 0, "td": 0, "th": 0} - self.suppressed = [] - self.nodes = 0 - - def handle_starttag(self, tag, attrs): - self.nodes += 1 - if self.nodes > MAX_NODES: - raise AdapterBlocked("document-node-limit") - if tag in {"script", "style"}: - self.suppressed.append(tag) - if not self.suppressed and tag in self.tags: - self.tags[tag] += 1 - - def handle_endtag(self, tag): - if self.suppressed and self.suppressed[-1] == tag: - self.suppressed.pop() - - def handle_data(self, data): - if not self.suppressed: - self.characters += len(data) - - -def assess_text(data, hint): - if len(data) > MAX_TEXT: - raise AdapterBlocked("text-input-limit") - try: - text = data.decode("utf-8-sig", errors="strict") - except UnicodeError: - raise AdapterBlocked("unsupported-text-encoding-or-format") from None - if any(ord(char) < 32 and char not in "\t\r\n" for char in text): - raise AdapterBlocked("unsupported-binary-format") - kind = "text" if hint == "auto" else hint - if kind not in {"text", "markdown", "html", "json"}: - raise AdapterBlocked("format-hint-mismatch") - if kind == "json": - def reject_constant(value): - raise AdapterBlocked("nonstandard-json") - - def unique_object(pairs): - result = {} - for key, value in pairs: - if key in result: - raise AdapterBlocked("duplicate-json-key") - result[key] = value - return result - - try: - value = json.loads(text, parse_constant=reject_constant, object_pairs_hook=unique_object) - except AdapterBlocked: - raise - except (ValueError, UnicodeError): - raise AdapterBlocked("invalid-json") from None - count = nodes = objects = arrays = keys = 0 - pending = [(value, 1)] - while pending: - value, depth = pending.pop() - nodes += 1 - if nodes > MAX_NODES or depth > MAX_DEPTH: - raise AdapterBlocked("document-node-or-depth-limit") - if isinstance(value, str): - count += len(value) - elif isinstance(value, dict): - objects += 1 - keys += len(value) - pending.extend((item, depth + 1) for item in value.values()) - elif isinstance(value, list): - arrays += 1 - pending.extend((item, depth + 1) for item in value) - return kind, "stdlib-json", text_facts(count, "string-values-only", - objects=objects, arrays=arrays, keys=keys) - if kind == "html": - parser = HTMLCounts() - parser.feed(text) - parser.close() - return kind, "stdlib-HTMLParser", text_facts( - parser.characters, "data-events-excluding-script-style", - html_elements=parser.tags, validation="tolerant-syntax-not-rendering") - return kind, "stdlib-utf8", text_facts( - len(text), "decoded-characters-including-whitespace-and-markup", - markdown_structure="not-assessed" if kind == "markdown" else "not-applicable") - - -def xml_parser(): - try: - import defusedxml - from defusedxml.ElementTree import fromstring - from defusedxml.common import DefusedXmlException - from xml.etree.ElementTree import ParseError - except ImportError: - raise AdapterBlocked("office-parser-dependency-missing") from None - if defusedxml.__version__ != XML_VERSION: - raise AdapterBlocked("office-parser-version-unsupported") - return fromstring, (DefusedXmlException, ParseError) - - -def bounded_xml(data, parse, errors): - if len(data) > MAX_TEXT: - raise AdapterBlocked("archive-entry-limit") - try: - root = parse(data, forbid_dtd=True, forbid_entities=True, forbid_external=True) - except errors: - raise AdapterBlocked("unsafe-or-invalid-office-xml") from None - pending, nodes = [(root, 1)], 0 - while pending: - element, depth = pending.pop() - nodes += 1 - if nodes > MAX_NODES or depth > MAX_DEPTH: - raise AdapterBlocked("document-node-or-depth-limit") - pending.extend((child, depth + 1) for child in element) - return root, nodes - - -def assess_office(data, hint, parse, errors): - try: - with zipfile.ZipFile(io.BytesIO(data)) as archive: - entries = archive.infolist() - if len(entries) > MAX_ENTRIES: - raise AdapterBlocked("archive-entry-count-limit") - names, total = set(), 0 - for entry in entries: - path = PurePosixPath(entry.filename) - if (entry.filename in names or path.is_absolute() or ".." in path.parts - or "\\" in entry.filename or ":" in entry.filename): - raise AdapterBlocked("unsafe-or-duplicate-archive-entry") - names.add(entry.filename) - total += entry.file_size - if entry.flag_bits & 1: - raise AdapterBlocked("encrypted-office-archive") - if entry.compress_type not in {zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED}: - raise AdapterBlocked("unsupported-archive-compression") - if (entry.file_size > MAX_TEXT or total > MAX_INFLATED - or entry.file_size > MAX_RATIO * max(entry.compress_size, 1)): - raise AdapterBlocked("archive-inflation-limit") - if entry.filename.lower().endswith("vbaproject.bin"): - raise AdapterBlocked("macro-package-not-assessed") - - def read_xml(part): - if part not in names: - raise AdapterBlocked("incomplete-office-package") - with archive.open(part) as source: - content = source.read(MAX_TEXT + 1) - return bounded_xml(content, parse, errors) - - if "[Content_Types].xml" not in names: - raise AdapterBlocked("unsupported-zip-format") - types, nodes = read_xml("[Content_Types].xml") - if types.tag != f"{{{CT}}}Types": - raise AdapterBlocked("unsupported-office-namespace") - declared = [(child.get("PartName"), child.get("ContentType")) for child in types] - if any("macroenabled" in (kind or "").lower() for _, kind in declared): - raise AdapterBlocked("macro-package-not-assessed") - kinds = [kind for kind, definition in OFFICE.items() if definition in declared] - if len(kinds) != 1: - raise AdapterBlocked("unsupported-office-profile") - kind = kinds[0] - if hint not in {"auto", kind}: - raise AdapterBlocked("format-hint-mismatch") - main = OFFICE[kind][0][1:] - root, used = read_xml(main) - nodes += used - expected = {"docx": f"{{{W}}}document", "pptx": f"{{{P}}}presentation", - "xlsx": f"{{{S}}}workbook"}[kind] - if root.tag != expected: - raise AdapterBlocked("unsupported-office-namespace") - patterns = { - "docx": r"word/document\.xml", - "pptx": r"ppt/slides/slide[0-9]+\.xml", - "xlsx": r"xl/(worksheets/sheet[0-9]+|sharedStrings|tables/table[0-9]+)\.xml", - } - parts = sorted(part for part in names if re.fullmatch(patterns[kind], part)) - if not parts: - raise AdapterBlocked("office-content-parts-unavailable") - characters = tables = image_refs = paragraphs = formulas = cells = 0 - for part in parts: - tree, used = (root, 0) if part == main else read_xml(part) - nodes += used - if nodes > MAX_NODES: - raise AdapterBlocked("document-node-limit") - allowed_roots = { - "docx": {f"{{{W}}}document"}, "pptx": {f"{{{P}}}sld"}, - "xlsx": {f"{{{S}}}worksheet", f"{{{S}}}sst", f"{{{S}}}table"}, - }[kind] - if tree.tag not in allowed_roots: - raise AdapterBlocked("unsupported-office-namespace") - for element in tree.iter(): - if element.tag == {"docx": f"{{{W}}}t", "pptx": f"{{{A}}}t", - "xlsx": f"{{{S}}}t"}[kind]: - characters += len(element.text or "") - tables += element.tag in {f"{{{W}}}tbl", f"{{{A}}}tbl", f"{{{S}}}table"} - image_refs += element.tag == f"{{{A}}}blip" - paragraphs += element.tag in {f"{{{W}}}p", f"{{{A}}}p"} - formulas += element.tag == f"{{{S}}}f" - cells += element.tag == f"{{{S}}}c" - result = text_facts( - characters, "stored-text-elements-not-rendered-or-cell-occurrences", - page_count="not-assessed", inspected_xml_parts=len(parts), - table_elements=tables, image_reference_elements=image_refs, - paragraphs=paragraphs if kind != "xlsx" else "not-applicable", - formula_elements=formulas if kind == "xlsx" else "not-applicable", - cell_elements=cells if kind == "xlsx" else "not-applicable", - slide_parts=len(parts) if kind == "pptx" else "not-applicable", - declared_sheets=sum(e.tag == f"{{{S}}}sheet" for e in root.iter()) - if kind == "xlsx" else "not-applicable", - coverage={"docx": "main-document-only", "pptx": "all-standard-slide-parts", - "xlsx": "standard-worksheet-shared-string-table-parts"}[kind], - relationships="not-resolved", rendering="not-assessed", - ) - return kind, f"stdlib-zipfile+defusedxml=={XML_VERSION}", result - except (zipfile.BadZipFile, zipfile.LargeZipFile, EOFError, zlib.error): - raise AdapterBlocked("invalid-office-archive") from None - - -def image_facts(kind, width, height, **values): - if not width or not height or width * height > MAX_IMAGE_PIXELS: - raise AdapterBlocked("image-dimension-limit") - return kind, "stdlib-struct-header", facts( - width=width, height=height, text_characters="not-assessed", - text_available="not-assessed", validation="header-only-pixels-not-assessed", - frame_count="not-assessed", **values) - - -def assess_png(data): - if len(data) < 33 or data[8:16] != b"\0\0\0\rIHDR": - raise AdapterBlocked("invalid-png-header") - width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", data[16:29]) - if (zlib.crc32(data[12:29]) != struct.unpack(">I", data[29:33])[0] - or compression or filtering or interlace not in (0, 1) - or depth not in {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, - 4: {8, 16}, 6: {8, 16}}.get(color, set())): - raise AdapterBlocked("invalid-png-header") - return image_facts("png", width, height, bit_depth=depth, color_type=color) - - -def assess_jpeg(data): - position = 2 - for _ in range(4096): - if position >= len(data) or data[position] != 255: - raise AdapterBlocked("invalid-jpeg-header") - while position < len(data) and data[position] == 255: - position += 1 - if position >= len(data): - raise AdapterBlocked("invalid-jpeg-header") - marker = data[position] - position += 1 - if marker in (0xDA, 0xD9) or position + 2 > len(data): - raise AdapterBlocked("jpeg-frame-header-unavailable") - length = int.from_bytes(data[position:position + 2], "big") - if length < 2 or position + length > len(data): - raise AdapterBlocked("invalid-jpeg-header") - if marker in (0xC0, 0xC1, 0xC2): - if length < 8: - raise AdapterBlocked("invalid-jpeg-header") - depth, height, width, components = struct.unpack(">BHHB", data[position + 2:position + 8]) - if length != 8 + 3 * components or not 1 <= components <= 4 or depth not in (8, 12): - raise AdapterBlocked("invalid-jpeg-header") - return image_facts("jpeg", width, height, precision=depth, components=components) - position += length - raise AdapterBlocked("image-marker-limit") - - -def assess_document(data, hint, deny_reads): - for encoding in ("utf-8-sig", "cp437", "ascii", "utf-8"): - codecs.lookup(encoding) - office = data.startswith((b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08")) - parse, errors = xml_parser() if office else (None, ()) - sys.addaudithook(deny_reads) - if office: - return assess_office(data, hint, parse, errors) - if data.startswith(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"): - raise AdapterBlocked("legacy-or-encrypted-office-not-assessed") - if data.startswith(b"\x89PNG\r\n\x1a\n"): - if hint not in {"auto", "png"}: - raise AdapterBlocked("format-hint-mismatch") - return assess_png(data) - if data.startswith(b"\xff\xd8"): - if hint not in {"auto", "jpeg"}: - raise AdapterBlocked("format-hint-mismatch") - return assess_jpeg(data) - return assess_text(data, hint) +"""Bounded lexical, OOXML-part and image-header adapters; no conversion.""" +from __future__ import annotations + +import codecs +from html.parser import HTMLParser +import io +import json +from pathlib import PurePosixPath +import re +import struct +import sys +import zipfile +import zlib + +XML_VERSION = "0.7.1" +MAX_TEXT = 2 * 1024 * 1024 +MAX_ENTRIES = 256 +MAX_INFLATED = 16 * 1024 * 1024 +MAX_RATIO = 100 +MAX_NODES = 100_000 +MAX_DEPTH = 64 +MAX_IMAGE_PIXELS = 100_000_000 +CT = "http://schemas.openxmlformats.org/package/2006/content-types" +W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" +P = "http://schemas.openxmlformats.org/presentationml/2006/main" +A = "http://schemas.openxmlformats.org/drawingml/2006/main" +S = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" +OFFICE = { + "docx": ("/word/document.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"), + "pptx": ("/ppt/presentation.xml", "application/vnd.openxmlformats-officedocument.presentationml.presentation.main+xml"), + "xlsx": ("/xl/workbook.xml", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"), +} + + +class AdapterBlocked(ValueError): + """Fixed content-free reason; callers never forward parser exceptions.""" + + +def facts(**values): + return { + "page_count": "not-applicable", + "text_characters": "not-assessed", "text_available": "not-assessed", + "total_images": "not-assessed", "total_drawings": "not-assessed", + "table_structure": "not-assessed", "layout_relationships": "not-assessed", + "input_profile": "unknown", "answerability": "not-assessed", + "ocr_need": "not-assessed", **values, + } + + +def text_facts(count, basis, **values): + return facts(text_characters=count, text_available=count > 0, text_basis=basis, **values) + + +class HTMLCounts(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.characters = 0 + self.tags = {"img": 0, "table": 0, "tr": 0, "td": 0, "th": 0} + self.suppressed = [] + self.nodes = 0 + + def handle_starttag(self, tag, attrs): + self.nodes += 1 + if self.nodes > MAX_NODES: + raise AdapterBlocked("document-node-limit") + if tag in {"script", "style"}: + self.suppressed.append(tag) + if not self.suppressed and tag in self.tags: + self.tags[tag] += 1 + + def handle_endtag(self, tag): + if self.suppressed and self.suppressed[-1] == tag: + self.suppressed.pop() + + def handle_data(self, data): + if not self.suppressed: + self.characters += len(data) + + +def assess_text(data, hint): + if len(data) > MAX_TEXT: + raise AdapterBlocked("text-input-limit") + try: + text = data.decode("utf-8-sig", errors="strict") + except UnicodeError: + raise AdapterBlocked("unsupported-text-encoding-or-format") from None + if any(ord(char) < 32 and char not in "\t\r\n" for char in text): + raise AdapterBlocked("unsupported-binary-format") + kind = "text" if hint == "auto" else hint + if kind not in {"text", "markdown", "html", "json"}: + raise AdapterBlocked("format-hint-mismatch") + if kind == "json": + def reject_constant(value): + raise AdapterBlocked("nonstandard-json") + + def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise AdapterBlocked("duplicate-json-key") + result[key] = value + return result + + try: + value = json.loads(text, parse_constant=reject_constant, object_pairs_hook=unique_object) + except AdapterBlocked: + raise + except (ValueError, UnicodeError): + raise AdapterBlocked("invalid-json") from None + count = nodes = objects = arrays = keys = 0 + pending = [(value, 1)] + while pending: + value, depth = pending.pop() + nodes += 1 + if nodes > MAX_NODES or depth > MAX_DEPTH: + raise AdapterBlocked("document-node-or-depth-limit") + if isinstance(value, str): + count += len(value) + elif isinstance(value, dict): + objects += 1 + keys += len(value) + pending.extend((item, depth + 1) for item in value.values()) + elif isinstance(value, list): + arrays += 1 + pending.extend((item, depth + 1) for item in value) + return kind, "stdlib-json", text_facts(count, "string-values-only", + objects=objects, arrays=arrays, keys=keys) + if kind == "html": + parser = HTMLCounts() + parser.feed(text) + parser.close() + return kind, "stdlib-HTMLParser", text_facts( + parser.characters, "data-events-excluding-script-style", + html_elements=parser.tags, validation="tolerant-syntax-not-rendering") + return kind, "stdlib-utf8", text_facts( + len(text), "decoded-characters-including-whitespace-and-markup", + markdown_structure="not-assessed" if kind == "markdown" else "not-applicable") + + +def xml_parser(): + try: + import defusedxml + from defusedxml.ElementTree import fromstring + from defusedxml.common import DefusedXmlException + from xml.etree.ElementTree import ParseError + except ImportError: + raise AdapterBlocked("office-parser-dependency-missing") from None + if defusedxml.__version__ != XML_VERSION: + raise AdapterBlocked("office-parser-version-unsupported") + return fromstring, (DefusedXmlException, ParseError) + + +def bounded_xml(data, parse, errors): + if len(data) > MAX_TEXT: + raise AdapterBlocked("archive-entry-limit") + try: + root = parse(data, forbid_dtd=True, forbid_entities=True, forbid_external=True) + except errors: + raise AdapterBlocked("unsafe-or-invalid-office-xml") from None + pending, nodes = [(root, 1)], 0 + while pending: + element, depth = pending.pop() + nodes += 1 + if nodes > MAX_NODES or depth > MAX_DEPTH: + raise AdapterBlocked("document-node-or-depth-limit") + pending.extend((child, depth + 1) for child in element) + return root, nodes + + +def assess_office(data, hint, parse, errors): + try: + with zipfile.ZipFile(io.BytesIO(data)) as archive: + entries = archive.infolist() + if len(entries) > MAX_ENTRIES: + raise AdapterBlocked("archive-entry-count-limit") + names, total = set(), 0 + for entry in entries: + path = PurePosixPath(entry.filename) + if (entry.filename in names or path.is_absolute() or ".." in path.parts + or "\\" in entry.filename or ":" in entry.filename): + raise AdapterBlocked("unsafe-or-duplicate-archive-entry") + names.add(entry.filename) + total += entry.file_size + if entry.flag_bits & 1: + raise AdapterBlocked("encrypted-office-archive") + if entry.compress_type not in {zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED}: + raise AdapterBlocked("unsupported-archive-compression") + if (entry.file_size > MAX_TEXT or total > MAX_INFLATED + or entry.file_size > MAX_RATIO * max(entry.compress_size, 1)): + raise AdapterBlocked("archive-inflation-limit") + if entry.filename.lower().endswith("vbaproject.bin"): + raise AdapterBlocked("macro-package-not-assessed") + + def read_xml(part): + if part not in names: + raise AdapterBlocked("incomplete-office-package") + with archive.open(part) as source: + content = source.read(MAX_TEXT + 1) + return bounded_xml(content, parse, errors) + + if "[Content_Types].xml" not in names: + raise AdapterBlocked("unsupported-zip-format") + types, nodes = read_xml("[Content_Types].xml") + if types.tag != f"{{{CT}}}Types": + raise AdapterBlocked("unsupported-office-namespace") + declared = [(child.get("PartName"), child.get("ContentType")) for child in types] + if any("macroenabled" in (kind or "").lower() for _, kind in declared): + raise AdapterBlocked("macro-package-not-assessed") + kinds = [kind for kind, definition in OFFICE.items() if definition in declared] + if len(kinds) != 1: + raise AdapterBlocked("unsupported-office-profile") + kind = kinds[0] + if hint not in {"auto", kind}: + raise AdapterBlocked("format-hint-mismatch") + main = OFFICE[kind][0][1:] + root, used = read_xml(main) + nodes += used + expected = {"docx": f"{{{W}}}document", "pptx": f"{{{P}}}presentation", + "xlsx": f"{{{S}}}workbook"}[kind] + if root.tag != expected: + raise AdapterBlocked("unsupported-office-namespace") + patterns = { + "docx": r"word/document\.xml", + "pptx": r"ppt/slides/slide[0-9]+\.xml", + "xlsx": r"xl/(worksheets/sheet[0-9]+|sharedStrings|tables/table[0-9]+)\.xml", + } + parts = sorted(part for part in names if re.fullmatch(patterns[kind], part)) + if not parts: + raise AdapterBlocked("office-content-parts-unavailable") + characters = tables = image_refs = paragraphs = formulas = cells = 0 + for part in parts: + tree, used = (root, 0) if part == main else read_xml(part) + nodes += used + if nodes > MAX_NODES: + raise AdapterBlocked("document-node-limit") + allowed_roots = { + "docx": {f"{{{W}}}document"}, "pptx": {f"{{{P}}}sld"}, + "xlsx": {f"{{{S}}}worksheet", f"{{{S}}}sst", f"{{{S}}}table"}, + }[kind] + if tree.tag not in allowed_roots: + raise AdapterBlocked("unsupported-office-namespace") + for element in tree.iter(): + if element.tag == {"docx": f"{{{W}}}t", "pptx": f"{{{A}}}t", + "xlsx": f"{{{S}}}t"}[kind]: + characters += len(element.text or "") + tables += element.tag in {f"{{{W}}}tbl", f"{{{A}}}tbl", f"{{{S}}}table"} + image_refs += element.tag == f"{{{A}}}blip" + paragraphs += element.tag in {f"{{{W}}}p", f"{{{A}}}p"} + formulas += element.tag == f"{{{S}}}f" + cells += element.tag == f"{{{S}}}c" + result = text_facts( + characters, "stored-text-elements-not-rendered-or-cell-occurrences", + page_count="not-assessed", inspected_xml_parts=len(parts), + table_elements=tables, image_reference_elements=image_refs, + paragraphs=paragraphs if kind != "xlsx" else "not-applicable", + formula_elements=formulas if kind == "xlsx" else "not-applicable", + cell_elements=cells if kind == "xlsx" else "not-applicable", + slide_parts=len(parts) if kind == "pptx" else "not-applicable", + declared_sheets=sum(e.tag == f"{{{S}}}sheet" for e in root.iter()) + if kind == "xlsx" else "not-applicable", + coverage={"docx": "main-document-only", "pptx": "all-standard-slide-parts", + "xlsx": "standard-worksheet-shared-string-table-parts"}[kind], + relationships="not-resolved", rendering="not-assessed", + ) + return kind, f"stdlib-zipfile+defusedxml=={XML_VERSION}", result + except (zipfile.BadZipFile, zipfile.LargeZipFile, EOFError, zlib.error): + raise AdapterBlocked("invalid-office-archive") from None + + +def image_facts(kind, width, height, **values): + if not width or not height or width * height > MAX_IMAGE_PIXELS: + raise AdapterBlocked("image-dimension-limit") + return kind, "stdlib-struct-header", facts( + width=width, height=height, text_characters="not-assessed", + text_available="not-assessed", validation="header-only-pixels-not-assessed", + frame_count="not-assessed", **values) + + +def assess_png(data): + if len(data) < 33 or data[8:16] != b"\0\0\0\rIHDR": + raise AdapterBlocked("invalid-png-header") + width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", data[16:29]) + if (zlib.crc32(data[12:29]) != struct.unpack(">I", data[29:33])[0] + or compression or filtering or interlace not in (0, 1) + or depth not in {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, + 4: {8, 16}, 6: {8, 16}}.get(color, set())): + raise AdapterBlocked("invalid-png-header") + return image_facts("png", width, height, bit_depth=depth, color_type=color) + + +def assess_jpeg(data): + position = 2 + for _ in range(4096): + if position >= len(data) or data[position] != 255: + raise AdapterBlocked("invalid-jpeg-header") + while position < len(data) and data[position] == 255: + position += 1 + if position >= len(data): + raise AdapterBlocked("invalid-jpeg-header") + marker = data[position] + position += 1 + if marker in (0xDA, 0xD9) or position + 2 > len(data): + raise AdapterBlocked("jpeg-frame-header-unavailable") + length = int.from_bytes(data[position:position + 2], "big") + if length < 2 or position + length > len(data): + raise AdapterBlocked("invalid-jpeg-header") + if marker in (0xC0, 0xC1, 0xC2): + if length < 8: + raise AdapterBlocked("invalid-jpeg-header") + depth, height, width, components = struct.unpack(">BHHB", data[position + 2:position + 8]) + if length != 8 + 3 * components or not 1 <= components <= 4 or depth not in (8, 12): + raise AdapterBlocked("invalid-jpeg-header") + return image_facts("jpeg", width, height, precision=depth, components=components) + position += length + raise AdapterBlocked("image-marker-limit") + + +def assess_document(data, hint, deny_reads): + for encoding in ("utf-8-sig", "cp437", "ascii", "utf-8"): + codecs.lookup(encoding) + office = data.startswith((b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08")) + parse, errors = xml_parser() if office else (None, ()) + sys.addaudithook(deny_reads) + if office: + return assess_office(data, hint, parse, errors) + if data.startswith(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"): + raise AdapterBlocked("legacy-or-encrypted-office-not-assessed") + if data.startswith(b"\x89PNG\r\n\x1a\n"): + if hint not in {"auto", "png"}: + raise AdapterBlocked("format-hint-mismatch") + return assess_png(data) + if data.startswith(b"\xff\xd8"): + if hint not in {"auto", "jpeg"}: + raise AdapterBlocked("format-hint-mismatch") + return assess_jpeg(data) + return assess_text(data, hint) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_document_limits.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_document_limits.py index 38059779..35ba1517 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_document_limits.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_document_limits.py @@ -1,95 +1,95 @@ -"""OS-enforced limits installed in the document worker before parser imports.""" -from __future__ import annotations - -import os -import sys - -MEMORY_BYTES = 256 * 1024 * 1024 -CPU_SECONDS = 10 - - -def install_limits(): - """Return a live Windows job handle, or None on Linux; fail closed elsewhere.""" - if sys.platform == "linux": - import resource - - resource.setrlimit(resource.RLIMIT_AS, (MEMORY_BYTES, MEMORY_BYTES)) - resource.setrlimit(resource.RLIMIT_CPU, (CPU_SECONDS, CPU_SECONDS)) - resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) - resource.setrlimit(resource.RLIMIT_FSIZE, (0, 0)) - return None - if sys.platform != "win32": - raise OSError("unsupported limit platform") - - import ctypes - from ctypes import wintypes as w - - class Basic(ctypes.Structure): - _fields_ = [ - ("process_time", ctypes.c_longlong), ("job_time", ctypes.c_longlong), - ("flags", w.DWORD), ("min_ws", ctypes.c_size_t), - ("max_ws", ctypes.c_size_t), ("active", w.DWORD), - ("affinity", ctypes.c_size_t), ("priority", w.DWORD), - ("scheduling", w.DWORD), - ] - - class IO(ctypes.Structure): - _fields_ = [(name, ctypes.c_ulonglong) for name in - ("read_ops", "write_ops", "other_ops", "read_bytes", - "write_bytes", "other_bytes")] - - class Extended(ctypes.Structure): - _fields_ = [ - ("basic", Basic), ("io", IO), ("process_memory", ctypes.c_size_t), - ("job_memory", ctypes.c_size_t), ("peak_process", ctypes.c_size_t), - ("peak_job", ctypes.c_size_t), - ] - - kernel = ctypes.WinDLL("kernel32", use_last_error=True) - kernel.CreateJobObjectW.argtypes = [ctypes.c_void_p, w.LPCWSTR] - kernel.CreateJobObjectW.restype = w.HANDLE - kernel.SetInformationJobObject.argtypes = [w.HANDLE, ctypes.c_int, - ctypes.c_void_p, w.DWORD] - kernel.SetInformationJobObject.restype = w.BOOL - kernel.GetCurrentProcess.restype = w.HANDLE - kernel.AssignProcessToJobObject.argtypes = [w.HANDLE, w.HANDLE] - kernel.AssignProcessToJobObject.restype = w.BOOL - kernel.CloseHandle.argtypes = [w.HANDLE] - kernel.CloseHandle.restype = w.BOOL - job = kernel.CreateJobObjectW(None, None) - if not job: - raise ctypes.WinError(ctypes.get_last_error()) - limits = Extended() - # PROCESS_TIME | ACTIVE_PROCESS | PROCESS_MEMORY | KILL_ON_JOB_CLOSE. - limits.basic.flags = 0x2 | 0x8 | 0x100 | 0x2000 - limits.basic.process_time = CPU_SECONDS * 10_000_000 - limits.basic.active = 1 - limits.process_memory = MEMORY_BYTES - if not kernel.SetInformationJobObject(job, 9, ctypes.byref(limits), - ctypes.sizeof(limits)): - error = ctypes.get_last_error() - kernel.CloseHandle(job) - raise ctypes.WinError(error) - if not kernel.AssignProcessToJobObject(job, kernel.GetCurrentProcess()): - error = ctypes.get_last_error() - kernel.CloseHandle(job) - raise ctypes.WinError(error) - # Keep the handle until process exit: closing it terminates this worker. - return job - - -def deny_side_effects(event, args): - """Defense in depth, not a sandbox for arbitrary Python/native code.""" - if event.startswith("socket.") or event in { - "subprocess.Popen", "os.system", "os.posix_spawn", "os.fork", - "os.remove", "os.unlink", "os.rename", "os.replace", "os.mkdir", "os.rmdir", - "os.truncate", "os.chmod", "os.chown", "os.lchown", "os.utime", - "os.link", "os.symlink", "os.chflags", "os.setxattr", "os.removexattr", - }: - raise PermissionError("Document side effect denied") - if event == "open": - _, mode, flags = args - if (mode and any(char in mode for char in "wax+")) or ( - flags & (os.O_WRONLY | os.O_RDWR | os.O_CREAT | os.O_TRUNC) - ): - raise PermissionError("Document write denied") +"""OS-enforced limits installed in the document worker before parser imports.""" +from __future__ import annotations + +import os +import sys + +MEMORY_BYTES = 256 * 1024 * 1024 +CPU_SECONDS = 10 + + +def install_limits(): + """Return a live Windows job handle, or None on Linux; fail closed elsewhere.""" + if sys.platform == "linux": + import resource + + resource.setrlimit(resource.RLIMIT_AS, (MEMORY_BYTES, MEMORY_BYTES)) + resource.setrlimit(resource.RLIMIT_CPU, (CPU_SECONDS, CPU_SECONDS)) + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + resource.setrlimit(resource.RLIMIT_FSIZE, (0, 0)) + return None + if sys.platform != "win32": + raise OSError("unsupported limit platform") + + import ctypes + from ctypes import wintypes as w + + class Basic(ctypes.Structure): + _fields_ = [ + ("process_time", ctypes.c_longlong), ("job_time", ctypes.c_longlong), + ("flags", w.DWORD), ("min_ws", ctypes.c_size_t), + ("max_ws", ctypes.c_size_t), ("active", w.DWORD), + ("affinity", ctypes.c_size_t), ("priority", w.DWORD), + ("scheduling", w.DWORD), + ] + + class IO(ctypes.Structure): + _fields_ = [(name, ctypes.c_ulonglong) for name in + ("read_ops", "write_ops", "other_ops", "read_bytes", + "write_bytes", "other_bytes")] + + class Extended(ctypes.Structure): + _fields_ = [ + ("basic", Basic), ("io", IO), ("process_memory", ctypes.c_size_t), + ("job_memory", ctypes.c_size_t), ("peak_process", ctypes.c_size_t), + ("peak_job", ctypes.c_size_t), + ] + + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.CreateJobObjectW.argtypes = [ctypes.c_void_p, w.LPCWSTR] + kernel.CreateJobObjectW.restype = w.HANDLE + kernel.SetInformationJobObject.argtypes = [w.HANDLE, ctypes.c_int, + ctypes.c_void_p, w.DWORD] + kernel.SetInformationJobObject.restype = w.BOOL + kernel.GetCurrentProcess.restype = w.HANDLE + kernel.AssignProcessToJobObject.argtypes = [w.HANDLE, w.HANDLE] + kernel.AssignProcessToJobObject.restype = w.BOOL + kernel.CloseHandle.argtypes = [w.HANDLE] + kernel.CloseHandle.restype = w.BOOL + job = kernel.CreateJobObjectW(None, None) + if not job: + raise ctypes.WinError(ctypes.get_last_error()) + limits = Extended() + # PROCESS_TIME | ACTIVE_PROCESS | PROCESS_MEMORY | KILL_ON_JOB_CLOSE. + limits.basic.flags = 0x2 | 0x8 | 0x100 | 0x2000 + limits.basic.process_time = CPU_SECONDS * 10_000_000 + limits.basic.active = 1 + limits.process_memory = MEMORY_BYTES + if not kernel.SetInformationJobObject(job, 9, ctypes.byref(limits), + ctypes.sizeof(limits)): + error = ctypes.get_last_error() + kernel.CloseHandle(job) + raise ctypes.WinError(error) + if not kernel.AssignProcessToJobObject(job, kernel.GetCurrentProcess()): + error = ctypes.get_last_error() + kernel.CloseHandle(job) + raise ctypes.WinError(error) + # Keep the handle until process exit: closing it terminates this worker. + return job + + +def deny_side_effects(event, args): + """Defense in depth, not a sandbox for arbitrary Python/native code.""" + if event.startswith("socket.") or event in { + "subprocess.Popen", "os.system", "os.posix_spawn", "os.fork", + "os.remove", "os.unlink", "os.rename", "os.replace", "os.mkdir", "os.rmdir", + "os.truncate", "os.chmod", "os.chown", "os.lchown", "os.utime", + "os.link", "os.symlink", "os.chflags", "os.setxattr", "os.removexattr", + }: + raise PermissionError("Document side effect denied") + if event == "open": + _, mode, flags = args + if (mode and any(char in mode for char in "wax+")) or ( + flags & (os.O_WRONLY | os.O_RDWR | os.O_CREAT | os.O_TRUNC) + ): + raise PermissionError("Document write denied") diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_indexer_observation.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_indexer_observation.py index 0e20ee22..32a0d4c3 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_indexer_observation.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_indexer_observation.py @@ -1,135 +1,135 @@ -"""Private, versioned evidence for generated indexers; no service operations.""" -from __future__ import annotations - -import re -from datetime import datetime, timedelta, timezone - -try: - from ._common import HelperFailure, digest -except ImportError: - from _common import HelperFailure, digest - - -PROJECTION_FIELDS = {"non_schedule_digest", "schedule_digest", "schedule_raw_digest"} -TICKS = 10_000_000 -DURATION = re.compile(r"P(?:(?P[0-9]{1,9})D)?(?:T(?:(?P[0-9]{1,9})H)?" - r"(?:(?P[0-9]{1,9})M)?(?:(?P[0-9]{1,9})(?:\.(?P[0-9]{1,7}))?S)?)?") -INSTANT = re.compile(r"([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})" - r"(?:\.([0-9]{1,7}))?(Z|[+-][0-9]{2}:[0-9]{2})") - - -def note(diagnostics, severity, code, field, message, request_id): - entry = dict(severity=severity, code=code, field=field, message=message, request_id=request_id) - if entry not in diagnostics: - diagnostics.append(entry) - - -def failure(diagnostics, code, field, message, request_id): - note(diagnostics, "error", code, field, message, request_id) - return HelperFailure(code, message, blocked_at="indexer-verification", request_id=request_id) - - -def interval(value): - match = DURATION.fullmatch(value) if isinstance(value, str) and len(value) <= 96 else None - if not match or not any(match.group(k) for k in ("days", "hours", "minutes", "seconds")): - raise ValueError("Unsupported duration") - if "T" in value and not any(match.group(k) for k in ("hours", "minutes", "seconds")): - raise ValueError("Empty time component") - parts = match.groupdict() - seconds = sum(int(parts[k] or 0) * scale for k, scale in - (("days", 86400), ("hours", 3600), ("minutes", 60), ("seconds", 1))) - ticks = seconds * TICKS + int((parts["fraction"] or "").ljust(7, "0")) - if not 300 * TICKS <= ticks <= 86400 * TICKS: - raise ValueError("Indexer interval outside documented bounds") - return ticks - - -def instant(value): - match = INSTANT.fullmatch(value) if isinstance(value, str) and len(value) <= 40 else None - if not match or match[8] == "-00:00": - raise ValueError("Unknown or unsupported timezone") - year, month, day, hour, minute, second = map(int, match.groups()[:6]) - zone = match[8] - offset = 0 - if zone != "Z": - hours, minutes = int(zone[1:3]), int(zone[4:6]) - if hours > 14 or minutes > 59 or hours == 14 and minutes: - raise ValueError("Invalid offset") - offset = (hours * 60 + minutes) * (-1 if zone[0] == "-" else 1) - stamp = datetime(year, month, day, hour, minute, second, - tzinfo=timezone(timedelta(minutes=offset))).astimezone(timezone.utc) - elapsed = stamp - datetime(1, 1, 1, tzinfo=timezone.utc) - return (elapsed.days * 86400 + elapsed.seconds) * TICKS + int((match[7] or "").ljust(7, "0")) - - -def schedule(value, *, present=True, diagnostics=None, request_id=None): - diagnostics = diagnostics if diagnostics is not None else [] - if value is None: - return {"state": "null" if present else "missing"} - field = "schedule" - try: - if not isinstance(value, dict) or set(value) - {"interval", "startTime"}: - raise ValueError("Unknown shape") - field = "schedule.interval" - period = interval(value.get("interval")) - field = "schedule.startTime" - start = instant(value["startTime"]) if value.get("startTime") is not None else None - return {"state": "object", "interval": period, "startTime": start} - except (ValueError, OverflowError) as exc: - raise failure(diagnostics, "indexer-schedule-evidence-invalid", field, - f"Generated indexer {field} is malformed or unsupported; no default is inferred.", - request_id) from exc - - -def observe(child, approved, diagnostics, request_id): - observed = schedule(child.get("schedule"), present="schedule" in child, - diagnostics=diagnostics, request_id=request_id) - expected = schedule(approved, diagnostics=diagnostics, request_id=request_id) - if approved is not None: - for field in ("interval", "startTime"): - if field == "startTime" and approved.get(field) is None: - continue - if observed.get(field) != expected[field]: - raise failure(diagnostics, "indexer-schedule-constraint-violation", f"schedule.{field}", - f"Generated indexer schedule.{field} violates the approved constraint; this is not an ingestion failure.", - request_id) - if child["schedule"].get(field) != approved.get(field): - note(diagnostics, "info", "indexer-schedule-format-equivalent", f"schedule.{field}", - "Schedule formats represent the same duration or instant.", request_id) - if observed != expected: - note(diagnostics, "warning", "indexer-schedule-unconstrained", "schedule", - "Generated schedule differs on unconstrained timing; recurring work/cost is possible. No schedule was changed or default assumed.", - request_id) - return { - "etag": child["@odata.etag"], "digest": digest(child), - "non_schedule_digest": digest({k: v for k, v in child.items() if k not in {"schedule", "@odata.etag"}}), - "schedule_digest": digest(observed), - "schedule_raw_digest": digest({"present": "schedule" in child, "value": child.get("schedule")}), - } - - -def compare(current, expected, diagnostics, request_id): - if current == expected: - return - if not PROJECTION_FIELDS <= set(expected): - raise failure(diagnostics, "indexer-legacy-evidence-insufficient", "indexer", - "Legacy full-hash evidence differs; no schedule preimage/projection exists. Retain it; do not auto-upgrade.", - request_id) - if current["digest"] == expected["digest"]: - raise failure(diagnostics, "indexer-evidence-inconsistent", "indexer", - "Equal full hashes have inconsistent version/projection evidence.", request_id) - if current["non_schedule_digest"] != expected["non_schedule_digest"]: - raise failure(diagnostics, "indexer-definition-drift", "indexer", - "A non-schedule indexer field changed; schedule policy cannot admit it.", request_id) - if current["schedule_raw_digest"] == expected["schedule_raw_digest"]: - raise failure(diagnostics, "indexer-version-unexplained", "@odata.etag", - "Indexer version/full hash changed without an observed schedule change; the revision is unproven.", request_id) - equivalent = current["schedule_digest"] == expected["schedule_digest"] - note(diagnostics, "info" if equivalent else "warning", - "indexer-schedule-format-equivalent" if equivalent else "indexer-schedule-unconstrained", - "schedule", "Only schedule changed; every other observed field matches the retained projection and approved timing constraints hold.", - request_id) - - -def warnings(diagnostics): - return list(dict.fromkeys(item["message"] for item in diagnostics if item["severity"] == "warning")) +"""Private, versioned evidence for generated indexers; no service operations.""" +from __future__ import annotations + +import re +from datetime import datetime, timedelta, timezone + +try: + from ._common import HelperFailure, digest +except ImportError: + from _common import HelperFailure, digest + + +PROJECTION_FIELDS = {"non_schedule_digest", "schedule_digest", "schedule_raw_digest"} +TICKS = 10_000_000 +DURATION = re.compile(r"P(?:(?P[0-9]{1,9})D)?(?:T(?:(?P[0-9]{1,9})H)?" + r"(?:(?P[0-9]{1,9})M)?(?:(?P[0-9]{1,9})(?:\.(?P[0-9]{1,7}))?S)?)?") +INSTANT = re.compile(r"([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})" + r"(?:\.([0-9]{1,7}))?(Z|[+-][0-9]{2}:[0-9]{2})") + + +def note(diagnostics, severity, code, field, message, request_id): + entry = dict(severity=severity, code=code, field=field, message=message, request_id=request_id) + if entry not in diagnostics: + diagnostics.append(entry) + + +def failure(diagnostics, code, field, message, request_id): + note(diagnostics, "error", code, field, message, request_id) + return HelperFailure(code, message, blocked_at="indexer-verification", request_id=request_id) + + +def interval(value): + match = DURATION.fullmatch(value) if isinstance(value, str) and len(value) <= 96 else None + if not match or not any(match.group(k) for k in ("days", "hours", "minutes", "seconds")): + raise ValueError("Unsupported duration") + if "T" in value and not any(match.group(k) for k in ("hours", "minutes", "seconds")): + raise ValueError("Empty time component") + parts = match.groupdict() + seconds = sum(int(parts[k] or 0) * scale for k, scale in + (("days", 86400), ("hours", 3600), ("minutes", 60), ("seconds", 1))) + ticks = seconds * TICKS + int((parts["fraction"] or "").ljust(7, "0")) + if not 300 * TICKS <= ticks <= 86400 * TICKS: + raise ValueError("Indexer interval outside documented bounds") + return ticks + + +def instant(value): + match = INSTANT.fullmatch(value) if isinstance(value, str) and len(value) <= 40 else None + if not match or match[8] == "-00:00": + raise ValueError("Unknown or unsupported timezone") + year, month, day, hour, minute, second = map(int, match.groups()[:6]) + zone = match[8] + offset = 0 + if zone != "Z": + hours, minutes = int(zone[1:3]), int(zone[4:6]) + if hours > 14 or minutes > 59 or hours == 14 and minutes: + raise ValueError("Invalid offset") + offset = (hours * 60 + minutes) * (-1 if zone[0] == "-" else 1) + stamp = datetime(year, month, day, hour, minute, second, + tzinfo=timezone(timedelta(minutes=offset))).astimezone(timezone.utc) + elapsed = stamp - datetime(1, 1, 1, tzinfo=timezone.utc) + return (elapsed.days * 86400 + elapsed.seconds) * TICKS + int((match[7] or "").ljust(7, "0")) + + +def schedule(value, *, present=True, diagnostics=None, request_id=None): + diagnostics = diagnostics if diagnostics is not None else [] + if value is None: + return {"state": "null" if present else "missing"} + field = "schedule" + try: + if not isinstance(value, dict) or set(value) - {"interval", "startTime"}: + raise ValueError("Unknown shape") + field = "schedule.interval" + period = interval(value.get("interval")) + field = "schedule.startTime" + start = instant(value["startTime"]) if value.get("startTime") is not None else None + return {"state": "object", "interval": period, "startTime": start} + except (ValueError, OverflowError) as exc: + raise failure(diagnostics, "indexer-schedule-evidence-invalid", field, + f"Generated indexer {field} is malformed or unsupported; no default is inferred.", + request_id) from exc + + +def observe(child, approved, diagnostics, request_id): + observed = schedule(child.get("schedule"), present="schedule" in child, + diagnostics=diagnostics, request_id=request_id) + expected = schedule(approved, diagnostics=diagnostics, request_id=request_id) + if approved is not None: + for field in ("interval", "startTime"): + if field == "startTime" and approved.get(field) is None: + continue + if observed.get(field) != expected[field]: + raise failure(diagnostics, "indexer-schedule-constraint-violation", f"schedule.{field}", + f"Generated indexer schedule.{field} violates the approved constraint; this is not an ingestion failure.", + request_id) + if child["schedule"].get(field) != approved.get(field): + note(diagnostics, "info", "indexer-schedule-format-equivalent", f"schedule.{field}", + "Schedule formats represent the same duration or instant.", request_id) + if observed != expected: + note(diagnostics, "warning", "indexer-schedule-unconstrained", "schedule", + "Generated schedule differs on unconstrained timing; recurring work/cost is possible. No schedule was changed or default assumed.", + request_id) + return { + "etag": child["@odata.etag"], "digest": digest(child), + "non_schedule_digest": digest({k: v for k, v in child.items() if k not in {"schedule", "@odata.etag"}}), + "schedule_digest": digest(observed), + "schedule_raw_digest": digest({"present": "schedule" in child, "value": child.get("schedule")}), + } + + +def compare(current, expected, diagnostics, request_id): + if current == expected: + return + if not PROJECTION_FIELDS <= set(expected): + raise failure(diagnostics, "indexer-legacy-evidence-insufficient", "indexer", + "Legacy full-hash evidence differs; no schedule preimage/projection exists. Retain it; do not auto-upgrade.", + request_id) + if current["digest"] == expected["digest"]: + raise failure(diagnostics, "indexer-evidence-inconsistent", "indexer", + "Equal full hashes have inconsistent version/projection evidence.", request_id) + if current["non_schedule_digest"] != expected["non_schedule_digest"]: + raise failure(diagnostics, "indexer-definition-drift", "indexer", + "A non-schedule indexer field changed; schedule policy cannot admit it.", request_id) + if current["schedule_raw_digest"] == expected["schedule_raw_digest"]: + raise failure(diagnostics, "indexer-version-unexplained", "@odata.etag", + "Indexer version/full hash changed without an observed schedule change; the revision is unproven.", request_id) + equivalent = current["schedule_digest"] == expected["schedule_digest"] + note(diagnostics, "info" if equivalent else "warning", + "indexer-schedule-format-equivalent" if equivalent else "indexer-schedule-unconstrained", + "schedule", "Only schedule changed; every other observed field matches the retained projection and approved timing constraints hold.", + request_id) + + +def warnings(diagnostics): + return list(dict.fromkeys(item["message"] for item in diagnostics if item["severity"] == "warning")) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_initial_prompt.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_initial_prompt.py index 5db316b1..cf3e1314 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_initial_prompt.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_initial_prompt.py @@ -1,181 +1,181 @@ -"""Receipt-producing form of the existing local Foundry v1 create-agent fallback.""" -from __future__ import annotations - -import re -from urllib.parse import quote - -try: - from . import prompt_connect, _cleanup_dependencies as dependencies, _cleanup_receipts as receipts - from ._common import HelperFailure, canonical_bytes, digest, reject_secrets, require_allowed_fields, sdk_error_status, sdk_error_metadata -except ImportError: - import prompt_connect, _cleanup_dependencies as dependencies, _cleanup_receipts as receipts - from _common import HelperFailure, canonical_bytes, digest, reject_secrets, require_allowed_fields, sdk_error_status, sdk_error_metadata - - -def fail(code, message): - return HelperFailure(code, message, blocked_at="initial-prompt-creation") - - -def validate(plan): - reject_secrets(plan) - require_allowed_fields(plan, {"operation", "sdk_major", "project_resource_id", "project_endpoint", - "agent", "owner", "cleanup_approved", "inventory_limits", "absence_inventory_digest", - "prerequisites"}, label="Initial Prompt creation") - prompt_connect._project_identity(plan) - if (plan.get("operation") != "create-initial-prompt-agent" or plan.get("sdk_major") != 2 - or plan.get("cleanup_approved") is not False or not isinstance(plan.get("owner"), str) or not plan["owner"].strip()): - raise fail("input-schema-invalid", "Initial Prompt creation needs separate approved creation intent and owner.") - agent = plan.get("agent") - if not isinstance(agent, dict) or set(agent) != {"name", "definition"}: - raise fail("input-schema-invalid", "Select one exact initial agent name and complete definition.") - if not isinstance(agent["name"], str) or not re.fullmatch(r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?", agent["name"]): - raise fail("input-schema-invalid", "Select a v1 agent name, never a suffix or latest.") - definition = agent["definition"] - if (not isinstance(definition, dict) or set(definition) != {"kind", "model", "instructions", "tools"} - or definition["kind"] != "prompt" or not isinstance(definition["model"], str) or not definition["model"].strip() - or not isinstance(definition["instructions"], str) or definition["tools"] != []): - raise fail("input-schema-invalid", "Initial creation permits only an explicit Prompt model/instructions with no tools.") - prerequisites = plan.get("prerequisites") - if (not isinstance(prerequisites, dict) or set(prerequisites) != {"project", "model", "identity", "network"} - or any(not isinstance(v, str) or not v.strip() for v in prerequisites.values())): - raise fail("initial-prerequisites-unverified", "Retain the existing local-creation procedure's project/model/identity/network evidence.") - dependencies.limits(plan.get("inventory_limits")) - if not isinstance(plan.get("absence_inventory_digest"), str) or not dependencies.SHA.fullmatch(plan["absence_inventory_digest"]): - raise fail("input-schema-invalid", "The complete scoped absence inventory must be fingerprinted.") - - -def absence(plan, sdk_loader): - Client, _, _, _, extras = sdk_loader() - Credential, AzureError = extras - client = Client(endpoint=plan["project_endpoint"], credential=Credential()) - try: - names = [] - for item in dependencies._paged(client.agents.list(), dependencies.limits(plan["inventory_limits"])): - name = getattr(item, "name", None) - if not isinstance(name, str) or not name.strip() or name in names: - raise fail("initial-inventory-incomplete", "Agent absence needs a complete, unambiguous project inventory.") - names.append(name) - if plan["agent"]["name"] in names: - raise fail("initial-agent-exists", "Retain the existing agent; return to Connect rather than creating a version.") - try: - client.agents.get(agent_name=plan["agent"]["name"]) - except AzureError as exc: - if sdk_error_status(exc) == 404: - return digest(sorted(names)) - raise - raise fail("initial-agent-exists", "Exact agent GET found an existing identity.") - except AzureError as exc: - raise HelperFailure(message="Complete initial-agent absence discovery failed.", blocked_at="initial-prompt-creation", - **sdk_error_metadata(exc, "initial-inventory-failed")) from exc - except (AttributeError, TypeError, ValueError) as exc: - raise fail("initial-api-unavailable", "SDK complete agent listing and exact agent GET are required.") from exc - finally: - client.close() - - -def plan(request, sdk_loader): - if not isinstance(request, dict): - raise fail("input-schema-invalid", "Initial creation request must be a closed object.") - require_allowed_fields(request, {"schema_version", "project_resource_id", "project_endpoint", "agent", - "owner", "inventory_limits", "prerequisites"}, label="Initial Prompt request") - if request.get("schema_version") != "1.0": - raise fail("input-schema-invalid", "Initial request requires schema_version 1.0.") - proposed = {k: v for k, v in request.items() if k != "schema_version"} - proposed.update(operation="create-initial-prompt-agent", sdk_major=2, cleanup_approved=False, - inventory_limits=dependencies.limits(request.get("inventory_limits")), - absence_inventory_digest="sha256:" + "0" * 64) - validate(proposed) - proposed["absence_inventory_digest"] = absence(proposed, sdk_loader) - fingerprint = digest(proposed) - return {"status": "planned", "outcome": "create-initial-prompt-agent", "plan_fingerprint": fingerprint, - "execution_required": True, "mutation_approval_required": True, - "execution_input": {"schema_version": "1.0", "plan": proposed, - "approval": {"confirmed": False, "fingerprint": fingerprint}}, - "approval_summary": {"create": [proposed["agent"]["name"]], "retain": ["existing agents", "projects", "models", "grants"], - "model_readiness": "Provided prerequisite evidence must be independently verified before approval.", - "invocation": "not performed", "cleanup": "separate owned-version approval; container retained"}} - - -def verify_created(plan, target, birth, sdk_loader): - Client, _, _, _, extras = sdk_loader() - Credential, AzureError = extras - client = Client(endpoint=plan["project_endpoint"], credential=Credential()) - try: - agent = client.agents.get(agent_name=target["name"]) - if agent.name != target["name"]: - raise fail("initial-readback-invalid", "Initial agent identity readback changed.") - versions = [ - str(item.version) for item in dependencies._paged( - client.agents.list_versions(agent_name=target["name"], include_drafts=True), - dependencies.limits(plan["inventory_limits"])) - ] - if versions != [target["version"]]: - raise fail("initial-version-inventory-changed", "Initial creation must retain exactly its acknowledged version, with no duplicates or unexpected versions.") - current = client.agents.get_version(agent_name=target["name"], agent_version=target["version"]) - if (current.name != target["name"] or str(current.version) != target["version"] - or current.definition.as_dict() != plan["agent"]["definition"] - or receipts.version_identity(current) != birth): - raise fail("definition-drift", "Exact initial version differs from its native birth snapshot.") - except AzureError as exc: - raise HelperFailure(message="Initial agent/version verification failed.", blocked_at="initial-prompt-creation", - **sdk_error_metadata(exc, "initial-readback-failed")) from exc - except (AttributeError, TypeError, ValueError) as exc: - raise fail("initial-api-unavailable", "Complete draft-inclusive agent/version readback is required.") from exc - finally: - client.close() - - -def execute(document, *, capture, token_provider, transport, sdk_loader): - plan = document["plan"] - validate(plan) - if capture is None or capture.plan_digest != document["_computed_fingerprint"] or capture.owner != plan["owner"]: - raise fail("cleanup-receipt-required", "Initial creation requires its original approved input and protected receipt directory.") - if absence(plan, sdk_loader) != plan["absence_inventory_digest"]: - raise fail("initial-inventory-drift", "Project inventory changed; replan rather than weakening absence protection.") - token = token_provider("https://ai.azure.com/") - url = plan["project_endpoint"].rstrip("/") + "/agents?api-version=v1" - target = receipts.project_target(plan, "prompt-agent-version", name=plan["agent"]["name"], version="unverified") - try: - result = transport("POST", url, token, body=canonical_bytes(plan["agent"]), headers={"Content-Type": "application/json"}) - except HelperFailure as failure: - if failure.http_status in (None, 408, 429) or (failure.http_status or 0) >= 500: - failure.partial = True - failure.warnings.append("Initial creation outcome unproven; no replay, ownership adoption or automatic cleanup.") - raise - write = {"action": "created", "type": "prompt-agent-version", "name": plan["agent"]["name"]} - if result.status != 200: - raise HelperFailure("initial-create-failed", "Create-agent returned an unexpected status; no replay or ownership adoption.", - blocked_at="initial-prompt-creation", status=result.status, - request_id=result.request_id, partial=(200 <= result.status < 300 or result.status in (408, 429) or result.status >= 500)) - try: - if not isinstance(result.body, dict): - raise fail("initial-create-response-invalid", "Foundry v1 create-agent did not return the documented HTTP 200 AgentObject.") - versions = result.body.get("versions") - latest = versions.get("latest") if isinstance(versions, dict) else None - latest = latest if isinstance(latest, dict) else {} - target["version"] = latest.get("version", "unverified") - capture.start(target, {"operation": "agents.create", "status": result.status, "request_id": result.request_id, - "definition_digest": digest(latest.get("definition")), "version": target["version"], - "etag_evidence": None, "generated": None, "version_identity": receipts.version_identity(latest)}) - if (result.body.get("name") != plan["agent"]["name"] or latest.get("name") != plan["agent"]["name"] - or not isinstance(target["version"], str) or not re.fullmatch(r"[1-9][0-9]*", target["version"]) - or latest.get("definition") != plan["agent"]["definition"] or receipts.version_identity(latest) is None): - raise fail("initial-create-response-invalid", "Native created agent/version differs from approved initial intent.") - write["version"] = target["version"] - readback = transport("GET", plan["project_endpoint"].rstrip("/") + "/agents/" + quote(target["name"], safe="") - + "/versions/" + quote(target["version"], safe="") + "?api-version=v1", token) - if (readback.status != 200 or not isinstance(readback.body, dict) - or readback.body.get("name") != target["name"] or readback.body.get("version") != target["version"] - or readback.body.get("definition") != latest["definition"]): - raise fail("definition-drift", "Initial version readback differs from the original successful create-agent response.") - verify_created(plan, target, receipts.version_identity(latest), sdk_loader) - capture.finish(target, {"definition_digest": digest(latest["definition"]), "etag": None, "generated": [], - "version_identity": receipts.version_identity(readback.body)}) - except HelperFailure as failure: - failure.partial = True - failure.writes.insert(0, write) - raise - return {"status": "completed", "outcome": "create-initial-prompt-agent", - "approved_plan": document["approval"], "resources": {"created": [target], "reused": []}, - "ownership": {"owner": plan["owner"], "run_owned": [target], "reused_not_owned": []}, - "agent_invocation": "not-run", "cleanup": "separate version-only approval; never delete the container"} +"""Receipt-producing form of the existing local Foundry v1 create-agent fallback.""" +from __future__ import annotations + +import re +from urllib.parse import quote + +try: + from . import prompt_connect, _cleanup_dependencies as dependencies, _cleanup_receipts as receipts + from ._common import HelperFailure, canonical_bytes, digest, reject_secrets, require_allowed_fields, sdk_error_status, sdk_error_metadata +except ImportError: + import prompt_connect, _cleanup_dependencies as dependencies, _cleanup_receipts as receipts + from _common import HelperFailure, canonical_bytes, digest, reject_secrets, require_allowed_fields, sdk_error_status, sdk_error_metadata + + +def fail(code, message): + return HelperFailure(code, message, blocked_at="initial-prompt-creation") + + +def validate(plan): + reject_secrets(plan) + require_allowed_fields(plan, {"operation", "sdk_major", "project_resource_id", "project_endpoint", + "agent", "owner", "cleanup_approved", "inventory_limits", "absence_inventory_digest", + "prerequisites"}, label="Initial Prompt creation") + prompt_connect._project_identity(plan) + if (plan.get("operation") != "create-initial-prompt-agent" or plan.get("sdk_major") != 2 + or plan.get("cleanup_approved") is not False or not isinstance(plan.get("owner"), str) or not plan["owner"].strip()): + raise fail("input-schema-invalid", "Initial Prompt creation needs separate approved creation intent and owner.") + agent = plan.get("agent") + if not isinstance(agent, dict) or set(agent) != {"name", "definition"}: + raise fail("input-schema-invalid", "Select one exact initial agent name and complete definition.") + if not isinstance(agent["name"], str) or not re.fullmatch(r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?", agent["name"]): + raise fail("input-schema-invalid", "Select a v1 agent name, never a suffix or latest.") + definition = agent["definition"] + if (not isinstance(definition, dict) or set(definition) != {"kind", "model", "instructions", "tools"} + or definition["kind"] != "prompt" or not isinstance(definition["model"], str) or not definition["model"].strip() + or not isinstance(definition["instructions"], str) or definition["tools"] != []): + raise fail("input-schema-invalid", "Initial creation permits only an explicit Prompt model/instructions with no tools.") + prerequisites = plan.get("prerequisites") + if (not isinstance(prerequisites, dict) or set(prerequisites) != {"project", "model", "identity", "network"} + or any(not isinstance(v, str) or not v.strip() for v in prerequisites.values())): + raise fail("initial-prerequisites-unverified", "Retain the existing local-creation procedure's project/model/identity/network evidence.") + dependencies.limits(plan.get("inventory_limits")) + if not isinstance(plan.get("absence_inventory_digest"), str) or not dependencies.SHA.fullmatch(plan["absence_inventory_digest"]): + raise fail("input-schema-invalid", "The complete scoped absence inventory must be fingerprinted.") + + +def absence(plan, sdk_loader): + Client, _, _, _, extras = sdk_loader() + Credential, AzureError = extras + client = Client(endpoint=plan["project_endpoint"], credential=Credential()) + try: + names = [] + for item in dependencies._paged(client.agents.list(), dependencies.limits(plan["inventory_limits"])): + name = getattr(item, "name", None) + if not isinstance(name, str) or not name.strip() or name in names: + raise fail("initial-inventory-incomplete", "Agent absence needs a complete, unambiguous project inventory.") + names.append(name) + if plan["agent"]["name"] in names: + raise fail("initial-agent-exists", "Retain the existing agent; return to Connect rather than creating a version.") + try: + client.agents.get(agent_name=plan["agent"]["name"]) + except AzureError as exc: + if sdk_error_status(exc) == 404: + return digest(sorted(names)) + raise + raise fail("initial-agent-exists", "Exact agent GET found an existing identity.") + except AzureError as exc: + raise HelperFailure(message="Complete initial-agent absence discovery failed.", blocked_at="initial-prompt-creation", + **sdk_error_metadata(exc, "initial-inventory-failed")) from exc + except (AttributeError, TypeError, ValueError) as exc: + raise fail("initial-api-unavailable", "SDK complete agent listing and exact agent GET are required.") from exc + finally: + client.close() + + +def plan(request, sdk_loader): + if not isinstance(request, dict): + raise fail("input-schema-invalid", "Initial creation request must be a closed object.") + require_allowed_fields(request, {"schema_version", "project_resource_id", "project_endpoint", "agent", + "owner", "inventory_limits", "prerequisites"}, label="Initial Prompt request") + if request.get("schema_version") != "1.0": + raise fail("input-schema-invalid", "Initial request requires schema_version 1.0.") + proposed = {k: v for k, v in request.items() if k != "schema_version"} + proposed.update(operation="create-initial-prompt-agent", sdk_major=2, cleanup_approved=False, + inventory_limits=dependencies.limits(request.get("inventory_limits")), + absence_inventory_digest="sha256:" + "0" * 64) + validate(proposed) + proposed["absence_inventory_digest"] = absence(proposed, sdk_loader) + fingerprint = digest(proposed) + return {"status": "planned", "outcome": "create-initial-prompt-agent", "plan_fingerprint": fingerprint, + "execution_required": True, "mutation_approval_required": True, + "execution_input": {"schema_version": "1.0", "plan": proposed, + "approval": {"confirmed": False, "fingerprint": fingerprint}}, + "approval_summary": {"create": [proposed["agent"]["name"]], "retain": ["existing agents", "projects", "models", "grants"], + "model_readiness": "Provided prerequisite evidence must be independently verified before approval.", + "invocation": "not performed", "cleanup": "separate owned-version approval; container retained"}} + + +def verify_created(plan, target, birth, sdk_loader): + Client, _, _, _, extras = sdk_loader() + Credential, AzureError = extras + client = Client(endpoint=plan["project_endpoint"], credential=Credential()) + try: + agent = client.agents.get(agent_name=target["name"]) + if agent.name != target["name"]: + raise fail("initial-readback-invalid", "Initial agent identity readback changed.") + versions = [ + str(item.version) for item in dependencies._paged( + client.agents.list_versions(agent_name=target["name"], include_drafts=True), + dependencies.limits(plan["inventory_limits"])) + ] + if versions != [target["version"]]: + raise fail("initial-version-inventory-changed", "Initial creation must retain exactly its acknowledged version, with no duplicates or unexpected versions.") + current = client.agents.get_version(agent_name=target["name"], agent_version=target["version"]) + if (current.name != target["name"] or str(current.version) != target["version"] + or current.definition.as_dict() != plan["agent"]["definition"] + or receipts.version_identity(current) != birth): + raise fail("definition-drift", "Exact initial version differs from its native birth snapshot.") + except AzureError as exc: + raise HelperFailure(message="Initial agent/version verification failed.", blocked_at="initial-prompt-creation", + **sdk_error_metadata(exc, "initial-readback-failed")) from exc + except (AttributeError, TypeError, ValueError) as exc: + raise fail("initial-api-unavailable", "Complete draft-inclusive agent/version readback is required.") from exc + finally: + client.close() + + +def execute(document, *, capture, token_provider, transport, sdk_loader): + plan = document["plan"] + validate(plan) + if capture is None or capture.plan_digest != document["_computed_fingerprint"] or capture.owner != plan["owner"]: + raise fail("cleanup-receipt-required", "Initial creation requires its original approved input and protected receipt directory.") + if absence(plan, sdk_loader) != plan["absence_inventory_digest"]: + raise fail("initial-inventory-drift", "Project inventory changed; replan rather than weakening absence protection.") + token = token_provider("https://ai.azure.com/") + url = plan["project_endpoint"].rstrip("/") + "/agents?api-version=v1" + target = receipts.project_target(plan, "prompt-agent-version", name=plan["agent"]["name"], version="unverified") + try: + result = transport("POST", url, token, body=canonical_bytes(plan["agent"]), headers={"Content-Type": "application/json"}) + except HelperFailure as failure: + if failure.http_status in (None, 408, 429) or (failure.http_status or 0) >= 500: + failure.partial = True + failure.warnings.append("Initial creation outcome unproven; no replay, ownership adoption or automatic cleanup.") + raise + write = {"action": "created", "type": "prompt-agent-version", "name": plan["agent"]["name"]} + if result.status != 200: + raise HelperFailure("initial-create-failed", "Create-agent returned an unexpected status; no replay or ownership adoption.", + blocked_at="initial-prompt-creation", status=result.status, + request_id=result.request_id, partial=(200 <= result.status < 300 or result.status in (408, 429) or result.status >= 500)) + try: + if not isinstance(result.body, dict): + raise fail("initial-create-response-invalid", "Foundry v1 create-agent did not return the documented HTTP 200 AgentObject.") + versions = result.body.get("versions") + latest = versions.get("latest") if isinstance(versions, dict) else None + latest = latest if isinstance(latest, dict) else {} + target["version"] = latest.get("version", "unverified") + capture.start(target, {"operation": "agents.create", "status": result.status, "request_id": result.request_id, + "definition_digest": digest(latest.get("definition")), "version": target["version"], + "etag_evidence": None, "generated": None, "version_identity": receipts.version_identity(latest)}) + if (result.body.get("name") != plan["agent"]["name"] or latest.get("name") != plan["agent"]["name"] + or not isinstance(target["version"], str) or not re.fullmatch(r"[1-9][0-9]*", target["version"]) + or latest.get("definition") != plan["agent"]["definition"] or receipts.version_identity(latest) is None): + raise fail("initial-create-response-invalid", "Native created agent/version differs from approved initial intent.") + write["version"] = target["version"] + readback = transport("GET", plan["project_endpoint"].rstrip("/") + "/agents/" + quote(target["name"], safe="") + + "/versions/" + quote(target["version"], safe="") + "?api-version=v1", token) + if (readback.status != 200 or not isinstance(readback.body, dict) + or readback.body.get("name") != target["name"] or readback.body.get("version") != target["version"] + or readback.body.get("definition") != latest["definition"]): + raise fail("definition-drift", "Initial version readback differs from the original successful create-agent response.") + verify_created(plan, target, receipts.version_identity(latest), sdk_loader) + capture.finish(target, {"definition_digest": digest(latest["definition"]), "etag": None, "generated": [], + "version_identity": receipts.version_identity(readback.body)}) + except HelperFailure as failure: + failure.partial = True + failure.writes.insert(0, write) + raise + return {"status": "completed", "outcome": "create-initial-prompt-agent", + "approved_plan": document["approval"], "resources": {"created": [target], "reused": []}, + "ownership": {"owner": plan["owner"], "run_owned": [target], "reused_not_owned": []}, + "agent_invocation": "not-run", "cleanup": "separate version-only approval; never delete the container"} diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_progress.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_progress.py index f9e08c92..8d1e0ee7 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_progress.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_progress.py @@ -1,259 +1,259 @@ -"""Bounded, content-free observations; never a service execution controller.""" -from __future__ import annotations - -from functools import wraps -from contextlib import contextmanager -import copy -from datetime import datetime -import json -import math -import os -import sys -import time -import threading - -try: - from ._common import HelperFailure -except ImportError: - from _common import HelperFailure - - -STAGES = { - "file-source": ("validation", "source-reconciliation", "file-inventory", "file-upload", "file-readback"), - "file-upload": ("file-inventory", "file-upload", "file-readback"), - "blob-source": ("validation", "blob-inventory", "source-reconciliation", "ingestion-cycle", - "blob-readback", "source-readback"), - "blob-monitor": ("ingestion-cycle",), - "blob-capture": ("evidence-validation", "context-check", "source-binding", "blob-inventory", "checkpoint"), - "blob-recheck": ("evidence-validation", "context-check", "source-binding", "blob-inventory", - "ingestion-cycle", "blob-readback", "source-readback", "context-readback"), - "search-bootstrap": ("validation", "context-check", "region-check", "absence-check", "search-submit", - "arm-wait", "arm-readback"), -} -COUNTS = frozenset(("uploads_acknowledged", "files_reused", "files_verified", - "status_checks", "cycle_updates_processed", "cycle_items_skipped", - "files_failed", "files_unverified", "files_not_attempted", "files_pending", "files_ingested")) -FILE_HEARTBEAT_SECONDS = 5 -IO_WARNING = "progress-output-failed: stderr progress could not be written; execution result is authoritative." -SHUTDOWN_WARNING = "progress-shutdown-flush-unresolved: failed stderr could not be redirected; shutdown may override the exit code." - - -def validate_blob_progress(value): - fields = {"schema_version", "phase", "run_start", "processed", "failed", "skipped", - "unit", "total", "remaining", "denominator", "synchronization_status", - "elapsed_seconds", "next_check_seconds"} - if not isinstance(value, dict) or set(value) != fields: - raise ValueError("Invalid Blob progress shape.") - if (value["schema_version"] != "1.0" or value["unit"] != "item-updates" - or value["denominator"] != "not-comparable-to-files" - or value["total"] is not None or value["remaining"] is not None - or value["phase"] not in ("ingesting", "waiting", "throttled", "paused", "completed", "failed") - or value["synchronization_status"] not in ("not-reported", "active", "creating", "deleting")): - raise ValueError("Invalid Blob progress labels.") - for field in ("processed", "failed", "skipped"): - count = value[field] - if count is not None and (type(count) is not int or count < 0): - raise ValueError("Invalid Blob progress count.") - for field in ("elapsed_seconds", "next_check_seconds"): - number = value[field] - if field == "next_check_seconds" and number is None: - continue - if type(number) not in (int, float) or not math.isfinite(number) or number < 0: - raise ValueError("Invalid Blob progress timing.") - if value["next_check_seconds"] is not None and value["next_check_seconds"] > 60: - raise ValueError("Invalid Blob progress wait.") - start = value["run_start"] - if start is not None: - if not isinstance(start, str) or len(start) > 40: - raise ValueError("Invalid Blob progress run.") - parsed = datetime.fromisoformat(start.replace("Z", "+00:00")) - if parsed.tzinfo is None or parsed.isoformat() != start: - raise ValueError("Blob progress run must be a canonical timestamp.") - - -class Progress: - def __init__(self, workflow, *, enabled=True, stream=None, clock=time.monotonic): - self.stages = STAGES[workflow] - self.workflow = workflow - self.enabled = enabled - self.stream = stream - self.clock = clock - self.depth = 0 - self.stage = self.stages[0] - self.counts = {} - self.started = None - self.elapsed = 0.0 - self.last_sent = None - self.last_stage = None - self.output_failed = False - self.shutdown_flush_unresolved = False - self.blob_progress = None - self.last_blob_phase = None - - def blob_update(self, value): - if not self.workflow.startswith("blob-"): - raise ValueError("Blob progress cannot describe a File or ARM workflow.") - validate_blob_progress(value) - self.blob_progress = copy.deepcopy(value) - for field, target in (("processed", "cycle_updates_processed"), ("skipped", "cycle_items_skipped")): - self.counts.pop(target, None) - if value[field] is not None: - self.counts[target] = value[field] - self.update("ingestion-cycle") - - def update(self, stage, **counts): - if stage not in self.stages or self.stages.index(stage) < self.stages.index(self.stage): - raise ValueError("Invalid progress stage transition.") - if any(key not in COUNTS or type(value) is not int or value < 0 for key, value in counts.items()): - raise ValueError("Invalid progress count.") - self.stage = stage - self.counts.update(counts) - self._emit("running") - - def waiting(self, seconds): - if not isinstance(seconds, (int, float)) or not math.isfinite(seconds) or not 0 <= seconds <= 30: - raise ValueError("Invalid recovery wait.") - self._emit("waiting", wait={"reason": "http-429", "seconds": seconds}) - - @contextmanager - def processing_file(self, ordinal, total, attempt): - if (self.workflow not in {"file-source", "file-upload"} or type(ordinal) is not int - or type(total) is not int or not 1 <= ordinal <= total <= 200 or attempt not in (1, 2)): - raise ValueError("Invalid active file.") - active = {"ordinal": ordinal, "total": total, "attempt": attempt} - if not self.enabled or self.output_failed: - yield - return - started = self.clock() - stop = threading.Event() - - def pulse(): - while not stop.wait(FILE_HEARTBEAT_SECONDS): - if self.output_failed: - return - self._emit("processing", active=active, active_started=started) - - self._emit("processing", active=active, active_started=started) - worker = threading.Thread(target=pulse, name="foundry-file-progress", daemon=True) - worker.start() - try: - yield - finally: - stop.set() - worker.join() - - def _emit(self, state, *, wait=None, active=None, active_started=None): - if not self.enabled or self.output_failed: - return - observed = self.clock() - # Clock regressions/nonfinite observations must not create negative time - # or bypass throttling. Progress never shares the service deadline clock. - if math.isfinite(observed): - if self.started is None: - self.started = observed - delta = observed - self.started - if math.isfinite(delta): - self.elapsed = max(self.elapsed, delta) - phase = self.blob_progress["phase"] if self.blob_progress is not None else None - if (state == "running" and self.stage == self.last_stage and phase == self.last_blob_phase - and self.last_sent is not None and self.elapsed - self.last_sent < 1.0): - return - event = { - "event": "progress", "workflow": self.workflow, "activity": self.stage, - "state": state, "elapsed_seconds": round(self.elapsed, 3), - "remaining_checks": [] if state == "completed" else list(self.stages[self.stages.index(self.stage) + 1:]), - } - if self.counts: - event["completed_counts"] = dict(self.counts) - if wait is not None: - event["wait"] = wait - if active is not None: - duration = observed - active_started - event["processing_file"] = {**active, "elapsed_seconds": round(max(0, duration), 3) if math.isfinite(duration) else 0} - if self.blob_progress is not None: - value = self.blob_progress - event["blob_progress"] = copy.deepcopy(value) - counts = "; ".join(f"{field}={value[field] if value[field] is not None else 'unknown'}" - for field in ("processed", "failed", "skipped")) - next_check = ("paused/resumable" if value["phase"] == "paused" else "none") if value["next_check_seconds"] is None else f"{value['next_check_seconds']:g}s" - event["message"] = ( - f"Blob ingestion observation {value['phase']}: item updates {counts}; file total/remaining unknown " - f"(not comparable); elapsed={value['elapsed_seconds']:g}s; next check={next_check}." - ) - stream = self.stream if self.stream is not None else sys.stderr - try: - text = json.dumps(event, sort_keys=True, separators=(",", ":")) + "\n" - if stream.write(text) != len(text): - raise OSError("Short progress write.") - stream.flush() - except (OSError, UnicodeError, ValueError): - # Continue the approved operation, retaining a fixed secondary warning - # even when stderr itself is broken. Never replace a primary failure. - self.output_failed = True - self._disable_failed_default_stderr(stream) - self.last_sent, self.last_stage = self.elapsed, self.stage - self.last_blob_phase = phase - - def _disable_failed_default_stderr(self, stream): - if self.stream is not None or stream is not sys.__stderr__ or stream.closed: - return - try: - if stream.fileno() == 2: - # A failed TextIOWrapper flush can retain pending bytes. Redirect - # only the failed native stderr so shutdown can drain that buffer - # without replacing the authoritative process exit status. - sink = os.open(os.devnull, os.O_WRONLY) - try: - os.dup2(sink, 2) - finally: - if sink != 2: - os.close(sink) - except (OSError, ValueError): - self.shutdown_flush_unresolved = True - - def finish(self, result=None, failure=None): - if self.blob_progress is not None and failure is not None: - self.blob_progress.update(phase="failed", next_check_seconds=None) - if failure is not None: - state = "partial" if failure.partial or failure.writes else "blocked" - else: - state = {"verified": "completed", "unverified": "blocked"}.get(result["status"], result["status"]) - if state not in {"completed", "blocked", "partial"}: - raise ValueError("Invalid progress terminal state.") - self._emit(state) - if self.output_failed: - warnings = failure.warnings if failure is not None else result.setdefault("warnings", []) - if IO_WARNING not in warnings: - warnings.append(IO_WARNING) - if self.shutdown_flush_unresolved and SHUTDOWN_WARNING not in warnings: - warnings.append(SHUTDOWN_WARNING) - - -def reporting(workflow): - """One reporter and terminal event across nested public entrypoints.""" - def decorate(function): - @wraps(function) - def wrapped(*args, progress=None, **kwargs): - if progress is None: - progress = Progress(workflow, enabled=False) - outer = progress.depth == 0 - progress.depth += 1 - try: - result = function(*args, progress=progress, **kwargs) - except HelperFailure as failure: - if outer: - progress.finish(failure=failure) - raise - else: - if outer: - progress.finish(result=result) - return result - finally: - progress.depth -= 1 - return wrapped - return decorate - - -def add_progress_argument(parser): - parser.add_argument("--no-progress", dest="progress", action="store_false", - help="Suppress content-free execution progress on stderr.") +"""Bounded, content-free observations; never a service execution controller.""" +from __future__ import annotations + +from functools import wraps +from contextlib import contextmanager +import copy +from datetime import datetime +import json +import math +import os +import sys +import time +import threading + +try: + from ._common import HelperFailure +except ImportError: + from _common import HelperFailure + + +STAGES = { + "file-source": ("validation", "source-reconciliation", "file-inventory", "file-upload", "file-readback"), + "file-upload": ("file-inventory", "file-upload", "file-readback"), + "blob-source": ("validation", "blob-inventory", "source-reconciliation", "ingestion-cycle", + "blob-readback", "source-readback"), + "blob-monitor": ("ingestion-cycle",), + "blob-capture": ("evidence-validation", "context-check", "source-binding", "blob-inventory", "checkpoint"), + "blob-recheck": ("evidence-validation", "context-check", "source-binding", "blob-inventory", + "ingestion-cycle", "blob-readback", "source-readback", "context-readback"), + "search-bootstrap": ("validation", "context-check", "region-check", "absence-check", "search-submit", + "arm-wait", "arm-readback"), +} +COUNTS = frozenset(("uploads_acknowledged", "files_reused", "files_verified", + "status_checks", "cycle_updates_processed", "cycle_items_skipped", + "files_failed", "files_unverified", "files_not_attempted", "files_pending", "files_ingested")) +FILE_HEARTBEAT_SECONDS = 5 +IO_WARNING = "progress-output-failed: stderr progress could not be written; execution result is authoritative." +SHUTDOWN_WARNING = "progress-shutdown-flush-unresolved: failed stderr could not be redirected; shutdown may override the exit code." + + +def validate_blob_progress(value): + fields = {"schema_version", "phase", "run_start", "processed", "failed", "skipped", + "unit", "total", "remaining", "denominator", "synchronization_status", + "elapsed_seconds", "next_check_seconds"} + if not isinstance(value, dict) or set(value) != fields: + raise ValueError("Invalid Blob progress shape.") + if (value["schema_version"] != "1.0" or value["unit"] != "item-updates" + or value["denominator"] != "not-comparable-to-files" + or value["total"] is not None or value["remaining"] is not None + or value["phase"] not in ("ingesting", "waiting", "throttled", "paused", "completed", "failed") + or value["synchronization_status"] not in ("not-reported", "active", "creating", "deleting")): + raise ValueError("Invalid Blob progress labels.") + for field in ("processed", "failed", "skipped"): + count = value[field] + if count is not None and (type(count) is not int or count < 0): + raise ValueError("Invalid Blob progress count.") + for field in ("elapsed_seconds", "next_check_seconds"): + number = value[field] + if field == "next_check_seconds" and number is None: + continue + if type(number) not in (int, float) or not math.isfinite(number) or number < 0: + raise ValueError("Invalid Blob progress timing.") + if value["next_check_seconds"] is not None and value["next_check_seconds"] > 60: + raise ValueError("Invalid Blob progress wait.") + start = value["run_start"] + if start is not None: + if not isinstance(start, str) or len(start) > 40: + raise ValueError("Invalid Blob progress run.") + parsed = datetime.fromisoformat(start.replace("Z", "+00:00")) + if parsed.tzinfo is None or parsed.isoformat() != start: + raise ValueError("Blob progress run must be a canonical timestamp.") + + +class Progress: + def __init__(self, workflow, *, enabled=True, stream=None, clock=time.monotonic): + self.stages = STAGES[workflow] + self.workflow = workflow + self.enabled = enabled + self.stream = stream + self.clock = clock + self.depth = 0 + self.stage = self.stages[0] + self.counts = {} + self.started = None + self.elapsed = 0.0 + self.last_sent = None + self.last_stage = None + self.output_failed = False + self.shutdown_flush_unresolved = False + self.blob_progress = None + self.last_blob_phase = None + + def blob_update(self, value): + if not self.workflow.startswith("blob-"): + raise ValueError("Blob progress cannot describe a File or ARM workflow.") + validate_blob_progress(value) + self.blob_progress = copy.deepcopy(value) + for field, target in (("processed", "cycle_updates_processed"), ("skipped", "cycle_items_skipped")): + self.counts.pop(target, None) + if value[field] is not None: + self.counts[target] = value[field] + self.update("ingestion-cycle") + + def update(self, stage, **counts): + if stage not in self.stages or self.stages.index(stage) < self.stages.index(self.stage): + raise ValueError("Invalid progress stage transition.") + if any(key not in COUNTS or type(value) is not int or value < 0 for key, value in counts.items()): + raise ValueError("Invalid progress count.") + self.stage = stage + self.counts.update(counts) + self._emit("running") + + def waiting(self, seconds): + if not isinstance(seconds, (int, float)) or not math.isfinite(seconds) or not 0 <= seconds <= 30: + raise ValueError("Invalid recovery wait.") + self._emit("waiting", wait={"reason": "http-429", "seconds": seconds}) + + @contextmanager + def processing_file(self, ordinal, total, attempt): + if (self.workflow not in {"file-source", "file-upload"} or type(ordinal) is not int + or type(total) is not int or not 1 <= ordinal <= total <= 200 or attempt not in (1, 2)): + raise ValueError("Invalid active file.") + active = {"ordinal": ordinal, "total": total, "attempt": attempt} + if not self.enabled or self.output_failed: + yield + return + started = self.clock() + stop = threading.Event() + + def pulse(): + while not stop.wait(FILE_HEARTBEAT_SECONDS): + if self.output_failed: + return + self._emit("processing", active=active, active_started=started) + + self._emit("processing", active=active, active_started=started) + worker = threading.Thread(target=pulse, name="foundry-file-progress", daemon=True) + worker.start() + try: + yield + finally: + stop.set() + worker.join() + + def _emit(self, state, *, wait=None, active=None, active_started=None): + if not self.enabled or self.output_failed: + return + observed = self.clock() + # Clock regressions/nonfinite observations must not create negative time + # or bypass throttling. Progress never shares the service deadline clock. + if math.isfinite(observed): + if self.started is None: + self.started = observed + delta = observed - self.started + if math.isfinite(delta): + self.elapsed = max(self.elapsed, delta) + phase = self.blob_progress["phase"] if self.blob_progress is not None else None + if (state == "running" and self.stage == self.last_stage and phase == self.last_blob_phase + and self.last_sent is not None and self.elapsed - self.last_sent < 1.0): + return + event = { + "event": "progress", "workflow": self.workflow, "activity": self.stage, + "state": state, "elapsed_seconds": round(self.elapsed, 3), + "remaining_checks": [] if state == "completed" else list(self.stages[self.stages.index(self.stage) + 1:]), + } + if self.counts: + event["completed_counts"] = dict(self.counts) + if wait is not None: + event["wait"] = wait + if active is not None: + duration = observed - active_started + event["processing_file"] = {**active, "elapsed_seconds": round(max(0, duration), 3) if math.isfinite(duration) else 0} + if self.blob_progress is not None: + value = self.blob_progress + event["blob_progress"] = copy.deepcopy(value) + counts = "; ".join(f"{field}={value[field] if value[field] is not None else 'unknown'}" + for field in ("processed", "failed", "skipped")) + next_check = ("paused/resumable" if value["phase"] == "paused" else "none") if value["next_check_seconds"] is None else f"{value['next_check_seconds']:g}s" + event["message"] = ( + f"Blob ingestion observation {value['phase']}: item updates {counts}; file total/remaining unknown " + f"(not comparable); elapsed={value['elapsed_seconds']:g}s; next check={next_check}." + ) + stream = self.stream if self.stream is not None else sys.stderr + try: + text = json.dumps(event, sort_keys=True, separators=(",", ":")) + "\n" + if stream.write(text) != len(text): + raise OSError("Short progress write.") + stream.flush() + except (OSError, UnicodeError, ValueError): + # Continue the approved operation, retaining a fixed secondary warning + # even when stderr itself is broken. Never replace a primary failure. + self.output_failed = True + self._disable_failed_default_stderr(stream) + self.last_sent, self.last_stage = self.elapsed, self.stage + self.last_blob_phase = phase + + def _disable_failed_default_stderr(self, stream): + if self.stream is not None or stream is not sys.__stderr__ or stream.closed: + return + try: + if stream.fileno() == 2: + # A failed TextIOWrapper flush can retain pending bytes. Redirect + # only the failed native stderr so shutdown can drain that buffer + # without replacing the authoritative process exit status. + sink = os.open(os.devnull, os.O_WRONLY) + try: + os.dup2(sink, 2) + finally: + if sink != 2: + os.close(sink) + except (OSError, ValueError): + self.shutdown_flush_unresolved = True + + def finish(self, result=None, failure=None): + if self.blob_progress is not None and failure is not None: + self.blob_progress.update(phase="failed", next_check_seconds=None) + if failure is not None: + state = "partial" if failure.partial or failure.writes else "blocked" + else: + state = {"verified": "completed", "unverified": "blocked"}.get(result["status"], result["status"]) + if state not in {"completed", "blocked", "partial"}: + raise ValueError("Invalid progress terminal state.") + self._emit(state) + if self.output_failed: + warnings = failure.warnings if failure is not None else result.setdefault("warnings", []) + if IO_WARNING not in warnings: + warnings.append(IO_WARNING) + if self.shutdown_flush_unresolved and SHUTDOWN_WARNING not in warnings: + warnings.append(SHUTDOWN_WARNING) + + +def reporting(workflow): + """One reporter and terminal event across nested public entrypoints.""" + def decorate(function): + @wraps(function) + def wrapped(*args, progress=None, **kwargs): + if progress is None: + progress = Progress(workflow, enabled=False) + outer = progress.depth == 0 + progress.depth += 1 + try: + result = function(*args, progress=progress, **kwargs) + except HelperFailure as failure: + if outer: + progress.finish(failure=failure) + raise + else: + if outer: + progress.finish(result=result) + return result + finally: + progress.depth -= 1 + return wrapped + return decorate + + +def add_progress_argument(parser): + parser.add_argument("--no-progress", dest="progress", action="store_false", + help="Suppress content-free execution progress on stderr.") diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_prompt_read.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_prompt_read.py index 897f3e08..bf6f4f5f 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_prompt_read.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_prompt_read.py @@ -1,230 +1,230 @@ -"""Read-only, resource-specific prerequisites for Prompt connection planning/apply.""" -from __future__ import annotations - -import copy -import json -import re -from typing import Any -from urllib.parse import parse_qs, quote, unquote, urlsplit - -try: - from ._bootstrap_io import run_cli - from ._common import ( - MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, HelperFailure, digest, - require_allowed_fields, - ) -except ImportError: - from _bootstrap_io import run_cli - from _common import ( - MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, HelperFailure, digest, - require_allowed_fields, - ) - -PROJECT_API = "2025-10-01-preview" -SEARCH_API = "2025-05-01" -ROLE_API = "2022-04-01" -READER_ROLE = "1407120a-92aa-4202-b7e9-c0e197c71c8f" -NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}") -GUID = re.compile(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}") -SEARCH_ID = re.compile( - r"/subscriptions/[^/?#\s]+/resourceGroups/[^/?#\s]+/" - r"providers/Microsoft\.Search/searchServices/(?P[a-z0-9-]{2,60})", - re.IGNORECASE, -) - - -def fail(code: str, message: str, response=None, *, status=None, request_id=None) -> HelperFailure: - return HelperFailure( - code, message, blocked_at="reconciliation", - status=response.status if response is not None else status, - request_id=response.request_id if response is not None else request_id, - ) - - -def binding(plan: dict[str, Any]) -> tuple[str, str, str]: - scope = plan["rbac_verified"].get("scope") - match = SEARCH_ID.fullmatch(scope) if isinstance(scope, str) else None - target = urlsplit(plan["connection"]["target"]) - parts = target.path.split("/") - if ( - match is None or target.hostname != match["name"].lower() + ".search.windows.net" - or len(parts) != 4 or parts[1] != "knowledgebases" or parts[3] != "mcp" - or parse_qs(target.query) != {"api-version": ["2026-08-01-preview"]} - or not NAME.fullmatch(unquote(parts[2])) - ): - raise fail("kb-binding-invalid", "Bind an exact Search service ID and preview KB MCP name, not a generated index.") - assignment = plan["rbac_verified"].get("assignment_id") - prefix = scope + "/providers/Microsoft.Authorization/roleAssignments/" - if ( - not isinstance(assignment, str) or not assignment.casefold().startswith(prefix.casefold()) - or not GUID.fullmatch(assignment[len(prefix):]) - ): - raise fail("rbac-scope-invalid", "Select one exact Search-service-scoped role assignment resource ID.") - return scope, "https://" + target.hostname, unquote(parts[2]) - - -def get_object(url, token, transport, *, absent=False, label="resource"): - try: - result = transport( - "GET", url, token, follow_redirects=False, max_response_bytes=1024 * 1024, - ) - except HelperFailure as error: - if absent and error.http_status == 404 and not error.partial: - return None, [error.request_id] if error.request_id else [] - raise - if result.status == 404 and absent: - return None, [result.request_id] if result.request_id else [] - if result.status != 200: - raise fail(label + "-unavailable", "Exact " + label + " read failed; no fallback or inferred absence.", result) - if not isinstance(result.body, dict) or not result.body: - raise fail(label + "-readback-invalid", "Exact " + label + " readback must be a nonempty object.", result) - return result.body, [result.request_id] if result.request_id else [] - - -def cli_context(project_id, *, cli=run_cli): - rc, out, _ = cli(["account", "show"], 60) - if rc: - raise fail("cli-context-unavailable", "The selected signed-in Azure CLI context could not be read.") - try: - value = json.loads(out) - except (UnicodeError, ValueError) as error: - raise fail("cli-context-invalid", "Azure CLI context is malformed.") from error - user = value.get("user") if isinstance(value, dict) else None - if ( - not isinstance(user, dict) or user.get("type") not in {"user", "servicePrincipal"} - or not isinstance(user.get("name"), str) or not 1 <= len(user["name"]) <= 256 - or value.get("environmentName") != "AzureCloud" - or str(value.get("state", "")).casefold() != "enabled" - or str(value.get("id", "")).casefold() != project_id.split("/")[2].casefold() - or not isinstance(value.get("tenantId"), str) or not GUID.fullmatch(value["tenantId"]) - ): - raise fail("cli-context-conflict", "Select the project's enabled subscription and tenant without changing identity.") - return {"subscription_id": value["id"].lower(), "tenant_id": value["tenantId"].lower(), - "principal": user["name"], "principal_type": user["type"]} - - -def kb_state(value, name): - if value.get("name") != name: - raise fail("kb-identity-mismatch", "The exact knowledgebases API returned another KB identity.") - sources, models = value.get("knowledgeSources"), value.get("models", []) - if models is None: - models = [] - effort = value.get("retrievalReasoningEffort") - mode = value.get("outputMode") - if ( - not isinstance(sources, list) or not 1 <= len(sources) <= 200 - or any(not isinstance(item, dict) or not isinstance(item.get("name"), str) - or not NAME.fullmatch(item["name"]) for item in sources) - or len({item["name"] for item in sources}) != len(sources) - or not isinstance(models, list) or any( - not isinstance(item, dict) or not isinstance(item.get("@odata.type"), str) - or not item["@odata.type"].strip() - for item in models - ) - or not isinstance(effort, dict) or effort.get("kind") not in {"minimal", "low", "medium"} - or mode not in {"extractiveData", "answerSynthesis"} - ): - raise fail("kb-configuration-unresolved", "Read complete KB sources, model configuration, reasoning and output; never infer them from generated indexes.") - if not models and (effort["kind"] != "minimal" or mode != "extractiveData"): - raise fail("kb-model-required", "This existing KB configuration requires a KB chat model; connection planning never changes its mode or models.") - material = copy.deepcopy(value) - for field in ("@odata.etag", "description", "tags"): - material.pop(field, None) - return {"name": name, "definition_digest": digest(material)}, { - "source_count": len(sources), "model_configured": bool(models), - "reasoning": effort["kind"], "output": mode, - } - - -def read_dependencies(plan, *, token_provider, transport, cli=run_cli, capture_context=False): - scope, endpoint, kb_name = binding(plan) - project_id = plan["project_resource_id"] - context = cli_context(project_id, cli=cli) if capture_context else None - token = token_provider(MANAGEMENT_AUDIENCE) - request_ids = [] - project, ids = get_object(MANAGEMENT_AUDIENCE + project_id + "?api-version=" + PROJECT_API, - token, transport, label="project") - request_ids.extend(ids) - identity, props = project.get("identity"), project.get("properties") - endpoints = props.get("endpoints") if isinstance(props, dict) else None - identity_types = identity.get("type") if isinstance(identity, dict) else None - identity_types = {item.strip() for item in identity_types.split(",")} if isinstance(identity_types, str) else set() - if ( - str(project.get("id", "")).casefold() != project_id.casefold() - or not isinstance(identity, dict) or identity_types not in ({"SystemAssigned"}, {"SystemAssigned", "UserAssigned"}) - or not isinstance(identity.get("principalId"), str) or not GUID.fullmatch(identity["principalId"]) - or not isinstance(identity.get("tenantId"), str) or not GUID.fullmatch(identity["tenantId"]) - or not isinstance(props, dict) or str(props.get("provisioningState", "")).casefold() != "succeeded" - or not isinstance(endpoints, dict) or plan["project_endpoint"].rstrip("/") not in { - value.rstrip("/") for value in endpoints.values() if isinstance(value, str) - } - ): - raise fail("project-identity-unverified", "Require the selected ready Foundry PROJECT endpoint and its system-assigned principalId/tenantId.", - request_id=ids[-1] if ids else None) - if context is not None and context["tenant_id"] != identity["tenantId"].lower(): - raise fail("cli-context-conflict", "CLI tenant differs from the observed project identity tenant.") - project_state = {"id": project_id, "endpoint": plan["project_endpoint"].rstrip("/"), - "principal_id": identity["principalId"].lower(), "tenant_id": identity["tenantId"].lower()} - - search, ids = get_object(MANAGEMENT_AUDIENCE + scope + "?api-version=" + SEARCH_API, - token, transport, label="search") - request_ids.extend(ids) - props = search.get("properties") - if str(search.get("id", "")).casefold() != scope.casefold() or not isinstance(props, dict): - raise fail("search-identity-unverified", "The selected Search resource identity is unresolved.", request_id=ids[-1] if ids else None) - status, provisioning = str(props.get("status", "")).lower(), str(props.get("provisioningState", "")).lower() - if status not in {"running", "provisioning", "degraded"} or provisioning not in {"succeeded", "provisioning"}: - raise fail("search-operation-blocked", "Search is failed, disabled, deleting or unresolved; no connection write is allowed.", - request_id=ids[-1] if ids else None) - warnings = [] if status == "running" and provisioning == "succeeded" else [ - "Search is provisioning/degraded; healthy KB GET permits connection configuration only, not readiness or retrieval proof." - ] - search_state = {"id": scope, "endpoint": endpoint, "access_digest": digest({ - key: props.get(key) for key in ("disableLocalAuth", "authOptions", "publicNetworkAccess", "networkRuleSet", "privateEndpointConnections") - })} - - assignment_id = plan["rbac_verified"]["assignment_id"] - assignment, ids = get_object(MANAGEMENT_AUDIENCE + assignment_id + "?api-version=" + ROLE_API, - token, transport, label="role-assignment") - request_ids.extend(ids) - role = assignment.get("properties") - if ( - str(assignment.get("id", "")).casefold() != assignment_id.casefold() - or not isinstance(role, dict) - or str(role.get("principalId", "")).casefold() != project_state["principal_id"] - or str(role.get("scope", "")).casefold() != scope.casefold() - or str(role.get("roleDefinitionId", "")).casefold() not in { - "/providers/microsoft.authorization/roledefinitions/" + READER_ROLE, - "/subscriptions/" + scope.split("/")[2].lower() + "/providers/microsoft.authorization/roledefinitions/" + READER_ROLE, - } - or role.get("principalType", "ServicePrincipal") != "ServicePrincipal" - or role.get("condition") not in (None, "") - ): - raise fail("project-reader-role-unverified", "Search Index Data Reader must be an unconditional exact-scope grant to the observed PROJECT principal, not the agent identity.", - request_id=ids[-1] if ids else None) - rbac = {"assignment_id": assignment_id, "principal_id": project_state["principal_id"], - "scope": scope, "role_definition_id": READER_ROLE} - url = endpoint + "/knowledgebases('" + quote(kb_name.replace("'", "''"), safe="") + "')?api-version=2026-08-01-preview" - kb, ids = get_object(url, token_provider(SEARCH_AUDIENCE), transport, absent=True, label="knowledge-base") - request_ids.extend(ids) - if kb is None: - raise fail("knowledge-base-absent", "The exact KB is absent; a generated index is not a knowledge base.", - status=404, request_id=ids[-1] if ids else None) - try: - knowledge_base, profile = kb_state(kb, kb_name) - except HelperFailure as error: - error.request_id = ids[-1] if ids else None - raise - state = {"project": project_state, "search": search_state, "rbac": rbac, "knowledge_base": knowledge_base} - if context is not None: - state["cli_context"] = context - expected = plan.get("verified_dependencies") - if expected is not None: - require_allowed_fields(expected, set(state), label="Verified Prompt dependencies") - if expected != state: - raise fail("connection-prerequisite-drift", "Project principal, CLI context, KB binding or exact role changed since planning; refresh before approval.") - elif plan["rbac_verified"].get("verified") is True and ( - str(plan["rbac_verified"]["principal_id"]).lower() != project_state["principal_id"] - ): - raise fail("project-reader-role-unverified", "The approved principal is not the observed Foundry PROJECT identity.") - return state, profile, warnings, request_ids +"""Read-only, resource-specific prerequisites for Prompt connection planning/apply.""" +from __future__ import annotations + +import copy +import json +import re +from typing import Any +from urllib.parse import parse_qs, quote, unquote, urlsplit + +try: + from ._bootstrap_io import run_cli + from ._common import ( + MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, HelperFailure, digest, + require_allowed_fields, + ) +except ImportError: + from _bootstrap_io import run_cli + from _common import ( + MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, HelperFailure, digest, + require_allowed_fields, + ) + +PROJECT_API = "2025-10-01-preview" +SEARCH_API = "2025-05-01" +ROLE_API = "2022-04-01" +READER_ROLE = "1407120a-92aa-4202-b7e9-c0e197c71c8f" +NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}") +GUID = re.compile(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}") +SEARCH_ID = re.compile( + r"/subscriptions/[^/?#\s]+/resourceGroups/[^/?#\s]+/" + r"providers/Microsoft\.Search/searchServices/(?P[a-z0-9-]{2,60})", + re.IGNORECASE, +) + + +def fail(code: str, message: str, response=None, *, status=None, request_id=None) -> HelperFailure: + return HelperFailure( + code, message, blocked_at="reconciliation", + status=response.status if response is not None else status, + request_id=response.request_id if response is not None else request_id, + ) + + +def binding(plan: dict[str, Any]) -> tuple[str, str, str]: + scope = plan["rbac_verified"].get("scope") + match = SEARCH_ID.fullmatch(scope) if isinstance(scope, str) else None + target = urlsplit(plan["connection"]["target"]) + parts = target.path.split("/") + if ( + match is None or target.hostname != match["name"].lower() + ".search.windows.net" + or len(parts) != 4 or parts[1] != "knowledgebases" or parts[3] != "mcp" + or parse_qs(target.query) != {"api-version": ["2026-08-01-preview"]} + or not NAME.fullmatch(unquote(parts[2])) + ): + raise fail("kb-binding-invalid", "Bind an exact Search service ID and preview KB MCP name, not a generated index.") + assignment = plan["rbac_verified"].get("assignment_id") + prefix = scope + "/providers/Microsoft.Authorization/roleAssignments/" + if ( + not isinstance(assignment, str) or not assignment.casefold().startswith(prefix.casefold()) + or not GUID.fullmatch(assignment[len(prefix):]) + ): + raise fail("rbac-scope-invalid", "Select one exact Search-service-scoped role assignment resource ID.") + return scope, "https://" + target.hostname, unquote(parts[2]) + + +def get_object(url, token, transport, *, absent=False, label="resource"): + try: + result = transport( + "GET", url, token, follow_redirects=False, max_response_bytes=1024 * 1024, + ) + except HelperFailure as error: + if absent and error.http_status == 404 and not error.partial: + return None, [error.request_id] if error.request_id else [] + raise + if result.status == 404 and absent: + return None, [result.request_id] if result.request_id else [] + if result.status != 200: + raise fail(label + "-unavailable", "Exact " + label + " read failed; no fallback or inferred absence.", result) + if not isinstance(result.body, dict) or not result.body: + raise fail(label + "-readback-invalid", "Exact " + label + " readback must be a nonempty object.", result) + return result.body, [result.request_id] if result.request_id else [] + + +def cli_context(project_id, *, cli=run_cli): + rc, out, _ = cli(["account", "show"], 60) + if rc: + raise fail("cli-context-unavailable", "The selected signed-in Azure CLI context could not be read.") + try: + value = json.loads(out) + except (UnicodeError, ValueError) as error: + raise fail("cli-context-invalid", "Azure CLI context is malformed.") from error + user = value.get("user") if isinstance(value, dict) else None + if ( + not isinstance(user, dict) or user.get("type") not in {"user", "servicePrincipal"} + or not isinstance(user.get("name"), str) or not 1 <= len(user["name"]) <= 256 + or value.get("environmentName") != "AzureCloud" + or str(value.get("state", "")).casefold() != "enabled" + or str(value.get("id", "")).casefold() != project_id.split("/")[2].casefold() + or not isinstance(value.get("tenantId"), str) or not GUID.fullmatch(value["tenantId"]) + ): + raise fail("cli-context-conflict", "Select the project's enabled subscription and tenant without changing identity.") + return {"subscription_id": value["id"].lower(), "tenant_id": value["tenantId"].lower(), + "principal": user["name"], "principal_type": user["type"]} + + +def kb_state(value, name): + if value.get("name") != name: + raise fail("kb-identity-mismatch", "The exact knowledgebases API returned another KB identity.") + sources, models = value.get("knowledgeSources"), value.get("models", []) + if models is None: + models = [] + effort = value.get("retrievalReasoningEffort") + mode = value.get("outputMode") + if ( + not isinstance(sources, list) or not 1 <= len(sources) <= 200 + or any(not isinstance(item, dict) or not isinstance(item.get("name"), str) + or not NAME.fullmatch(item["name"]) for item in sources) + or len({item["name"] for item in sources}) != len(sources) + or not isinstance(models, list) or any( + not isinstance(item, dict) or not isinstance(item.get("@odata.type"), str) + or not item["@odata.type"].strip() + for item in models + ) + or not isinstance(effort, dict) or effort.get("kind") not in {"minimal", "low", "medium"} + or mode not in {"extractiveData", "answerSynthesis"} + ): + raise fail("kb-configuration-unresolved", "Read complete KB sources, model configuration, reasoning and output; never infer them from generated indexes.") + if not models and (effort["kind"] != "minimal" or mode != "extractiveData"): + raise fail("kb-model-required", "This existing KB configuration requires a KB chat model; connection planning never changes its mode or models.") + material = copy.deepcopy(value) + for field in ("@odata.etag", "description", "tags"): + material.pop(field, None) + return {"name": name, "definition_digest": digest(material)}, { + "source_count": len(sources), "model_configured": bool(models), + "reasoning": effort["kind"], "output": mode, + } + + +def read_dependencies(plan, *, token_provider, transport, cli=run_cli, capture_context=False): + scope, endpoint, kb_name = binding(plan) + project_id = plan["project_resource_id"] + context = cli_context(project_id, cli=cli) if capture_context else None + token = token_provider(MANAGEMENT_AUDIENCE) + request_ids = [] + project, ids = get_object(MANAGEMENT_AUDIENCE + project_id + "?api-version=" + PROJECT_API, + token, transport, label="project") + request_ids.extend(ids) + identity, props = project.get("identity"), project.get("properties") + endpoints = props.get("endpoints") if isinstance(props, dict) else None + identity_types = identity.get("type") if isinstance(identity, dict) else None + identity_types = {item.strip() for item in identity_types.split(",")} if isinstance(identity_types, str) else set() + if ( + str(project.get("id", "")).casefold() != project_id.casefold() + or not isinstance(identity, dict) or identity_types not in ({"SystemAssigned"}, {"SystemAssigned", "UserAssigned"}) + or not isinstance(identity.get("principalId"), str) or not GUID.fullmatch(identity["principalId"]) + or not isinstance(identity.get("tenantId"), str) or not GUID.fullmatch(identity["tenantId"]) + or not isinstance(props, dict) or str(props.get("provisioningState", "")).casefold() != "succeeded" + or not isinstance(endpoints, dict) or plan["project_endpoint"].rstrip("/") not in { + value.rstrip("/") for value in endpoints.values() if isinstance(value, str) + } + ): + raise fail("project-identity-unverified", "Require the selected ready Foundry PROJECT endpoint and its system-assigned principalId/tenantId.", + request_id=ids[-1] if ids else None) + if context is not None and context["tenant_id"] != identity["tenantId"].lower(): + raise fail("cli-context-conflict", "CLI tenant differs from the observed project identity tenant.") + project_state = {"id": project_id, "endpoint": plan["project_endpoint"].rstrip("/"), + "principal_id": identity["principalId"].lower(), "tenant_id": identity["tenantId"].lower()} + + search, ids = get_object(MANAGEMENT_AUDIENCE + scope + "?api-version=" + SEARCH_API, + token, transport, label="search") + request_ids.extend(ids) + props = search.get("properties") + if str(search.get("id", "")).casefold() != scope.casefold() or not isinstance(props, dict): + raise fail("search-identity-unverified", "The selected Search resource identity is unresolved.", request_id=ids[-1] if ids else None) + status, provisioning = str(props.get("status", "")).lower(), str(props.get("provisioningState", "")).lower() + if status not in {"running", "provisioning", "degraded"} or provisioning not in {"succeeded", "provisioning"}: + raise fail("search-operation-blocked", "Search is failed, disabled, deleting or unresolved; no connection write is allowed.", + request_id=ids[-1] if ids else None) + warnings = [] if status == "running" and provisioning == "succeeded" else [ + "Search is provisioning/degraded; healthy KB GET permits connection configuration only, not readiness or retrieval proof." + ] + search_state = {"id": scope, "endpoint": endpoint, "access_digest": digest({ + key: props.get(key) for key in ("disableLocalAuth", "authOptions", "publicNetworkAccess", "networkRuleSet", "privateEndpointConnections") + })} + + assignment_id = plan["rbac_verified"]["assignment_id"] + assignment, ids = get_object(MANAGEMENT_AUDIENCE + assignment_id + "?api-version=" + ROLE_API, + token, transport, label="role-assignment") + request_ids.extend(ids) + role = assignment.get("properties") + if ( + str(assignment.get("id", "")).casefold() != assignment_id.casefold() + or not isinstance(role, dict) + or str(role.get("principalId", "")).casefold() != project_state["principal_id"] + or str(role.get("scope", "")).casefold() != scope.casefold() + or str(role.get("roleDefinitionId", "")).casefold() not in { + "/providers/microsoft.authorization/roledefinitions/" + READER_ROLE, + "/subscriptions/" + scope.split("/")[2].lower() + "/providers/microsoft.authorization/roledefinitions/" + READER_ROLE, + } + or role.get("principalType", "ServicePrincipal") != "ServicePrincipal" + or role.get("condition") not in (None, "") + ): + raise fail("project-reader-role-unverified", "Search Index Data Reader must be an unconditional exact-scope grant to the observed PROJECT principal, not the agent identity.", + request_id=ids[-1] if ids else None) + rbac = {"assignment_id": assignment_id, "principal_id": project_state["principal_id"], + "scope": scope, "role_definition_id": READER_ROLE} + url = endpoint + "/knowledgebases('" + quote(kb_name.replace("'", "''"), safe="") + "')?api-version=2026-08-01-preview" + kb, ids = get_object(url, token_provider(SEARCH_AUDIENCE), transport, absent=True, label="knowledge-base") + request_ids.extend(ids) + if kb is None: + raise fail("knowledge-base-absent", "The exact KB is absent; a generated index is not a knowledge base.", + status=404, request_id=ids[-1] if ids else None) + try: + knowledge_base, profile = kb_state(kb, kb_name) + except HelperFailure as error: + error.request_id = ids[-1] if ids else None + raise + state = {"project": project_state, "search": search_state, "rbac": rbac, "knowledge_base": knowledge_base} + if context is not None: + state["cli_context"] = context + expected = plan.get("verified_dependencies") + if expected is not None: + require_allowed_fields(expected, set(state), label="Verified Prompt dependencies") + if expected != state: + raise fail("connection-prerequisite-drift", "Project principal, CLI context, KB binding or exact role changed since planning; refresh before approval.") + elif plan["rbac_verified"].get("verified") is True and ( + str(plan["rbac_verified"]["principal_id"]).lower() != project_state["principal_id"] + ): + raise fail("project-reader-role-unverified", "The approved principal is not the observed Foundry PROJECT identity.") + return state, profile, warnings, request_ids diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob-contracts.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob-contracts.md index 29865368..ba35dee1 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob-contracts.md +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob-contracts.md @@ -1,142 +1,142 @@ -# Blob/ADLS contract - -[Blob progress](../references/blob-ingestion-progress.md): `--no-progress` disables stderr. - -## Planning - -```text -python helpers/blob_source.py --plan intent.json -``` - -Intent: - -```json -{"schema_version":"1.0","endpoint":"https://svc.search.windows.net","name":"manuals","owner":"owner@example.com","storage_id":"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/Example/providers/Microsoft.Storage/storageAccounts/example","container":"documents","prefix":"Reports/","is_adls":false,"api_version":"2026-04-01","processing":"minimal-lexical","network_access":"public","identity":"system-assigned","permission_options":[],"ingestion_schedule":null,"description":null,"rbac":{"assignments":[{"id":"","principalId":""}]},"network":{"posture":"public","evidence":""},"inventory_limits":{"max_pages":100,"max_objects":10000,"max_requests":25000,"deadline_seconds":600},"poll":{"deadline_seconds":600,"max_requests":120,"interval_seconds":5}} -``` - -Content fit (no default): `minimal-lexical`=minimal/no vectors, -`minimal-vector`=minimal/[vectors](vector-contracts.md), -`standard-cu`=standard/optional vectors; not KB reasoning. -Public/system-assigned; no schedule settings/permission ingestion. -Blob: `2026-04-01` or `2026-08-01-preview`; ADLS: preview, `is_adls: true`, -directory prefix without trailing `/`. Root: explicit `prefix: ""`, no widening. -Description: text/null; other modes: `planning-processing-unsupported`. - -No writes/auth changes/installs/polling/cleanup. Verify access/cost/data preapproval. - -Two exact Search GETs bracket two-pass Storage observation. Reuse -requires matching definition/ETag, Storage binding and four generated identities. -`reuse_input_file`/`reuse_result_file`: retained approved input/successful creation -result. Validate pairing/types before auth; load only for redacted binding. -Required fingerprints/boundaries/source ETags/definitions/generated identities -must match fresh readback. Unproven binding: `source-binding-unverified`. -Old consent/readiness never authorizes new work. - -Exit `0`: `status: planned`, `plan_fingerprint`, `execution_input`, -`approval_summary`, request IDs and `writes_performed: []`. Save only -`execution_input`; its approval is false and the executor rejects it until -actual unchanged-plan approval. Private artifacts preserve ResourceId connection -strings/fingerprints; ordinary result redaction is unchanged. -Summaries omit paths/ACLs/hashes; no readiness. Exact reuse sets `execution_required` and -`mutation_approval_required` false: no executor or approval needed. -Drift: rerun `--plan`, replace the artifact, discard consent; never hand-edit -nested plans/hashes. Cleanup stays separate and run-owned-only. - -For `standard-cu`, read [CU choices/wire](blob-cu-contracts.md). -`content_understanding` binds CU; optional `embedding` selects source vectors. - -## Discovery - -`python helpers/blob_source.py --discover ` accepts this closed -JSON (no approval): - -```json -{"boundary":{"storage_id":"","container":"","prefix":"Reports/","is_adls":false},"inventory_limits":{"max_pages":100,"max_objects":10000,"max_requests":25000,"deadline_seconds":600}} -``` - -Arbitrary partial-name prefixes: `boundary-ambiguous`. -Public cloud/private DNS. - -Read all pages twice including empty continuations. Exit `0` returns -`status: discovered`, `mutation: none`, `writes_performed: []`, boundary/account, -sorted objects (URL/path/size/ETag/version), ADLS path/property digests, -request IDs, `inventory_digest`; not approval/ingestion. -Incomplete/duplicate/out-of-scope/unreadable/unstable evidence blocks. -ADLS HEADs path/ancestor ACLs; no raw ACL/content. Operator access is not Search access. -Every path has an ACL HEAD; every non-root path also has a properties HEAD for -type and matching ETag. Root has no type header. Preserve ancestors and normalize -quoted/unquoted Blob/DFS ETags only for comparison; never skip required reads. - -Inventory bounds apply per discovery: pages/objects per pass; requests/time -across both passes, including empty pages and all HEADs. Maximum accepted limits -are 1000 pages, 100000 objects, 200000 requests and 3600 seconds; exceeding any -bound blocks, never truncates. Planning adds at most two Search GETs and shares -its time budget with discovery. Bodies are capped at 8 MiB plus one overflow -probe byte. Socket watchdog interrupts body reads; DNS/connect/header -acquisition and scheduling are not hard wall-clock bounded. Redirects and -unsupported deadline readers block. Limits are safety bounds, not spending consent. - -## Apply - -`python helpers/blob_source.py --input ` uses: - -```json -{"schema_version":"1.0","plan":{},"approval":{"confirmed":true,"fingerprint":"sha256:"}} -``` - -Canonical JSON: UTF-8, sorted keys, ASCII-escaped/compact. -`operation: reconcile-and-monitor`, `owner`, -`cleanup_approved: false`, discovered `boundary`, `inventory_digest`, -`inventory_limits`, `source`, -`poll: {"deadline_seconds":600,"max_requests":120,"interval_seconds":5}`. -Planner reuse: `expected_generated`, four typed/name/service-managed identities, -checked before writes; legacy artifacts remain valid. -Planner creation sets `expected_source_absent: true`; an intervening source -blocks. Redacted GET requires the PUT ETag (body/header); conflicts block. -ambiguous unproven ownership remains partial, never adopted or cleaned up. - -`source`: `operation: reconcile`, `resource_type: -knowledge-source`, exact `endpoint`/`name`/`api_version`, `action: create` -or `reuse`, matching `owner`, `cleanup_approved: false`, approved -`desired`, and `source_evidence` with `verified: true` and the same -`inventory_digest`. ADLS also requires `path_verified: true`, `acl_verified: -true` attest metadata, not effective Search permissions. - -`desired`: matching `name`, `kind: azureBlob`; `azureBlobParameters`: -`connectionString: ResourceId=`, `containerName`, -exact `folderPath` (`null` for explicit root), boolean `isADLSGen2`, -`ingestionParameters`. Optional [shared controls](contracts.md). -No keys/SAS/updates/credential environments or supplied `createdResources`/`assetStore`. - -Recompute approval/evidence; block drift. Create conditionally/reuse zero-write; -supplied ETag must match. -Poll GET `knowledgesources('{name}')/status` within bounds. -Only 408/429/500/502/503/504/transport-ambiguous reads retry; 403/404 block. -Watch expiry pauses; it does not fail ingestion. Backoff and explicit GET-only -continuation/compact output: [recheck](../references/blob-readiness-recheck.md). - -Require completed start/end times at or after the pre-write cutoff, -nonnegative integer `itemsUpdatesProcessed`, `itemsUpdatesFailed`, and -`itemsSkipped`, zero failures, no observed relevant errors or active cycle. -Missing counts are not zero. -Completed `status` may be absent; reject `partialSuccess`/`failure`/malformed/unknown. -Never trigger indexers/edit schedules; stale success cannot pass. -Reuse also excludes the first observed completion. - -## Results - -`--receipt-dir`: PUT acknowledgement before GET. -[Binding/recovery](../references/blob-binding-evidence.md): redaction, recovery, -reuse; never reconstruct cutoffs. -Timing differences: [indexer checks](../references/blob-indexer-observation.md). - -Exit `0`: `completed`; unverified zero-write reuse is exit `2`, -`blocked` with `reconciliation: completed`. After a write, exit `3` / `partial` -retains approval/first failure/`resources_remaining.run_owned`. -Conflicting children aren't owned. Return generated identities/readiness/digests/ -warnings/cleanup; errors retain status/request ID, not sensitive text. -No rollback/source writes/child edits. - -Owner handles RBAC/private links. CLI Entra: ARM/Storage/Search audiences. -Probes: ARM GET `2025-06-01`, Storage `2025-05-05` listing, -ADLS HEAD `action=getAccessControl&upn=false`. +# Blob/ADLS contract + +[Blob progress](../references/blob-ingestion-progress.md): `--no-progress` disables stderr. + +## Planning + +```text +python helpers/blob_source.py --plan intent.json +``` + +Intent: + +```json +{"schema_version":"1.0","endpoint":"https://svc.search.windows.net","name":"manuals","owner":"owner@example.com","storage_id":"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/Example/providers/Microsoft.Storage/storageAccounts/example","container":"documents","prefix":"Reports/","is_adls":false,"api_version":"2026-04-01","processing":"minimal-lexical","network_access":"public","identity":"system-assigned","permission_options":[],"ingestion_schedule":null,"description":null,"rbac":{"assignments":[{"id":"","principalId":""}]},"network":{"posture":"public","evidence":""},"inventory_limits":{"max_pages":100,"max_objects":10000,"max_requests":25000,"deadline_seconds":600},"poll":{"deadline_seconds":600,"max_requests":120,"interval_seconds":5}} +``` + +Content fit (no default): `minimal-lexical`=minimal/no vectors, +`minimal-vector`=minimal/[vectors](vector-contracts.md), +`standard-cu`=standard/optional vectors; not KB reasoning. +Public/system-assigned; no schedule settings/permission ingestion. +Blob: `2026-04-01` or `2026-08-01-preview`; ADLS: preview, `is_adls: true`, +directory prefix without trailing `/`. Root: explicit `prefix: ""`, no widening. +Description: text/null; other modes: `planning-processing-unsupported`. + +No writes/auth changes/installs/polling/cleanup. Verify access/cost/data preapproval. + +Two exact Search GETs bracket two-pass Storage observation. Reuse +requires matching definition/ETag, Storage binding and four generated identities. +`reuse_input_file`/`reuse_result_file`: retained approved input/successful creation +result. Validate pairing/types before auth; load only for redacted binding. +Required fingerprints/boundaries/source ETags/definitions/generated identities +must match fresh readback. Unproven binding: `source-binding-unverified`. +Old consent/readiness never authorizes new work. + +Exit `0`: `status: planned`, `plan_fingerprint`, `execution_input`, +`approval_summary`, request IDs and `writes_performed: []`. Save only +`execution_input`; its approval is false and the executor rejects it until +actual unchanged-plan approval. Private artifacts preserve ResourceId connection +strings/fingerprints; ordinary result redaction is unchanged. +Summaries omit paths/ACLs/hashes; no readiness. Exact reuse sets `execution_required` and +`mutation_approval_required` false: no executor or approval needed. +Drift: rerun `--plan`, replace the artifact, discard consent; never hand-edit +nested plans/hashes. Cleanup stays separate and run-owned-only. + +For `standard-cu`, read [CU choices/wire](blob-cu-contracts.md). +`content_understanding` binds CU; optional `embedding` selects source vectors. + +## Discovery + +`python helpers/blob_source.py --discover ` accepts this closed +JSON (no approval): + +```json +{"boundary":{"storage_id":"","container":"","prefix":"Reports/","is_adls":false},"inventory_limits":{"max_pages":100,"max_objects":10000,"max_requests":25000,"deadline_seconds":600}} +``` + +Arbitrary partial-name prefixes: `boundary-ambiguous`. +Public cloud/private DNS. + +Read all pages twice including empty continuations. Exit `0` returns +`status: discovered`, `mutation: none`, `writes_performed: []`, boundary/account, +sorted objects (URL/path/size/ETag/version), ADLS path/property digests, +request IDs, `inventory_digest`; not approval/ingestion. +Incomplete/duplicate/out-of-scope/unreadable/unstable evidence blocks. +ADLS HEADs path/ancestor ACLs; no raw ACL/content. Operator access is not Search access. +Every path has an ACL HEAD; every non-root path also has a properties HEAD for +type and matching ETag. Root has no type header. Preserve ancestors and normalize +quoted/unquoted Blob/DFS ETags only for comparison; never skip required reads. + +Inventory bounds apply per discovery: pages/objects per pass; requests/time +across both passes, including empty pages and all HEADs. Maximum accepted limits +are 1000 pages, 100000 objects, 200000 requests and 3600 seconds; exceeding any +bound blocks, never truncates. Planning adds at most two Search GETs and shares +its time budget with discovery. Bodies are capped at 8 MiB plus one overflow +probe byte. Socket watchdog interrupts body reads; DNS/connect/header +acquisition and scheduling are not hard wall-clock bounded. Redirects and +unsupported deadline readers block. Limits are safety bounds, not spending consent. + +## Apply + +`python helpers/blob_source.py --input ` uses: + +```json +{"schema_version":"1.0","plan":{},"approval":{"confirmed":true,"fingerprint":"sha256:"}} +``` + +Canonical JSON: UTF-8, sorted keys, ASCII-escaped/compact. +`operation: reconcile-and-monitor`, `owner`, +`cleanup_approved: false`, discovered `boundary`, `inventory_digest`, +`inventory_limits`, `source`, +`poll: {"deadline_seconds":600,"max_requests":120,"interval_seconds":5}`. +Planner reuse: `expected_generated`, four typed/name/service-managed identities, +checked before writes; legacy artifacts remain valid. +Planner creation sets `expected_source_absent: true`; an intervening source +blocks. Redacted GET requires the PUT ETag (body/header); conflicts block. +ambiguous unproven ownership remains partial, never adopted or cleaned up. + +`source`: `operation: reconcile`, `resource_type: +knowledge-source`, exact `endpoint`/`name`/`api_version`, `action: create` +or `reuse`, matching `owner`, `cleanup_approved: false`, approved +`desired`, and `source_evidence` with `verified: true` and the same +`inventory_digest`. ADLS also requires `path_verified: true`, `acl_verified: +true` attest metadata, not effective Search permissions. + +`desired`: matching `name`, `kind: azureBlob`; `azureBlobParameters`: +`connectionString: ResourceId=`, `containerName`, +exact `folderPath` (`null` for explicit root), boolean `isADLSGen2`, +`ingestionParameters`. Optional [shared controls](contracts.md). +No keys/SAS/updates/credential environments or supplied `createdResources`/`assetStore`. + +Recompute approval/evidence; block drift. Create conditionally/reuse zero-write; +supplied ETag must match. +Poll GET `knowledgesources('{name}')/status` within bounds. +Only 408/429/500/502/503/504/transport-ambiguous reads retry; 403/404 block. +Watch expiry pauses; it does not fail ingestion. Backoff and explicit GET-only +continuation/compact output: [recheck](../references/blob-readiness-recheck.md). + +Require completed start/end times at or after the pre-write cutoff, +nonnegative integer `itemsUpdatesProcessed`, `itemsUpdatesFailed`, and +`itemsSkipped`, zero failures, no observed relevant errors or active cycle. +Missing counts are not zero. +Completed `status` may be absent; reject `partialSuccess`/`failure`/malformed/unknown. +Never trigger indexers/edit schedules; stale success cannot pass. +Reuse also excludes the first observed completion. + +## Results + +`--receipt-dir`: PUT acknowledgement before GET. +[Binding/recovery](../references/blob-binding-evidence.md): redaction, recovery, +reuse; never reconstruct cutoffs. +Timing differences: [indexer checks](../references/blob-indexer-observation.md). + +Exit `0`: `completed`; unverified zero-write reuse is exit `2`, +`blocked` with `reconciliation: completed`. After a write, exit `3` / `partial` +retains approval/first failure/`resources_remaining.run_owned`. +Conflicting children aren't owned. Return generated identities/readiness/digests/ +warnings/cleanup; errors retain status/request ID, not sensitive text. +No rollback/source writes/child edits. + +Owner handles RBAC/private links. CLI Entra: ARM/Storage/Search audiences. +Probes: ARM GET `2025-06-01`, Storage `2025-05-05` listing, +ADLS HEAD `action=getAccessControl&upn=false`. diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob-cu-contracts.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob-cu-contracts.md index 160e9f8a..192f7eaf 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob-cu-contracts.md +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob-cu-contracts.md @@ -1,132 +1,132 @@ -# Optional Blob Content Understanding branch - -Read only for `processing: standard-cu` in the [Blob procedure](../knowledge-sources/create-azure-blob.md). -Selected Search/Storage/AI Services only; no provisioning, uploads, asset store, -permission changes or KB creation. Resolve processing before planning; never -downgrade CU or add it to minimal-vector. - -## Availability and dependencies - -Read [shared CU ingestion](../references/cu-ingestion.md) for independent model -purposes, necessary checks and non-executable pending-dependency drafts. -For portal/required-field questions, -use the [API/service/helper settings matrix](../references/standard-cu.md#source-settings-api-versus-helper). - -Blob requires CU-capable `AIServices` in a supported region with an embedding deployment, -not Search `embeddingModel` or source vectors. -Image verbalization/source chat are disabled. -Ingestion does not choose KB reasoning, reranking, query mode or synthesis. - -Use the existing Search system-assigned identity with Cognitive Services User -on that account; Storage stays ResourceId/keyless with Storage -Blob Data Reader. Do not copy File CU credential channels or enable local auth. -The Blob documentation's `aiServices` keyless configuration omits `apiKey`. -Keys, user-assigned identities, private networking, schedules, asset stores and -permission ingestion are outside this standard branch. - -Verify CU capability/region/deployments, effective Search identity/RBAC and -reachability through exact readbacks. -`configuration` references selected processing/required deployment evidence. -Existing dependencies permit planning; no setup-owner or `azd` prerequisite. -Do not ask for a separate account-configuration confirmation. -Default mappings need contract/observed-error evidence; never change them. -`cu-prerequisite-missing`: name missing evidence, not a generic setup gate. -No provisioning/model tests or default extraction downgrade. - -## Closed intent - -Use the ordinary [Blob intent](blob-contracts.md#planning), set -`processing: standard-cu` and add this required object: - -```json -{"content_understanding":{"endpoint":"https://models.services.ai.azure.com","auth":"system-assigned","prerequisites":{"resource":"","configuration":"","identity":"","network":""}}} -``` - -Fields required; references: 1–4096 characters. -Omit `embedding` (or use null in the intent) to leave source vectors off. -Only when selected, add the closed [embedding choice](vector-contracts.md). -CU and source-vector endpoints are independently validated; they need not match. -Combine CU/auth/source/vector costs and data consent; no manual credential/MI -confirmation. Missing setup uses approved native bootstrap. -A single trailing slash is immaterial. -Custom CU endpoints, URL paths/queries/ports and credentials block. -Minimal modes omit `content_understanding`. -Private artifacts retain choices; summaries omit references, which are attestations, not CU calls. - -New CU plans emit fingerprint-bound `cu_plan_version: "1.0"`. This CU-specific -marker requires the CU choice and exact presence/value binding of optional embeddings. -Legacy approved wire-only standard artifacts, including `expected_source_absent` -and `expected_generated`, retain their original admission, guards and fingerprints. -Those generic guards predate CU planning; they do not select the new contract. -Never rewrite historical approved artifacts/receipts. Removing choices from a -marked CU artifact blocks if CU is missing or the wire still requests removed -embeddings; changing/removing its marker invalidates existing -approval. No new field is sent in the Search request. - -## Wire and approval - -Both GA `2026-04-01` and preview `2026-08-01-preview` -expose `azureBlobParameters.ingestionParameters` with -`contentExtractionMode: standard`, `aiServices: {"uri": ""}`, -`disableImageVerbalization: true`, null identity/schedule. -Set `embeddingModel` only for selected source vectorization; otherwise omit it. -Preview also sets public networking and empty permission options. -ADLS remains preview-only in this helper with unchanged path/ACL controls. - -Use the same `blob_source.py --plan` → private unapproved `execution_input` → -one explicit unchanged-plan approval → `--input` flow. Execution uses conditional -PUT `/knowledgesources('')?api-version=` with -`If-None-Match: *`, exact definition GET and status GET; no direct CU API call. -Never overwrite or suffix around a collision. - -CU creation also retains approved input and uses `--receipt-dir`. -Timeout/interruption: [read-only recheck](../references/blob-readiness-recheck.md), -not another PUT or direct CU call. Fresh reuse capture carries no creation ownership. - -Show combined source/CU/embedding cost, access and processing changes before -approval. CU is billable with no free document allowance; selected documents -move to CU and, only if selected, source embeddings, possibly across regions. -Search retains outputs. -The CU skill uses underlying CU `2025-11-01`; Search manages that call, not a user-selectable -version. Both GA and preview Swagger -declare `embeddingModel` optional/nullable, with no required or conditional -standard-extraction vectorizer constraint. CU deployment prerequisites -must not be substituted for Search wire-field requirements. -Search and CU limits apply, including five-minute analysis timeout with possible charges. -No universal quality claim. Acceptance does not prove CU support; see -[format limits](../references/platform-interfaces.md#source-formats). - -## Verification and failure - -Require exact CU mode/endpoint/auth readback. Allow only absent/null/empty-string -`aiServices.apiKey` and null/absent ingestion identity. Masked/nonempty/malformed -keys or another identity return `cu-auth-conflict`, without echoing values. -Known null metadata, a single endpoint trailing slash and the documented -`resultsProcessing: rerank` default do not require manual plan edits. -Different endpoints, models, extraction, auth, scope, ETags or generated identities -block; rerun planning and discard old consent. - -Acceptance proves configuration only. The owning executor requires a relevant -completed zero-failure ingestion cycle, not `active` or old success. -[Vector verification](vector-contracts.md#read-only-verification-plan), only when configured, additionally -observes generated dimensions/profile/vectorizer and nonempty zero-skip readiness. -Retrieval needs its separately approved query; neither proves extraction quality -or corpus-wide relevance. CU-only vector probes return `embedding-not-configured`, -not “CU unsupported.” Fresh exact reuse remains mutation-approval-free, -not ingestion/retrieval proof. Tests/mocks are not live evidence. - -Keep the first failure/request ID. A failed post-write check is partial with -run-owned source/children retained; never automatically delete anything. -Use existing separately approved source cleanup only, never Storage, -dependencies, shared resources or role assignments. - -## Authorities - -Authorities: failure/conflict/uncertainty only. - -[Blob prerequisites](https://learn.microsoft.com/azure/search/agentic-knowledge-source-how-to-blob#prerequisites), -[GA create](https://learn.microsoft.com/rest/api/searchservice/knowledge-sources/create?view=rest-searchservice-2026-04-01), -[preview create](https://learn.microsoft.com/rest/api/searchservice/knowledge-sources/create?view=rest-searchservice-2026-08-01-preview), -[CU skill](https://learn.microsoft.com/azure/search/cognitive-search-skill-content-understanding), -[GA Swagger](https://github.com/Azure/azure-rest-api-specs/blob/main/specification/search/data-plane/Search/stable/2026-04-01/search.json), -[preview Swagger](https://github.com/Azure/azure-rest-api-specs/blob/main/specification/search/data-plane/Search/preview/2026-08-01-preview/search.json). +# Optional Blob Content Understanding branch + +Read only for `processing: standard-cu` in the [Blob procedure](../knowledge-sources/create-azure-blob.md). +Selected Search/Storage/AI Services only; no provisioning, uploads, asset store, +permission changes or KB creation. Resolve processing before planning; never +downgrade CU or add it to minimal-vector. + +## Availability and dependencies + +Read [shared CU ingestion](../references/cu-ingestion.md) for independent model +purposes, necessary checks and non-executable pending-dependency drafts. +For portal/required-field questions, +use the [API/service/helper settings matrix](../references/standard-cu.md#source-settings-api-versus-helper). + +Blob requires CU-capable `AIServices` in a supported region with an embedding deployment, +not Search `embeddingModel` or source vectors. +Image verbalization/source chat are disabled. +Ingestion does not choose KB reasoning, reranking, query mode or synthesis. + +Use the existing Search system-assigned identity with Cognitive Services User +on that account; Storage stays ResourceId/keyless with Storage +Blob Data Reader. Do not copy File CU credential channels or enable local auth. +The Blob documentation's `aiServices` keyless configuration omits `apiKey`. +Keys, user-assigned identities, private networking, schedules, asset stores and +permission ingestion are outside this standard branch. + +Verify CU capability/region/deployments, effective Search identity/RBAC and +reachability through exact readbacks. +`configuration` references selected processing/required deployment evidence. +Existing dependencies permit planning; no setup-owner or `azd` prerequisite. +Do not ask for a separate account-configuration confirmation. +Default mappings need contract/observed-error evidence; never change them. +`cu-prerequisite-missing`: name missing evidence, not a generic setup gate. +No provisioning/model tests or default extraction downgrade. + +## Closed intent + +Use the ordinary [Blob intent](blob-contracts.md#planning), set +`processing: standard-cu` and add this required object: + +```json +{"content_understanding":{"endpoint":"https://models.services.ai.azure.com","auth":"system-assigned","prerequisites":{"resource":"","configuration":"","identity":"","network":""}}} +``` + +Fields required; references: 1–4096 characters. +Omit `embedding` (or use null in the intent) to leave source vectors off. +Only when selected, add the closed [embedding choice](vector-contracts.md). +CU and source-vector endpoints are independently validated; they need not match. +Combine CU/auth/source/vector costs and data consent; no manual credential/MI +confirmation. Missing setup uses approved native bootstrap. +A single trailing slash is immaterial. +Custom CU endpoints, URL paths/queries/ports and credentials block. +Minimal modes omit `content_understanding`. +Private artifacts retain choices; summaries omit references, which are attestations, not CU calls. + +New CU plans emit fingerprint-bound `cu_plan_version: "1.0"`. This CU-specific +marker requires the CU choice and exact presence/value binding of optional embeddings. +Legacy approved wire-only standard artifacts, including `expected_source_absent` +and `expected_generated`, retain their original admission, guards and fingerprints. +Those generic guards predate CU planning; they do not select the new contract. +Never rewrite historical approved artifacts/receipts. Removing choices from a +marked CU artifact blocks if CU is missing or the wire still requests removed +embeddings; changing/removing its marker invalidates existing +approval. No new field is sent in the Search request. + +## Wire and approval + +Both GA `2026-04-01` and preview `2026-08-01-preview` +expose `azureBlobParameters.ingestionParameters` with +`contentExtractionMode: standard`, `aiServices: {"uri": ""}`, +`disableImageVerbalization: true`, null identity/schedule. +Set `embeddingModel` only for selected source vectorization; otherwise omit it. +Preview also sets public networking and empty permission options. +ADLS remains preview-only in this helper with unchanged path/ACL controls. + +Use the same `blob_source.py --plan` → private unapproved `execution_input` → +one explicit unchanged-plan approval → `--input` flow. Execution uses conditional +PUT `/knowledgesources('')?api-version=` with +`If-None-Match: *`, exact definition GET and status GET; no direct CU API call. +Never overwrite or suffix around a collision. + +CU creation also retains approved input and uses `--receipt-dir`. +Timeout/interruption: [read-only recheck](../references/blob-readiness-recheck.md), +not another PUT or direct CU call. Fresh reuse capture carries no creation ownership. + +Show combined source/CU/embedding cost, access and processing changes before +approval. CU is billable with no free document allowance; selected documents +move to CU and, only if selected, source embeddings, possibly across regions. +Search retains outputs. +The CU skill uses underlying CU `2025-11-01`; Search manages that call, not a user-selectable +version. Both GA and preview Swagger +declare `embeddingModel` optional/nullable, with no required or conditional +standard-extraction vectorizer constraint. CU deployment prerequisites +must not be substituted for Search wire-field requirements. +Search and CU limits apply, including five-minute analysis timeout with possible charges. +No universal quality claim. Acceptance does not prove CU support; see +[format limits](../references/platform-interfaces.md#source-formats). + +## Verification and failure + +Require exact CU mode/endpoint/auth readback. Allow only absent/null/empty-string +`aiServices.apiKey` and null/absent ingestion identity. Masked/nonempty/malformed +keys or another identity return `cu-auth-conflict`, without echoing values. +Known null metadata, a single endpoint trailing slash and the documented +`resultsProcessing: rerank` default do not require manual plan edits. +Different endpoints, models, extraction, auth, scope, ETags or generated identities +block; rerun planning and discard old consent. + +Acceptance proves configuration only. The owning executor requires a relevant +completed zero-failure ingestion cycle, not `active` or old success. +[Vector verification](vector-contracts.md#read-only-verification-plan), only when configured, additionally +observes generated dimensions/profile/vectorizer and nonempty zero-skip readiness. +Retrieval needs its separately approved query; neither proves extraction quality +or corpus-wide relevance. CU-only vector probes return `embedding-not-configured`, +not “CU unsupported.” Fresh exact reuse remains mutation-approval-free, +not ingestion/retrieval proof. Tests/mocks are not live evidence. + +Keep the first failure/request ID. A failed post-write check is partial with +run-owned source/children retained; never automatically delete anything. +Use existing separately approved source cleanup only, never Storage, +dependencies, shared resources or role assignments. + +## Authorities + +Authorities: failure/conflict/uncertainty only. + +[Blob prerequisites](https://learn.microsoft.com/azure/search/agentic-knowledge-source-how-to-blob#prerequisites), +[GA create](https://learn.microsoft.com/rest/api/searchservice/knowledge-sources/create?view=rest-searchservice-2026-04-01), +[preview create](https://learn.microsoft.com/rest/api/searchservice/knowledge-sources/create?view=rest-searchservice-2026-08-01-preview), +[CU skill](https://learn.microsoft.com/azure/search/cognitive-search-skill-content-understanding), +[GA Swagger](https://github.com/Azure/azure-rest-api-specs/blob/main/specification/search/data-plane/Search/stable/2026-04-01/search.json), +[preview Swagger](https://github.com/Azure/azure-rest-api-specs/blob/main/specification/search/data-plane/Search/preview/2026-08-01-preview/search.json). diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_inventory.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_inventory.py index 9814c864..5e3eb8ee 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_inventory.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_inventory.py @@ -1,330 +1,330 @@ -from __future__ import annotations - -import re -import time -from typing import Any, Callable -from urllib.parse import quote, unquote, urlencode -from xml.etree import ElementTree - -try: - from ._common import ( - MANAGEMENT_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, - digest, http_request, reject_secrets, require_allowed_fields, - ) -except ImportError: - from _common import ( # type: ignore[no-redef] - MANAGEMENT_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, - digest, http_request, reject_secrets, require_allowed_fields, - ) - - -STORAGE_AUDIENCE = "https://storage.azure.com/" -ARM_VERSION = "2025-06-01" -STORAGE_VERSION = "2025-05-05" -ACCOUNT_ID = re.compile( - r"^/subscriptions/[0-9a-fA-F-]{36}/resourceGroups/[^/;?#\r\n]+/" - r"providers/Microsoft\.Storage/storageAccounts/([a-z0-9]{3,24})$" -) -CONTAINER = re.compile(r"^[a-z0-9](?:[a-z0-9-]{1,61}[a-z0-9])$") - - -def _fail(code: str, message: str, request_id: str | None = None) -> HelperFailure: - return HelperFailure(code, message, blocked_at="source-preflight", request_id=request_id) - - -def _strip_etag_quotes(value: str) -> str: - # Blob List XML returns unquoted ETags; DFS HTTP responses return - # RFC 7232-quoted ETags for the identical underlying value. - return value[1:-1] if len(value) >= 2 and value[0] == '"' and value[-1] == '"' else value - - -def validate_boundary(value: Any) -> dict[str, Any]: - if not isinstance(value, dict): - raise _fail("boundary-invalid", "An explicitly selected Storage boundary is required.") - reject_secrets(value) - require_allowed_fields(value, {"storage_id", "container", "prefix", "is_adls"}, label="boundary") - if ( - not isinstance(value.get("storage_id"), str) - or ACCOUNT_ID.fullmatch(value["storage_id"]) is None - or not isinstance(value.get("container"), str) - or CONTAINER.fullmatch(value["container"]) is None - or "--" in value["container"] - or not isinstance(value.get("prefix"), str) - or type(value.get("is_adls")) is not bool - ): - raise _fail("boundary-invalid", "Exact Storage ID, container, explicit prefix (including empty root), and subtype are required.") - prefix = value["prefix"] - try: - value["storage_id"].encode("utf-8") - prefix.encode("utf-8") - except UnicodeError as exc: - raise _fail("boundary-invalid", "Storage ID and prefix must be valid UTF-8.") from exc - if ( - prefix.startswith("/") - or "\\" in prefix - or any(ord(c) < 32 for c in prefix) - or (any(part in {".", "..", ""} for part in prefix.rstrip("/").split("/")) and prefix != "") - or "//" in prefix - or value["is_adls"] and prefix.endswith("/") - or not value["is_adls"] and prefix and not prefix.endswith("/") - ): - raise _fail( - "boundary-ambiguous", - "Use explicit empty root, a Blob folder prefix ending in '/', or an ADLS directory path without a trailing '/'. Arbitrary partial-name prefixes are not verified.", - ) - return dict(value) - - -def validate_limits(value: Any) -> dict[str, int]: - if not isinstance(value, dict): - raise _fail("input-schema-invalid", "inventory_limits must be an object.") - require_allowed_fields(value, {"max_pages", "max_objects", "max_requests", "deadline_seconds"}, - label="inventory limits") - for field, maximum in ( - ("max_pages", 1000), ("max_objects", 100000), ("max_requests", 200000), - ("deadline_seconds", 3600), - ): - if type(value.get(field)) is not int or not 1 <= value[field] <= maximum: - raise _fail("input-schema-invalid", "Inventory limits must be bounded positive integers.") - return value - - -class _Reader: - def __init__( - self, limits: dict[str, int], token_provider: TokenProvider, - transport: Transport, monotonic: Callable[[], float], - deadline: float | None = None, - ) -> None: - self.limits = limits - self.transport = transport - self.monotonic = monotonic - self.deadline = monotonic() + limits["deadline_seconds"] - if deadline is not None: - self.deadline = min(self.deadline, deadline) - self.token_provider = token_provider - self.tokens: dict[str, str] = {} - self.request_ids: list[str] = [] - self.requests = 0 - - def read(self, method: str, url: str, *, arm: bool = False, raw: bool = False) -> Any: - if method not in {"GET", "HEAD"}: - raise _fail("source-write-forbidden", "Storage probes may only read.") - remaining = self.deadline - self.monotonic() - if self.requests >= self.limits["max_requests"] or remaining <= 0: - raise _fail("inventory-incomplete", "Inventory exceeded the approved request or time bound.") - audience = MANAGEMENT_AUDIENCE if arm else STORAGE_AUDIENCE - if audience not in self.tokens: - self.tokens[audience] = self.token_provider(audience) - remaining = self.deadline - self.monotonic() - if remaining <= 0: - raise _fail("inventory-incomplete", "Inventory deadline elapsed during authentication.") - self.requests += 1 - try: - result = self.transport( - method, url, self.tokens[audience], timeout=min(30, remaining), - headers={} if arm else {"x-ms-version": STORAGE_VERSION, "Accept": "application/xml"}, - raw_response=raw, max_response_bytes=8 * 1024 * 1024, follow_redirects=False, - response_deadline=time.monotonic() + remaining, - ) - except HelperFailure as failure: - if failure.code in {"response-too-large", "response-deadline-exceeded"}: - raise HelperFailure( - "inventory-incomplete", "Source evidence exceeded the body or time bound.", - blocked_at="source-preflight", request_id=failure.request_id, status=failure.http_status, - ) from failure - raise HelperFailure( - "source-inaccessible", "Source evidence could not be read; inaccessible is not absent.", - blocked_at="source-preflight", request_id=failure.request_id, status=failure.http_status, - ) from failure - if result.request_id: - self.request_ids.append(result.request_id) - if result.status != 200: - raise HelperFailure( - "source-inaccessible", "Source evidence could not be read; inaccessible is not absent.", - blocked_at="source-preflight", request_id=result.request_id, status=result.status, - ) - if self.monotonic() >= self.deadline: - raise _fail("inventory-incomplete", "Inventory deadline elapsed during a read.", result.request_id) - return result - - -def _account(boundary: dict[str, Any], reader: _Reader) -> dict[str, Any]: - response = reader.read( - "GET", f"https://management.azure.com{quote(boundary['storage_id'], safe='/')}?api-version={ARM_VERSION}", arm=True, - ) - body = response.body - if not isinstance(body, dict) or body.get("id") != boundary["storage_id"]: - raise _fail("storage-identity-mismatch", "Storage readback does not match the selected resource.", response.request_id) - properties = body.get("properties") - if not isinstance(properties, dict): - raise _fail("hns-unverified", "Storage readback must explicitly establish HNS.", response.request_id) - # isHnsEnabled is set only at account creation and is immutable thereafter; ARM omits it - # from readback whenever it was never explicitly enabled, which documented behavior treats - # as a permanent, unambiguous false (Blob, not ADLS) rather than an unknown/undetermined state. - raw_hns = properties.get("isHnsEnabled") - if type(raw_hns) is not bool: - if raw_hns is not None: - raise _fail("hns-unverified", "Storage readback must explicitly establish HNS.", response.request_id) - hns_value = False - else: - hns_value = raw_hns - if hns_value != boundary["is_adls"]: - raise _fail("source-drift", "Storage HNS differs from the selected subtype.", response.request_id) - if properties.get("provisioningState") != "Succeeded": - raise _fail("storage-not-ready", "Storage account provisioning is not complete.", response.request_id) - endpoints = properties.get("primaryEndpoints") - account = boundary["storage_id"].rsplit("/", 1)[1] - selected_endpoints = {} - for kind in ("blob", "dfs") if boundary["is_adls"] else ("blob",): - expected = f"https://{account}.{kind}.core.windows.net/" - if not isinstance(endpoints, dict) or endpoints.get(kind) != expected: - raise _fail("storage-endpoint-unverified", "Storage service endpoint is not the selected public-cloud account endpoint.", response.request_id) - selected_endpoints[kind] = expected - return { - "id": body["id"], "hns": hns_value, "endpoints": selected_endpoints, - "network_digest": digest({k: properties.get(k) for k in - ("publicNetworkAccess", "networkAcls", "privateEndpointConnections")}), - } - - -def _object_name(element: Any) -> str: - if element is None or not isinstance(element.text, str) or not element.text: - raise _fail("inventory-invalid", "Listed object has no exact name.") - name = element.text - if element.get("Encoded") == "true": - if re.search(r"%(?![0-9A-Fa-f]{2})", name): - raise _fail("inventory-invalid", "Listed encoded name is ambiguous.") - try: - name = unquote(name, errors="strict") - except UnicodeError as exc: - raise _fail("inventory-invalid", "Listed encoded name is invalid.") from exc - elif element.get("Encoded") not in {None, "false"}: - raise _fail("inventory-invalid", "Unknown listed-name encoding.") - if any(ord(c) < 32 for c in name): - raise _fail("inventory-invalid", "Control characters in object names are unsupported.") - return name - - -def _objects( - boundary: dict[str, Any], account: dict[str, Any], reader: _Reader, -) -> list[dict[str, Any]]: - prefix = boundary["prefix"] - if boundary["is_adls"] and prefix: - prefix += "/" - base = account["endpoints"]["blob"] + boundary["container"] - marker = "" - markers: set[str] = set() - objects: dict[str, dict[str, Any]] = {} - for _ in range(reader.limits["max_pages"]): - query = {"restype": "container", "comp": "list", "prefix": prefix, - "maxresults": str(min(5000, reader.limits["max_objects"])), "marker": marker} - response = reader.read("GET", base + "?" + urlencode(query), raw=True) - payload = response.body - if not isinstance(payload, bytes) or b" reader.limits["max_objects"]: - raise _fail("inventory-incomplete", "Inventory exceeds the approved object bound.", response.request_id) - marker = root.findtext("NextMarker") or "" - if not marker: - return [objects[name] for name in sorted(objects)] - if marker in markers: - raise _fail("inventory-incomplete", "Storage repeated a continuation token.", response.request_id) - markers.add(marker) - raise _fail("inventory-incomplete", "Unconsumed pages exceed the approved page bound.") - - -def _adls_paths( - boundary: dict[str, Any], account: dict[str, Any], objects: list[dict[str, Any]], reader: _Reader, -) -> list[dict[str, Any]]: - paths: dict[str, str] = {"": "directory", boundary["prefix"]: "directory"} - identities = {item["path"]: item for item in objects} - for name, kind in [(boundary["prefix"], "directory")] + [(item["path"], item["kind"]) for item in objects]: - paths[name] = kind - parts = name.split("/") - for index in range(1, len(parts)): - paths["/".join(parts[:index])] = "directory" - records = [] - for path in sorted(paths): - base_url = account["endpoints"]["dfs"] + boundary["container"] + "/" + quote(path, safe="/") - acl_response = reader.read("HEAD", base_url + "?action=getAccessControl&upn=false") - headers = {key.lower(): value for key, value in acl_response.headers.items()} - fields = ("x-ms-owner", "x-ms-group", "x-ms-permissions", "x-ms-acl", "etag") - if any(not headers.get(field) for field in fields): - raise _fail("adls-evidence-unverified", "Exact ADLS path type and ACL/property readback are required.", acl_response.request_id) - # getAccessControl never returns x-ms-resource-type; the filesystem root itself has - # no resource type either, so only non-root paths can be, and must be, type-verified - # via a separate plain getProperties HEAD. - if path: - props_response = reader.read("HEAD", base_url) - properties = {key.lower(): value for key, value in props_response.headers.items()} - resource_type = properties.get("x-ms-resource-type") - if resource_type != paths[path]: - raise _fail("adls-evidence-unverified", "Exact ADLS path type and ACL/property readback are required.", props_response.request_id) - if ( - not properties.get("etag") - or _strip_etag_quotes(properties["etag"]) != _strip_etag_quotes(headers["etag"]) - ): - raise _fail("source-drift", "ADLS access-control and properties ETags disagree.", props_response.request_id) - if path in identities and _strip_etag_quotes(headers["etag"]) != _strip_etag_quotes(identities[path]["etag"]): - raise _fail("source-drift", "Blob and DFS path ETags disagree.", acl_response.request_id) - records.append({"path": path, "kind": paths[path], - "properties_digest": digest({field: headers[field] for field in fields})}) - return records - - -def _snapshot(boundary: dict[str, Any], reader: _Reader) -> dict[str, Any]: - account = _account(boundary, reader) - objects = _objects(boundary, account, reader) - adls = _adls_paths(boundary, account, objects, reader) if boundary["is_adls"] else [] - return {"boundary": boundary, "account": account, "objects": objects, "adls_paths": adls} - - -def discover( - boundary: Any, limits: Any, *, - token_provider: TokenProvider = azure_cli_token, transport: Transport = http_request, - monotonic: Callable[[], float] = time.monotonic, - deadline: float | None = None, -) -> dict[str, Any]: - boundary = validate_boundary(boundary) - reader = _Reader(validate_limits(limits), token_provider, transport, monotonic, deadline) - first = _snapshot(boundary, reader) - second = _snapshot(boundary, reader) - if digest(first) != digest(second): - raise _fail("source-drift", "Consecutive complete source observations differ; no stable evidence is available.") - return { - "status": "discovered", **second, "inventory_digest": digest(second), - "mutation": "none", "writes_performed": [], "request_ids": reader.request_ids, - "operator_reachability": "verified", "managed_ingestion_reachability": "not-proven", - "warnings": [ - "Observations are not an atomic Storage snapshot or a lock; source objects can change afterward.", - "ACL readback establishes observed metadata, not effective Search principal permissions.", - ], - } +from __future__ import annotations + +import re +import time +from typing import Any, Callable +from urllib.parse import quote, unquote, urlencode +from xml.etree import ElementTree + +try: + from ._common import ( + MANAGEMENT_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, + digest, http_request, reject_secrets, require_allowed_fields, + ) +except ImportError: + from _common import ( # type: ignore[no-redef] + MANAGEMENT_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, + digest, http_request, reject_secrets, require_allowed_fields, + ) + + +STORAGE_AUDIENCE = "https://storage.azure.com/" +ARM_VERSION = "2025-06-01" +STORAGE_VERSION = "2025-05-05" +ACCOUNT_ID = re.compile( + r"^/subscriptions/[0-9a-fA-F-]{36}/resourceGroups/[^/;?#\r\n]+/" + r"providers/Microsoft\.Storage/storageAccounts/([a-z0-9]{3,24})$" +) +CONTAINER = re.compile(r"^[a-z0-9](?:[a-z0-9-]{1,61}[a-z0-9])$") + + +def _fail(code: str, message: str, request_id: str | None = None) -> HelperFailure: + return HelperFailure(code, message, blocked_at="source-preflight", request_id=request_id) + + +def _strip_etag_quotes(value: str) -> str: + # Blob List XML returns unquoted ETags; DFS HTTP responses return + # RFC 7232-quoted ETags for the identical underlying value. + return value[1:-1] if len(value) >= 2 and value[0] == '"' and value[-1] == '"' else value + + +def validate_boundary(value: Any) -> dict[str, Any]: + if not isinstance(value, dict): + raise _fail("boundary-invalid", "An explicitly selected Storage boundary is required.") + reject_secrets(value) + require_allowed_fields(value, {"storage_id", "container", "prefix", "is_adls"}, label="boundary") + if ( + not isinstance(value.get("storage_id"), str) + or ACCOUNT_ID.fullmatch(value["storage_id"]) is None + or not isinstance(value.get("container"), str) + or CONTAINER.fullmatch(value["container"]) is None + or "--" in value["container"] + or not isinstance(value.get("prefix"), str) + or type(value.get("is_adls")) is not bool + ): + raise _fail("boundary-invalid", "Exact Storage ID, container, explicit prefix (including empty root), and subtype are required.") + prefix = value["prefix"] + try: + value["storage_id"].encode("utf-8") + prefix.encode("utf-8") + except UnicodeError as exc: + raise _fail("boundary-invalid", "Storage ID and prefix must be valid UTF-8.") from exc + if ( + prefix.startswith("/") + or "\\" in prefix + or any(ord(c) < 32 for c in prefix) + or (any(part in {".", "..", ""} for part in prefix.rstrip("/").split("/")) and prefix != "") + or "//" in prefix + or value["is_adls"] and prefix.endswith("/") + or not value["is_adls"] and prefix and not prefix.endswith("/") + ): + raise _fail( + "boundary-ambiguous", + "Use explicit empty root, a Blob folder prefix ending in '/', or an ADLS directory path without a trailing '/'. Arbitrary partial-name prefixes are not verified.", + ) + return dict(value) + + +def validate_limits(value: Any) -> dict[str, int]: + if not isinstance(value, dict): + raise _fail("input-schema-invalid", "inventory_limits must be an object.") + require_allowed_fields(value, {"max_pages", "max_objects", "max_requests", "deadline_seconds"}, + label="inventory limits") + for field, maximum in ( + ("max_pages", 1000), ("max_objects", 100000), ("max_requests", 200000), + ("deadline_seconds", 3600), + ): + if type(value.get(field)) is not int or not 1 <= value[field] <= maximum: + raise _fail("input-schema-invalid", "Inventory limits must be bounded positive integers.") + return value + + +class _Reader: + def __init__( + self, limits: dict[str, int], token_provider: TokenProvider, + transport: Transport, monotonic: Callable[[], float], + deadline: float | None = None, + ) -> None: + self.limits = limits + self.transport = transport + self.monotonic = monotonic + self.deadline = monotonic() + limits["deadline_seconds"] + if deadline is not None: + self.deadline = min(self.deadline, deadline) + self.token_provider = token_provider + self.tokens: dict[str, str] = {} + self.request_ids: list[str] = [] + self.requests = 0 + + def read(self, method: str, url: str, *, arm: bool = False, raw: bool = False) -> Any: + if method not in {"GET", "HEAD"}: + raise _fail("source-write-forbidden", "Storage probes may only read.") + remaining = self.deadline - self.monotonic() + if self.requests >= self.limits["max_requests"] or remaining <= 0: + raise _fail("inventory-incomplete", "Inventory exceeded the approved request or time bound.") + audience = MANAGEMENT_AUDIENCE if arm else STORAGE_AUDIENCE + if audience not in self.tokens: + self.tokens[audience] = self.token_provider(audience) + remaining = self.deadline - self.monotonic() + if remaining <= 0: + raise _fail("inventory-incomplete", "Inventory deadline elapsed during authentication.") + self.requests += 1 + try: + result = self.transport( + method, url, self.tokens[audience], timeout=min(30, remaining), + headers={} if arm else {"x-ms-version": STORAGE_VERSION, "Accept": "application/xml"}, + raw_response=raw, max_response_bytes=8 * 1024 * 1024, follow_redirects=False, + response_deadline=time.monotonic() + remaining, + ) + except HelperFailure as failure: + if failure.code in {"response-too-large", "response-deadline-exceeded"}: + raise HelperFailure( + "inventory-incomplete", "Source evidence exceeded the body or time bound.", + blocked_at="source-preflight", request_id=failure.request_id, status=failure.http_status, + ) from failure + raise HelperFailure( + "source-inaccessible", "Source evidence could not be read; inaccessible is not absent.", + blocked_at="source-preflight", request_id=failure.request_id, status=failure.http_status, + ) from failure + if result.request_id: + self.request_ids.append(result.request_id) + if result.status != 200: + raise HelperFailure( + "source-inaccessible", "Source evidence could not be read; inaccessible is not absent.", + blocked_at="source-preflight", request_id=result.request_id, status=result.status, + ) + if self.monotonic() >= self.deadline: + raise _fail("inventory-incomplete", "Inventory deadline elapsed during a read.", result.request_id) + return result + + +def _account(boundary: dict[str, Any], reader: _Reader) -> dict[str, Any]: + response = reader.read( + "GET", f"https://management.azure.com{quote(boundary['storage_id'], safe='/')}?api-version={ARM_VERSION}", arm=True, + ) + body = response.body + if not isinstance(body, dict) or body.get("id") != boundary["storage_id"]: + raise _fail("storage-identity-mismatch", "Storage readback does not match the selected resource.", response.request_id) + properties = body.get("properties") + if not isinstance(properties, dict): + raise _fail("hns-unverified", "Storage readback must explicitly establish HNS.", response.request_id) + # isHnsEnabled is set only at account creation and is immutable thereafter; ARM omits it + # from readback whenever it was never explicitly enabled, which documented behavior treats + # as a permanent, unambiguous false (Blob, not ADLS) rather than an unknown/undetermined state. + raw_hns = properties.get("isHnsEnabled") + if type(raw_hns) is not bool: + if raw_hns is not None: + raise _fail("hns-unverified", "Storage readback must explicitly establish HNS.", response.request_id) + hns_value = False + else: + hns_value = raw_hns + if hns_value != boundary["is_adls"]: + raise _fail("source-drift", "Storage HNS differs from the selected subtype.", response.request_id) + if properties.get("provisioningState") != "Succeeded": + raise _fail("storage-not-ready", "Storage account provisioning is not complete.", response.request_id) + endpoints = properties.get("primaryEndpoints") + account = boundary["storage_id"].rsplit("/", 1)[1] + selected_endpoints = {} + for kind in ("blob", "dfs") if boundary["is_adls"] else ("blob",): + expected = f"https://{account}.{kind}.core.windows.net/" + if not isinstance(endpoints, dict) or endpoints.get(kind) != expected: + raise _fail("storage-endpoint-unverified", "Storage service endpoint is not the selected public-cloud account endpoint.", response.request_id) + selected_endpoints[kind] = expected + return { + "id": body["id"], "hns": hns_value, "endpoints": selected_endpoints, + "network_digest": digest({k: properties.get(k) for k in + ("publicNetworkAccess", "networkAcls", "privateEndpointConnections")}), + } + + +def _object_name(element: Any) -> str: + if element is None or not isinstance(element.text, str) or not element.text: + raise _fail("inventory-invalid", "Listed object has no exact name.") + name = element.text + if element.get("Encoded") == "true": + if re.search(r"%(?![0-9A-Fa-f]{2})", name): + raise _fail("inventory-invalid", "Listed encoded name is ambiguous.") + try: + name = unquote(name, errors="strict") + except UnicodeError as exc: + raise _fail("inventory-invalid", "Listed encoded name is invalid.") from exc + elif element.get("Encoded") not in {None, "false"}: + raise _fail("inventory-invalid", "Unknown listed-name encoding.") + if any(ord(c) < 32 for c in name): + raise _fail("inventory-invalid", "Control characters in object names are unsupported.") + return name + + +def _objects( + boundary: dict[str, Any], account: dict[str, Any], reader: _Reader, +) -> list[dict[str, Any]]: + prefix = boundary["prefix"] + if boundary["is_adls"] and prefix: + prefix += "/" + base = account["endpoints"]["blob"] + boundary["container"] + marker = "" + markers: set[str] = set() + objects: dict[str, dict[str, Any]] = {} + for _ in range(reader.limits["max_pages"]): + query = {"restype": "container", "comp": "list", "prefix": prefix, + "maxresults": str(min(5000, reader.limits["max_objects"])), "marker": marker} + response = reader.read("GET", base + "?" + urlencode(query), raw=True) + payload = response.body + if not isinstance(payload, bytes) or b" reader.limits["max_objects"]: + raise _fail("inventory-incomplete", "Inventory exceeds the approved object bound.", response.request_id) + marker = root.findtext("NextMarker") or "" + if not marker: + return [objects[name] for name in sorted(objects)] + if marker in markers: + raise _fail("inventory-incomplete", "Storage repeated a continuation token.", response.request_id) + markers.add(marker) + raise _fail("inventory-incomplete", "Unconsumed pages exceed the approved page bound.") + + +def _adls_paths( + boundary: dict[str, Any], account: dict[str, Any], objects: list[dict[str, Any]], reader: _Reader, +) -> list[dict[str, Any]]: + paths: dict[str, str] = {"": "directory", boundary["prefix"]: "directory"} + identities = {item["path"]: item for item in objects} + for name, kind in [(boundary["prefix"], "directory")] + [(item["path"], item["kind"]) for item in objects]: + paths[name] = kind + parts = name.split("/") + for index in range(1, len(parts)): + paths["/".join(parts[:index])] = "directory" + records = [] + for path in sorted(paths): + base_url = account["endpoints"]["dfs"] + boundary["container"] + "/" + quote(path, safe="/") + acl_response = reader.read("HEAD", base_url + "?action=getAccessControl&upn=false") + headers = {key.lower(): value for key, value in acl_response.headers.items()} + fields = ("x-ms-owner", "x-ms-group", "x-ms-permissions", "x-ms-acl", "etag") + if any(not headers.get(field) for field in fields): + raise _fail("adls-evidence-unverified", "Exact ADLS path type and ACL/property readback are required.", acl_response.request_id) + # getAccessControl never returns x-ms-resource-type; the filesystem root itself has + # no resource type either, so only non-root paths can be, and must be, type-verified + # via a separate plain getProperties HEAD. + if path: + props_response = reader.read("HEAD", base_url) + properties = {key.lower(): value for key, value in props_response.headers.items()} + resource_type = properties.get("x-ms-resource-type") + if resource_type != paths[path]: + raise _fail("adls-evidence-unverified", "Exact ADLS path type and ACL/property readback are required.", props_response.request_id) + if ( + not properties.get("etag") + or _strip_etag_quotes(properties["etag"]) != _strip_etag_quotes(headers["etag"]) + ): + raise _fail("source-drift", "ADLS access-control and properties ETags disagree.", props_response.request_id) + if path in identities and _strip_etag_quotes(headers["etag"]) != _strip_etag_quotes(identities[path]["etag"]): + raise _fail("source-drift", "Blob and DFS path ETags disagree.", acl_response.request_id) + records.append({"path": path, "kind": paths[path], + "properties_digest": digest({field: headers[field] for field in fields})}) + return records + + +def _snapshot(boundary: dict[str, Any], reader: _Reader) -> dict[str, Any]: + account = _account(boundary, reader) + objects = _objects(boundary, account, reader) + adls = _adls_paths(boundary, account, objects, reader) if boundary["is_adls"] else [] + return {"boundary": boundary, "account": account, "objects": objects, "adls_paths": adls} + + +def discover( + boundary: Any, limits: Any, *, + token_provider: TokenProvider = azure_cli_token, transport: Transport = http_request, + monotonic: Callable[[], float] = time.monotonic, + deadline: float | None = None, +) -> dict[str, Any]: + boundary = validate_boundary(boundary) + reader = _Reader(validate_limits(limits), token_provider, transport, monotonic, deadline) + first = _snapshot(boundary, reader) + second = _snapshot(boundary, reader) + if digest(first) != digest(second): + raise _fail("source-drift", "Consecutive complete source observations differ; no stable evidence is available.") + return { + "status": "discovered", **second, "inventory_digest": digest(second), + "mutation": "none", "writes_performed": [], "request_ids": reader.request_ids, + "operator_reachability": "verified", "managed_ingestion_reachability": "not-proven", + "warnings": [ + "Observations are not an atomic Storage snapshot or a lock; source objects can change afterward.", + "ACL readback establishes observed metadata, not effective Search principal permissions.", + ], + } diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_recheck.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_recheck.py index 2b474bde..f49d06d9 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_recheck.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_recheck.py @@ -1,1131 +1,1131 @@ -"""Retained Blob creation/reuse observations and read-only readiness follow-up.""" -from __future__ import annotations - -import argparse -import copy -import json -import os -import re -import stat -import sys -import time -import uuid -from datetime import datetime, timezone -from pathlib import Path -from urllib.parse import urlsplit - -try: - from . import _bootstrap_io as private_io - from ._progress import Progress, add_progress_argument, reporting - from . import blob_inventory, blob_source, search_reconcile, source_vector, _indexer_observation as indexer - from . import _blob_observation as semantic - from ._common import ( - HelperFailure, SEARCH_AUDIENCE, azure_cli_token, blocked_result, digest, - emit_result, http_request, reject_secrets, - ) -except ImportError: - import _bootstrap_io as private_io - from _progress import Progress, add_progress_argument, reporting - import blob_inventory, blob_source, search_reconcile, source_vector - import _indexer_observation as indexer - import _blob_observation as semantic - from _common import ( - HelperFailure, SEARCH_AUDIENCE, azure_cli_token, blocked_result, digest, - emit_result, http_request, reject_secrets, - ) - - -fail = blob_source._failure -COLLECTIONS = {"datasource": "datasources", "indexer": "indexers", - "skillset": "skillsets", "index": "indexes"} - - -def _json(raw): - def unique(pairs): - value = {} - for key, child in pairs: - if key in value: - raise ValueError("Duplicate field") - value[key] = child - return value - try: - value = json.loads(raw, object_pairs_hook=unique) - json.dumps(value, allow_nan=False, ensure_ascii=False).encode("utf-8") - if not isinstance(value, dict): - raise ValueError("Expected object") - return value - except (ValueError, UnicodeError, RecursionError) as exc: - raise fail("recheck-evidence-invalid", "Retain bounded, unmodified UTF-8 object evidence.") from exc - - -def read_private(path): - path = Path(path) - private_io.private_directory(str(path.parent)) - try: - selected = path.lstat() - if (not stat.S_ISREG(selected.st_mode) or selected.st_nlink != 1 - or getattr(selected, "st_file_attributes", 0) & 0x400): - raise OSError("Not an ordinary private file") - if os.name == "nt": - private_io._windows_private(path) - elif selected.st_uid != os.getuid() or stat.S_IMODE(selected.st_mode) & 0o077: - raise OSError("Not private") - descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) - with os.fdopen(descriptor, "rb") as handle: - opened = os.fstat(handle.fileno()) - if (selected.st_dev, selected.st_ino) != (opened.st_dev, opened.st_ino): - raise OSError("Evidence changed identity") - raw = handle.read(private_io.MAX_BYTES + 1) - if len(raw) > private_io.MAX_BYTES: - raise OSError("Evidence exceeds bound") - return _json(raw.decode("utf-8")) - except (OSError, UnicodeError) as exc: - raise fail("recheck-evidence-unreadable", "Select existing private, unlinked evidence files; no permissions were changed.") from exc - - -def account_context(): - code, stdout, _ = private_io.run_cli(["account", "show"], 30) - if code: - raise fail("recheck-auth-context-unavailable", "Current signed-in CLI context is inaccessible; details withheld.") - account = _json(stdout) - user = account.get("user") - if (account.get("environmentName") != "AzureCloud" or account.get("state") != "Enabled" - or not isinstance(user, dict) or user.get("type") not in ("user", "servicePrincipal") - or any(not isinstance(value, str) or not value.strip() for value in - (account.get("id"), account.get("tenantId"), user.get("name")))): - raise fail("recheck-auth-context-unavailable", "An enabled public-cloud tenant/subscription/principal context is required.") - return digest({key: account[key] for key in ("id", "tenantId", "environmentName")} - | {"principal": {"name": user["name"], "type": user["type"]}}) - - -def _supported(plan): - try: - source, _ = blob_source._validate_plan(plan) - ingestion = source["desired"]["azureBlobParameters"]["ingestionParameters"] - except (KeyError, TypeError, AttributeError, RecursionError) as exc: - raise fail("recheck-evidence-invalid", "Original Blob creation plan is malformed.") from exc - if (ingestion.get("contentExtractionMode") not in ("minimal", "standard") or ingestion.get("identity") is not None - or source["api_version"] not in {"2026-04-01", "2026-08-01-preview"} - or plan["boundary"]["is_adls"] and source["api_version"] != "2026-08-01-preview"): - raise fail("recheck-scope-unsupported", "Checkpointing requires supported Blob extraction and provable system-assigned authentication.") - schedule = ingestion.get("ingestionSchedule") - indexer.schedule(schedule) - - -def _etag(value): - etag = value.get("@odata.etag") if isinstance(value, dict) else None - if not isinstance(etag, str) or not etag.strip(): - raise fail("recheck-evidence-missing", "Every source/generated readback requires a nonempty string ETag.") - return etag - - -def _processing_readback(plan, current): - desired = plan["source"]["desired"]["azureBlobParameters"]["ingestionParameters"] - if desired.get("contentExtractionMode") == "standard": - ai = desired.get("aiServices") - if not isinstance(ai, dict) or not isinstance(ai.get("uri"), str) or not ai["uri"].strip(): - raise fail("recheck-processing-unverified", "Standard extraction needs its original CU endpoint evidence.") - blob_source.verify_content_understanding_readback({"endpoint": ai["uri"]}, current) - source_vector.verify_source_readback(plan.get("embedding"), current) - model = desired.get("embeddingModel") - if model is not None and "embedding" not in plan: - parameters = model.get("azureOpenAIParameters") if isinstance(model, dict) else None - if (not isinstance(parameters, dict) or model.get("kind") != "azureOpenAI" - or any(not isinstance(parameters.get(key), str) or not parameters[key].strip() - for key in ("resourceUri", "deploymentId", "modelName"))): - raise fail("recheck-processing-unverified", "Legacy embedding configuration cannot be verified by this client.") - source_vector.verify_source_readback({ - "endpoint": parameters["resourceUri"], "deployment": parameters["deploymentId"], - "model": parameters["modelName"], - }, current) - - -def _binding(plan, record, current, generated, binding_receipts=None): - source = plan["source"] - observed = {"type": "knowledge-source", "name": source["name"], "etag": _etag(current), - "definition_digest": digest(search_reconcile._definition(current))} - if source["action"] == "create": - blob_source._verify_creation_binding(source, plan["boundary"], current, generated, (plan, record)) - elif (record["verification"]["readback"] != observed - or not search_reconcile.definitions_match(source["desired"], current) - or source.get("expected_etag", observed["etag"]) != observed["etag"]): - raise fail("source-binding-unverified", "Reused source does not match its retained read-only identity/configuration.") - if (blob_source.generated_resources(current, strict=True) != generated - or plan.get("expected_generated", generated) != generated): - raise fail("definition-drift", "Generated source identities changed.") - if binding_receipts is not None and ( - binding_receipts[0]["owner"] != plan["owner"] - or binding_receipts[0]["inventory_digest"] != plan["inventory_digest"] - ): - raise fail("source-binding-unverified", "Retained creation must bind the same owner and Storage inventory.") - blob_source._verify_storage_binding( - source, plan["boundary"], current, generated, - lambda: (plan, record) if source["action"] == "create" else binding_receipts, - ) - _processing_readback(plan, current) - - -def _sanitized_datasource_url(url): - parsed = urlsplit(url) - search_reconcile.validate_search_endpoint(f"{parsed.scheme}://{parsed.netloc}") - if (parsed.fragment or parsed.query not in { - f"api-version={version}" for version in search_reconcile.SUPPORTED_API_VERSIONS} - or re.fullmatch(r"/datasources\('[a-zA-Z0-9][a-zA-Z0-9_-]{0,127}'\)", parsed.path) is None): - raise fail("recheck-read-url-invalid", "Sanitized binding requires one exact datasource identity and the bound supported API version.") - return url + "&includeConnectionString=true" - - -def _reader(transport, token_provider, *, warnings=None): - recovery = None - recovery_warnings = warnings if warnings is not None else [] - - def tracked(method, url, token, **kwargs): - try: - parsed = urlsplit(url) - search_reconcile.validate_search_endpoint(f"{parsed.scheme}://{parsed.netloc}") - if parsed.fragment: - raise fail("recheck-read-url-invalid", "Read-only Search URLs cannot contain fragments.") - if parsed.query not in { - f"api-version={version}" for version in search_reconcile.SUPPORTED_API_VERSIONS}: - suffix = "&includeConnectionString=true" - if not url.endswith(suffix) or _sanitized_datasource_url(url[:-len(suffix)]) != url: - raise fail("recheck-read-url-invalid", "Only the exact sanitized datasource GET option is permitted.") - kwargs["timeout"] = min(60, kwargs.get("timeout", 60)) - if recovery is None: - response = transport(method, url, token, **kwargs) - else: - response = recovery.get( - url, token, transport=transport, - max_requests=1 if url.endswith("&includeConnectionString=true") else 2, - **kwargs, - ) - ids.extend(recovery.request_ids[:-1]) - if method == "GET" and response.status == 200 and isinstance(response.body, dict): - etag = search_reconcile.resolve_etag(search_reconcile.response_etags(response), response.request_id) - if etag is not None: - response = search_reconcile.HttpResult( - response.status, {**response.body, "@odata.etag": etag}, response.headers, response.etag_values, - ) - return response - except HelperFailure as failure: - if recovery is not None: - ids.extend(recovery.request_ids) - elif failure.request_id: - ids.append(failure.request_id) - raise - read, original_get, ids = source_vector._reader(tracked, token_provider) - - def get(url): - nonlocal recovery - recovery = search_reconcile.ReadRecovery() - try: - result = original_get(url) - get.request_id = ids[-1] if ids else None - return result - except HelperFailure as failure: - if failure.code in {"vector-resource-missing", "vector-evidence-missing", "etag-invalid"}: - missing = failure.code == "vector-resource-missing" - raise HelperFailure( - "recheck-resource-missing" if missing else "recheck-evidence-missing", - "An exact original source/generated definition or ETag is unavailable.", - blocked_at="verification", status=404 if missing else failure.http_status, - request_id=ids[-1] if ids else failure.request_id, - ) from failure - raise - finally: - recovery_warnings.extend(recovery.warnings) - recovery = None - get.recovery_warnings = recovery_warnings - return read, get, ids - - -def _configuration(plan, creation, generated, get, *, binding_receipts=None, expected=None, - diagnostics=None, observations=None, binding_observations=None): - diagnostics = diagnostics if diagnostics is not None else [] - source = plan["source"] - current = get(search_reconcile.resource_url(source)) - _binding(plan, creation, current, generated, binding_receipts) - names = {item["type"]: item["name"] for item in generated} - snapshots = {} - failures = [] - for kind, collection in COLLECTIONS.items(): - name = names[kind] - url = f"{source['endpoint'].rstrip('/')}/{collection}('{name}')?api-version={source['api_version']}" - try: - snapshots[kind] = _child_configuration( - plan, kind, name, names, get, url, expected, diagnostics, observations, binding_observations, - ) - except HelperFailure as failure: - if failure.request_id is None: - failure.request_id = getattr(get, "request_id", None) - semantic.note(diagnostics, kind, "error", failure.code, "definition", failure.request_id) - failures.append(failure) - else: - semantic.note(diagnostics, kind, "info", "generated-configuration-verified", "definition", - getattr(get, "request_id", None)) - refreshed = get(search_reconcile.resource_url(source)) - _binding(plan, creation, refreshed, generated, binding_receipts) - if failures: - raise failures[0] - return snapshots - - -def _child_configuration(plan, kind, name, names, get, url, expected, diagnostics, observations, - binding_observations=None): - source = plan["source"] - child = get(url) - etag = _etag(child) - if child.get("name") != name: - raise fail("definition-drift", "Generated configuration returned an unrelated identity.") - if kind == "datasource": - boundary = plan["boundary"] - container, credentials = child.get("container"), child.get("credentials") - invariants = { - "type": child.get("type") == ("adlsgen2" if boundary["is_adls"] else "azureblob"), - "container.name": isinstance(container, dict) and container.get("name") == boundary["container"], - "container.query": isinstance(container, dict) and container.get("query") in - (boundary["prefix"], None if not boundary["prefix"] else boundary["prefix"]), - "credentials": credentials is None or - isinstance(credentials, dict) and not set(credentials) - {"connectionString"}, - "identity": child.get("identity") is None, - } - for field, valid in invariants.items(): - if not valid: - semantic.note(diagnostics, kind, "error", "source-binding-unverified", field, - getattr(get, "request_id", None)) - if not all(invariants.values()): - raise fail("source-binding-unverified", "Generated datasource type/container/prefix/identity or credential shape conflicts with the bound source.", - request_id=getattr(get, "request_id", None)) - try: - binding = blob_source._connection_binding( - credentials.get("connectionString") if isinstance(credentials, dict) else None, boundary, - ) - except HelperFailure as failure: - failure.request_id = getattr(get, "request_id", None) - semantic.note(diagnostics, kind, "error", failure.code, "credentials.connectionString", failure.request_id) - raise - if kind == "indexer": - request_id = getattr(get, "request_id", None) - for field, target in (("dataSourceName", "datasource"), ("targetIndexName", "index"), ("skillsetName", "skillset")): - if child.get(field) != names[target]: - raise indexer.failure(diagnostics, "indexer-binding-mismatch", field, - f"Generated indexer {field} does not match the bound resource; details withheld.", request_id) - observed = indexer.observe(child, source["desired"]["azureBlobParameters"]["ingestionParameters"].get("ingestionSchedule"), - diagnostics, request_id) - if observations is not None: - observations.append({"request_id": request_id, "etag_digest": digest(etag), - **{key: value for key, value in observed.items() if key != "etag"}}) - snapshot = observed if expected is None or indexer.PROJECTION_FIELDS <= set(expected[kind]) else { - "etag": etag, "digest": digest(child), - } - if expected is None or semantic.PROJECTION_FIELDS <= set(expected[kind]): - snapshot.update(semantic.observe(child, kind)) - if expected is not None: - if semantic.PROJECTION_FIELDS <= set(expected[kind]): - semantic.compare(snapshot, expected[kind], kind, diagnostics, request_id) - # A core-equal ETag-only revision is not schedule or configuration drift. - if snapshot["schedule_raw_digest"] != expected[kind]["schedule_raw_digest"]: - indexer.compare(snapshot, expected[kind], diagnostics, request_id) - else: - indexer.compare(snapshot, expected[kind], diagnostics, request_id) - return snapshot - # Keep only integrity observations, never generated credentials or skill text. - snapshot = {"etag": etag, "digest": digest(child)} - if expected is None or semantic.PROJECTION_FIELDS <= set(expected[kind]): - snapshot.update(semantic.observe(child, kind)) - if kind == "datasource": - snapshot["binding_proof"] = "resource-id" if binding == "visible" else "unverified" - if expected is not None: - semantic.compare(snapshot, expected[kind], kind, diagnostics, getattr(get, "request_id", None)) - if kind == "datasource" and binding == "concealed": - _current_datasource_binding(get, url, snapshot, plan["boundary"], diagnostics, - binding_observations) - elif kind == "datasource" and binding == "concealed": - semantic.note(diagnostics, kind, "info", "datasource-credential-projection-concealed", - "credentials.connectionString", getattr(get, "request_id", None)) - return snapshot - visible_preimage = False - if kind == "datasource" and expected and expected[kind]["etag"] == etag: - # Only concealed/verified credentials may vary; every other byte and the ETag stay bound. - for credentials in ( - None, {}, {"connectionString": None}, {"connectionString": ""}, - {"connectionString": ""}, {"connectionString": ""}, - {"connectionString": f"ResourceId={plan['boundary']['storage_id']}"}, - {"connectionString": f"ResourceId={plan['boundary']['storage_id']};"}, - ): - candidate = {**child, "credentials": credentials} - if digest(candidate) == expected[kind]["digest"]: - snapshot["digest"] = expected[kind]["digest"] - if isinstance(credentials, dict) and isinstance(credentials.get("connectionString"), str): - visible_preimage = credentials["connectionString"].startswith("ResourceId=") - candidate = {key: value for key, value in child.items() if key != "credentials"} - if digest(candidate) == expected[kind]["digest"]: - snapshot["digest"] = expected[kind]["digest"] - if snapshot != expected[kind]: - raise fail("generated-legacy-evidence-insufficient", - "Legacy generated full-hash evidence differs; no core projection was retained.", - request_id=getattr(get, "request_id", None)) - if kind == "datasource" and binding == "concealed": - if visible_preimage: - semantic.note(diagnostics, kind, "warning", "datasource-credential-projection-changed", - "credentials", getattr(get, "request_id", None)) - else: - _current_datasource_binding(get, url, {"etag": etag, "digest": digest(child), **semantic.observe(child, kind)}, - plan["boundary"], diagnostics, binding_observations) - return snapshot - - -def _current_datasource_binding(get, url, snapshot, boundary, diagnostics, observations=None): - observation = { - "schema_version": "1.0", "status": "unverified", "request_id": None, - "plain": {"etag_digest": digest(snapshot["etag"]), "definition_digest": snapshot["digest"], - "core_digest": snapshot["core_digest"]}, - "sanitized": None, - } - if observations is not None: - observations.append(observation) - # The service sanitizes internally; this option never requests raw keys or SAS. - try: - current = get(_sanitized_datasource_url(url)) - except HelperFailure as failure: - observation["request_id"] = failure.request_id - raise - credentials = current.get("credentials") - request_id = getattr(get, "request_id", None) - observation["request_id"] = request_id - projection = semantic.observe(current, "datasource") - observation["sanitized"] = { - "etag_digest": digest(_etag(current)), "definition_digest": digest(current), - "core_digest": projection["core_digest"], - } - if (_etag(current) != snapshot["etag"] - or observation["sanitized"]["core_digest"] != snapshot["core_digest"]): - fields = projection["field_digests"].keys() | snapshot["field_digests"].keys() - changed = [field for field in sorted(fields) - if projection["field_digests"].get(field) != snapshot["field_digests"].get(field)] - if _etag(current) != snapshot["etag"]: - changed.append("@odata.etag") - for field in changed: - semantic.note(diagnostics, "datasource", "error", "datasource-binding-unverified", field, request_id) - raise fail("datasource-binding-unverified", - "Independent datasource binding readback changed revision or configuration; no stable current proof is available.", - request_id=request_id) - try: - visible = (isinstance(credentials, dict) and set(credentials) == {"connectionString"} - and blob_source._connection_binding(credentials["connectionString"], boundary) == "visible") - except HelperFailure: - visible = False - if not visible: - semantic.note(diagnostics, "datasource", "error", "datasource-binding-unverified", - "credentials.connectionString", request_id) - raise fail("datasource-binding-unverified", - "Sanitized current datasource readback does not prove the exact selected keyless ResourceId; values withheld. Retain resources and investigate this evidence gap read-only.", - request_id=request_id) - observation["status"] = "verified" - semantic.note(diagnostics, "datasource", "info", "datasource-current-binding-verified", - "credentials.connectionString", request_id) - - -def _baseline_cycle(source, read, token_provider): - url = search_reconcile.resource_url(source).replace(")?", ")/status?") - response = read("GET", url, token_provider(SEARCH_AUDIENCE)) - body = response.body - if response.status != 200 or not isinstance(body, dict) or body.get("kind") != "azureBlob": - raise HelperFailure("ingestion-inaccessible", "Initial reuse status is unavailable.", - blocked_at="verification", status=response.status, request_id=response.request_id) - last = body.get("lastSynchronizationState") - if last is None: - return None - if not isinstance(last, dict): - raise fail("ingestion-status-invalid", "Initial reuse synchronization must be an object.") - if last.get("endTime") is None: - blob_source._timestamp(last.get("startTime")) - return None - cycle = [last.get("startTime"), last.get("endTime")] - _validate_cycle(cycle) - return cycle - - -def _validate_cycle(cycle): - if cycle is None: - return - if not isinstance(cycle, list) or len(cycle) != 2: - raise fail("recheck-evidence-invalid", "Retain the original observed reuse cycle, not a reconstructed bound.") - if blob_source._timestamp(cycle[1]) < blob_source._timestamp(cycle[0]): - raise fail("ingestion-status-invalid", "Initial synchronization interval is invalid.") - - -def _binding_receipts(paths): - if paths is None: - return None - if (not isinstance(paths, (tuple, list)) or len(paths) != 2 - or any(not isinstance(path, (str, Path)) or not str(path).strip() for path in paths)): - raise fail("reuse-evidence-invalid", "Select both existing private creation evidence files.") - _, prior, fingerprint = _document(paths[0]) - result = read_private(paths[1]) - blob_source._validate_reuse_receipts(prior, fingerprint, result) - return prior, result - - -class Checkpoint: - def __init__(self, directory, plan, *, context_provider=account_context, binding_receipts=None): - _supported(plan) - self.directory = private_io.private_directory(str(directory)) - self.context_provider = context_provider - self.context = context_provider() - self.plan_digest = digest(plan) - self.operation_id = uuid.uuid4().hex - self.summary = None - self.excluded_cycle = None - self.request_ids = [] - self.recovery_warnings = [] - self.acknowledgement = None - self.write_acknowledgement = None - self.write_observation = None - self.binding_receipts = binding_receipts - self.diagnostics = [] - self.observations = [] - self.binding_observations = [] - self.configuration = None - self.creation = None - - def acknowledge(self, plan, response, not_before, *, url, body, headers): - if (digest(plan) != self.plan_digest or plan["source"]["action"] != "create" - or response.status not in {200, 201} - or url != search_reconcile.resource_url(plan["source"]) - or headers.get("If-None-Match") != "*" or "If-Match" in headers - or body != search_reconcile.canonical_bytes(plan["source"]["desired"])): - raise fail("recheck-ownership-unproven", "Only the exact successful conditional create can retain a write acknowledgement.") - self.write_observation = { - "schema_version": "1.0", "state": "acknowledged", "type": "knowledge-source", - "name": plan["source"]["name"], "http_status": response.status, "request_id": response.request_id, - } - if self.context_provider() != self.context: - raise fail("recheck-auth-context-drift", "CLI context changed before write acknowledgement retention.") - receipt = { - "schema_version": "1.0", "kind": "blob-write-acknowledgement", - "operation_id": self.operation_id, "plan_digest": self.plan_digest, - "not_before": not_before.isoformat(), "context_digest": self.context, - "write": { - "method": "PUT", "url": url, "if_none_match": headers["If-None-Match"], - "body_digest": digest(plan["source"]["desired"]), "status": response.status, - "request_id": response.request_id, "response_etags": search_reconcile.response_etags(response), - "generated": blob_source.generated_resources(response.body) if isinstance(response.body, dict) else [], - }, - } - reject_secrets(receipt) - receipt["integrity"] = digest(receipt) - path = private_io.private_file(self.directory, self.operation_id + ".blob-write.json", receipt) - self.write_acknowledgement = {"operation_id": self.operation_id, "receipt_file": path.name, - "evidence_digest": receipt["integrity"], "status": "retained"} - - def persist(self, plan, result, generated, not_before, *, token_provider, transport): - reused = plan["source"]["action"] == "reuse" - selected, other = ("reused", "created") if reused else ("created", "reused") - owned = "reused_not_owned" if reused else "run_owned" - if (digest(plan) != self.plan_digest or len(result["resources"][selected]) != 1 - or result["resources"][other] or result["resources"][selected] != result["ownership"][owned]): - raise fail("recheck-ownership-unproven", "Retain exact acknowledged creation or read-only reuse; never adopt shared resources.") - generated = copy.deepcopy(generated) - creation = {key: copy.deepcopy(result[key]) for key in - ("approved_plan", "resources", "verification", "ownership")} - creation["source"] = {"generated": generated} - seed = { - "schema_version": "1.0", "kind": "blob-source-acknowledgement", - "operation_id": self.operation_id, "plan_digest": digest(plan), - "not_before": not_before.isoformat(), "context_digest": self.context, - "creation": creation, "request_ids": [], - } - if not reused: - if self.context_provider() != self.context: - raise fail("recheck-auth-context-drift", "CLI context changed before acknowledgement retention.") - seed["integrity"] = digest(seed) - path = private_io.private_file(self.directory, self.operation_id + ".blob-ack.json", seed) - self.acknowledgement = {"operation_id": self.operation_id, "receipt_file": path.name, - "evidence_digest": seed["integrity"], "status": "retained"} - read, get, ids = _reader(transport, token_provider, warnings=self.recovery_warnings) - self.request_ids = ids - configuration = _configuration(plan, creation, generated, get, binding_receipts=self.binding_receipts, - diagnostics=self.diagnostics, observations=self.observations) - if reused: - self.excluded_cycle = _baseline_cycle(plan["source"], read, token_provider) - if self.context_provider() != self.context: - raise fail("recheck-auth-context-drift", "CLI context changed during checkpoint capture; no checkpoint was retained.") - receipt = { - "schema_version": "3.1" if reused else "3.0", "kind": "blob-readiness-checkpoint", - "operation_id": self.operation_id, "plan_digest": digest(plan), - "not_before": not_before.isoformat(), "context_digest": self.context, - "creation": creation, "configuration": configuration, "request_ids": ids, - } - if reused: - receipt["excluded_cycle"] = self.excluded_cycle - if self.binding_receipts is not None: - receipt["schema_version"] = "3.2" - receipt["binding_digest"] = digest(self.binding_receipts) - receipt["integrity"] = digest(receipt) - path = private_io.private_file(self.directory, self.operation_id + ".blob-readiness.json", receipt) - self.request_ids = ids - self.summary = {"operation_id": self.operation_id, "receipt_file": path.name, - "evidence_digest": receipt["integrity"], "status": "retained"} - self.configuration, self.creation = configuration, creation - - def verify(self, plan, *, token_provider, transport): - _, get, ids = _reader(transport, token_provider, warnings=self.recovery_warnings) - try: - _configuration(plan, self.creation, self.creation["source"]["generated"], get, - expected=self.configuration, binding_receipts=self.binding_receipts, - diagnostics=self.diagnostics, observations=self.observations, - binding_observations=self.binding_observations) - finally: - self.request_ids.extend(ids) - if self.context_provider() != self.context: - raise fail("recheck-auth-context-drift", "CLI context changed during generated readback.") - - def finish(self, result): - record = {"schema_version": "1.0", "kind": "blob-operation-result", - "operation_id": self.operation_id, "plan_digest": self.plan_digest, - "result": copy.deepcopy(result)} - reject_secrets(record) - record["integrity"] = digest(record) - path = private_io.private_file(self.directory, self.operation_id + ".blob-result.json", record) - result["result_evidence"] = {"receipt_file": path.name, "status": "retained"} - - -def _document(input_path): - document = read_private(input_path) - reject_secrets(document) - if set(document) != {"schema_version", "plan", "approval"} or document["schema_version"] != "1.0": - raise fail("recheck-evidence-invalid", "Retain the original source envelope.") - plan = document["plan"] - if not isinstance(plan, dict): - raise fail("recheck-evidence-invalid", "The original plan is unavailable.") - _supported(plan) - fingerprint = digest(plan) - approval = document["approval"] - if (not isinstance(approval, dict) or set(approval) != {"confirmed", "fingerprint"} - or type(approval["confirmed"]) is not bool or approval["fingerprint"] != fingerprint - or plan["source"]["action"] == "create" and approval["confirmed"] is not True): - raise fail("approval-mismatch", "Retain unchanged creation consent or the fingerprinted read-only reuse plan.") - return document, plan, fingerprint - - -def _load(input_path, receipt_path, *, acknowledgement=False, binding_receipts=None): - document, plan, fingerprint = _document(input_path) - receipt = read_private(receipt_path) - reject_secrets(receipt) - if acknowledgement and receipt.get("kind") == "blob-write-acknowledgement": - _validate_write(plan, receipt) - return plan, receipt - reused = plan["source"]["action"] == "reuse" - version = receipt.get("schema_version") - if not isinstance(version, str): - raise fail("recheck-evidence-invalid", "Checkpoint schema version must be a supported string.") - core_projected = version in {"3.0", "3.1", "3.2"} and not acknowledgement - projected = (version in {"2.0", "2.1", "2.2"} or core_projected) and not acknowledgement - bound = reused and version in {"1.2", "2.2", "3.2"} - fields = {"schema_version", "kind", "operation_id", "plan_digest", "not_before", - "context_digest", "creation", "configuration", "request_ids", "integrity"} - if reused: - fields.add("excluded_cycle") - if bound: - fields.add("binding_digest") - if acknowledgement: - fields.remove("configuration") - expected_version = ("3" if core_projected else "2" if projected else "1") + (".2" if bound else ".1" if reused else ".0") - if (set(receipt) != fields or version != expected_version - or receipt["kind"] != ("blob-source-acknowledgement" if acknowledgement else "blob-readiness-checkpoint") - or acknowledgement and reused - or bound and (binding_receipts is None or receipt["binding_digest"] != digest(binding_receipts)) - or receipt["plan_digest"] != fingerprint - or not isinstance(receipt["operation_id"], str) - or re.fullmatch("[0-9a-f]{32}", receipt["operation_id"]) is None - or receipt["integrity"] != digest({k: v for k, v in receipt.items() if k != "integrity"})): - raise fail("recheck-evidence-invalid", "Original operation/cutoff/checkpoint integrity is missing or changed; never reconstruct it.") - blob_source._timestamp(receipt["not_before"]) - if reused: - _validate_cycle(receipt["excluded_cycle"]) - creation = receipt["creation"] - try: - observed = creation["verification"]["readback"] - generated = creation["source"]["generated"] - expected = {"type": "knowledge-source", "name": plan["source"]["name"], - "etag": observed["etag"], - "definition_digest": digest(search_reconcile._definition(plan["source"]["desired"]))} - valid = ( - set(creation) == {"approved_plan", "resources", "verification", "ownership", "source"} - and set(creation["source"]) == {"generated"} - and set(creation["verification"]) == {"readback", "absence", "request_ids", "idempotency"} - and creation["verification"]["absence"] is False - and creation["verification"]["idempotency"] == "exact readback is zero-write" - and isinstance(creation["verification"]["request_ids"], list) - and all(isinstance(item, str) for item in creation["verification"]["request_ids"]) - and creation["approved_plan"] == document["approval"] - and creation["resources"] == {"created": [] if reused else [expected], - "reused": [expected] if reused else [], "updated": [], "skipped": []} - and creation["ownership"] == {"run_owned": [] if reused else [expected], - "reused_not_owned": [expected] if reused else [], "owner": plan["owner"]} - and observed == expected and isinstance(expected["etag"], str) and bool(expected["etag"].strip()) - and generated == blob_source.generated_resources( - {"azureBlobParameters": {"createdResources": {item["type"]: item["name"] for item in generated}}}, - strict=True, - ) - and (acknowledgement or set(receipt["configuration"]) == set(COLLECTIONS)) - and isinstance(receipt["request_ids"], list) - and all(isinstance(item, str) for item in receipt["request_ids"]) - and isinstance(receipt["context_digest"], str) - and search_reconcile.SHA256.fullmatch(receipt["context_digest"]) - ) - for kind, item in ({} if acknowledgement else receipt["configuration"]).items(): - if core_projected: - valid = valid and semantic.valid(item, kind, search_reconcile.SHA256) - if kind == "indexer": - valid = valid and item.get("core_digest") == item.get("non_schedule_digest") - if kind == "datasource" and item.get("binding_proof") == "resource-id": - valid = valid and item["credential_digest"] in { - digest({"present": True, "value": {"connectionString": f"ResourceId={plan['boundary']['storage_id']}{suffix}"}}) - for suffix in ("", ";") - } - continue - hashes = {"digest"} | (indexer.PROJECTION_FIELDS if projected and kind == "indexer" else set()) - valid = valid and set(item) == {"etag"} | hashes and isinstance(item["etag"], str) and bool(item["etag"].strip()) - valid = valid and all(isinstance(item[key], str) and search_reconcile.SHA256.fullmatch(item[key]) for key in hashes) - except (KeyError, TypeError, AttributeError): - valid = False - if not valid: - raise fail("recheck-ownership-unproven", "Checkpoint must retain exact acknowledged ownership, generated configuration and provenance.") - return plan, receipt - - -def _historical_result(receipt_path, receipt): - path = Path(receipt_path).parent / (receipt["operation_id"] + ".blob-result.json") - if not path.exists(): - return "not-recorded-by-pre-monitor-checkpoint" - record = read_private(path) - reject_secrets(record) - if (set(record) != {"schema_version", "kind", "operation_id", "plan_digest", "result", "integrity"} - or record["schema_version"] != "1.0" or record["kind"] != "blob-operation-result" - or record["operation_id"] != receipt["operation_id"] or record["plan_digest"] != receipt["plan_digest"] - or record["integrity"] != digest({key: value for key, value in record.items() if key != "integrity"}) - or not isinstance(record["result"], dict)): - raise fail("recheck-evidence-invalid", "Historical operation result is malformed or changed; preserve original evidence.") - result = record["result"] - readiness = result.get("readiness", {}) - if not isinstance(readiness, dict): - raise fail("recheck-evidence-invalid", "Historical readiness must be an object.") - watch = readiness.get("watch") - if watch is not None and ( - not isinstance(watch, dict) or watch.get("schema_version") != "1.0" - or not isinstance(watch.get("state"), str) - or watch.get("state") not in {"paused", "completed", "blocked"}): - raise fail("recheck-evidence-invalid", "Historical watch metadata is malformed or unsupported.") - return {"receipt_file": path.name, "status": result.get("status"), - "first_failure": result.get("first_failure", result.get("first_blocker")), - "watch": copy.deepcopy(watch), - "writes_performed": result.get("writes_performed", result.get("completed_writes", []))} - - -def _validate_write(plan, receipt): - try: - write = receipt["write"] - valid = ( - set(receipt) == {"schema_version", "kind", "operation_id", "plan_digest", "not_before", - "context_digest", "write", "integrity"} - and receipt["schema_version"] == "1.0" and plan["source"]["action"] == "create" - and receipt["plan_digest"] == digest(plan) - and isinstance(receipt["operation_id"], str) - and re.fullmatch("[0-9a-f]{32}", receipt["operation_id"]) is not None - and isinstance(receipt["context_digest"], str) - and search_reconcile.SHA256.fullmatch(receipt["context_digest"]) is not None - and receipt["integrity"] == digest({k: v for k, v in receipt.items() if k != "integrity"}) - and set(write) == {"method", "url", "if_none_match", "body_digest", "status", - "request_id", "response_etags", "generated"} - and write["method"] == "PUT" and write["if_none_match"] == "*" - and type(write["status"]) is int and write["status"] in {200, 201} - and write["url"] == search_reconcile.resource_url(plan["source"]) - and write["body_digest"] == digest(plan["source"]["desired"]) - and isinstance(write["request_id"], str) and bool(write["request_id"].strip()) - and set(write["response_etags"]) == {"body", "headers"} - and isinstance(write["response_etags"]["headers"], list) - and isinstance(write["generated"], list) - ) - if write["generated"]: - valid = valid and write["generated"] == blob_source.generated_resources( - {"azureBlobParameters": {"createdResources": { - item["type"]: item["name"] for item in write["generated"]}}}, strict=True, - ) - except (KeyError, TypeError, AttributeError): - valid = False - if not valid: - raise fail("recheck-ownership-unproven", "Retain the private authenticated conditional-write receipt; input booleans or observed existence cannot replace it.") - blob_source._timestamp(receipt["not_before"]) - if search_reconcile.resolve_etag(write["response_etags"], write["request_id"]) is None: - raise fail("creation-version-unproven", "Write acknowledgement has no response ETag; never borrow a later GET version.", - request_id=write["request_id"]) - - -@reporting("blob-capture") -def capture(input_path, directory, *, token_provider=azure_cli_token, transport=http_request, - storage_transport=http_request, context_provider=account_context, - now=lambda: datetime.now(timezone.utc), progress: Progress | None = None, - binding_paths=None): - progress.update("evidence-validation") - document, plan, fingerprint = _document(input_path) - if plan["source"]["action"] != "reuse": - raise fail("recheck-scope-unsupported", "Fresh capture accepts only a reuse plan; it cannot recover missing original creation proof.") - binding_receipts = _binding_receipts(binding_paths) - progress.update("context-check") - checkpoint = Checkpoint(directory, plan, context_provider=context_provider, binding_receipts=binding_receipts) - cutoff = now() - progress.update("source-binding") - _, get, ids = _reader(transport, token_provider) - current = get(search_reconcile.resource_url(plan["source"])) - generated = blob_source.generated_resources(current, strict=True) - result = search_reconcile._completed( - "blob-readiness-capture", fingerprint, plan["source"], action="reused", - readback=current, request_ids=ids, absence=False, - ) - result["approved_plan"] = document["approval"] - progress.update("blob-inventory") - inventory = blob_inventory.discover(plan["boundary"], plan["inventory_limits"], - token_provider=token_provider, transport=storage_transport) - if inventory["inventory_digest"] != plan["inventory_digest"]: - raise fail("source-drift", "Selected Storage/ACL evidence differs from the reuse plan.") - result["verification"]["request_ids"].extend(inventory["request_ids"]) - progress.update("checkpoint") - checkpoint.persist(plan, result, generated, cutoff, token_provider=token_provider, transport=transport) - return { - "status": "completed", "outcome": "blob-readiness-capture", - "recheck_checkpoint": checkpoint.summary, "writes_performed": [], - "readiness": {"status": "unverified"}, "retrieval": "unverified", "knowledge_base": "not-verified", - "ownership": result["ownership"], "cleanup": {"separate_confirmation_required": True}, - "read_only_evidence": {"request_ids": result["verification"]["request_ids"] + checkpoint.request_ids}, - "indexer_diagnostics": semantic.indexer_only(checkpoint.diagnostics), - "generated_diagnostics": checkpoint.diagnostics, "indexer_observations": checkpoint.observations, - "warnings": [blob_source.SNAPSHOT_WARNING, "Fresh reuse observation is not recovered creation ownership or ingestion proof.", - *get.recovery_warnings, *checkpoint.recovery_warnings, - *indexer.warnings(checkpoint.diagnostics)], - } - - -@reporting("blob-capture") -def recover(input_path, acknowledgement_path, directory, *, token_provider=azure_cli_token, - transport=http_request, storage_transport=http_request, context_provider=account_context, - progress: Progress | None = None): - progress.update("evidence-validation") - plan, acknowledgement = _load(input_path, acknowledgement_path, acknowledgement=True) - historical = _historical_result(acknowledgement_path, acknowledgement) - directory = private_io.private_directory(str(directory)) - progress.update("context-check") - if context_provider() != acknowledgement["context_digest"]: - raise fail("recheck-auth-context-drift", "Current context differs from the acknowledged original run.") - progress.update("source-binding") - _, get, ids = _reader(transport, token_provider) - diagnostics, observations, binding_observations = [], [], [] - if acknowledgement["kind"] == "blob-write-acknowledgement": - write = acknowledgement["write"] - current = get(search_reconcile.resource_url(plan["source"])) - if (_etag(current) != search_reconcile.resolve_etag(write["response_etags"], write["request_id"]) - or search_reconcile._definition(current) != search_reconcile._definition(plan["source"]["desired"])): - raise fail("definition-drift", "Current source differs from the acknowledged conditional-write version/definition.", - request_id=getattr(get, "request_id", None)) - generated = blob_source.generated_resources(current, strict=True) - if write["generated"] and generated != write["generated"]: - raise fail("definition-drift", "Generated identities differ from the create response.") - verified = search_reconcile._completed( - "create-blob-knowledge-source", acknowledgement["plan_digest"], plan["source"], - action="created", readback=current, request_ids=[write["request_id"], *ids], absence=False, - ) - creation = {key: verified[key] for key in ("approved_plan", "resources", "verification", "ownership")} - creation["source"] = {"generated": generated} - else: - creation = acknowledgement["creation"] - configuration = _configuration(plan, creation, creation["source"]["generated"], get, - diagnostics=diagnostics, observations=observations) - progress.update("blob-inventory") - inventory = blob_inventory.discover(plan["boundary"], plan["inventory_limits"], - token_provider=token_provider, transport=storage_transport) - ids.extend(inventory["request_ids"]) - if inventory["inventory_digest"] != plan["inventory_digest"]: - raise fail("source-drift", "Storage/ACL inventory differs from the acknowledged original run.") - progress.update("checkpoint") - _configuration(plan, creation, creation["source"]["generated"], get, expected=configuration, - diagnostics=diagnostics, observations=observations, binding_observations=binding_observations) - if context_provider() != acknowledgement["context_digest"]: - raise fail("recheck-auth-context-drift", "CLI context changed during recovery observation.") - receipt = {**acknowledgement, "schema_version": "3.0", "kind": "blob-readiness-checkpoint", "configuration": configuration, - "creation": creation, "request_ids": ids} - receipt.pop("write", None) - receipt.pop("integrity") - receipt["integrity"] = digest(receipt) - path = private_io.private_file(directory, receipt["operation_id"] + ".blob-readiness.json", receipt) - return { - "status": "completed", "outcome": "blob-readiness-recovery-capture", "writes_performed": [], - "recheck_checkpoint": {"operation_id": receipt["operation_id"], "receipt_file": path.name, - "evidence_digest": receipt["integrity"], "status": "retained"}, - "readiness": {"status": "unverified"}, "retrieval": "unverified", "knowledge_base": "not-verified", - "ownership": {"run_owned": [], "reused_not_owned": []}, - "original_run": {"ownership": creation["ownership"], "not_before": receipt["not_before"], - "historical_created": copy.deepcopy(creation["resources"]["created"]), - "original_failure": historical, - "acknowledgement_digest": acknowledgement["integrity"]}, - "read_only_evidence": {"request_ids": ids}, - "indexer_diagnostics": semantic.indexer_only(diagnostics), - "generated_diagnostics": diagnostics, "indexer_observations": observations, - "datasource_binding_observations": binding_observations, - "safe_next_decision": "Run blob_recheck.py --input with the unchanged original input and --receipt with this checkpoint; then return to the KB/retrieval owner. Preserve the original first failure separately.", - "warnings": [blob_source.SNAPSHOT_WARNING, "Configuration was observed during recovery; no earlier generated revision or new ownership is asserted.", - *get.recovery_warnings, - *indexer.warnings(diagnostics)], - } - - -@reporting("blob-recheck") -def recheck(input_path, receipt_path, *, token_provider=azure_cli_token, - transport=http_request, storage_transport=http_request, - context_provider=account_context, monotonic=time.monotonic, sleep=time.sleep, - now=lambda: datetime.now(timezone.utc), progress: Progress | None = None, - binding_paths=None, watch_limits=None, cancelled=lambda: False): - progress.update("evidence-validation") - binding_receipts = _binding_receipts(binding_paths) - plan, receipt = _load(input_path, receipt_path, binding_receipts=binding_receipts) - limits = blob_source._poll_limits(watch_limits if watch_limits is not None else plan["poll"]) - creation = receipt["creation"] - generated = creation["source"]["generated"] - readiness = {"status": "unverified"} - historical = "not-recorded-by-pre-monitor-checkpoint" - ids = [] - recovery_warnings = [] - diagnostics, observations, binding_observations = [], [], [] - try: - historical = _historical_result(receipt_path, receipt) - progress.update("context-check") - if context_provider() != receipt["context_digest"]: - raise fail("recheck-auth-context-drift", "Current CLI tenant/subscription/principal differs from the original run; no auth changes were made.") - read, get, ids = _reader(transport, token_provider) - recovery_warnings = get.recovery_warnings - - def inventory(): - inventory = blob_inventory.discover( - plan["boundary"], plan["inventory_limits"], token_provider=token_provider, - transport=storage_transport, - ) - ids.extend(inventory["request_ids"]) - if inventory["inventory_digest"] != plan["inventory_digest"]: - raise fail("source-drift", "Selected Storage/ACL evidence differs from original creation.") - - for stage in ("before", "after"): - if stage == "after": - progress.update("blob-readback") - inventory() - progress.update("source-binding" if stage == "before" else "source-readback") - _configuration(plan, creation, generated, get, binding_receipts=binding_receipts, - expected=receipt["configuration"], diagnostics=diagnostics, observations=observations, - binding_observations=binding_observations) - if stage == "before": - progress.update("blob-inventory") - inventory() - readiness = blob_source.monitor( - plan["source"], not_before=blob_source._timestamp(receipt["not_before"]), - limits=limits, token_provider=token_provider, transport=read, - monotonic=monotonic, sleep=sleep, progress=progress, - excluded_cycle=receipt.get("excluded_cycle"), - cancelled=cancelled, - indexer_name=next(item["name"] for item in generated if item["type"] == "indexer"), - ) - if readiness["status"] != "verified": - raise HelperFailure( - readiness["code"], "Original-run ingestion remains unverified.", - blocked_at="verification", request_id=readiness.get("request_id"), - status=readiness.get("http_status"), - ) - cycle = readiness["synchronization"] - if (cycle["itemsUpdatesProcessed"] == 0 or cycle["itemsSkipped"] - or blob_source._timestamp(cycle["endTime"]) > now()): - raise fail("ingestion-unverified", "A checkpoint is not prior ingestion proof; nonempty zero-skip completion is required.") - progress.update("context-readback") - if context_provider() != receipt["context_digest"]: - raise fail("recheck-auth-context-drift", "CLI context changed during recheck.") - except HelperFailure as failure: - recovery_warnings.extend(failure.warnings) - if failure.request_id and failure.request_id not in ids: - ids.append(failure.request_id) - safe_failure = HelperFailure( - failure.code, failure.message if failure.blocked_at == "indexer-verification" else - "Read-only evidence could not verify readiness; service details withheld.", - blocked_at=failure.blocked_at, status=failure.http_status, request_id=failure.request_id, - ) - result = blocked_result(safe_failure, outcome="blob-readiness-recheck", fingerprint=None) - result["safe_next_decision"] = "Preserve the first failure. GET the exact source/generated definitions; for a concealed source binding supply --reuse-input-file/--reuse-result-file from its successful creation. Fix access or investigate actual drift read-only; do not replay creation, run/reset an indexer or default to cleanup." - if failure.code == "datasource-binding-unverified": - result["first_blocker"]["message"] = ( - "The datasource revision lacks independently observed current Storage binding; " - "this is unverified, not evidence of misconfiguration." - ) - result["safe_next_decision"] = ( - "Retain source and checkpoint. One exact datasource GET with includeConnectionString=true did not " - "provide stable sanitized ResourceId proof. " - "Ask the service owner for authoritative current child binding evidence; root identity and historical " - "receipts are insufficient. Resume the same GET-only recheck if that readback becomes available. " - "Do not retrieve keys, patch/recreate the source, run/reset an indexer, or delete resources." - ) - if failure.code == "indexer-legacy-evidence-insufficient": - result["safe_next_decision"] = "Retain the legacy checkpoint unchanged; GET current definitions and compare any legitimately retained preimage. This helper cannot infer its missing projection. A separately planned --capture reuse operation may observe fresh readiness, not recover historical configuration/ownership. Do not replay writes." - readiness = {**readiness, "status": "unverified"} - if readiness.get("watch", {}).get("state") == "paused": - result["safe_next_decision"] = readiness["safe_next_decision"] - else: - result = {"status": "completed", "outcome": "blob-readiness-recheck", "writes_performed": []} - result.update( - readiness=readiness, retrieval="unverified", knowledge_base="not-verified", - original_run={"operation_id": receipt["operation_id"], "plan_digest": receipt["plan_digest"], - "source_action": plan["source"]["action"], - "evidence_digest": receipt["integrity"], "not_before": receipt["not_before"], - "request_ids": creation["verification"]["request_ids"], - "checkpoint_request_ids": receipt["request_ids"], - "original_failure": historical, - "historical_created": copy.deepcopy(creation["resources"]["created"]), - "ownership": copy.deepcopy(creation["ownership"]), "generated": generated}, - ownership={"run_owned": [], "reused_not_owned": []}, - read_only_evidence={"request_ids": ids}, - indexer_diagnostics=semantic.indexer_only(diagnostics), - generated_diagnostics=diagnostics, indexer_observations=observations, - datasource_binding_observations=binding_observations, - cleanup={"status": "not-requested", "separate_confirmation_required": True}, - recheck_checkpoint={"operation_id": receipt["operation_id"], "receipt_file": Path(receipt_path).name, - "evidence_digest": receipt["integrity"], "status": "retained"}, - warnings=[blob_source.SNAPSHOT_WARNING, - *recovery_warnings, - "Local checkpoint integrity is not a service signature or new ownership/cleanup authorization.", - *indexer.warnings(diagnostics)], - ) - return result - - -def compact_result(result, directory): - """Opt-in versioned presentation; retain the native result privately first.""" - reject_secrets(result) - directory = private_io.private_directory(str(directory)) - path = private_io.private_file(directory, uuid.uuid4().hex + ".blob-result.json", result) - ids = [] - - def collect(value, key=None): - if isinstance(value, dict): - for field, child in value.items(): - collect(child, field) - elif isinstance(value, list): - for child in value: - collect(child, key) - elif (key in {"request_id", "request_ids"} or isinstance(key, str) and key.endswith("_request_ids")) and isinstance(value, str): - ids.append(value) - - collect(result) - failure = result.get("first_failure", result.get("first_blocker")) - original = result.get("original_run", {}) - writes = result.get("writes_performed", result.get("completed_writes", [])) - readiness = result.get("readiness", {"status": "unverified"}) - item_error = readiness.get("first_error") - if readiness.get("code") == "ingestion-timeout" and readiness.get("watch", {}).get("state") == "paused": - failure = None - elif failure and isinstance(item_error, dict) and item_error.get("request_id"): - failure = {**failure, "request_id": item_error["request_id"]} - historical = original.get("original_failure") - historical_failure = historical.get("first_failure") if isinstance(historical, dict) else None - historical_watch = historical.get("watch") if isinstance(historical, dict) else None - if (historical_failure and historical_failure.get("code") == "ingestion-timeout" - and isinstance(historical_watch, dict) and historical_watch.get("schema_version") == "1.0" - and historical_watch.get("state") == "paused"): - historical_failure = None - generated = original.get("generated", result.get("source", {}).get("generated", [])) - diagnostic_keys = ("severity", "code", "field") - diagnostics = dict.fromkeys( - tuple(item[key] for key in diagnostic_keys) - for item in result.get("generated_diagnostics", result.get("indexer_diagnostics", [])) - ) - return { - "schema_version": "1.1" if "progress" in readiness else "1.0", "kind": "blob-operation-summary", - "status": result["status"], "outcome": result["outcome"], - "writes_performed": writes, - "creation_acknowledgement": result.get("creation_acknowledgement"), - "historical_created": [ - {"type": item["type"], "name": item["name"]} - for item in original.get("historical_created", []) - ], - "generated_resources": [{"type": item["type"], "name": item["name"]} for item in generated], - "readiness": {"status": readiness["status"], "watch": readiness.get("watch"), - **({"progress": readiness["progress"]} if "progress" in readiness else {})}, - "first_failure": ({key: failure.get(key) for key in ("code", "status", "request_id")} - if failure else None), - "historical_failure": ({key: historical_failure.get(key) for key in ("code", "status", "request_id")} - if historical_failure else None), - "first_retry": readiness.get("first_retry"), - "diagnostics": [dict(zip(diagnostic_keys, item)) for item in diagnostics], - "request_id_count": len(set(ids)), "evidence_file": path.name, - "checkpoint_file": result.get("recheck_checkpoint", {}).get("receipt_file"), - "retrieval": "unverified", "knowledge_base": "not-verified", - "safe_next_decision": result.get("safe_next_decision", - "Return to the KB/retrieval owner; no cleanup or new writes are authorized." - if readiness["status"] == "verified" else - "Retain resources and inspect the named source/indexer and private evidence. " - "Use receipt-backed GET-only recheck when evidence is available; no write replay or cleanup."), - } - - -def main(argv=None): - parser = argparse.ArgumentParser() - modes = parser.add_mutually_exclusive_group(required=True) - modes.add_argument("--input", type=Path) - modes.add_argument("--capture", type=Path) - modes.add_argument("--recover", type=Path) - parser.add_argument("--receipt", type=Path) - parser.add_argument("--receipt-dir", type=Path) - parser.add_argument("--reuse-input-file", type=Path) - parser.add_argument("--reuse-result-file", type=Path) - parser.add_argument("--watch-seconds", type=int) - parser.add_argument("--watch-max-requests", type=int) - parser.add_argument("--watch-interval", type=int) - parser.add_argument("--compact", action="store_true") - add_progress_argument(parser) - args = parser.parse_args(argv) - if (args.capture and (not args.receipt_dir or args.receipt) - or args.input and (not args.receipt or args.receipt_dir) - or args.recover and (not args.receipt or not args.receipt_dir)): - parser.error("--capture needs --receipt-dir; --input needs --receipt; --recover needs both.") - if bool(args.reuse_input_file) != bool(args.reuse_result_file) or args.recover and args.reuse_input_file: - parser.error("Select both reuse evidence files, only with --capture or --input.") - options = {"binding_paths": (args.reuse_input_file, args.reuse_result_file)} if args.reuse_input_file else {} - watch = (args.watch_seconds, args.watch_max_requests, args.watch_interval) - if any(value is not None for value in watch): - if not args.input or any(value is None for value in watch): - parser.error("Explicit GET-only watch needs --input and all three --watch-* limits.") - options["watch_limits"] = dict(zip(("deadline_seconds", "max_requests", "interval_seconds"), watch)) - try: - if args.capture: - result = capture(args.capture, args.receipt_dir, progress=Progress("blob-capture", enabled=args.progress), **options) - elif args.recover: - result = recover(args.recover, args.receipt, args.receipt_dir, - progress=Progress("blob-capture", enabled=args.progress)) - else: - result = recheck(args.input, args.receipt, progress=Progress("blob-recheck", enabled=args.progress), **options) - except HelperFailure as failure: - outcome = "blob-readiness-recovery-capture" if args.recover else "blob-readiness-capture" if args.capture else "blob-readiness-recheck" - result = blocked_result(failure, outcome=outcome, fingerprint=None) - result["safe_next_decision"] = ( - "Preserve original error/resources. Inspect the exact source and genuine private evidence: " - "--recover needs its retained acknowledgement; --capture needs a reuse plan and any required " - "creation proof. If a checkpoint already exists, inspect it and use --input/--receipt. " - "No write replay, new ownership or cleanup is authorized." - ) - if args.compact: - try: - result = compact_result(result, args.receipt_dir or args.receipt.parent) - except HelperFailure as failure: - result.setdefault("warnings", []).append("compact-evidence-persistence-failed: full native result retained in output.") - result["presentation_failure"] = {"code": failure.code} - emit_result(result) - return 2 - emit_result(result) - return 0 if result["status"] == "completed" else 2 - - -if __name__ == "__main__": - sys.exit(main()) +"""Retained Blob creation/reuse observations and read-only readiness follow-up.""" +from __future__ import annotations + +import argparse +import copy +import json +import os +import re +import stat +import sys +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlsplit + +try: + from . import _bootstrap_io as private_io + from ._progress import Progress, add_progress_argument, reporting + from . import blob_inventory, blob_source, search_reconcile, source_vector, _indexer_observation as indexer + from . import _blob_observation as semantic + from ._common import ( + HelperFailure, SEARCH_AUDIENCE, azure_cli_token, blocked_result, digest, + emit_result, http_request, reject_secrets, + ) +except ImportError: + import _bootstrap_io as private_io + from _progress import Progress, add_progress_argument, reporting + import blob_inventory, blob_source, search_reconcile, source_vector + import _indexer_observation as indexer + import _blob_observation as semantic + from _common import ( + HelperFailure, SEARCH_AUDIENCE, azure_cli_token, blocked_result, digest, + emit_result, http_request, reject_secrets, + ) + + +fail = blob_source._failure +COLLECTIONS = {"datasource": "datasources", "indexer": "indexers", + "skillset": "skillsets", "index": "indexes"} + + +def _json(raw): + def unique(pairs): + value = {} + for key, child in pairs: + if key in value: + raise ValueError("Duplicate field") + value[key] = child + return value + try: + value = json.loads(raw, object_pairs_hook=unique) + json.dumps(value, allow_nan=False, ensure_ascii=False).encode("utf-8") + if not isinstance(value, dict): + raise ValueError("Expected object") + return value + except (ValueError, UnicodeError, RecursionError) as exc: + raise fail("recheck-evidence-invalid", "Retain bounded, unmodified UTF-8 object evidence.") from exc + + +def read_private(path): + path = Path(path) + private_io.private_directory(str(path.parent)) + try: + selected = path.lstat() + if (not stat.S_ISREG(selected.st_mode) or selected.st_nlink != 1 + or getattr(selected, "st_file_attributes", 0) & 0x400): + raise OSError("Not an ordinary private file") + if os.name == "nt": + private_io._windows_private(path) + elif selected.st_uid != os.getuid() or stat.S_IMODE(selected.st_mode) & 0o077: + raise OSError("Not private") + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + with os.fdopen(descriptor, "rb") as handle: + opened = os.fstat(handle.fileno()) + if (selected.st_dev, selected.st_ino) != (opened.st_dev, opened.st_ino): + raise OSError("Evidence changed identity") + raw = handle.read(private_io.MAX_BYTES + 1) + if len(raw) > private_io.MAX_BYTES: + raise OSError("Evidence exceeds bound") + return _json(raw.decode("utf-8")) + except (OSError, UnicodeError) as exc: + raise fail("recheck-evidence-unreadable", "Select existing private, unlinked evidence files; no permissions were changed.") from exc + + +def account_context(): + code, stdout, _ = private_io.run_cli(["account", "show"], 30) + if code: + raise fail("recheck-auth-context-unavailable", "Current signed-in CLI context is inaccessible; details withheld.") + account = _json(stdout) + user = account.get("user") + if (account.get("environmentName") != "AzureCloud" or account.get("state") != "Enabled" + or not isinstance(user, dict) or user.get("type") not in ("user", "servicePrincipal") + or any(not isinstance(value, str) or not value.strip() for value in + (account.get("id"), account.get("tenantId"), user.get("name")))): + raise fail("recheck-auth-context-unavailable", "An enabled public-cloud tenant/subscription/principal context is required.") + return digest({key: account[key] for key in ("id", "tenantId", "environmentName")} + | {"principal": {"name": user["name"], "type": user["type"]}}) + + +def _supported(plan): + try: + source, _ = blob_source._validate_plan(plan) + ingestion = source["desired"]["azureBlobParameters"]["ingestionParameters"] + except (KeyError, TypeError, AttributeError, RecursionError) as exc: + raise fail("recheck-evidence-invalid", "Original Blob creation plan is malformed.") from exc + if (ingestion.get("contentExtractionMode") not in ("minimal", "standard") or ingestion.get("identity") is not None + or source["api_version"] not in {"2026-04-01", "2026-08-01-preview"} + or plan["boundary"]["is_adls"] and source["api_version"] != "2026-08-01-preview"): + raise fail("recheck-scope-unsupported", "Checkpointing requires supported Blob extraction and provable system-assigned authentication.") + schedule = ingestion.get("ingestionSchedule") + indexer.schedule(schedule) + + +def _etag(value): + etag = value.get("@odata.etag") if isinstance(value, dict) else None + if not isinstance(etag, str) or not etag.strip(): + raise fail("recheck-evidence-missing", "Every source/generated readback requires a nonempty string ETag.") + return etag + + +def _processing_readback(plan, current): + desired = plan["source"]["desired"]["azureBlobParameters"]["ingestionParameters"] + if desired.get("contentExtractionMode") == "standard": + ai = desired.get("aiServices") + if not isinstance(ai, dict) or not isinstance(ai.get("uri"), str) or not ai["uri"].strip(): + raise fail("recheck-processing-unverified", "Standard extraction needs its original CU endpoint evidence.") + blob_source.verify_content_understanding_readback({"endpoint": ai["uri"]}, current) + source_vector.verify_source_readback(plan.get("embedding"), current) + model = desired.get("embeddingModel") + if model is not None and "embedding" not in plan: + parameters = model.get("azureOpenAIParameters") if isinstance(model, dict) else None + if (not isinstance(parameters, dict) or model.get("kind") != "azureOpenAI" + or any(not isinstance(parameters.get(key), str) or not parameters[key].strip() + for key in ("resourceUri", "deploymentId", "modelName"))): + raise fail("recheck-processing-unverified", "Legacy embedding configuration cannot be verified by this client.") + source_vector.verify_source_readback({ + "endpoint": parameters["resourceUri"], "deployment": parameters["deploymentId"], + "model": parameters["modelName"], + }, current) + + +def _binding(plan, record, current, generated, binding_receipts=None): + source = plan["source"] + observed = {"type": "knowledge-source", "name": source["name"], "etag": _etag(current), + "definition_digest": digest(search_reconcile._definition(current))} + if source["action"] == "create": + blob_source._verify_creation_binding(source, plan["boundary"], current, generated, (plan, record)) + elif (record["verification"]["readback"] != observed + or not search_reconcile.definitions_match(source["desired"], current) + or source.get("expected_etag", observed["etag"]) != observed["etag"]): + raise fail("source-binding-unverified", "Reused source does not match its retained read-only identity/configuration.") + if (blob_source.generated_resources(current, strict=True) != generated + or plan.get("expected_generated", generated) != generated): + raise fail("definition-drift", "Generated source identities changed.") + if binding_receipts is not None and ( + binding_receipts[0]["owner"] != plan["owner"] + or binding_receipts[0]["inventory_digest"] != plan["inventory_digest"] + ): + raise fail("source-binding-unverified", "Retained creation must bind the same owner and Storage inventory.") + blob_source._verify_storage_binding( + source, plan["boundary"], current, generated, + lambda: (plan, record) if source["action"] == "create" else binding_receipts, + ) + _processing_readback(plan, current) + + +def _sanitized_datasource_url(url): + parsed = urlsplit(url) + search_reconcile.validate_search_endpoint(f"{parsed.scheme}://{parsed.netloc}") + if (parsed.fragment or parsed.query not in { + f"api-version={version}" for version in search_reconcile.SUPPORTED_API_VERSIONS} + or re.fullmatch(r"/datasources\('[a-zA-Z0-9][a-zA-Z0-9_-]{0,127}'\)", parsed.path) is None): + raise fail("recheck-read-url-invalid", "Sanitized binding requires one exact datasource identity and the bound supported API version.") + return url + "&includeConnectionString=true" + + +def _reader(transport, token_provider, *, warnings=None): + recovery = None + recovery_warnings = warnings if warnings is not None else [] + + def tracked(method, url, token, **kwargs): + try: + parsed = urlsplit(url) + search_reconcile.validate_search_endpoint(f"{parsed.scheme}://{parsed.netloc}") + if parsed.fragment: + raise fail("recheck-read-url-invalid", "Read-only Search URLs cannot contain fragments.") + if parsed.query not in { + f"api-version={version}" for version in search_reconcile.SUPPORTED_API_VERSIONS}: + suffix = "&includeConnectionString=true" + if not url.endswith(suffix) or _sanitized_datasource_url(url[:-len(suffix)]) != url: + raise fail("recheck-read-url-invalid", "Only the exact sanitized datasource GET option is permitted.") + kwargs["timeout"] = min(60, kwargs.get("timeout", 60)) + if recovery is None: + response = transport(method, url, token, **kwargs) + else: + response = recovery.get( + url, token, transport=transport, + max_requests=1 if url.endswith("&includeConnectionString=true") else 2, + **kwargs, + ) + ids.extend(recovery.request_ids[:-1]) + if method == "GET" and response.status == 200 and isinstance(response.body, dict): + etag = search_reconcile.resolve_etag(search_reconcile.response_etags(response), response.request_id) + if etag is not None: + response = search_reconcile.HttpResult( + response.status, {**response.body, "@odata.etag": etag}, response.headers, response.etag_values, + ) + return response + except HelperFailure as failure: + if recovery is not None: + ids.extend(recovery.request_ids) + elif failure.request_id: + ids.append(failure.request_id) + raise + read, original_get, ids = source_vector._reader(tracked, token_provider) + + def get(url): + nonlocal recovery + recovery = search_reconcile.ReadRecovery() + try: + result = original_get(url) + get.request_id = ids[-1] if ids else None + return result + except HelperFailure as failure: + if failure.code in {"vector-resource-missing", "vector-evidence-missing", "etag-invalid"}: + missing = failure.code == "vector-resource-missing" + raise HelperFailure( + "recheck-resource-missing" if missing else "recheck-evidence-missing", + "An exact original source/generated definition or ETag is unavailable.", + blocked_at="verification", status=404 if missing else failure.http_status, + request_id=ids[-1] if ids else failure.request_id, + ) from failure + raise + finally: + recovery_warnings.extend(recovery.warnings) + recovery = None + get.recovery_warnings = recovery_warnings + return read, get, ids + + +def _configuration(plan, creation, generated, get, *, binding_receipts=None, expected=None, + diagnostics=None, observations=None, binding_observations=None): + diagnostics = diagnostics if diagnostics is not None else [] + source = plan["source"] + current = get(search_reconcile.resource_url(source)) + _binding(plan, creation, current, generated, binding_receipts) + names = {item["type"]: item["name"] for item in generated} + snapshots = {} + failures = [] + for kind, collection in COLLECTIONS.items(): + name = names[kind] + url = f"{source['endpoint'].rstrip('/')}/{collection}('{name}')?api-version={source['api_version']}" + try: + snapshots[kind] = _child_configuration( + plan, kind, name, names, get, url, expected, diagnostics, observations, binding_observations, + ) + except HelperFailure as failure: + if failure.request_id is None: + failure.request_id = getattr(get, "request_id", None) + semantic.note(diagnostics, kind, "error", failure.code, "definition", failure.request_id) + failures.append(failure) + else: + semantic.note(diagnostics, kind, "info", "generated-configuration-verified", "definition", + getattr(get, "request_id", None)) + refreshed = get(search_reconcile.resource_url(source)) + _binding(plan, creation, refreshed, generated, binding_receipts) + if failures: + raise failures[0] + return snapshots + + +def _child_configuration(plan, kind, name, names, get, url, expected, diagnostics, observations, + binding_observations=None): + source = plan["source"] + child = get(url) + etag = _etag(child) + if child.get("name") != name: + raise fail("definition-drift", "Generated configuration returned an unrelated identity.") + if kind == "datasource": + boundary = plan["boundary"] + container, credentials = child.get("container"), child.get("credentials") + invariants = { + "type": child.get("type") == ("adlsgen2" if boundary["is_adls"] else "azureblob"), + "container.name": isinstance(container, dict) and container.get("name") == boundary["container"], + "container.query": isinstance(container, dict) and container.get("query") in + (boundary["prefix"], None if not boundary["prefix"] else boundary["prefix"]), + "credentials": credentials is None or + isinstance(credentials, dict) and not set(credentials) - {"connectionString"}, + "identity": child.get("identity") is None, + } + for field, valid in invariants.items(): + if not valid: + semantic.note(diagnostics, kind, "error", "source-binding-unverified", field, + getattr(get, "request_id", None)) + if not all(invariants.values()): + raise fail("source-binding-unverified", "Generated datasource type/container/prefix/identity or credential shape conflicts with the bound source.", + request_id=getattr(get, "request_id", None)) + try: + binding = blob_source._connection_binding( + credentials.get("connectionString") if isinstance(credentials, dict) else None, boundary, + ) + except HelperFailure as failure: + failure.request_id = getattr(get, "request_id", None) + semantic.note(diagnostics, kind, "error", failure.code, "credentials.connectionString", failure.request_id) + raise + if kind == "indexer": + request_id = getattr(get, "request_id", None) + for field, target in (("dataSourceName", "datasource"), ("targetIndexName", "index"), ("skillsetName", "skillset")): + if child.get(field) != names[target]: + raise indexer.failure(diagnostics, "indexer-binding-mismatch", field, + f"Generated indexer {field} does not match the bound resource; details withheld.", request_id) + observed = indexer.observe(child, source["desired"]["azureBlobParameters"]["ingestionParameters"].get("ingestionSchedule"), + diagnostics, request_id) + if observations is not None: + observations.append({"request_id": request_id, "etag_digest": digest(etag), + **{key: value for key, value in observed.items() if key != "etag"}}) + snapshot = observed if expected is None or indexer.PROJECTION_FIELDS <= set(expected[kind]) else { + "etag": etag, "digest": digest(child), + } + if expected is None or semantic.PROJECTION_FIELDS <= set(expected[kind]): + snapshot.update(semantic.observe(child, kind)) + if expected is not None: + if semantic.PROJECTION_FIELDS <= set(expected[kind]): + semantic.compare(snapshot, expected[kind], kind, diagnostics, request_id) + # A core-equal ETag-only revision is not schedule or configuration drift. + if snapshot["schedule_raw_digest"] != expected[kind]["schedule_raw_digest"]: + indexer.compare(snapshot, expected[kind], diagnostics, request_id) + else: + indexer.compare(snapshot, expected[kind], diagnostics, request_id) + return snapshot + # Keep only integrity observations, never generated credentials or skill text. + snapshot = {"etag": etag, "digest": digest(child)} + if expected is None or semantic.PROJECTION_FIELDS <= set(expected[kind]): + snapshot.update(semantic.observe(child, kind)) + if kind == "datasource": + snapshot["binding_proof"] = "resource-id" if binding == "visible" else "unverified" + if expected is not None: + semantic.compare(snapshot, expected[kind], kind, diagnostics, getattr(get, "request_id", None)) + if kind == "datasource" and binding == "concealed": + _current_datasource_binding(get, url, snapshot, plan["boundary"], diagnostics, + binding_observations) + elif kind == "datasource" and binding == "concealed": + semantic.note(diagnostics, kind, "info", "datasource-credential-projection-concealed", + "credentials.connectionString", getattr(get, "request_id", None)) + return snapshot + visible_preimage = False + if kind == "datasource" and expected and expected[kind]["etag"] == etag: + # Only concealed/verified credentials may vary; every other byte and the ETag stay bound. + for credentials in ( + None, {}, {"connectionString": None}, {"connectionString": ""}, + {"connectionString": ""}, {"connectionString": ""}, + {"connectionString": f"ResourceId={plan['boundary']['storage_id']}"}, + {"connectionString": f"ResourceId={plan['boundary']['storage_id']};"}, + ): + candidate = {**child, "credentials": credentials} + if digest(candidate) == expected[kind]["digest"]: + snapshot["digest"] = expected[kind]["digest"] + if isinstance(credentials, dict) and isinstance(credentials.get("connectionString"), str): + visible_preimage = credentials["connectionString"].startswith("ResourceId=") + candidate = {key: value for key, value in child.items() if key != "credentials"} + if digest(candidate) == expected[kind]["digest"]: + snapshot["digest"] = expected[kind]["digest"] + if snapshot != expected[kind]: + raise fail("generated-legacy-evidence-insufficient", + "Legacy generated full-hash evidence differs; no core projection was retained.", + request_id=getattr(get, "request_id", None)) + if kind == "datasource" and binding == "concealed": + if visible_preimage: + semantic.note(diagnostics, kind, "warning", "datasource-credential-projection-changed", + "credentials", getattr(get, "request_id", None)) + else: + _current_datasource_binding(get, url, {"etag": etag, "digest": digest(child), **semantic.observe(child, kind)}, + plan["boundary"], diagnostics, binding_observations) + return snapshot + + +def _current_datasource_binding(get, url, snapshot, boundary, diagnostics, observations=None): + observation = { + "schema_version": "1.0", "status": "unverified", "request_id": None, + "plain": {"etag_digest": digest(snapshot["etag"]), "definition_digest": snapshot["digest"], + "core_digest": snapshot["core_digest"]}, + "sanitized": None, + } + if observations is not None: + observations.append(observation) + # The service sanitizes internally; this option never requests raw keys or SAS. + try: + current = get(_sanitized_datasource_url(url)) + except HelperFailure as failure: + observation["request_id"] = failure.request_id + raise + credentials = current.get("credentials") + request_id = getattr(get, "request_id", None) + observation["request_id"] = request_id + projection = semantic.observe(current, "datasource") + observation["sanitized"] = { + "etag_digest": digest(_etag(current)), "definition_digest": digest(current), + "core_digest": projection["core_digest"], + } + if (_etag(current) != snapshot["etag"] + or observation["sanitized"]["core_digest"] != snapshot["core_digest"]): + fields = projection["field_digests"].keys() | snapshot["field_digests"].keys() + changed = [field for field in sorted(fields) + if projection["field_digests"].get(field) != snapshot["field_digests"].get(field)] + if _etag(current) != snapshot["etag"]: + changed.append("@odata.etag") + for field in changed: + semantic.note(diagnostics, "datasource", "error", "datasource-binding-unverified", field, request_id) + raise fail("datasource-binding-unverified", + "Independent datasource binding readback changed revision or configuration; no stable current proof is available.", + request_id=request_id) + try: + visible = (isinstance(credentials, dict) and set(credentials) == {"connectionString"} + and blob_source._connection_binding(credentials["connectionString"], boundary) == "visible") + except HelperFailure: + visible = False + if not visible: + semantic.note(diagnostics, "datasource", "error", "datasource-binding-unverified", + "credentials.connectionString", request_id) + raise fail("datasource-binding-unverified", + "Sanitized current datasource readback does not prove the exact selected keyless ResourceId; values withheld. Retain resources and investigate this evidence gap read-only.", + request_id=request_id) + observation["status"] = "verified" + semantic.note(diagnostics, "datasource", "info", "datasource-current-binding-verified", + "credentials.connectionString", request_id) + + +def _baseline_cycle(source, read, token_provider): + url = search_reconcile.resource_url(source).replace(")?", ")/status?") + response = read("GET", url, token_provider(SEARCH_AUDIENCE)) + body = response.body + if response.status != 200 or not isinstance(body, dict) or body.get("kind") != "azureBlob": + raise HelperFailure("ingestion-inaccessible", "Initial reuse status is unavailable.", + blocked_at="verification", status=response.status, request_id=response.request_id) + last = body.get("lastSynchronizationState") + if last is None: + return None + if not isinstance(last, dict): + raise fail("ingestion-status-invalid", "Initial reuse synchronization must be an object.") + if last.get("endTime") is None: + blob_source._timestamp(last.get("startTime")) + return None + cycle = [last.get("startTime"), last.get("endTime")] + _validate_cycle(cycle) + return cycle + + +def _validate_cycle(cycle): + if cycle is None: + return + if not isinstance(cycle, list) or len(cycle) != 2: + raise fail("recheck-evidence-invalid", "Retain the original observed reuse cycle, not a reconstructed bound.") + if blob_source._timestamp(cycle[1]) < blob_source._timestamp(cycle[0]): + raise fail("ingestion-status-invalid", "Initial synchronization interval is invalid.") + + +def _binding_receipts(paths): + if paths is None: + return None + if (not isinstance(paths, (tuple, list)) or len(paths) != 2 + or any(not isinstance(path, (str, Path)) or not str(path).strip() for path in paths)): + raise fail("reuse-evidence-invalid", "Select both existing private creation evidence files.") + _, prior, fingerprint = _document(paths[0]) + result = read_private(paths[1]) + blob_source._validate_reuse_receipts(prior, fingerprint, result) + return prior, result + + +class Checkpoint: + def __init__(self, directory, plan, *, context_provider=account_context, binding_receipts=None): + _supported(plan) + self.directory = private_io.private_directory(str(directory)) + self.context_provider = context_provider + self.context = context_provider() + self.plan_digest = digest(plan) + self.operation_id = uuid.uuid4().hex + self.summary = None + self.excluded_cycle = None + self.request_ids = [] + self.recovery_warnings = [] + self.acknowledgement = None + self.write_acknowledgement = None + self.write_observation = None + self.binding_receipts = binding_receipts + self.diagnostics = [] + self.observations = [] + self.binding_observations = [] + self.configuration = None + self.creation = None + + def acknowledge(self, plan, response, not_before, *, url, body, headers): + if (digest(plan) != self.plan_digest or plan["source"]["action"] != "create" + or response.status not in {200, 201} + or url != search_reconcile.resource_url(plan["source"]) + or headers.get("If-None-Match") != "*" or "If-Match" in headers + or body != search_reconcile.canonical_bytes(plan["source"]["desired"])): + raise fail("recheck-ownership-unproven", "Only the exact successful conditional create can retain a write acknowledgement.") + self.write_observation = { + "schema_version": "1.0", "state": "acknowledged", "type": "knowledge-source", + "name": plan["source"]["name"], "http_status": response.status, "request_id": response.request_id, + } + if self.context_provider() != self.context: + raise fail("recheck-auth-context-drift", "CLI context changed before write acknowledgement retention.") + receipt = { + "schema_version": "1.0", "kind": "blob-write-acknowledgement", + "operation_id": self.operation_id, "plan_digest": self.plan_digest, + "not_before": not_before.isoformat(), "context_digest": self.context, + "write": { + "method": "PUT", "url": url, "if_none_match": headers["If-None-Match"], + "body_digest": digest(plan["source"]["desired"]), "status": response.status, + "request_id": response.request_id, "response_etags": search_reconcile.response_etags(response), + "generated": blob_source.generated_resources(response.body) if isinstance(response.body, dict) else [], + }, + } + reject_secrets(receipt) + receipt["integrity"] = digest(receipt) + path = private_io.private_file(self.directory, self.operation_id + ".blob-write.json", receipt) + self.write_acknowledgement = {"operation_id": self.operation_id, "receipt_file": path.name, + "evidence_digest": receipt["integrity"], "status": "retained"} + + def persist(self, plan, result, generated, not_before, *, token_provider, transport): + reused = plan["source"]["action"] == "reuse" + selected, other = ("reused", "created") if reused else ("created", "reused") + owned = "reused_not_owned" if reused else "run_owned" + if (digest(plan) != self.plan_digest or len(result["resources"][selected]) != 1 + or result["resources"][other] or result["resources"][selected] != result["ownership"][owned]): + raise fail("recheck-ownership-unproven", "Retain exact acknowledged creation or read-only reuse; never adopt shared resources.") + generated = copy.deepcopy(generated) + creation = {key: copy.deepcopy(result[key]) for key in + ("approved_plan", "resources", "verification", "ownership")} + creation["source"] = {"generated": generated} + seed = { + "schema_version": "1.0", "kind": "blob-source-acknowledgement", + "operation_id": self.operation_id, "plan_digest": digest(plan), + "not_before": not_before.isoformat(), "context_digest": self.context, + "creation": creation, "request_ids": [], + } + if not reused: + if self.context_provider() != self.context: + raise fail("recheck-auth-context-drift", "CLI context changed before acknowledgement retention.") + seed["integrity"] = digest(seed) + path = private_io.private_file(self.directory, self.operation_id + ".blob-ack.json", seed) + self.acknowledgement = {"operation_id": self.operation_id, "receipt_file": path.name, + "evidence_digest": seed["integrity"], "status": "retained"} + read, get, ids = _reader(transport, token_provider, warnings=self.recovery_warnings) + self.request_ids = ids + configuration = _configuration(plan, creation, generated, get, binding_receipts=self.binding_receipts, + diagnostics=self.diagnostics, observations=self.observations) + if reused: + self.excluded_cycle = _baseline_cycle(plan["source"], read, token_provider) + if self.context_provider() != self.context: + raise fail("recheck-auth-context-drift", "CLI context changed during checkpoint capture; no checkpoint was retained.") + receipt = { + "schema_version": "3.1" if reused else "3.0", "kind": "blob-readiness-checkpoint", + "operation_id": self.operation_id, "plan_digest": digest(plan), + "not_before": not_before.isoformat(), "context_digest": self.context, + "creation": creation, "configuration": configuration, "request_ids": ids, + } + if reused: + receipt["excluded_cycle"] = self.excluded_cycle + if self.binding_receipts is not None: + receipt["schema_version"] = "3.2" + receipt["binding_digest"] = digest(self.binding_receipts) + receipt["integrity"] = digest(receipt) + path = private_io.private_file(self.directory, self.operation_id + ".blob-readiness.json", receipt) + self.request_ids = ids + self.summary = {"operation_id": self.operation_id, "receipt_file": path.name, + "evidence_digest": receipt["integrity"], "status": "retained"} + self.configuration, self.creation = configuration, creation + + def verify(self, plan, *, token_provider, transport): + _, get, ids = _reader(transport, token_provider, warnings=self.recovery_warnings) + try: + _configuration(plan, self.creation, self.creation["source"]["generated"], get, + expected=self.configuration, binding_receipts=self.binding_receipts, + diagnostics=self.diagnostics, observations=self.observations, + binding_observations=self.binding_observations) + finally: + self.request_ids.extend(ids) + if self.context_provider() != self.context: + raise fail("recheck-auth-context-drift", "CLI context changed during generated readback.") + + def finish(self, result): + record = {"schema_version": "1.0", "kind": "blob-operation-result", + "operation_id": self.operation_id, "plan_digest": self.plan_digest, + "result": copy.deepcopy(result)} + reject_secrets(record) + record["integrity"] = digest(record) + path = private_io.private_file(self.directory, self.operation_id + ".blob-result.json", record) + result["result_evidence"] = {"receipt_file": path.name, "status": "retained"} + + +def _document(input_path): + document = read_private(input_path) + reject_secrets(document) + if set(document) != {"schema_version", "plan", "approval"} or document["schema_version"] != "1.0": + raise fail("recheck-evidence-invalid", "Retain the original source envelope.") + plan = document["plan"] + if not isinstance(plan, dict): + raise fail("recheck-evidence-invalid", "The original plan is unavailable.") + _supported(plan) + fingerprint = digest(plan) + approval = document["approval"] + if (not isinstance(approval, dict) or set(approval) != {"confirmed", "fingerprint"} + or type(approval["confirmed"]) is not bool or approval["fingerprint"] != fingerprint + or plan["source"]["action"] == "create" and approval["confirmed"] is not True): + raise fail("approval-mismatch", "Retain unchanged creation consent or the fingerprinted read-only reuse plan.") + return document, plan, fingerprint + + +def _load(input_path, receipt_path, *, acknowledgement=False, binding_receipts=None): + document, plan, fingerprint = _document(input_path) + receipt = read_private(receipt_path) + reject_secrets(receipt) + if acknowledgement and receipt.get("kind") == "blob-write-acknowledgement": + _validate_write(plan, receipt) + return plan, receipt + reused = plan["source"]["action"] == "reuse" + version = receipt.get("schema_version") + if not isinstance(version, str): + raise fail("recheck-evidence-invalid", "Checkpoint schema version must be a supported string.") + core_projected = version in {"3.0", "3.1", "3.2"} and not acknowledgement + projected = (version in {"2.0", "2.1", "2.2"} or core_projected) and not acknowledgement + bound = reused and version in {"1.2", "2.2", "3.2"} + fields = {"schema_version", "kind", "operation_id", "plan_digest", "not_before", + "context_digest", "creation", "configuration", "request_ids", "integrity"} + if reused: + fields.add("excluded_cycle") + if bound: + fields.add("binding_digest") + if acknowledgement: + fields.remove("configuration") + expected_version = ("3" if core_projected else "2" if projected else "1") + (".2" if bound else ".1" if reused else ".0") + if (set(receipt) != fields or version != expected_version + or receipt["kind"] != ("blob-source-acknowledgement" if acknowledgement else "blob-readiness-checkpoint") + or acknowledgement and reused + or bound and (binding_receipts is None or receipt["binding_digest"] != digest(binding_receipts)) + or receipt["plan_digest"] != fingerprint + or not isinstance(receipt["operation_id"], str) + or re.fullmatch("[0-9a-f]{32}", receipt["operation_id"]) is None + or receipt["integrity"] != digest({k: v for k, v in receipt.items() if k != "integrity"})): + raise fail("recheck-evidence-invalid", "Original operation/cutoff/checkpoint integrity is missing or changed; never reconstruct it.") + blob_source._timestamp(receipt["not_before"]) + if reused: + _validate_cycle(receipt["excluded_cycle"]) + creation = receipt["creation"] + try: + observed = creation["verification"]["readback"] + generated = creation["source"]["generated"] + expected = {"type": "knowledge-source", "name": plan["source"]["name"], + "etag": observed["etag"], + "definition_digest": digest(search_reconcile._definition(plan["source"]["desired"]))} + valid = ( + set(creation) == {"approved_plan", "resources", "verification", "ownership", "source"} + and set(creation["source"]) == {"generated"} + and set(creation["verification"]) == {"readback", "absence", "request_ids", "idempotency"} + and creation["verification"]["absence"] is False + and creation["verification"]["idempotency"] == "exact readback is zero-write" + and isinstance(creation["verification"]["request_ids"], list) + and all(isinstance(item, str) for item in creation["verification"]["request_ids"]) + and creation["approved_plan"] == document["approval"] + and creation["resources"] == {"created": [] if reused else [expected], + "reused": [expected] if reused else [], "updated": [], "skipped": []} + and creation["ownership"] == {"run_owned": [] if reused else [expected], + "reused_not_owned": [expected] if reused else [], "owner": plan["owner"]} + and observed == expected and isinstance(expected["etag"], str) and bool(expected["etag"].strip()) + and generated == blob_source.generated_resources( + {"azureBlobParameters": {"createdResources": {item["type"]: item["name"] for item in generated}}}, + strict=True, + ) + and (acknowledgement or set(receipt["configuration"]) == set(COLLECTIONS)) + and isinstance(receipt["request_ids"], list) + and all(isinstance(item, str) for item in receipt["request_ids"]) + and isinstance(receipt["context_digest"], str) + and search_reconcile.SHA256.fullmatch(receipt["context_digest"]) + ) + for kind, item in ({} if acknowledgement else receipt["configuration"]).items(): + if core_projected: + valid = valid and semantic.valid(item, kind, search_reconcile.SHA256) + if kind == "indexer": + valid = valid and item.get("core_digest") == item.get("non_schedule_digest") + if kind == "datasource" and item.get("binding_proof") == "resource-id": + valid = valid and item["credential_digest"] in { + digest({"present": True, "value": {"connectionString": f"ResourceId={plan['boundary']['storage_id']}{suffix}"}}) + for suffix in ("", ";") + } + continue + hashes = {"digest"} | (indexer.PROJECTION_FIELDS if projected and kind == "indexer" else set()) + valid = valid and set(item) == {"etag"} | hashes and isinstance(item["etag"], str) and bool(item["etag"].strip()) + valid = valid and all(isinstance(item[key], str) and search_reconcile.SHA256.fullmatch(item[key]) for key in hashes) + except (KeyError, TypeError, AttributeError): + valid = False + if not valid: + raise fail("recheck-ownership-unproven", "Checkpoint must retain exact acknowledged ownership, generated configuration and provenance.") + return plan, receipt + + +def _historical_result(receipt_path, receipt): + path = Path(receipt_path).parent / (receipt["operation_id"] + ".blob-result.json") + if not path.exists(): + return "not-recorded-by-pre-monitor-checkpoint" + record = read_private(path) + reject_secrets(record) + if (set(record) != {"schema_version", "kind", "operation_id", "plan_digest", "result", "integrity"} + or record["schema_version"] != "1.0" or record["kind"] != "blob-operation-result" + or record["operation_id"] != receipt["operation_id"] or record["plan_digest"] != receipt["plan_digest"] + or record["integrity"] != digest({key: value for key, value in record.items() if key != "integrity"}) + or not isinstance(record["result"], dict)): + raise fail("recheck-evidence-invalid", "Historical operation result is malformed or changed; preserve original evidence.") + result = record["result"] + readiness = result.get("readiness", {}) + if not isinstance(readiness, dict): + raise fail("recheck-evidence-invalid", "Historical readiness must be an object.") + watch = readiness.get("watch") + if watch is not None and ( + not isinstance(watch, dict) or watch.get("schema_version") != "1.0" + or not isinstance(watch.get("state"), str) + or watch.get("state") not in {"paused", "completed", "blocked"}): + raise fail("recheck-evidence-invalid", "Historical watch metadata is malformed or unsupported.") + return {"receipt_file": path.name, "status": result.get("status"), + "first_failure": result.get("first_failure", result.get("first_blocker")), + "watch": copy.deepcopy(watch), + "writes_performed": result.get("writes_performed", result.get("completed_writes", []))} + + +def _validate_write(plan, receipt): + try: + write = receipt["write"] + valid = ( + set(receipt) == {"schema_version", "kind", "operation_id", "plan_digest", "not_before", + "context_digest", "write", "integrity"} + and receipt["schema_version"] == "1.0" and plan["source"]["action"] == "create" + and receipt["plan_digest"] == digest(plan) + and isinstance(receipt["operation_id"], str) + and re.fullmatch("[0-9a-f]{32}", receipt["operation_id"]) is not None + and isinstance(receipt["context_digest"], str) + and search_reconcile.SHA256.fullmatch(receipt["context_digest"]) is not None + and receipt["integrity"] == digest({k: v for k, v in receipt.items() if k != "integrity"}) + and set(write) == {"method", "url", "if_none_match", "body_digest", "status", + "request_id", "response_etags", "generated"} + and write["method"] == "PUT" and write["if_none_match"] == "*" + and type(write["status"]) is int and write["status"] in {200, 201} + and write["url"] == search_reconcile.resource_url(plan["source"]) + and write["body_digest"] == digest(plan["source"]["desired"]) + and isinstance(write["request_id"], str) and bool(write["request_id"].strip()) + and set(write["response_etags"]) == {"body", "headers"} + and isinstance(write["response_etags"]["headers"], list) + and isinstance(write["generated"], list) + ) + if write["generated"]: + valid = valid and write["generated"] == blob_source.generated_resources( + {"azureBlobParameters": {"createdResources": { + item["type"]: item["name"] for item in write["generated"]}}}, strict=True, + ) + except (KeyError, TypeError, AttributeError): + valid = False + if not valid: + raise fail("recheck-ownership-unproven", "Retain the private authenticated conditional-write receipt; input booleans or observed existence cannot replace it.") + blob_source._timestamp(receipt["not_before"]) + if search_reconcile.resolve_etag(write["response_etags"], write["request_id"]) is None: + raise fail("creation-version-unproven", "Write acknowledgement has no response ETag; never borrow a later GET version.", + request_id=write["request_id"]) + + +@reporting("blob-capture") +def capture(input_path, directory, *, token_provider=azure_cli_token, transport=http_request, + storage_transport=http_request, context_provider=account_context, + now=lambda: datetime.now(timezone.utc), progress: Progress | None = None, + binding_paths=None): + progress.update("evidence-validation") + document, plan, fingerprint = _document(input_path) + if plan["source"]["action"] != "reuse": + raise fail("recheck-scope-unsupported", "Fresh capture accepts only a reuse plan; it cannot recover missing original creation proof.") + binding_receipts = _binding_receipts(binding_paths) + progress.update("context-check") + checkpoint = Checkpoint(directory, plan, context_provider=context_provider, binding_receipts=binding_receipts) + cutoff = now() + progress.update("source-binding") + _, get, ids = _reader(transport, token_provider) + current = get(search_reconcile.resource_url(plan["source"])) + generated = blob_source.generated_resources(current, strict=True) + result = search_reconcile._completed( + "blob-readiness-capture", fingerprint, plan["source"], action="reused", + readback=current, request_ids=ids, absence=False, + ) + result["approved_plan"] = document["approval"] + progress.update("blob-inventory") + inventory = blob_inventory.discover(plan["boundary"], plan["inventory_limits"], + token_provider=token_provider, transport=storage_transport) + if inventory["inventory_digest"] != plan["inventory_digest"]: + raise fail("source-drift", "Selected Storage/ACL evidence differs from the reuse plan.") + result["verification"]["request_ids"].extend(inventory["request_ids"]) + progress.update("checkpoint") + checkpoint.persist(plan, result, generated, cutoff, token_provider=token_provider, transport=transport) + return { + "status": "completed", "outcome": "blob-readiness-capture", + "recheck_checkpoint": checkpoint.summary, "writes_performed": [], + "readiness": {"status": "unverified"}, "retrieval": "unverified", "knowledge_base": "not-verified", + "ownership": result["ownership"], "cleanup": {"separate_confirmation_required": True}, + "read_only_evidence": {"request_ids": result["verification"]["request_ids"] + checkpoint.request_ids}, + "indexer_diagnostics": semantic.indexer_only(checkpoint.diagnostics), + "generated_diagnostics": checkpoint.diagnostics, "indexer_observations": checkpoint.observations, + "warnings": [blob_source.SNAPSHOT_WARNING, "Fresh reuse observation is not recovered creation ownership or ingestion proof.", + *get.recovery_warnings, *checkpoint.recovery_warnings, + *indexer.warnings(checkpoint.diagnostics)], + } + + +@reporting("blob-capture") +def recover(input_path, acknowledgement_path, directory, *, token_provider=azure_cli_token, + transport=http_request, storage_transport=http_request, context_provider=account_context, + progress: Progress | None = None): + progress.update("evidence-validation") + plan, acknowledgement = _load(input_path, acknowledgement_path, acknowledgement=True) + historical = _historical_result(acknowledgement_path, acknowledgement) + directory = private_io.private_directory(str(directory)) + progress.update("context-check") + if context_provider() != acknowledgement["context_digest"]: + raise fail("recheck-auth-context-drift", "Current context differs from the acknowledged original run.") + progress.update("source-binding") + _, get, ids = _reader(transport, token_provider) + diagnostics, observations, binding_observations = [], [], [] + if acknowledgement["kind"] == "blob-write-acknowledgement": + write = acknowledgement["write"] + current = get(search_reconcile.resource_url(plan["source"])) + if (_etag(current) != search_reconcile.resolve_etag(write["response_etags"], write["request_id"]) + or search_reconcile._definition(current) != search_reconcile._definition(plan["source"]["desired"])): + raise fail("definition-drift", "Current source differs from the acknowledged conditional-write version/definition.", + request_id=getattr(get, "request_id", None)) + generated = blob_source.generated_resources(current, strict=True) + if write["generated"] and generated != write["generated"]: + raise fail("definition-drift", "Generated identities differ from the create response.") + verified = search_reconcile._completed( + "create-blob-knowledge-source", acknowledgement["plan_digest"], plan["source"], + action="created", readback=current, request_ids=[write["request_id"], *ids], absence=False, + ) + creation = {key: verified[key] for key in ("approved_plan", "resources", "verification", "ownership")} + creation["source"] = {"generated": generated} + else: + creation = acknowledgement["creation"] + configuration = _configuration(plan, creation, creation["source"]["generated"], get, + diagnostics=diagnostics, observations=observations) + progress.update("blob-inventory") + inventory = blob_inventory.discover(plan["boundary"], plan["inventory_limits"], + token_provider=token_provider, transport=storage_transport) + ids.extend(inventory["request_ids"]) + if inventory["inventory_digest"] != plan["inventory_digest"]: + raise fail("source-drift", "Storage/ACL inventory differs from the acknowledged original run.") + progress.update("checkpoint") + _configuration(plan, creation, creation["source"]["generated"], get, expected=configuration, + diagnostics=diagnostics, observations=observations, binding_observations=binding_observations) + if context_provider() != acknowledgement["context_digest"]: + raise fail("recheck-auth-context-drift", "CLI context changed during recovery observation.") + receipt = {**acknowledgement, "schema_version": "3.0", "kind": "blob-readiness-checkpoint", "configuration": configuration, + "creation": creation, "request_ids": ids} + receipt.pop("write", None) + receipt.pop("integrity") + receipt["integrity"] = digest(receipt) + path = private_io.private_file(directory, receipt["operation_id"] + ".blob-readiness.json", receipt) + return { + "status": "completed", "outcome": "blob-readiness-recovery-capture", "writes_performed": [], + "recheck_checkpoint": {"operation_id": receipt["operation_id"], "receipt_file": path.name, + "evidence_digest": receipt["integrity"], "status": "retained"}, + "readiness": {"status": "unverified"}, "retrieval": "unverified", "knowledge_base": "not-verified", + "ownership": {"run_owned": [], "reused_not_owned": []}, + "original_run": {"ownership": creation["ownership"], "not_before": receipt["not_before"], + "historical_created": copy.deepcopy(creation["resources"]["created"]), + "original_failure": historical, + "acknowledgement_digest": acknowledgement["integrity"]}, + "read_only_evidence": {"request_ids": ids}, + "indexer_diagnostics": semantic.indexer_only(diagnostics), + "generated_diagnostics": diagnostics, "indexer_observations": observations, + "datasource_binding_observations": binding_observations, + "safe_next_decision": "Run blob_recheck.py --input with the unchanged original input and --receipt with this checkpoint; then return to the KB/retrieval owner. Preserve the original first failure separately.", + "warnings": [blob_source.SNAPSHOT_WARNING, "Configuration was observed during recovery; no earlier generated revision or new ownership is asserted.", + *get.recovery_warnings, + *indexer.warnings(diagnostics)], + } + + +@reporting("blob-recheck") +def recheck(input_path, receipt_path, *, token_provider=azure_cli_token, + transport=http_request, storage_transport=http_request, + context_provider=account_context, monotonic=time.monotonic, sleep=time.sleep, + now=lambda: datetime.now(timezone.utc), progress: Progress | None = None, + binding_paths=None, watch_limits=None, cancelled=lambda: False): + progress.update("evidence-validation") + binding_receipts = _binding_receipts(binding_paths) + plan, receipt = _load(input_path, receipt_path, binding_receipts=binding_receipts) + limits = blob_source._poll_limits(watch_limits if watch_limits is not None else plan["poll"]) + creation = receipt["creation"] + generated = creation["source"]["generated"] + readiness = {"status": "unverified"} + historical = "not-recorded-by-pre-monitor-checkpoint" + ids = [] + recovery_warnings = [] + diagnostics, observations, binding_observations = [], [], [] + try: + historical = _historical_result(receipt_path, receipt) + progress.update("context-check") + if context_provider() != receipt["context_digest"]: + raise fail("recheck-auth-context-drift", "Current CLI tenant/subscription/principal differs from the original run; no auth changes were made.") + read, get, ids = _reader(transport, token_provider) + recovery_warnings = get.recovery_warnings + + def inventory(): + inventory = blob_inventory.discover( + plan["boundary"], plan["inventory_limits"], token_provider=token_provider, + transport=storage_transport, + ) + ids.extend(inventory["request_ids"]) + if inventory["inventory_digest"] != plan["inventory_digest"]: + raise fail("source-drift", "Selected Storage/ACL evidence differs from original creation.") + + for stage in ("before", "after"): + if stage == "after": + progress.update("blob-readback") + inventory() + progress.update("source-binding" if stage == "before" else "source-readback") + _configuration(plan, creation, generated, get, binding_receipts=binding_receipts, + expected=receipt["configuration"], diagnostics=diagnostics, observations=observations, + binding_observations=binding_observations) + if stage == "before": + progress.update("blob-inventory") + inventory() + readiness = blob_source.monitor( + plan["source"], not_before=blob_source._timestamp(receipt["not_before"]), + limits=limits, token_provider=token_provider, transport=read, + monotonic=monotonic, sleep=sleep, progress=progress, + excluded_cycle=receipt.get("excluded_cycle"), + cancelled=cancelled, + indexer_name=next(item["name"] for item in generated if item["type"] == "indexer"), + ) + if readiness["status"] != "verified": + raise HelperFailure( + readiness["code"], "Original-run ingestion remains unverified.", + blocked_at="verification", request_id=readiness.get("request_id"), + status=readiness.get("http_status"), + ) + cycle = readiness["synchronization"] + if (cycle["itemsUpdatesProcessed"] == 0 or cycle["itemsSkipped"] + or blob_source._timestamp(cycle["endTime"]) > now()): + raise fail("ingestion-unverified", "A checkpoint is not prior ingestion proof; nonempty zero-skip completion is required.") + progress.update("context-readback") + if context_provider() != receipt["context_digest"]: + raise fail("recheck-auth-context-drift", "CLI context changed during recheck.") + except HelperFailure as failure: + recovery_warnings.extend(failure.warnings) + if failure.request_id and failure.request_id not in ids: + ids.append(failure.request_id) + safe_failure = HelperFailure( + failure.code, failure.message if failure.blocked_at == "indexer-verification" else + "Read-only evidence could not verify readiness; service details withheld.", + blocked_at=failure.blocked_at, status=failure.http_status, request_id=failure.request_id, + ) + result = blocked_result(safe_failure, outcome="blob-readiness-recheck", fingerprint=None) + result["safe_next_decision"] = "Preserve the first failure. GET the exact source/generated definitions; for a concealed source binding supply --reuse-input-file/--reuse-result-file from its successful creation. Fix access or investigate actual drift read-only; do not replay creation, run/reset an indexer or default to cleanup." + if failure.code == "datasource-binding-unverified": + result["first_blocker"]["message"] = ( + "The datasource revision lacks independently observed current Storage binding; " + "this is unverified, not evidence of misconfiguration." + ) + result["safe_next_decision"] = ( + "Retain source and checkpoint. One exact datasource GET with includeConnectionString=true did not " + "provide stable sanitized ResourceId proof. " + "Ask the service owner for authoritative current child binding evidence; root identity and historical " + "receipts are insufficient. Resume the same GET-only recheck if that readback becomes available. " + "Do not retrieve keys, patch/recreate the source, run/reset an indexer, or delete resources." + ) + if failure.code == "indexer-legacy-evidence-insufficient": + result["safe_next_decision"] = "Retain the legacy checkpoint unchanged; GET current definitions and compare any legitimately retained preimage. This helper cannot infer its missing projection. A separately planned --capture reuse operation may observe fresh readiness, not recover historical configuration/ownership. Do not replay writes." + readiness = {**readiness, "status": "unverified"} + if readiness.get("watch", {}).get("state") == "paused": + result["safe_next_decision"] = readiness["safe_next_decision"] + else: + result = {"status": "completed", "outcome": "blob-readiness-recheck", "writes_performed": []} + result.update( + readiness=readiness, retrieval="unverified", knowledge_base="not-verified", + original_run={"operation_id": receipt["operation_id"], "plan_digest": receipt["plan_digest"], + "source_action": plan["source"]["action"], + "evidence_digest": receipt["integrity"], "not_before": receipt["not_before"], + "request_ids": creation["verification"]["request_ids"], + "checkpoint_request_ids": receipt["request_ids"], + "original_failure": historical, + "historical_created": copy.deepcopy(creation["resources"]["created"]), + "ownership": copy.deepcopy(creation["ownership"]), "generated": generated}, + ownership={"run_owned": [], "reused_not_owned": []}, + read_only_evidence={"request_ids": ids}, + indexer_diagnostics=semantic.indexer_only(diagnostics), + generated_diagnostics=diagnostics, indexer_observations=observations, + datasource_binding_observations=binding_observations, + cleanup={"status": "not-requested", "separate_confirmation_required": True}, + recheck_checkpoint={"operation_id": receipt["operation_id"], "receipt_file": Path(receipt_path).name, + "evidence_digest": receipt["integrity"], "status": "retained"}, + warnings=[blob_source.SNAPSHOT_WARNING, + *recovery_warnings, + "Local checkpoint integrity is not a service signature or new ownership/cleanup authorization.", + *indexer.warnings(diagnostics)], + ) + return result + + +def compact_result(result, directory): + """Opt-in versioned presentation; retain the native result privately first.""" + reject_secrets(result) + directory = private_io.private_directory(str(directory)) + path = private_io.private_file(directory, uuid.uuid4().hex + ".blob-result.json", result) + ids = [] + + def collect(value, key=None): + if isinstance(value, dict): + for field, child in value.items(): + collect(child, field) + elif isinstance(value, list): + for child in value: + collect(child, key) + elif (key in {"request_id", "request_ids"} or isinstance(key, str) and key.endswith("_request_ids")) and isinstance(value, str): + ids.append(value) + + collect(result) + failure = result.get("first_failure", result.get("first_blocker")) + original = result.get("original_run", {}) + writes = result.get("writes_performed", result.get("completed_writes", [])) + readiness = result.get("readiness", {"status": "unverified"}) + item_error = readiness.get("first_error") + if readiness.get("code") == "ingestion-timeout" and readiness.get("watch", {}).get("state") == "paused": + failure = None + elif failure and isinstance(item_error, dict) and item_error.get("request_id"): + failure = {**failure, "request_id": item_error["request_id"]} + historical = original.get("original_failure") + historical_failure = historical.get("first_failure") if isinstance(historical, dict) else None + historical_watch = historical.get("watch") if isinstance(historical, dict) else None + if (historical_failure and historical_failure.get("code") == "ingestion-timeout" + and isinstance(historical_watch, dict) and historical_watch.get("schema_version") == "1.0" + and historical_watch.get("state") == "paused"): + historical_failure = None + generated = original.get("generated", result.get("source", {}).get("generated", [])) + diagnostic_keys = ("severity", "code", "field") + diagnostics = dict.fromkeys( + tuple(item[key] for key in diagnostic_keys) + for item in result.get("generated_diagnostics", result.get("indexer_diagnostics", [])) + ) + return { + "schema_version": "1.1" if "progress" in readiness else "1.0", "kind": "blob-operation-summary", + "status": result["status"], "outcome": result["outcome"], + "writes_performed": writes, + "creation_acknowledgement": result.get("creation_acknowledgement"), + "historical_created": [ + {"type": item["type"], "name": item["name"]} + for item in original.get("historical_created", []) + ], + "generated_resources": [{"type": item["type"], "name": item["name"]} for item in generated], + "readiness": {"status": readiness["status"], "watch": readiness.get("watch"), + **({"progress": readiness["progress"]} if "progress" in readiness else {})}, + "first_failure": ({key: failure.get(key) for key in ("code", "status", "request_id")} + if failure else None), + "historical_failure": ({key: historical_failure.get(key) for key in ("code", "status", "request_id")} + if historical_failure else None), + "first_retry": readiness.get("first_retry"), + "diagnostics": [dict(zip(diagnostic_keys, item)) for item in diagnostics], + "request_id_count": len(set(ids)), "evidence_file": path.name, + "checkpoint_file": result.get("recheck_checkpoint", {}).get("receipt_file"), + "retrieval": "unverified", "knowledge_base": "not-verified", + "safe_next_decision": result.get("safe_next_decision", + "Return to the KB/retrieval owner; no cleanup or new writes are authorized." + if readiness["status"] == "verified" else + "Retain resources and inspect the named source/indexer and private evidence. " + "Use receipt-backed GET-only recheck when evidence is available; no write replay or cleanup."), + } + + +def main(argv=None): + parser = argparse.ArgumentParser() + modes = parser.add_mutually_exclusive_group(required=True) + modes.add_argument("--input", type=Path) + modes.add_argument("--capture", type=Path) + modes.add_argument("--recover", type=Path) + parser.add_argument("--receipt", type=Path) + parser.add_argument("--receipt-dir", type=Path) + parser.add_argument("--reuse-input-file", type=Path) + parser.add_argument("--reuse-result-file", type=Path) + parser.add_argument("--watch-seconds", type=int) + parser.add_argument("--watch-max-requests", type=int) + parser.add_argument("--watch-interval", type=int) + parser.add_argument("--compact", action="store_true") + add_progress_argument(parser) + args = parser.parse_args(argv) + if (args.capture and (not args.receipt_dir or args.receipt) + or args.input and (not args.receipt or args.receipt_dir) + or args.recover and (not args.receipt or not args.receipt_dir)): + parser.error("--capture needs --receipt-dir; --input needs --receipt; --recover needs both.") + if bool(args.reuse_input_file) != bool(args.reuse_result_file) or args.recover and args.reuse_input_file: + parser.error("Select both reuse evidence files, only with --capture or --input.") + options = {"binding_paths": (args.reuse_input_file, args.reuse_result_file)} if args.reuse_input_file else {} + watch = (args.watch_seconds, args.watch_max_requests, args.watch_interval) + if any(value is not None for value in watch): + if not args.input or any(value is None for value in watch): + parser.error("Explicit GET-only watch needs --input and all three --watch-* limits.") + options["watch_limits"] = dict(zip(("deadline_seconds", "max_requests", "interval_seconds"), watch)) + try: + if args.capture: + result = capture(args.capture, args.receipt_dir, progress=Progress("blob-capture", enabled=args.progress), **options) + elif args.recover: + result = recover(args.recover, args.receipt, args.receipt_dir, + progress=Progress("blob-capture", enabled=args.progress)) + else: + result = recheck(args.input, args.receipt, progress=Progress("blob-recheck", enabled=args.progress), **options) + except HelperFailure as failure: + outcome = "blob-readiness-recovery-capture" if args.recover else "blob-readiness-capture" if args.capture else "blob-readiness-recheck" + result = blocked_result(failure, outcome=outcome, fingerprint=None) + result["safe_next_decision"] = ( + "Preserve original error/resources. Inspect the exact source and genuine private evidence: " + "--recover needs its retained acknowledgement; --capture needs a reuse plan and any required " + "creation proof. If a checkpoint already exists, inspect it and use --input/--receipt. " + "No write replay, new ownership or cleanup is authorized." + ) + if args.compact: + try: + result = compact_result(result, args.receipt_dir or args.receipt.parent) + except HelperFailure as failure: + result.setdefault("warnings", []).append("compact-evidence-persistence-failed: full native result retained in output.") + result["presentation_failure"] = {"code": failure.code} + emit_result(result) + return 2 + emit_result(result) + return 0 if result["status"] == "completed" else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_source.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_source.py index cd4ba166..ddae123f 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_source.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_source.py @@ -1,1346 +1,1346 @@ -from __future__ import annotations - -import argparse -import copy -import json -import math -import re -import sys -import time -from datetime import datetime, timezone -from email.utils import parsedate_to_datetime -from pathlib import Path -from typing import Any, Callable -from urllib.parse import urlencode - -try: - from ._progress import Progress, add_progress_argument, reporting - from . import blob_inventory, search_reconcile, source_vector, cu_ingestion_auth - from ._common import ( - SEARCH_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, - blocked_result, canonical_bytes, digest, emit_result, http_request, load_approved_input, HttpResult, - reject_secrets, require_allowed_fields, - ) -except ImportError: - from _progress import Progress, add_progress_argument, reporting - import blob_inventory # type: ignore[no-redef] - import search_reconcile # type: ignore[no-redef] - import source_vector # type: ignore[no-redef] - import cu_ingestion_auth - from _common import ( # type: ignore[no-redef] - SEARCH_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, - blocked_result, canonical_bytes, digest, emit_result, http_request, load_approved_input, HttpResult, - reject_secrets, require_allowed_fields, - ) - - -SNAPSHOT_WARNING = ( - "Before/after inventories detect observed drift, not an atomic Storage snapshot " - "or a source lock. Scheduled sources can change after this run." -) -RETRY_STATUS = {408, 429, 500, 502, 503, 504} - - -def validate_content_understanding(value: Any, *, enabled: bool) -> dict[str, Any] | None: - if not enabled: - if value is not None: - raise _failure("cu-choice-conflict", "Minimal extraction must omit Content Understanding choices.") - return None - if not isinstance(value, dict): - raise _failure("cu-prerequisite-missing", "Standard extraction requires an existing CU-capable AIServices account.") - require_allowed_fields(value, {"endpoint", "auth", "prerequisites"}, label="Content Understanding choices") - endpoint = value.get("endpoint") - if ( - not isinstance(endpoint, str) - or re.fullmatch(r"https://[a-z0-9][a-z0-9-]{0,62}\.services\.ai\.azure\.com/?", endpoint) is None - or value.get("auth") != "system-assigned" - ): - raise _failure("cu-auth-unsupported", "Select an exact AIServices services.ai.azure.com endpoint and existing Search system-assigned identity; no keys.") - prerequisites = value.get("prerequisites") - if not isinstance(prerequisites, dict): - raise _failure("cu-prerequisite-missing", "Supply current CU resource, configuration, identity and network evidence references.") - fields = {"resource", "configuration", "identity", "network"} - require_allowed_fields(prerequisites, fields, label="Content Understanding prerequisites") - if any(not isinstance(prerequisites.get(key), str) or not prerequisites[key].strip() - or len(prerequisites[key]) > 4096 for key in fields): - raise _failure("cu-prerequisite-missing", "Owner-verified CU capability/region, configuration, Search role and reachability evidence is required.") - return copy.deepcopy(value) - - -def verify_content_understanding_readback(choice: dict[str, Any] | None, current: Any) -> None: - if choice is None or current is None: - return - parameters = current.get("azureBlobParameters") if isinstance(current, dict) else None - ingestion = parameters.get("ingestionParameters") if isinstance(parameters, dict) else None - ai = ingestion.get("aiServices") if isinstance(ingestion, dict) else None - if ( - not isinstance(ai, dict) or ingestion.get("contentExtractionMode") != "standard" - or not isinstance(ai.get("uri"), str) - or ai["uri"].rstrip("/") != choice["endpoint"].rstrip("/") - ): - raise _failure("cu-readback-mismatch", "Observed standard extraction and CU endpoint must match the selected configuration.") - key = ai.get("apiKey") - if (key is not None and not (isinstance(key, str) and key == "") - or ingestion.get("identity") is not None): - raise _failure("cu-auth-conflict", "Observed CU authentication does not prove system-assigned mode; credential details withheld.") - - -def generated_resources(current: dict[str, Any], *, strict: bool = False) -> list[dict[str, Any]]: - parameters = current.get("azureBlobParameters") - created = parameters.get("createdResources") if isinstance(parameters, dict) else None - generated = [] - if isinstance(created, dict): - for kind, name in created.items(): - if ( - kind in {"datasource", "dataSourceConnection", "indexer", "skillset", "index"} - and isinstance(name, str) and re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_-]{0,127}", name) - ): - generated.append({ - "type": "datasource" if kind == "dataSourceConnection" else kind, - "name": name, "service_managed": True, - }) - generated.sort(key=lambda item: (item["type"], item["name"])) - if strict and ( - not isinstance(created, dict) or len(created) != 4 or len(generated) != 4 - or {item["type"] for item in generated} != {"datasource", "indexer", "skillset", "index"} - ): - raise _failure("generated-resources-unverified", "Exact service-generated identities are required for reuse.") - return generated - - -def _read_intent(path: Path) -> dict[str, Any]: - try: - document = json.loads(path.read_text(encoding="utf-8")) - except (OSError, UnicodeError, json.JSONDecodeError) as exc: - raise _failure("input-unreadable", "Input must be readable UTF-8 JSON.") from exc - if not isinstance(document, dict): - raise _failure("input-schema-invalid", "Input must be an object.") - return document - - -def _receipt_paths(request: dict[str, Any]) -> tuple[str, str] | None: - paths = [request.get(field) for field in ("reuse_input_file", "reuse_result_file")] - if paths == [None, None]: - return None - if not all(isinstance(path, str) and path.strip() for path in paths): - raise _failure("reuse-evidence-invalid", "Select both retained approved input and creation result files.") - return paths[0], paths[1] - - -def _reuse_receipts(paths: tuple[str, str] | None) -> tuple[dict[str, Any], dict[str, Any]] | None: - if paths is None: - return None - try: - _, prior, fingerprint = load_approved_input(Path(paths[0])) - except UnicodeError as exc: - raise _failure("input-unreadable", "Retained input must be readable UTF-8 JSON.") from exc - _validate_plan(prior) - result = _read_intent(Path(paths[1])) - _validate_reuse_receipts(prior, fingerprint, result) - return prior, result - - -def _validate_reuse_receipts(prior, fingerprint, result): - reject_secrets(result) - if not all(isinstance(result.get(field), dict) for field in ( - "approved_plan", "source_evidence", "resources", "verification", "source" - )): - raise _failure("reuse-evidence-invalid", "Retained creation result sections must be objects.") - if ( - prior["source"]["action"] != "create" or result.get("status") != "completed" - or result["approved_plan"].get("confirmed") is not True - or result.get("approved_plan") != {"confirmed": True, "fingerprint": fingerprint} - or result.get("source_evidence", {}).get("boundary") != prior["boundary"] - or result.get("source_evidence", {}).get("inventory_digest") != prior["inventory_digest"] - ): - raise _failure("reuse-evidence-invalid", "Retained records must prove the exact approved source creation.") - - -def _connection_binding(connection, boundary): - if connection is None or isinstance(connection, str) and connection.lower() in ("", ""): - return "concealed" - if isinstance(connection, str) and connection.startswith("ResourceId="): - if connection.removesuffix(";") != f"ResourceId={boundary['storage_id']}": - raise _failure("boundary-mismatch", "Observed connection targets a different Storage account.") - return "visible" - raise _failure("source-binding-conflict", "Observed credentials are not the selected keyless binding or a recognized concealed value; details withheld.") - - -def _verify_storage_binding( - source: dict[str, Any], boundary: dict[str, Any], current: dict[str, Any], - generated: list[dict[str, Any]], - load_receipts: Callable[[], tuple[dict[str, Any], dict[str, Any]] | None], -) -> None: - connection = current["azureBlobParameters"].get("connectionString") - if _connection_binding(connection, boundary) == "visible": - return - receipts = load_receipts() - if receipts is None: - raise _failure( - "source-binding-unverified", - "Source account binding is unknown: GET the exact source definition or select its retained approved input/successful creation result pair. Redaction is not a mismatch; snippets are not binding proof.", - ) - _verify_creation_binding(source, boundary, current, generated, receipts) - - -def _verify_creation_binding( - source: dict[str, Any], boundary: dict[str, Any], current: dict[str, Any], - generated: list[dict[str, Any]], receipts: tuple[dict[str, Any], dict[str, Any]], -) -> None: - prior, result = receipts - observed = { - "type": "knowledge-source", "name": source["name"], "etag": current["@odata.etag"], - "definition_digest": digest(search_reconcile._definition(current)), - } - if ( - prior["boundary"] != boundary - or any(prior["source"].get(field) != source[field] for field in ("endpoint", "name", "api_version")) - or not search_reconcile.definitions_match(prior["source"]["desired"], current) - or result.get("resources", {}).get("created") != [observed] - or result.get("verification", {}).get("readback") != observed - or result.get("source", {}).get("generated") != generated - ): - raise _failure("source-binding-unverified", "Retained creation evidence does not match the fresh source identity/ETag.") - - -def plan_source( - request: dict[str, Any], *, - token_provider: TokenProvider = azure_cli_token, transport: Transport = http_request, - storage_transport: Transport = http_request, monotonic: Callable[[], float] = time.monotonic, -) -> dict[str, Any]: - if not isinstance(request, dict): - raise _failure("input-schema-invalid", "Planning input must be an object.") - reject_secrets(request) - require_allowed_fields( - request, - {"schema_version", "endpoint", "name", "owner", "storage_id", "container", "prefix", "is_adls", - "api_version", "processing", "network_access", "identity", "permission_options", - "ingestion_schedule", "description", "rbac", "network", "inventory_limits", "poll", - "reuse_input_file", "reuse_result_file", "embedding", "content_understanding"}, - label="Blob planning input", - ) - try: - json.dumps(request, ensure_ascii=False, allow_nan=False).encode("utf-8") - except (UnicodeError, TypeError, ValueError) as exc: - raise _failure("input-schema-invalid", "Planning choices must be valid UTF-8 JSON.") from exc - if request.get("schema_version") != "1.0" or not isinstance(request.get("owner"), str) or not request["owner"].strip(): - raise _failure("input-schema-invalid", "schema_version 1.0 and an explicit owner are required.") - boundary = blob_inventory.validate_boundary({ - field: request.get(field) for field in ("storage_id", "container", "prefix", "is_adls") - }) - limits = copy.deepcopy(blob_inventory.validate_limits(request.get("inventory_limits"))) - poll = copy.deepcopy(_poll_limits(request.get("poll"))) - if ( - request.get("processing") not in ("minimal-lexical", "minimal-vector", "standard-cu") or request.get("network_access") != "public" - or request.get("identity") != "system-assigned" or request.get("permission_options") != [] - or "ingestion_schedule" not in request or request["ingestion_schedule"] is not None - or not isinstance(request.get("api_version"), str) - or request["api_version"] not in {"2026-04-01", "2026-08-01-preview"} - or boundary["is_adls"] and request["api_version"] != "2026-08-01-preview" - ): - raise _failure( - "planning-processing-unsupported", - "This planner supports Blob 2026-04-01/2026-08-01-preview and ADLS 2026-08-01-preview: " - "Internal presets: minimal-lexical = minimal extraction without vectors; " - "minimal-vector = minimal extraction with embeddings; standard-cu = standard CU with optional embeddings. " - "These are not API enums or KB reasoning modes. Requires public, system-assigned, no permissions/schedule. " - "Other requested versions/features need a compatible owner; never change them silently.", - ) - embedding = source_vector.validate_choice( - request.get("embedding"), enabled=( - request["processing"] == "minimal-vector" - or request["processing"] == "standard-cu" and request.get("embedding") is not None - ), - api_version=request["api_version"], - ) - cu = validate_content_understanding( - request.get("content_understanding"), enabled=request["processing"] == "standard-cu", - ) - if "description" not in request or not ( - request["description"] is None or isinstance(request["description"], str) - ): - raise _failure("input-schema-invalid", "description must be explicit text or null.") - rbac, network = request.get("rbac"), request.get("network") - if ( - not isinstance(rbac, dict) or not isinstance(rbac.get("assignments"), list) - or not rbac["assignments"] or not all(isinstance(item, dict) and item for item in rbac["assignments"]) - or not isinstance(network, dict) or network.get("posture") != "public" - or not isinstance(network.get("evidence"), str) or not network["evidence"].strip() - ): - raise _failure("planning-evidence-missing", "Supply observed RBAC assignments and public network evidence; the owner verifies effective access and policy.") - ingestion = { - "contentExtractionMode": "minimal", "disableImageVerbalization": True, - "identity": None, "ingestionSchedule": None, - } - if request["api_version"].endswith("-preview"): - ingestion.update(networkAccessMode="public", ingestionPermissionOptions=[]) - if embedding is not None: - ingestion["embeddingModel"] = source_vector.model_definition(embedding) - if cu is not None: - ingestion.update(contentExtractionMode="standard", aiServices={"uri": cu["endpoint"].rstrip("/")}) - source = { - "operation": "reconcile", "resource_type": "knowledge-source", - "outcome": "create-blob-knowledge-source", - "endpoint": request.get("endpoint"), "name": request.get("name"), - "api_version": request["api_version"], "action": "create", - "owner": request["owner"], "cleanup_approved": False, - "rbac": copy.deepcopy(rbac), "network": copy.deepcopy(network), - "desired": { - "name": request.get("name"), "kind": "azureBlob", "description": request["description"], - "azureBlobParameters": { - "connectionString": f"ResourceId={boundary['storage_id']}", - "containerName": boundary["container"], "folderPath": boundary["prefix"] or None, - "isADLSGen2": boundary["is_adls"], "ingestionParameters": ingestion, - }, - }, - } - # Validate the shared control sections and exact address before any authentication. - for section, fields in ((rbac, {"assignments"}), (network, {"posture", "evidence"})): - require_allowed_fields(section, fields, label="planning controls") - url = search_reconcile.resource_url(source) - receipt_paths = _receipt_paths(request) - receipts = None - - def load_receipts() -> tuple[dict[str, Any], dict[str, Any]] | None: - nonlocal receipts - if receipts is None: - receipts = _reuse_receipts(receipt_paths) - return receipts - - deadline = monotonic() + limits["deadline_seconds"] - - def read_search() -> tuple[dict[str, Any] | None, str | None]: - remaining = deadline - monotonic() - if remaining <= 0: - raise _failure("planning-deadline-exceeded", "Planning read deadline elapsed.") - - def bounded(method: str, target: str, token: str, **kwargs: Any) -> Any: - if method != "GET" or target != url: - raise _failure("planning-write-forbidden", "Planning reads only the exact Search source.") - remaining = deadline - monotonic() - if remaining <= 0: - raise _failure("planning-deadline-exceeded", "Planning read deadline elapsed during authentication.") - return transport( - method, target, token, timeout=min(30, remaining), follow_redirects=False, - max_response_bytes=8 * 1024 * 1024, response_deadline=time.monotonic() + remaining, - ) - - current, request_id = search_reconcile.read_resource(url, token_provider(SEARCH_AUDIENCE), transport=bounded) - try: - source_vector.verify_source_readback(embedding, current) - verify_content_understanding_readback(cu, current) - except HelperFailure as failure: - failure.request_id = request_id - raise - return current, request_id - - current, first_id = read_search() - generated = [] - if current is not None: - if not search_reconcile.definitions_match(source["desired"], current): - raise _failure("definition-conflict", "The exact source has a different definition; do not overwrite, suffix or repair it.") - if not isinstance(current.get("@odata.etag"), str) or not current["@odata.etag"]: - raise _failure("definition-evidence-missing", "Exact reuse requires the current source ETag.") - generated = generated_resources(current, strict=True) - _verify_storage_binding(source, boundary, current, generated, load_receipts) - snapshot = blob_inventory.discover( - boundary, limits, token_provider=token_provider, transport=storage_transport, - monotonic=monotonic, deadline=deadline, - ) - refreshed, last_id = read_search() - if (current is None) != (refreshed is None) or current is not None and ( - refreshed.get("@odata.etag") != current["@odata.etag"] - or not search_reconcile.definitions_match(source["desired"], refreshed) - or generated_resources(refreshed, strict=True) != generated - ): - raise _failure("definition-drift", "Source identity/ETag or generated resources changed during Storage observation.") - if refreshed is not None: - _verify_storage_binding(source, boundary, refreshed, generated, load_receipts) - source.update(action="reuse", expected_etag=refreshed["@odata.etag"]) - if monotonic() >= deadline: - raise _failure("planning-deadline-exceeded", "Planning read deadline elapsed during Search readback.") - evidence = {"verified": True, "inventory_digest": snapshot["inventory_digest"]} - if boundary["is_adls"]: - evidence.update(path_verified=True, acl_verified=True) - source["source_evidence"] = evidence - plan = { - "operation": "reconcile-and-monitor", "owner": request["owner"], "cleanup_approved": False, - "boundary": boundary, "inventory_digest": snapshot["inventory_digest"], - "inventory_limits": limits, "poll": poll, "source": source, - } - if embedding is not None: - plan["embedding"] = embedding - if cu is not None: - plan["content_understanding"] = cu - plan["cu_plan_version"] = "1.0" - if current is not None: - plan["expected_generated"] = generated - else: - plan["expected_source_absent"] = True - _validate_plan(plan) - fingerprint = digest(plan) - mutation = source["action"] == "create" - return { - "status": "planned", "outcome": "create-blob-knowledge-source", - "plan_fingerprint": fingerprint, - "execution_input": { - "schema_version": "1.0", "plan": plan, - "approval": {"confirmed": False, "fingerprint": fingerprint}, - }, - "approval_summary": { - "target": {"endpoint": source["endpoint"], "name": source["name"], "api_version": source["api_version"]}, - "storage_account": boundary["storage_id"].rsplit("/", 1)[1], - "source_kind": "adls-gen2" if boundary["is_adls"] else "azure-blob", - "scope": "selected folder/directory" if boundary["prefix"] else "explicit container/filesystem root", - "object_count": len(snapshot["objects"]), "total_bytes": sum(item["size"] for item in snapshot["objects"]), - "adls_paths_observed": len(snapshot["adls_paths"]), - "processing": ( - f"standard Content Understanding extraction {'with selected embeddings' if embedding else 'without source vectors'}; no image verbalization, chat, permission ingestion or schedule" - if cu else - "minimal extraction with embeddings; no image verbalization, permission ingestion or schedule" - if embedding else "minimal lexical; image verbalization disabled; no models, permission ingestion or schedule" - ), - **({"embedding": source_vector.summary(embedding)} if embedding else {}), - **({"content_understanding": { - "purpose": "Standard document extraction only; not source vectorization or KB answer synthesis.", - "endpoint": cu["endpoint"].rstrip("/"), "auth": cu["auth"], - "authentication": cu_ingestion_auth.approval_summary( - "adlsGen2" if request["is_adls"] else "azureBlob", cu["auth"], creating=mutation, - ), - "kb_reasoning": "Unchanged; KB chat requires separate selection, access and approval.", - "cost_and_data": "Creation sends selected documents to billable CU processing (no free document allowance); generated Search content is retained. Cross-region processing may apply. Selected source embeddings have separate costs.", - "source_vectorization": "azureOpenAI" if embedding else "none", - "prerequisites": "Owner-verified references only, not effective access or successful processing proof. No local auth, role, model or defaults changes.", - }} if cu else {}), - "network": "public; supplied access evidence remains owner-verified", - "supplied_role_assignments": len(rbac["assignments"]), - "source_action": source["action"], "execution_required": mutation, - "mutation_approval_required": mutation, - "ownership": "Only a newly created source and its service-generated children become run-owned; existing Search/Storage/objects/roles remain shared.", - "verification": "Fresh identity/inventory and ADLS owner/ACL metadata only; effective Search access, ingestion readiness and retrieval are unverified.", - "cost_and_retention": "Creation starts indexing and retains generated Search resources; existing charges/schedules continue on reuse.", - "cleanup": "Separate run-owned source cleanup only; never delete Storage objects or shared resources.", - "next_step": ( - "Owner refreshes identity, RBAC, network, source state and cost/data consent, then approves these changes before applying the unchanged private artifact." - if mutation else "Reuse without mutation approval or executor invocation. Refresh discovery before later use; evidence is not future consent." - ), - }, - "read_only_evidence": {"request_ids": [item for item in [first_id, *snapshot["request_ids"], last_id] if item]}, - "writes_performed": [], "warnings": [SNAPSHOT_WARNING], - } - - -def _failure(code: str, message: str, *, request_id: str | None = None) -> HelperFailure: - return HelperFailure(code, message, blocked_at="verification", request_id=request_id) - - -def _timestamp(value: Any) -> datetime: - if not isinstance(value, str): - raise _failure("ingestion-status-invalid", "Synchronization timestamp is missing.") - try: - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError as exc: - raise _failure("ingestion-status-invalid", "Synchronization timestamp is invalid.") from exc - if parsed.tzinfo is None: - raise _failure("ingestion-status-invalid", "Synchronization timestamp needs a time zone.") - return parsed - - -def _poll_limits(value: Any) -> dict[str, int]: - if not isinstance(value, dict): - raise _failure("input-schema-invalid", "poll must be an object.") - require_allowed_fields( - value, {"deadline_seconds", "max_requests", "interval_seconds"}, label="poll" - ) - for field, maximum in ( - ("deadline_seconds", 3600), ("max_requests", 1000), ("interval_seconds", 60) - ): - if type(value.get(field)) is not int or not 1 <= value[field] <= maximum: - raise _failure("input-schema-invalid", "Polling limits must be bounded positive integers.") - return value - - -def _validate_plan(plan: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: - reject_secrets(plan) - require_allowed_fields( - plan, - {"operation", "owner", "cleanup_approved", "source", "boundary", "inventory_digest", - "inventory_limits", "poll", "expected_generated", "expected_source_absent", "embedding", - "content_understanding", "cu_plan_version"}, - label="Blob source plan", - ) - if plan.get("operation") != "reconcile-and-monitor" or plan.get("cleanup_approved") is not False: - raise _failure("operation-invalid", "Blob application requires reconcile-and-monitor without cleanup.") - source = plan.get("source") - if not isinstance(source, dict) or not isinstance(plan.get("owner"), str) or not plan["owner"]: - raise _failure("input-schema-invalid", "A source plan and owner are required.") - if ( - source.get("operation") != "reconcile" - or source.get("resource_type") != "knowledge-source" - or source.get("action") not in ("create", "reuse") - or not isinstance(source.get("api_version"), str) - or source.get("owner") != plan["owner"] - or not isinstance(source.get("desired"), dict) - or source["desired"].get("kind") != "azureBlob" - or source.get("ai_services_api_key_environment") is not None - or source.get("ai_services_key_acquisition") is not None - ): - raise _failure("step-contract-mismatch", "Only same-owner Blob creation or exact reuse is supported.") - search_reconcile._validate_plan(source) - search_reconcile._resource_url(source) - boundary = blob_inventory.validate_boundary(plan.get("boundary")) - blob_inventory.validate_limits(plan.get("inventory_limits")) - _poll_limits(plan.get("poll")) - if "expected_source_absent" in plan and ( - plan["expected_source_absent"] is not True or source["action"] != "create" - or source.get("expected_etag") is not None - ): - raise _failure("step-contract-mismatch", "Expected absence is only valid for a creation plan.") - if "expected_generated" in plan and ( - source["action"] != "reuse" - or not isinstance(plan["expected_generated"], list) - or len(plan["expected_generated"]) != 4 - or any(not isinstance(item, dict) or set(item) != {"type", "name", "service_managed"} - or item["service_managed"] is not True - or not isinstance(item["type"], str) - or not isinstance(item["name"], str) - or re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_-]{0,127}", item["name"]) is None - for item in plan["expected_generated"]) - or {item["type"] for item in plan["expected_generated"]} != {"datasource", "indexer", "skillset", "index"} - ): - raise _failure("generated-resources-unverified", "Expected generated identities require a complete reuse plan.") - parameters = source["desired"]["azureBlobParameters"] - if ( - parameters["connectionString"].removesuffix(";") != f"ResourceId={boundary['storage_id']}" - or parameters["containerName"] != boundary["container"] - or parameters["folderPath"] != (boundary["prefix"] or None) - or parameters["isADLSGen2"] != boundary["is_adls"] - or "createdResources" in parameters - or source["source_evidence"]["inventory_digest"] != plan.get("inventory_digest") - ): - raise _failure("boundary-mismatch", "Source definition must bind the exact discovered boundary and inventory.") - if ( - not isinstance(plan.get("inventory_digest"), str) - or search_reconcile.SHA256.fullmatch(plan["inventory_digest"]) is None - ): - raise _failure("source-evidence-invalid", "The approved inventory digest is required.") - ingestion = parameters.get("ingestionParameters") - if not isinstance(ingestion, dict) or ingestion.get("assetStore") is not None: - raise _failure( - "source-write-forbidden", - "Ingestion parameters are required; asset-store writes are outside this read-only-source workflow.", - ) - source_vector.validate_plan_choice(plan) - standard = ingestion.get("contentExtractionMode") == "standard" - if "cu_plan_version" in plan and (plan["cu_plan_version"] != "1.0" or not standard): - raise _failure("cu-plan-version-invalid", "CU planner artifacts require version 1.0 and standard extraction.") - # Absence/generated guards predate CU planning and remain valid on legacy - # wire-only artifacts. Only CU-specific provenance selects the new contract. - planner_cu = standard and any(field in plan for field in ( - "cu_plan_version", "embedding", "content_understanding", - )) - cu = validate_content_understanding( - plan.get("content_understanding"), enabled=planner_cu, - ) - if cu is not None: - expected = { - "contentExtractionMode": "standard", "disableImageVerbalization": True, - "identity": None, "ingestionSchedule": None, - "aiServices": {"uri": cu["endpoint"].rstrip("/")}, - } - if "embedding" in plan: - expected["embeddingModel"] = source_vector.model_definition(plan["embedding"]) - if source["api_version"].endswith("-preview"): - expected.update(networkAccessMode="public", ingestionPermissionOptions=[]) - if ingestion != expected or boundary["is_adls"] and source["api_version"] != "2026-08-01-preview": - raise _failure("cu-plan-mismatch", "Standard CU requires the unchanged keyless public definition, matching optional embeddings and no chat, asset store, permissions or schedule.") - return source, boundary - - -def _first_error(state: dict[str, Any]) -> dict[str, Any] | None: - errors = state.get("errors") - if errors is None or errors == []: - return None - if not isinstance(errors, list) or any(not isinstance(error, dict) for error in errors): - raise _failure("ingestion-status-invalid", "Ingestion errors have an invalid shape.") - error = errors[0] - # Document errors can contain content, SAS URLs, and credentials in free text. - return { - "status": error.get("statusCode") if type(error.get("statusCode")) is int else None, - "message": "Document-level ingestion error; sensitive service text withheld.", - "diagnostic_digest": digest(error), - } - - -def _synchronization_state(state, request_id=None): - if state is None: - return None - if not isinstance(state, dict): - raise _failure("ingestion-status-invalid", "Synchronization state must be an object.", request_id=request_id) - return dict(state) - - -def _retry_after(value, *, now=None): - now = now or datetime.now(timezone.utc) - kind = getattr(value, "kind", None) - try: - if kind is not None: - if kind == "overlong": - return math.inf - payload = getattr(value, "value", None) - if (not isinstance(kind, str) or kind not in {"seconds", "date", "date-rfc850"} - or type(payload) not in (int, float) - or (type(payload) is float and not math.isfinite(payload))): - return None - if kind == "seconds": - return payload if payload >= 0 and payload == int(payload) else None - parsed = datetime.fromtimestamp(payload, timezone.utc) - obsolete_date = kind == "date-rfc850" - else: - if not isinstance(value, str) or len(value) > 128: - return None - if value.isascii() and value.isdigit() and len(value) <= 10: - return int(value) - parsed = parsedate_to_datetime(value) - obsolete_date = bool(re.fullmatch(r"[A-Za-z]+, \d{2}-[A-Za-z]{3}-\d{2} \d{2}:\d{2}:\d{2} GMT", value)) - if obsolete_date: - year = now.year // 100 * 100 + parsed.year % 100 - if year > now.year + 50: - year -= 100 - parsed = parsed.replace(year=year) - if parsed.tzinfo is not None: - return max(0, (parsed - now).total_seconds()) - except (ValueError, OverflowError, TypeError, OSError): - pass - return None - - -def _execution_progress(body, not_before): - if (not isinstance(body, dict) or not isinstance(body.get("status"), str) - or body["status"] not in {"running", "error", "unknown"}): - raise _failure("indexer-status-invalid", "Indexer status must be an object.") - if body["status"] == "error": - raise _failure("indexer-status-error", "Indexer availability reports an error; execution success is not inferred.") - run = body.get("lastResult") - if run is None: - return None - if not isinstance(run, dict): - raise _failure("indexer-status-invalid", "Latest indexer execution must be an object.") - start = _timestamp(run.get("startTime")) - if start < not_before: - return None - status = run.get("status") - if not isinstance(status, str) or status not in {"inProgress", "success", "transientFailure", "persistentFailure", "reset"}: - raise _failure("indexer-status-invalid", "Unknown latest execution status; top-level running is not execution proof.") - result = {"startTime": run["startTime"], "run_status": status} - if run.get("endTime") is not None: - if _timestamp(run["endTime"]) < start: - raise _failure("indexer-status-invalid", "Indexer execution interval is invalid.") - result["endTime"] = run["endTime"] - for field, target in (("itemsProcessed", "items_attempted"), ("itemsFailed", "items_failed")): - if field in run: - if type(run[field]) is not int or run[field] < 0: - raise _failure("indexer-status-invalid", "Indexer execution counters are invalid.") - result[target] = run[field] - if result.get("items_failed", 0) > result.get("items_attempted", result.get("items_failed", 0)): - raise _failure("indexer-status-invalid", "Failed item count exceeds attempted items.") - errors = run.get("errors", []) - if errors is not None and (not isinstance(errors, list) or any(not isinstance(item, dict) for item in errors)): - raise _failure("indexer-status-invalid", "Indexer execution errors are malformed.") - result["error_count"] = len(errors or []) - return result - - -def _execution_completed_by(execution, end, not_before): - required = {"startTime", "endTime", "run_status", "error_count"} - if (not isinstance(execution, dict) or not required <= execution.keys() - or execution.keys() - required - {"items_attempted", "items_failed"} - or execution["run_status"] != "success" - or type(execution["error_count"]) is not int or execution["error_count"] != 0 - or any(type(execution[key]) is not int or execution[key] < 0 - for key in ("items_attempted", "items_failed") if key in execution) - or execution.get("items_failed", 0)): - return False - try: - return not_before <= _timestamp(execution["startTime"]) <= _timestamp(execution["endTime"]) <= end - except HelperFailure: - return False - - -@reporting("blob-monitor") -def monitor( - source: dict[str, Any], - *, - not_before: datetime, - limits: dict[str, int], - token_provider: TokenProvider, - transport: Transport, - require_new_cycle: bool = False, - excluded_cycle: list[str] | None = None, - monotonic: Callable[[], float] = time.monotonic, - sleep: Callable[[float], None] = time.sleep, - progress: Progress | None = None, - cancelled: Callable[[], bool] = lambda: False, - indexer_name: str | None = None, -) -> dict[str, Any]: - _poll_limits(limits) - progress.update("ingestion-cycle") - excluded = tuple(_timestamp(value) for value in excluded_cycle) if excluded_cycle is not None else None - url = ( - f"{source['endpoint'].rstrip('/')}/knowledgesources('{search_reconcile._odata_name(source['name'])}')/status?" - + urlencode({"api-version": source["api_version"]}) - ) - token = token_provider(SEARCH_AUDIENCE) - started = monotonic() - deadline = started + limits["deadline_seconds"] - request_ids: list[str] = [] - first_error: dict[str, Any] | None = None - first_retry: dict[str, Any] | None = None - observed = "no-completed-cycle" - initial_cycle: tuple[Any, Any] | None = None - first_response = True - checks = 0 - delay = limits["interval_seconds"] - latest = {} - previous = None - previous_execution = None - execution_guard = None - pause_reason = "request-limit" - newest_run = None - synchronization_status = "not-reported" - - def report(phase, next_check=None): - value = { - "schema_version": "1.0", "phase": phase, - "run_start": _timestamp(latest["startTime"]).isoformat() if latest.get("startTime") else None, - "processed": latest.get("itemsUpdatesProcessed"), "failed": latest.get("itemsUpdatesFailed"), - "skipped": latest.get("itemsSkipped"), "unit": "item-updates", - "total": None, "remaining": None, "denominator": "not-comparable-to-files", - "synchronization_status": synchronization_status, - "elapsed_seconds": max(0, monotonic() - started), "next_check_seconds": next_check, - } - progress.counts["status_checks"] = checks - progress.blob_update(value) - return value - - def failure_result(*args): - result = _readiness_failure(*args) - result["progress"] = report("failed") - result["watch"] = {"schema_version": "1.0", "state": "blocked", "reason": "evidence-failure", - "elapsed_seconds": max(0, monotonic() - started), - "status_checks": checks, "latest": copy.deepcopy(latest)} - return result - - def read_status(target): - nonlocal checks, retry_after - remaining = deadline - monotonic() - if remaining <= 0: - raise _failure("ingestion-watch-expired", "The client watch window elapsed before this read.") - checks += 1 - response = transport( - "GET", target, token, timeout=min(30, remaining), - max_response_bytes=8 * 1024 * 1024, follow_redirects=False, - response_deadline=monotonic() + min(30, remaining), - ) - if response.request_id: - request_ids.append(response.request_id) - if response.status != 200: - value = getattr(response, "retry_after", None) - if value is None: - value = next((v for k, v in response.headers.items() if k.lower() == "retry-after"), None) - retry_after = _retry_after(value) - raise HelperFailure("ingestion-inaccessible", "Status read is inaccessible.", - blocked_at="verification", status=response.status, request_id=response.request_id) - return response - - while checks < limits["max_requests"]: - remaining = deadline - monotonic() - if cancelled(): - pause_reason = "cancelled" - break - if remaining <= 0: - pause_reason = "deadline" - break - retry_after = None - throttled = False - try: - response = read_status(url) - except KeyboardInterrupt: - pause_reason = "cancelled" - break - except HelperFailure as failure: - if failure.request_id and failure.request_id not in request_ids: - request_ids.append(failure.request_id) - if failure.code == "response-deadline-exceeded": - first_retry = first_retry or {"code": failure.code, "status": failure.http_status, - "request_id": failure.request_id} - if monotonic() >= deadline: - pause_reason = "deadline" - break - if failure.code == "ingestion-watch-expired": - pause_reason = "deadline" - break - if retry_after is None: - retry_after = _retry_after(failure.retry_after) - if (failure.http_status not in RETRY_STATUS - and failure.code not in {"azure-response-ambiguous", "response-deadline-exceeded"}): - return failure_result("ingestion-inaccessible", failure.request_id, request_ids, - first_error, first_retry, failure.http_status) - if first_retry is None: - first_retry = {"code": failure.code, "status": failure.http_status, - "request_id": failure.request_id} - throttled = failure.http_status == 429 - delay = min(60, max(limits["interval_seconds"], delay * 2)) - else: - if monotonic() >= deadline: - pause_reason = "deadline" - break - body = response.body - if not isinstance(body, dict) or body.get("kind") != "azureBlob": - raise _failure("ingestion-status-invalid", "Expected azureBlob status.", request_id=response.request_id) - synchronization_status = body.get("synchronizationStatus", "not-reported") - if synchronization_status not in ("not-reported", "active", "creating", "deleting"): - raise _failure("ingestion-status-invalid", "Unknown knowledge-source synchronization availability.", - request_id=response.request_id) - current = _synchronization_state(body.get("currentSynchronizationState"), response.request_id) - last = _synchronization_state(body.get("lastSynchronizationState"), response.request_id) - for state in (current, last): - if state is not None: - _timestamp(state.get("startTime")) - _first_error(state) - state = current or last - start = _timestamp(state["startTime"]) if state else None - if start is not None and start >= not_before and (newest_run is None or start >= newest_run): - newest_run = start - error = _first_error(state) - if first_error is None and error: - first_error = {**error, "request_id": response.request_id} - latest = {"startTime": state["startTime"], - "state": "in-progress" if current else "completed-cycle-observed"} - for field in ("itemsUpdatesProcessed", "itemsUpdatesFailed", "itemsSkipped"): - if field in state: - if type(state[field]) is not int or state[field] < 0: - raise _failure("ingestion-status-invalid", "Observed synchronization counters are invalid.", - request_id=response.request_id) - latest[field] = state[field] - if current: - observed = "in-progress" - signature = digest(latest) - delay = (limits["interval_seconds"] if signature != previous - else min(60, delay * 2)) - previous = signature - else: - latest = {} - observed = "stale-success-or-unrelated-cycle" if state else "no-completed-cycle" - signature = digest({}) - delay = limits["interval_seconds"] if signature != previous else min(60, delay * 2) - previous = signature - if execution_guard is not None: - latest["indexer_execution"] = copy.deepcopy(execution_guard) - if first_response and require_new_cycle and last: - initial_cycle = (last.get("startTime"), last.get("endTime")) - first_response = False - if ( - last and last.get("endTime") is not None - and _timestamp(last.get("startTime")) >= max(not_before, newest_run or not_before) - and (last.get("startTime"), last.get("endTime")) != initial_cycle - and (excluded is None or (_timestamp(last.get("startTime")), _timestamp(last.get("endTime"))) != excluded) - ): - status = last.get("status") - if status is not None and not isinstance(status, str): - raise _failure("ingestion-status-invalid", "Synchronization status must be a string when present.") - if status in {"failure", "partialSuccess"}: - return failure_result(f"ingestion-{status}", response.request_id, - request_ids, first_error, first_retry) - if status in {None, "success"}: - start = _timestamp(last.get("startTime")) - end = _timestamp(last.get("endTime")) - failed = last.get("itemsUpdatesFailed") - processed = last.get("itemsUpdatesProcessed") - skipped = last.get("itemsSkipped") - if end < start or any(type(count) is not int or count < 0 for count in (failed, processed, skipped)): - raise _failure("ingestion-status-invalid", "Completed synchronization counters or interval are invalid.") - if failed or first_error: - return failure_result("ingestion-partialSuccess", response.request_id, - request_ids, first_error, first_retry) - if (not current and synchronization_status in ("active", "not-reported") - and (execution_guard is None - or _execution_completed_by(execution_guard, end, not_before))): - return { - "status": "verified", - "progress": report("completed"), - "synchronization": { - field: last[field] for field in - ("startTime", "endTime", "itemsUpdatesProcessed", "itemsUpdatesFailed", "itemsSkipped") - }, - "not_before": not_before.isoformat(), - "request_ids": request_ids, - "first_retry": first_retry, - "watch": {"schema_version": "1.0", "state": "completed", - "elapsed_seconds": max(0, monotonic() - started), - "status_checks": checks, "latest": latest}, - } - else: - raise _failure("ingestion-status-invalid", "Unknown synchronization completion status.") - elif last and not current: - observed = "stale-success-or-unrelated-cycle" - if indexer_name and checks < limits["max_requests"] and monotonic() < deadline and not cancelled(): - target = (f"{source['endpoint'].rstrip('/')}/indexers('{search_reconcile._odata_name(indexer_name)}')/status?" - + urlencode({"api-version": source["api_version"]})) - run_response = None - try: - run_response = read_status(target) - cutoff = max(not_before, newest_run or not_before) - if execution_guard is not None: - cutoff = min(cutoff, _timestamp(execution_guard["startTime"])) - execution = _execution_progress(run_response.body, cutoff) - if (execution is not None and execution_guard is not None - and _timestamp(execution["startTime"]) < _timestamp(execution_guard["startTime"])): - execution = None - if execution is not None: - if execution_guard is not None or execution["run_status"] == "inProgress": - execution_guard = copy.deepcopy(execution) - execution_signature = digest(execution) - if execution_signature != previous_execution: - delay = limits["interval_seconds"] - previous_execution = execution_signature - latest["indexer_execution"] = execution - if (execution.get("items_failed", 0) or execution["error_count"] - or execution["run_status"] in {"transientFailure", "persistentFailure"}): - first_error = first_error or { - "status": None, "message": "Latest relevant indexer execution reports failures; details withheld.", - "request_id": run_response.request_id, - } - except KeyboardInterrupt: - pause_reason = "cancelled" - break - except HelperFailure as failure: - if failure.request_id is None and run_response is not None: - failure.request_id = run_response.request_id - if failure.request_id and failure.request_id not in request_ids: - request_ids.append(failure.request_id) - if failure.code == "response-deadline-exceeded": - first_retry = first_retry or {"code": failure.code, "status": failure.http_status, - "request_id": failure.request_id} - if monotonic() >= deadline: - pause_reason = "deadline" - break - if failure.code == "ingestion-watch-expired": - pause_reason = "deadline" - break - if (failure.http_status not in RETRY_STATUS - and failure.code not in {"azure-response-ambiguous", "response-deadline-exceeded"}): - code = failure.code if failure.code in {"indexer-status-invalid", "indexer-status-error"} else "indexer-status-unverified" - return failure_result(code, failure.request_id, request_ids, - first_error, first_retry, failure.http_status) - first_retry = first_retry or {"code": failure.code, "status": failure.http_status, - "request_id": failure.request_id} - throttled = failure.http_status == 429 - if retry_after is None: - retry_after = _retry_after(failure.retry_after) - delay = min(60, delay * 2) - remaining = deadline - monotonic() - if cancelled(): - pause_reason = "cancelled" - break - if retry_after is not None and retry_after > min(60, remaining): - report("throttled" if throttled else "waiting") - pause_reason = "retry-after" - break - if remaining > 0 and checks < limits["max_requests"]: - try: - wait = min(max(delay, retry_after or 0), remaining) - report("throttled" if throttled else - "ingesting" if observed == "in-progress" else "waiting", wait) - sleep(wait) - except KeyboardInterrupt: - pause_reason = "cancelled" - break - if monotonic() >= deadline: - pause_reason = "deadline" - result = _readiness_failure( - "ingestion-timeout", request_ids[-1] if request_ids else None, - request_ids, first_error, first_retry, - ) - result["observed"] = observed - result["watch"] = {"schema_version": "1.0", "state": "paused", "reason": pause_reason, - "elapsed_seconds": max(0, monotonic() - started), - "status_checks": checks, "latest": latest, - "service_execution": "not-modified"} - if first_error or latest.get("itemsUpdatesFailed"): - result["code"] = "ingestion-partialSuccess" - result["progress"] = report("failed" if result["code"] == "ingestion-partialSuccess" else "paused") - result["safe_next_decision"] = ( - "Service reports item failures; retain the partial work and inspect exact private error evidence read-only. " - "A new watch does not repair failures; do not replay writes or default to cleanup." - ) if result["code"] == "ingestion-partialSuccess" else ( - "Client watch paused; this is not an ingestion failure. Resume GET-only with the unchanged input " - "and readiness receipt using blob_recheck.py --input --receipt . " - "Do not recreate, run/reset, delete, or infer corpus completion." - ) - return result - - -def _readiness_failure( - code: str, request_id: str | None, request_ids: list[str], - first_error: dict[str, Any] | None, first_retry: dict[str, Any] | None, - status: int | None = None, -) -> dict[str, Any]: - return { - "status": "unverified", "code": code, "request_id": request_id, - "http_status": status, "request_ids": request_ids, - "first_error": first_error, "first_retry": first_retry, - } - - -@reporting("blob-source") -def execute( - document: dict[str, Any], - *, - token_provider: TokenProvider = azure_cli_token, - transport: Transport = http_request, - storage_transport: Transport = http_request, - now: Callable[[], datetime] = lambda: datetime.now(timezone.utc), - monotonic: Callable[[], float] = time.monotonic, - sleep: Callable[[float], None] = time.sleep, - checkpoint: Any = None, - progress: Progress | None = None, -) -> dict[str, Any]: - progress.update("validation") - reject_secrets(document) - require_allowed_fields(document, {"schema_version", "plan", "approval", "_computed_fingerprint"}, - label="input envelope") - plan = document.get("plan") - approval = document.get("approval") - if document.get("schema_version") != "1.0" or not isinstance(plan, dict) or not isinstance(approval, dict): - raise _failure("input-schema-invalid", "A typed user-approved envelope is required.") - require_allowed_fields(approval, {"confirmed", "fingerprint"}, label="approval") - fingerprint = digest(plan) - if approval.get("confirmed") is not True or approval.get("fingerprint") != fingerprint: - raise _failure("approval-mismatch", "The exact recomputed plan fingerprint must be approved.") - source, boundary = _validate_plan(plan) - progress.update("blob-inventory") - before = blob_inventory.discover( - boundary, plan["inventory_limits"], token_provider=token_provider, transport=storage_transport, - ) - if before["inventory_digest"] != plan["inventory_digest"]: - raise _failure("source-drift", "Current source evidence differs from approval; reconfirmation is required.") - not_before = now() - generated: list[dict[str, Any]] = [] - write_generated: list[dict[str, Any]] = [] - initial_read = True - creation_etag = None - embedding_transport = source_vector.guard_readback_transport(plan, transport) - - def reconcile_transport(method: str, url: str, token: str, **kwargs: Any) -> Any: - nonlocal initial_read - first_read = method == "GET" and initial_read - if first_read: - initial_read = False - kwargs.update(max_response_bytes=8 * 1024 * 1024, follow_redirects=False) - try: - response = embedding_transport(method, url, token, **kwargs) - except HelperFailure as failure: - if ( - first_read and failure.http_status == 404 and source["action"] == "create" - and source.get("expected_etag") is not None - ): - raise _failure( - "definition-drift", "The source bound by the approved ETag is absent; rebuild the plan.", - request_id=failure.request_id, - ) from failure - raise - if method == "GET" and response.status == 200 and isinstance(response.body, dict): - etag = search_reconcile.resolve_etag(search_reconcile.response_etags(response), response.request_id) - if etag is not None: - response = HttpResult(response.status, {**response.body, "@odata.etag": etag}, - response.headers, response.etag_values) - if creation_etag is not None and etag != creation_etag: - raise _failure("definition-drift", "Source version differs from the acknowledged create response.", - request_id=response.request_id) - if isinstance(response.body, dict): - if method == "GET" and response.status == 200: - parameters = response.body.get("azureBlobParameters") - if isinstance(parameters, dict): - try: - _connection_binding(parameters.get("connectionString"), boundary) - except HelperFailure as failure: - failure.request_id = response.request_id - raise - if first_read and response.status == 200: - if plan.get("expected_source_absent"): - raise _failure("definition-drift", "A source appeared after planning; rebuild the plan.", request_id=response.request_id) - if source.get("expected_etag") is not None and response.body.get("@odata.etag") != source["expected_etag"]: - raise _failure("definition-drift", "Source ETag differs from the approved readback.", request_id=response.request_id) - if method == "GET" and response.status == 200 and ( - plan.get("expected_source_absent") or "expected_generated" in plan - ): - if not isinstance(response.body.get("@odata.etag"), str) or not response.body["@odata.etag"]: - raise _failure("definition-evidence-missing", "Source readback must include its ETag.", request_id=response.request_id) - parameters = response.body.get("azureBlobParameters") - connection = parameters.get("connectionString") if isinstance(parameters, dict) else None - if isinstance(connection, str) and connection.startswith("ResourceId="): - if connection.removesuffix(";") != f"ResourceId={boundary['storage_id']}": - raise _failure("boundary-mismatch", "Readback targets a different Storage account.", request_id=response.request_id) - elif plan.get("expected_source_absent") and ( - not creation_etag or response.body.get("@odata.etag") != creation_etag - ): - raise _failure("source-binding-unverified", "Redacted creation readback lacks a matching acknowledged PUT ETag.", request_id=response.request_id) - generated.clear() - generated.extend(generated_resources(response.body)) - if method == "GET" and "expected_generated" in plan and generated != plan["expected_generated"]: - raise _failure("definition-drift", "Generated identities differ from the approved reuse plan.", request_id=response.request_id) - if method == "PUT" and response.status in {200, 201}: - write_generated[:] = generated - return response - - def created(*, response, url, body, headers): - nonlocal creation_etag - if (source["action"] != "create" or url != search_reconcile.resource_url(source) - or headers.get("If-None-Match") != "*" or "If-Match" in headers - or body != canonical_bytes(source["desired"])): - raise _failure("recheck-ownership-unproven", "Creation acknowledgement must bind the exact approved conditional wire.") - if checkpoint is not None: - checkpoint.acknowledge(plan, response, not_before, url=url, body=body, headers=headers) - creation_etag = search_reconcile.resolve_etag(search_reconcile.response_etags(response), response.request_id) - if creation_etag is None: - raise _failure("creation-version-unproven", "Successful create returned no ETag in body or HTTP headers; a later GET cannot prove its version.", - request_id=response.request_id) - - progress.update("source-reconciliation") - try: - result = search_reconcile.execute( - {"plan": source, "_computed_fingerprint": fingerprint}, - token_provider=token_provider, transport=reconcile_transport, on_created=created, - ) - except HelperFailure as failure: - if failure.writes: - failure.resources_remaining.extend(write_generated or generated) - elif failure.partial: - failure.resources_reused.extend(generated) - if checkpoint is not None and failure.writes: - result = blocked_result(failure, outcome="create-blob-knowledge-source", - fingerprint=fingerprint, owner=plan["owner"]) - result.update(readiness={"status": "unverified"}, knowledge_base="not-verified", retrieval="unverified") - return _checkpoint_result(result, checkpoint) - raise - writes = [ - {"action": "created", "type": item["type"], "name": item["name"]} - for item in result["resources"]["created"] - ] - readiness: dict[str, Any] = {"status": "unverified"} - after = None - owned_generated = list(generated or write_generated) if writes else [] - try: - expected_generated = list(generated) - if checkpoint is not None: - if source["action"] == "create" and ( - not creation_etag or creation_etag != result["verification"]["readback"]["etag"] - or write_generated and generated != write_generated - ): - raise _failure("recheck-ownership-unproven", "Checkpoint requires an acknowledged PUT ETag and unchanged generated identities.") - checkpoint.persist(plan, result, generated, not_before, - token_provider=token_provider, transport=transport) - readiness = monitor( - source, not_before=not_before, limits=plan["poll"], token_provider=token_provider, - transport=transport, require_new_cycle=not bool(writes) and checkpoint is None, - excluded_cycle=checkpoint.excluded_cycle if checkpoint is not None else None, - monotonic=monotonic, sleep=sleep, - progress=progress, - indexer_name=next((item["name"] for item in generated if item["type"] == "indexer"), None), - ) - if readiness["status"] != "verified": - raise HelperFailure( - readiness["code"], "Source reconciliation completed but ingestion readiness is unverified.", - blocked_at="verification", request_id=readiness["request_id"], - status=readiness["http_status"], - ) - progress.update("blob-readback") - after = blob_inventory.discover( - boundary, plan["inventory_limits"], token_provider=token_provider, transport=storage_transport, - ) - if after["inventory_digest"] != before["inventory_digest"]: - raise _failure("source-drift", "Source changed during ingestion; reconfirmation is required.") - progress.update("source-readback") - current, request_id = search_reconcile._get( - search_reconcile._resource_url(source), token_provider(SEARCH_AUDIENCE), transport=reconcile_transport, - ) - if ( - current is None - or search_reconcile._definition(current) != search_reconcile._definition(source["desired"]) - or current.get("@odata.etag") != result["verification"]["readback"]["etag"] - or generated != expected_generated - ): - raise _failure("definition-drift", "Source definition or ETag changed while monitoring.", request_id=request_id) - if request_id: - result["verification"]["request_ids"].append(request_id) - if len(generated) != 4 or {item["type"] for item in generated} != {"datasource", "indexer", "skillset", "index"}: - raise _failure("generated-resources-unverified", "Exact service-generated resource identities could not be read back.") - if checkpoint is not None: - checkpoint.verify(plan, token_provider=token_provider, transport=transport) - except HelperFailure as failure: - confirmed = copy.deepcopy(result) - result = blocked_result( - HelperFailure( - failure.code, failure.message, blocked_at=failure.blocked_at, writes=writes, - resources_remaining=result["ownership"]["run_owned"] + owned_generated, - request_id=failure.request_id, status=failure.http_status, partial=bool(writes), - ), - outcome="create-blob-knowledge-source", fingerprint=fingerprint, owner=plan["owner"], - ) - result["reconciliation"] = "completed" - result["writes_performed"] = writes - result["read_only_evidence"] = { - "inventory_request_ids": before["request_ids"] + (after["request_ids"] if after else []), - "configuration_request_ids": checkpoint.request_ids if checkpoint is not None else [], - } - result["confirmed_reconciliation"] = { - key: confirmed[key] for key in ("resources", "verification", "ownership") - } - result["readiness"] = {**readiness, "status": "unverified"} - if readiness.get("watch", {}).get("state") == "paused": - result["safe_next_decision"] = readiness["safe_next_decision"] - if not writes: - result["ownership"]["reused_not_owned"] = [ - {"type": "knowledge-source", "name": source["name"]}, *generated - ] - else: - result["outcome"] = "create-blob-knowledge-source" - result["readiness"] = readiness - result["verification"]["source_digest"] = after["inventory_digest"] - result["source"] = { - "type": "knowledge-source", "name": source["name"], - "generated": owned_generated if writes else generated, - "observed_generated": generated, - } - result["writes_performed"] = writes - result["source_evidence"] = { - "inventory_digest": before["inventory_digest"], "boundary": boundary, - "operator_reachability": "verified", "managed_ingestion_reachability": result["readiness"]["status"], - } - result.setdefault("warnings", []).append(SNAPSHOT_WARNING) - return _checkpoint_result(result, checkpoint) - - -def _checkpoint_result(result, checkpoint): - if "completed_writes" in result: - result["writes_performed"] = copy.deepcopy(result["completed_writes"]) - if checkpoint is not None: - result["warnings"].extend(checkpoint.recovery_warnings) - result["creation_acknowledgement"] = checkpoint.write_observation or { - "schema_version": "1.0", "state": "unavailable", - } - result["indexer_diagnostics"] = [ - item for item in checkpoint.diagnostics - if not item["field"].startswith(("datasource.", "indexer.", "skillset.", "index.")) - ] - result["generated_diagnostics"] = checkpoint.diagnostics - result["indexer_observations"] = checkpoint.observations - result["datasource_binding_observations"] = checkpoint.binding_observations - for item in checkpoint.diagnostics: - if item["severity"] == "warning" and item["message"] not in result.setdefault("warnings", []): - result["warnings"].append(item["message"]) - result["recheck_checkpoint"] = checkpoint.summary or {"status": "unavailable"} - result["recheck_write_acknowledgement"] = checkpoint.write_acknowledgement or {"status": "unavailable"} - result["recheck_acknowledgement"] = checkpoint.acknowledgement or checkpoint.write_acknowledgement or {"status": "unavailable"} - if checkpoint.summary is None: - result["safe_next_decision"] = "Preserve the first failure and resources. Use blob_recheck.py --recover with unchanged approved input and the retained acknowledgement, then --input/--receipt. Missing/conflicting write ETags or absent private evidence remain blockers; never borrow a GET version, replay creation or default to cleanup." - try: - checkpoint.finish(result) - except HelperFailure as failure: - if result["status"] == "completed": - original = result - result = blocked_result( - HelperFailure(failure.code, failure.message, blocked_at="evidence-retention", - writes=original["writes_performed"], - resources_remaining=original["ownership"]["run_owned"], - partial=bool(original["writes_performed"])), - outcome=original["outcome"], fingerprint=checkpoint.plan_digest, - ) - result["confirmed_reconciliation"] = original - result["writes_performed"] = original["writes_performed"] - result["result_evidence"] = {"status": "unavailable", "code": failure.code} - result.setdefault("warnings", []).append("Final private evidence retention failed; preserve the native result and first failure. No retry was performed.") - return result - - -def main(argv: list[str] | None = None) -> int: - try: - from .private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode - except ImportError: - from private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode - parser = argparse.ArgumentParser() - modes = parser.add_mutually_exclusive_group(required=True) - modes.add_argument("--discover", type=Path) - modes.add_argument("--plan", type=Path) - add_execution_output_argument(parser) - modes.add_argument("--input", type=Path) - parser.add_argument("--receipt-dir", type=Path) - parser.add_argument("--compact", action="store_true") - add_progress_argument(parser) - args = parser.parse_args(argv) - if args.compact and (not args.input or not args.receipt_dir): - parser.error("--compact requires --input and --receipt-dir to retain full private evidence.") - fingerprint = None - owner = None - progress = Progress("blob-source", enabled=args.progress) if args.input else None - execution_started = False - try: - validate_execution_output_mode(args) - if progress is not None: - progress.update("validation") - if args.receipt_dir and not args.input: - raise _failure("input-schema-invalid", "--receipt-dir requires --input; read-only reuse capture uses blob_recheck.py.") - if args.plan: - result = plan_source(_read_intent(args.plan)) - emit_plan_result(result, args.execution_output, preserve_unapproved_input=result["status"] == "planned") - return 0 if result["status"] == "planned" else 2 - elif args.discover: - try: - document = json.loads(args.discover.read_text(encoding="utf-8")) - except (OSError, UnicodeError, json.JSONDecodeError) as exc: - raise _failure("input-unreadable", "Discovery input must be readable UTF-8 JSON.") from exc - if not isinstance(document, dict): - raise _failure("input-schema-invalid", "Discovery input must be an object.") - reject_secrets(document) - require_allowed_fields(document, {"boundary", "inventory_limits"}, label="discovery input") - result = blob_inventory.discover(document.get("boundary"), document.get("inventory_limits")) - else: - document, plan, fingerprint = load_approved_input(args.input) - owner = plan.get("owner") - checkpoint = None - if args.receipt_dir: - try: - from . import blob_recheck - except ImportError: - import blob_recheck - checkpoint = blob_recheck.Checkpoint(args.receipt_dir, plan) - execution_started = True - if checkpoint is not None: - result = execute(document, checkpoint=checkpoint, progress=progress) - else: - result = execute(document, progress=progress) - except HelperFailure as failure: - if progress is not None and not execution_started: - progress.finish(failure=failure) - result = blocked_result(failure, outcome="blob-source-lifecycle", fingerprint=fingerprint, owner=owner) - if failure.partial and not failure.writes: - result["safe_next_decision"] = "Original create outcome is unproven. Preserve the mutation error and inspect observed resources read-only; no creation ownership, write replay or cleanup is authorized." - if args.compact: - try: - try: - from . import blob_recheck - except ImportError: - import blob_recheck - result = blob_recheck.compact_result(result, args.receipt_dir) - except HelperFailure as failure: - result.setdefault("warnings", []).append("compact-evidence-persistence-failed: full native result retained in output.") - result["presentation_failure"] = {"code": failure.code} - emit_result(result) - return 3 if result["status"] == "partial" else 2 - emit_result(result, preserve_unapproved_input=result["status"] == "planned") - return {"completed": 0, "discovered": 0, "planned": 0, "blocked": 2, "partial": 3}[result["status"]] - - -if __name__ == "__main__": - sys.exit(main()) +from __future__ import annotations + +import argparse +import copy +import json +import math +import re +import sys +import time +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from pathlib import Path +from typing import Any, Callable +from urllib.parse import urlencode + +try: + from ._progress import Progress, add_progress_argument, reporting + from . import blob_inventory, search_reconcile, source_vector, cu_ingestion_auth + from ._common import ( + SEARCH_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, + blocked_result, canonical_bytes, digest, emit_result, http_request, load_approved_input, HttpResult, + reject_secrets, require_allowed_fields, + ) +except ImportError: + from _progress import Progress, add_progress_argument, reporting + import blob_inventory # type: ignore[no-redef] + import search_reconcile # type: ignore[no-redef] + import source_vector # type: ignore[no-redef] + import cu_ingestion_auth + from _common import ( # type: ignore[no-redef] + SEARCH_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, + blocked_result, canonical_bytes, digest, emit_result, http_request, load_approved_input, HttpResult, + reject_secrets, require_allowed_fields, + ) + + +SNAPSHOT_WARNING = ( + "Before/after inventories detect observed drift, not an atomic Storage snapshot " + "or a source lock. Scheduled sources can change after this run." +) +RETRY_STATUS = {408, 429, 500, 502, 503, 504} + + +def validate_content_understanding(value: Any, *, enabled: bool) -> dict[str, Any] | None: + if not enabled: + if value is not None: + raise _failure("cu-choice-conflict", "Minimal extraction must omit Content Understanding choices.") + return None + if not isinstance(value, dict): + raise _failure("cu-prerequisite-missing", "Standard extraction requires an existing CU-capable AIServices account.") + require_allowed_fields(value, {"endpoint", "auth", "prerequisites"}, label="Content Understanding choices") + endpoint = value.get("endpoint") + if ( + not isinstance(endpoint, str) + or re.fullmatch(r"https://[a-z0-9][a-z0-9-]{0,62}\.services\.ai\.azure\.com/?", endpoint) is None + or value.get("auth") != "system-assigned" + ): + raise _failure("cu-auth-unsupported", "Select an exact AIServices services.ai.azure.com endpoint and existing Search system-assigned identity; no keys.") + prerequisites = value.get("prerequisites") + if not isinstance(prerequisites, dict): + raise _failure("cu-prerequisite-missing", "Supply current CU resource, configuration, identity and network evidence references.") + fields = {"resource", "configuration", "identity", "network"} + require_allowed_fields(prerequisites, fields, label="Content Understanding prerequisites") + if any(not isinstance(prerequisites.get(key), str) or not prerequisites[key].strip() + or len(prerequisites[key]) > 4096 for key in fields): + raise _failure("cu-prerequisite-missing", "Owner-verified CU capability/region, configuration, Search role and reachability evidence is required.") + return copy.deepcopy(value) + + +def verify_content_understanding_readback(choice: dict[str, Any] | None, current: Any) -> None: + if choice is None or current is None: + return + parameters = current.get("azureBlobParameters") if isinstance(current, dict) else None + ingestion = parameters.get("ingestionParameters") if isinstance(parameters, dict) else None + ai = ingestion.get("aiServices") if isinstance(ingestion, dict) else None + if ( + not isinstance(ai, dict) or ingestion.get("contentExtractionMode") != "standard" + or not isinstance(ai.get("uri"), str) + or ai["uri"].rstrip("/") != choice["endpoint"].rstrip("/") + ): + raise _failure("cu-readback-mismatch", "Observed standard extraction and CU endpoint must match the selected configuration.") + key = ai.get("apiKey") + if (key is not None and not (isinstance(key, str) and key == "") + or ingestion.get("identity") is not None): + raise _failure("cu-auth-conflict", "Observed CU authentication does not prove system-assigned mode; credential details withheld.") + + +def generated_resources(current: dict[str, Any], *, strict: bool = False) -> list[dict[str, Any]]: + parameters = current.get("azureBlobParameters") + created = parameters.get("createdResources") if isinstance(parameters, dict) else None + generated = [] + if isinstance(created, dict): + for kind, name in created.items(): + if ( + kind in {"datasource", "dataSourceConnection", "indexer", "skillset", "index"} + and isinstance(name, str) and re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_-]{0,127}", name) + ): + generated.append({ + "type": "datasource" if kind == "dataSourceConnection" else kind, + "name": name, "service_managed": True, + }) + generated.sort(key=lambda item: (item["type"], item["name"])) + if strict and ( + not isinstance(created, dict) or len(created) != 4 or len(generated) != 4 + or {item["type"] for item in generated} != {"datasource", "indexer", "skillset", "index"} + ): + raise _failure("generated-resources-unverified", "Exact service-generated identities are required for reuse.") + return generated + + +def _read_intent(path: Path) -> dict[str, Any]: + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise _failure("input-unreadable", "Input must be readable UTF-8 JSON.") from exc + if not isinstance(document, dict): + raise _failure("input-schema-invalid", "Input must be an object.") + return document + + +def _receipt_paths(request: dict[str, Any]) -> tuple[str, str] | None: + paths = [request.get(field) for field in ("reuse_input_file", "reuse_result_file")] + if paths == [None, None]: + return None + if not all(isinstance(path, str) and path.strip() for path in paths): + raise _failure("reuse-evidence-invalid", "Select both retained approved input and creation result files.") + return paths[0], paths[1] + + +def _reuse_receipts(paths: tuple[str, str] | None) -> tuple[dict[str, Any], dict[str, Any]] | None: + if paths is None: + return None + try: + _, prior, fingerprint = load_approved_input(Path(paths[0])) + except UnicodeError as exc: + raise _failure("input-unreadable", "Retained input must be readable UTF-8 JSON.") from exc + _validate_plan(prior) + result = _read_intent(Path(paths[1])) + _validate_reuse_receipts(prior, fingerprint, result) + return prior, result + + +def _validate_reuse_receipts(prior, fingerprint, result): + reject_secrets(result) + if not all(isinstance(result.get(field), dict) for field in ( + "approved_plan", "source_evidence", "resources", "verification", "source" + )): + raise _failure("reuse-evidence-invalid", "Retained creation result sections must be objects.") + if ( + prior["source"]["action"] != "create" or result.get("status") != "completed" + or result["approved_plan"].get("confirmed") is not True + or result.get("approved_plan") != {"confirmed": True, "fingerprint": fingerprint} + or result.get("source_evidence", {}).get("boundary") != prior["boundary"] + or result.get("source_evidence", {}).get("inventory_digest") != prior["inventory_digest"] + ): + raise _failure("reuse-evidence-invalid", "Retained records must prove the exact approved source creation.") + + +def _connection_binding(connection, boundary): + if connection is None or isinstance(connection, str) and connection.lower() in ("", ""): + return "concealed" + if isinstance(connection, str) and connection.startswith("ResourceId="): + if connection.removesuffix(";") != f"ResourceId={boundary['storage_id']}": + raise _failure("boundary-mismatch", "Observed connection targets a different Storage account.") + return "visible" + raise _failure("source-binding-conflict", "Observed credentials are not the selected keyless binding or a recognized concealed value; details withheld.") + + +def _verify_storage_binding( + source: dict[str, Any], boundary: dict[str, Any], current: dict[str, Any], + generated: list[dict[str, Any]], + load_receipts: Callable[[], tuple[dict[str, Any], dict[str, Any]] | None], +) -> None: + connection = current["azureBlobParameters"].get("connectionString") + if _connection_binding(connection, boundary) == "visible": + return + receipts = load_receipts() + if receipts is None: + raise _failure( + "source-binding-unverified", + "Source account binding is unknown: GET the exact source definition or select its retained approved input/successful creation result pair. Redaction is not a mismatch; snippets are not binding proof.", + ) + _verify_creation_binding(source, boundary, current, generated, receipts) + + +def _verify_creation_binding( + source: dict[str, Any], boundary: dict[str, Any], current: dict[str, Any], + generated: list[dict[str, Any]], receipts: tuple[dict[str, Any], dict[str, Any]], +) -> None: + prior, result = receipts + observed = { + "type": "knowledge-source", "name": source["name"], "etag": current["@odata.etag"], + "definition_digest": digest(search_reconcile._definition(current)), + } + if ( + prior["boundary"] != boundary + or any(prior["source"].get(field) != source[field] for field in ("endpoint", "name", "api_version")) + or not search_reconcile.definitions_match(prior["source"]["desired"], current) + or result.get("resources", {}).get("created") != [observed] + or result.get("verification", {}).get("readback") != observed + or result.get("source", {}).get("generated") != generated + ): + raise _failure("source-binding-unverified", "Retained creation evidence does not match the fresh source identity/ETag.") + + +def plan_source( + request: dict[str, Any], *, + token_provider: TokenProvider = azure_cli_token, transport: Transport = http_request, + storage_transport: Transport = http_request, monotonic: Callable[[], float] = time.monotonic, +) -> dict[str, Any]: + if not isinstance(request, dict): + raise _failure("input-schema-invalid", "Planning input must be an object.") + reject_secrets(request) + require_allowed_fields( + request, + {"schema_version", "endpoint", "name", "owner", "storage_id", "container", "prefix", "is_adls", + "api_version", "processing", "network_access", "identity", "permission_options", + "ingestion_schedule", "description", "rbac", "network", "inventory_limits", "poll", + "reuse_input_file", "reuse_result_file", "embedding", "content_understanding"}, + label="Blob planning input", + ) + try: + json.dumps(request, ensure_ascii=False, allow_nan=False).encode("utf-8") + except (UnicodeError, TypeError, ValueError) as exc: + raise _failure("input-schema-invalid", "Planning choices must be valid UTF-8 JSON.") from exc + if request.get("schema_version") != "1.0" or not isinstance(request.get("owner"), str) or not request["owner"].strip(): + raise _failure("input-schema-invalid", "schema_version 1.0 and an explicit owner are required.") + boundary = blob_inventory.validate_boundary({ + field: request.get(field) for field in ("storage_id", "container", "prefix", "is_adls") + }) + limits = copy.deepcopy(blob_inventory.validate_limits(request.get("inventory_limits"))) + poll = copy.deepcopy(_poll_limits(request.get("poll"))) + if ( + request.get("processing") not in ("minimal-lexical", "minimal-vector", "standard-cu") or request.get("network_access") != "public" + or request.get("identity") != "system-assigned" or request.get("permission_options") != [] + or "ingestion_schedule" not in request or request["ingestion_schedule"] is not None + or not isinstance(request.get("api_version"), str) + or request["api_version"] not in {"2026-04-01", "2026-08-01-preview"} + or boundary["is_adls"] and request["api_version"] != "2026-08-01-preview" + ): + raise _failure( + "planning-processing-unsupported", + "This planner supports Blob 2026-04-01/2026-08-01-preview and ADLS 2026-08-01-preview: " + "Internal presets: minimal-lexical = minimal extraction without vectors; " + "minimal-vector = minimal extraction with embeddings; standard-cu = standard CU with optional embeddings. " + "These are not API enums or KB reasoning modes. Requires public, system-assigned, no permissions/schedule. " + "Other requested versions/features need a compatible owner; never change them silently.", + ) + embedding = source_vector.validate_choice( + request.get("embedding"), enabled=( + request["processing"] == "minimal-vector" + or request["processing"] == "standard-cu" and request.get("embedding") is not None + ), + api_version=request["api_version"], + ) + cu = validate_content_understanding( + request.get("content_understanding"), enabled=request["processing"] == "standard-cu", + ) + if "description" not in request or not ( + request["description"] is None or isinstance(request["description"], str) + ): + raise _failure("input-schema-invalid", "description must be explicit text or null.") + rbac, network = request.get("rbac"), request.get("network") + if ( + not isinstance(rbac, dict) or not isinstance(rbac.get("assignments"), list) + or not rbac["assignments"] or not all(isinstance(item, dict) and item for item in rbac["assignments"]) + or not isinstance(network, dict) or network.get("posture") != "public" + or not isinstance(network.get("evidence"), str) or not network["evidence"].strip() + ): + raise _failure("planning-evidence-missing", "Supply observed RBAC assignments and public network evidence; the owner verifies effective access and policy.") + ingestion = { + "contentExtractionMode": "minimal", "disableImageVerbalization": True, + "identity": None, "ingestionSchedule": None, + } + if request["api_version"].endswith("-preview"): + ingestion.update(networkAccessMode="public", ingestionPermissionOptions=[]) + if embedding is not None: + ingestion["embeddingModel"] = source_vector.model_definition(embedding) + if cu is not None: + ingestion.update(contentExtractionMode="standard", aiServices={"uri": cu["endpoint"].rstrip("/")}) + source = { + "operation": "reconcile", "resource_type": "knowledge-source", + "outcome": "create-blob-knowledge-source", + "endpoint": request.get("endpoint"), "name": request.get("name"), + "api_version": request["api_version"], "action": "create", + "owner": request["owner"], "cleanup_approved": False, + "rbac": copy.deepcopy(rbac), "network": copy.deepcopy(network), + "desired": { + "name": request.get("name"), "kind": "azureBlob", "description": request["description"], + "azureBlobParameters": { + "connectionString": f"ResourceId={boundary['storage_id']}", + "containerName": boundary["container"], "folderPath": boundary["prefix"] or None, + "isADLSGen2": boundary["is_adls"], "ingestionParameters": ingestion, + }, + }, + } + # Validate the shared control sections and exact address before any authentication. + for section, fields in ((rbac, {"assignments"}), (network, {"posture", "evidence"})): + require_allowed_fields(section, fields, label="planning controls") + url = search_reconcile.resource_url(source) + receipt_paths = _receipt_paths(request) + receipts = None + + def load_receipts() -> tuple[dict[str, Any], dict[str, Any]] | None: + nonlocal receipts + if receipts is None: + receipts = _reuse_receipts(receipt_paths) + return receipts + + deadline = monotonic() + limits["deadline_seconds"] + + def read_search() -> tuple[dict[str, Any] | None, str | None]: + remaining = deadline - monotonic() + if remaining <= 0: + raise _failure("planning-deadline-exceeded", "Planning read deadline elapsed.") + + def bounded(method: str, target: str, token: str, **kwargs: Any) -> Any: + if method != "GET" or target != url: + raise _failure("planning-write-forbidden", "Planning reads only the exact Search source.") + remaining = deadline - monotonic() + if remaining <= 0: + raise _failure("planning-deadline-exceeded", "Planning read deadline elapsed during authentication.") + return transport( + method, target, token, timeout=min(30, remaining), follow_redirects=False, + max_response_bytes=8 * 1024 * 1024, response_deadline=time.monotonic() + remaining, + ) + + current, request_id = search_reconcile.read_resource(url, token_provider(SEARCH_AUDIENCE), transport=bounded) + try: + source_vector.verify_source_readback(embedding, current) + verify_content_understanding_readback(cu, current) + except HelperFailure as failure: + failure.request_id = request_id + raise + return current, request_id + + current, first_id = read_search() + generated = [] + if current is not None: + if not search_reconcile.definitions_match(source["desired"], current): + raise _failure("definition-conflict", "The exact source has a different definition; do not overwrite, suffix or repair it.") + if not isinstance(current.get("@odata.etag"), str) or not current["@odata.etag"]: + raise _failure("definition-evidence-missing", "Exact reuse requires the current source ETag.") + generated = generated_resources(current, strict=True) + _verify_storage_binding(source, boundary, current, generated, load_receipts) + snapshot = blob_inventory.discover( + boundary, limits, token_provider=token_provider, transport=storage_transport, + monotonic=monotonic, deadline=deadline, + ) + refreshed, last_id = read_search() + if (current is None) != (refreshed is None) or current is not None and ( + refreshed.get("@odata.etag") != current["@odata.etag"] + or not search_reconcile.definitions_match(source["desired"], refreshed) + or generated_resources(refreshed, strict=True) != generated + ): + raise _failure("definition-drift", "Source identity/ETag or generated resources changed during Storage observation.") + if refreshed is not None: + _verify_storage_binding(source, boundary, refreshed, generated, load_receipts) + source.update(action="reuse", expected_etag=refreshed["@odata.etag"]) + if monotonic() >= deadline: + raise _failure("planning-deadline-exceeded", "Planning read deadline elapsed during Search readback.") + evidence = {"verified": True, "inventory_digest": snapshot["inventory_digest"]} + if boundary["is_adls"]: + evidence.update(path_verified=True, acl_verified=True) + source["source_evidence"] = evidence + plan = { + "operation": "reconcile-and-monitor", "owner": request["owner"], "cleanup_approved": False, + "boundary": boundary, "inventory_digest": snapshot["inventory_digest"], + "inventory_limits": limits, "poll": poll, "source": source, + } + if embedding is not None: + plan["embedding"] = embedding + if cu is not None: + plan["content_understanding"] = cu + plan["cu_plan_version"] = "1.0" + if current is not None: + plan["expected_generated"] = generated + else: + plan["expected_source_absent"] = True + _validate_plan(plan) + fingerprint = digest(plan) + mutation = source["action"] == "create" + return { + "status": "planned", "outcome": "create-blob-knowledge-source", + "plan_fingerprint": fingerprint, + "execution_input": { + "schema_version": "1.0", "plan": plan, + "approval": {"confirmed": False, "fingerprint": fingerprint}, + }, + "approval_summary": { + "target": {"endpoint": source["endpoint"], "name": source["name"], "api_version": source["api_version"]}, + "storage_account": boundary["storage_id"].rsplit("/", 1)[1], + "source_kind": "adls-gen2" if boundary["is_adls"] else "azure-blob", + "scope": "selected folder/directory" if boundary["prefix"] else "explicit container/filesystem root", + "object_count": len(snapshot["objects"]), "total_bytes": sum(item["size"] for item in snapshot["objects"]), + "adls_paths_observed": len(snapshot["adls_paths"]), + "processing": ( + f"standard Content Understanding extraction {'with selected embeddings' if embedding else 'without source vectors'}; no image verbalization, chat, permission ingestion or schedule" + if cu else + "minimal extraction with embeddings; no image verbalization, permission ingestion or schedule" + if embedding else "minimal lexical; image verbalization disabled; no models, permission ingestion or schedule" + ), + **({"embedding": source_vector.summary(embedding)} if embedding else {}), + **({"content_understanding": { + "purpose": "Standard document extraction only; not source vectorization or KB answer synthesis.", + "endpoint": cu["endpoint"].rstrip("/"), "auth": cu["auth"], + "authentication": cu_ingestion_auth.approval_summary( + "adlsGen2" if request["is_adls"] else "azureBlob", cu["auth"], creating=mutation, + ), + "kb_reasoning": "Unchanged; KB chat requires separate selection, access and approval.", + "cost_and_data": "Creation sends selected documents to billable CU processing (no free document allowance); generated Search content is retained. Cross-region processing may apply. Selected source embeddings have separate costs.", + "source_vectorization": "azureOpenAI" if embedding else "none", + "prerequisites": "Owner-verified references only, not effective access or successful processing proof. No local auth, role, model or defaults changes.", + }} if cu else {}), + "network": "public; supplied access evidence remains owner-verified", + "supplied_role_assignments": len(rbac["assignments"]), + "source_action": source["action"], "execution_required": mutation, + "mutation_approval_required": mutation, + "ownership": "Only a newly created source and its service-generated children become run-owned; existing Search/Storage/objects/roles remain shared.", + "verification": "Fresh identity/inventory and ADLS owner/ACL metadata only; effective Search access, ingestion readiness and retrieval are unverified.", + "cost_and_retention": "Creation starts indexing and retains generated Search resources; existing charges/schedules continue on reuse.", + "cleanup": "Separate run-owned source cleanup only; never delete Storage objects or shared resources.", + "next_step": ( + "Owner refreshes identity, RBAC, network, source state and cost/data consent, then approves these changes before applying the unchanged private artifact." + if mutation else "Reuse without mutation approval or executor invocation. Refresh discovery before later use; evidence is not future consent." + ), + }, + "read_only_evidence": {"request_ids": [item for item in [first_id, *snapshot["request_ids"], last_id] if item]}, + "writes_performed": [], "warnings": [SNAPSHOT_WARNING], + } + + +def _failure(code: str, message: str, *, request_id: str | None = None) -> HelperFailure: + return HelperFailure(code, message, blocked_at="verification", request_id=request_id) + + +def _timestamp(value: Any) -> datetime: + if not isinstance(value, str): + raise _failure("ingestion-status-invalid", "Synchronization timestamp is missing.") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise _failure("ingestion-status-invalid", "Synchronization timestamp is invalid.") from exc + if parsed.tzinfo is None: + raise _failure("ingestion-status-invalid", "Synchronization timestamp needs a time zone.") + return parsed + + +def _poll_limits(value: Any) -> dict[str, int]: + if not isinstance(value, dict): + raise _failure("input-schema-invalid", "poll must be an object.") + require_allowed_fields( + value, {"deadline_seconds", "max_requests", "interval_seconds"}, label="poll" + ) + for field, maximum in ( + ("deadline_seconds", 3600), ("max_requests", 1000), ("interval_seconds", 60) + ): + if type(value.get(field)) is not int or not 1 <= value[field] <= maximum: + raise _failure("input-schema-invalid", "Polling limits must be bounded positive integers.") + return value + + +def _validate_plan(plan: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: + reject_secrets(plan) + require_allowed_fields( + plan, + {"operation", "owner", "cleanup_approved", "source", "boundary", "inventory_digest", + "inventory_limits", "poll", "expected_generated", "expected_source_absent", "embedding", + "content_understanding", "cu_plan_version"}, + label="Blob source plan", + ) + if plan.get("operation") != "reconcile-and-monitor" or plan.get("cleanup_approved") is not False: + raise _failure("operation-invalid", "Blob application requires reconcile-and-monitor without cleanup.") + source = plan.get("source") + if not isinstance(source, dict) or not isinstance(plan.get("owner"), str) or not plan["owner"]: + raise _failure("input-schema-invalid", "A source plan and owner are required.") + if ( + source.get("operation") != "reconcile" + or source.get("resource_type") != "knowledge-source" + or source.get("action") not in ("create", "reuse") + or not isinstance(source.get("api_version"), str) + or source.get("owner") != plan["owner"] + or not isinstance(source.get("desired"), dict) + or source["desired"].get("kind") != "azureBlob" + or source.get("ai_services_api_key_environment") is not None + or source.get("ai_services_key_acquisition") is not None + ): + raise _failure("step-contract-mismatch", "Only same-owner Blob creation or exact reuse is supported.") + search_reconcile._validate_plan(source) + search_reconcile._resource_url(source) + boundary = blob_inventory.validate_boundary(plan.get("boundary")) + blob_inventory.validate_limits(plan.get("inventory_limits")) + _poll_limits(plan.get("poll")) + if "expected_source_absent" in plan and ( + plan["expected_source_absent"] is not True or source["action"] != "create" + or source.get("expected_etag") is not None + ): + raise _failure("step-contract-mismatch", "Expected absence is only valid for a creation plan.") + if "expected_generated" in plan and ( + source["action"] != "reuse" + or not isinstance(plan["expected_generated"], list) + or len(plan["expected_generated"]) != 4 + or any(not isinstance(item, dict) or set(item) != {"type", "name", "service_managed"} + or item["service_managed"] is not True + or not isinstance(item["type"], str) + or not isinstance(item["name"], str) + or re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_-]{0,127}", item["name"]) is None + for item in plan["expected_generated"]) + or {item["type"] for item in plan["expected_generated"]} != {"datasource", "indexer", "skillset", "index"} + ): + raise _failure("generated-resources-unverified", "Expected generated identities require a complete reuse plan.") + parameters = source["desired"]["azureBlobParameters"] + if ( + parameters["connectionString"].removesuffix(";") != f"ResourceId={boundary['storage_id']}" + or parameters["containerName"] != boundary["container"] + or parameters["folderPath"] != (boundary["prefix"] or None) + or parameters["isADLSGen2"] != boundary["is_adls"] + or "createdResources" in parameters + or source["source_evidence"]["inventory_digest"] != plan.get("inventory_digest") + ): + raise _failure("boundary-mismatch", "Source definition must bind the exact discovered boundary and inventory.") + if ( + not isinstance(plan.get("inventory_digest"), str) + or search_reconcile.SHA256.fullmatch(plan["inventory_digest"]) is None + ): + raise _failure("source-evidence-invalid", "The approved inventory digest is required.") + ingestion = parameters.get("ingestionParameters") + if not isinstance(ingestion, dict) or ingestion.get("assetStore") is not None: + raise _failure( + "source-write-forbidden", + "Ingestion parameters are required; asset-store writes are outside this read-only-source workflow.", + ) + source_vector.validate_plan_choice(plan) + standard = ingestion.get("contentExtractionMode") == "standard" + if "cu_plan_version" in plan and (plan["cu_plan_version"] != "1.0" or not standard): + raise _failure("cu-plan-version-invalid", "CU planner artifacts require version 1.0 and standard extraction.") + # Absence/generated guards predate CU planning and remain valid on legacy + # wire-only artifacts. Only CU-specific provenance selects the new contract. + planner_cu = standard and any(field in plan for field in ( + "cu_plan_version", "embedding", "content_understanding", + )) + cu = validate_content_understanding( + plan.get("content_understanding"), enabled=planner_cu, + ) + if cu is not None: + expected = { + "contentExtractionMode": "standard", "disableImageVerbalization": True, + "identity": None, "ingestionSchedule": None, + "aiServices": {"uri": cu["endpoint"].rstrip("/")}, + } + if "embedding" in plan: + expected["embeddingModel"] = source_vector.model_definition(plan["embedding"]) + if source["api_version"].endswith("-preview"): + expected.update(networkAccessMode="public", ingestionPermissionOptions=[]) + if ingestion != expected or boundary["is_adls"] and source["api_version"] != "2026-08-01-preview": + raise _failure("cu-plan-mismatch", "Standard CU requires the unchanged keyless public definition, matching optional embeddings and no chat, asset store, permissions or schedule.") + return source, boundary + + +def _first_error(state: dict[str, Any]) -> dict[str, Any] | None: + errors = state.get("errors") + if errors is None or errors == []: + return None + if not isinstance(errors, list) or any(not isinstance(error, dict) for error in errors): + raise _failure("ingestion-status-invalid", "Ingestion errors have an invalid shape.") + error = errors[0] + # Document errors can contain content, SAS URLs, and credentials in free text. + return { + "status": error.get("statusCode") if type(error.get("statusCode")) is int else None, + "message": "Document-level ingestion error; sensitive service text withheld.", + "diagnostic_digest": digest(error), + } + + +def _synchronization_state(state, request_id=None): + if state is None: + return None + if not isinstance(state, dict): + raise _failure("ingestion-status-invalid", "Synchronization state must be an object.", request_id=request_id) + return dict(state) + + +def _retry_after(value, *, now=None): + now = now or datetime.now(timezone.utc) + kind = getattr(value, "kind", None) + try: + if kind is not None: + if kind == "overlong": + return math.inf + payload = getattr(value, "value", None) + if (not isinstance(kind, str) or kind not in {"seconds", "date", "date-rfc850"} + or type(payload) not in (int, float) + or (type(payload) is float and not math.isfinite(payload))): + return None + if kind == "seconds": + return payload if payload >= 0 and payload == int(payload) else None + parsed = datetime.fromtimestamp(payload, timezone.utc) + obsolete_date = kind == "date-rfc850" + else: + if not isinstance(value, str) or len(value) > 128: + return None + if value.isascii() and value.isdigit() and len(value) <= 10: + return int(value) + parsed = parsedate_to_datetime(value) + obsolete_date = bool(re.fullmatch(r"[A-Za-z]+, \d{2}-[A-Za-z]{3}-\d{2} \d{2}:\d{2}:\d{2} GMT", value)) + if obsolete_date: + year = now.year // 100 * 100 + parsed.year % 100 + if year > now.year + 50: + year -= 100 + parsed = parsed.replace(year=year) + if parsed.tzinfo is not None: + return max(0, (parsed - now).total_seconds()) + except (ValueError, OverflowError, TypeError, OSError): + pass + return None + + +def _execution_progress(body, not_before): + if (not isinstance(body, dict) or not isinstance(body.get("status"), str) + or body["status"] not in {"running", "error", "unknown"}): + raise _failure("indexer-status-invalid", "Indexer status must be an object.") + if body["status"] == "error": + raise _failure("indexer-status-error", "Indexer availability reports an error; execution success is not inferred.") + run = body.get("lastResult") + if run is None: + return None + if not isinstance(run, dict): + raise _failure("indexer-status-invalid", "Latest indexer execution must be an object.") + start = _timestamp(run.get("startTime")) + if start < not_before: + return None + status = run.get("status") + if not isinstance(status, str) or status not in {"inProgress", "success", "transientFailure", "persistentFailure", "reset"}: + raise _failure("indexer-status-invalid", "Unknown latest execution status; top-level running is not execution proof.") + result = {"startTime": run["startTime"], "run_status": status} + if run.get("endTime") is not None: + if _timestamp(run["endTime"]) < start: + raise _failure("indexer-status-invalid", "Indexer execution interval is invalid.") + result["endTime"] = run["endTime"] + for field, target in (("itemsProcessed", "items_attempted"), ("itemsFailed", "items_failed")): + if field in run: + if type(run[field]) is not int or run[field] < 0: + raise _failure("indexer-status-invalid", "Indexer execution counters are invalid.") + result[target] = run[field] + if result.get("items_failed", 0) > result.get("items_attempted", result.get("items_failed", 0)): + raise _failure("indexer-status-invalid", "Failed item count exceeds attempted items.") + errors = run.get("errors", []) + if errors is not None and (not isinstance(errors, list) or any(not isinstance(item, dict) for item in errors)): + raise _failure("indexer-status-invalid", "Indexer execution errors are malformed.") + result["error_count"] = len(errors or []) + return result + + +def _execution_completed_by(execution, end, not_before): + required = {"startTime", "endTime", "run_status", "error_count"} + if (not isinstance(execution, dict) or not required <= execution.keys() + or execution.keys() - required - {"items_attempted", "items_failed"} + or execution["run_status"] != "success" + or type(execution["error_count"]) is not int or execution["error_count"] != 0 + or any(type(execution[key]) is not int or execution[key] < 0 + for key in ("items_attempted", "items_failed") if key in execution) + or execution.get("items_failed", 0)): + return False + try: + return not_before <= _timestamp(execution["startTime"]) <= _timestamp(execution["endTime"]) <= end + except HelperFailure: + return False + + +@reporting("blob-monitor") +def monitor( + source: dict[str, Any], + *, + not_before: datetime, + limits: dict[str, int], + token_provider: TokenProvider, + transport: Transport, + require_new_cycle: bool = False, + excluded_cycle: list[str] | None = None, + monotonic: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, + progress: Progress | None = None, + cancelled: Callable[[], bool] = lambda: False, + indexer_name: str | None = None, +) -> dict[str, Any]: + _poll_limits(limits) + progress.update("ingestion-cycle") + excluded = tuple(_timestamp(value) for value in excluded_cycle) if excluded_cycle is not None else None + url = ( + f"{source['endpoint'].rstrip('/')}/knowledgesources('{search_reconcile._odata_name(source['name'])}')/status?" + + urlencode({"api-version": source["api_version"]}) + ) + token = token_provider(SEARCH_AUDIENCE) + started = monotonic() + deadline = started + limits["deadline_seconds"] + request_ids: list[str] = [] + first_error: dict[str, Any] | None = None + first_retry: dict[str, Any] | None = None + observed = "no-completed-cycle" + initial_cycle: tuple[Any, Any] | None = None + first_response = True + checks = 0 + delay = limits["interval_seconds"] + latest = {} + previous = None + previous_execution = None + execution_guard = None + pause_reason = "request-limit" + newest_run = None + synchronization_status = "not-reported" + + def report(phase, next_check=None): + value = { + "schema_version": "1.0", "phase": phase, + "run_start": _timestamp(latest["startTime"]).isoformat() if latest.get("startTime") else None, + "processed": latest.get("itemsUpdatesProcessed"), "failed": latest.get("itemsUpdatesFailed"), + "skipped": latest.get("itemsSkipped"), "unit": "item-updates", + "total": None, "remaining": None, "denominator": "not-comparable-to-files", + "synchronization_status": synchronization_status, + "elapsed_seconds": max(0, monotonic() - started), "next_check_seconds": next_check, + } + progress.counts["status_checks"] = checks + progress.blob_update(value) + return value + + def failure_result(*args): + result = _readiness_failure(*args) + result["progress"] = report("failed") + result["watch"] = {"schema_version": "1.0", "state": "blocked", "reason": "evidence-failure", + "elapsed_seconds": max(0, monotonic() - started), + "status_checks": checks, "latest": copy.deepcopy(latest)} + return result + + def read_status(target): + nonlocal checks, retry_after + remaining = deadline - monotonic() + if remaining <= 0: + raise _failure("ingestion-watch-expired", "The client watch window elapsed before this read.") + checks += 1 + response = transport( + "GET", target, token, timeout=min(30, remaining), + max_response_bytes=8 * 1024 * 1024, follow_redirects=False, + response_deadline=monotonic() + min(30, remaining), + ) + if response.request_id: + request_ids.append(response.request_id) + if response.status != 200: + value = getattr(response, "retry_after", None) + if value is None: + value = next((v for k, v in response.headers.items() if k.lower() == "retry-after"), None) + retry_after = _retry_after(value) + raise HelperFailure("ingestion-inaccessible", "Status read is inaccessible.", + blocked_at="verification", status=response.status, request_id=response.request_id) + return response + + while checks < limits["max_requests"]: + remaining = deadline - monotonic() + if cancelled(): + pause_reason = "cancelled" + break + if remaining <= 0: + pause_reason = "deadline" + break + retry_after = None + throttled = False + try: + response = read_status(url) + except KeyboardInterrupt: + pause_reason = "cancelled" + break + except HelperFailure as failure: + if failure.request_id and failure.request_id not in request_ids: + request_ids.append(failure.request_id) + if failure.code == "response-deadline-exceeded": + first_retry = first_retry or {"code": failure.code, "status": failure.http_status, + "request_id": failure.request_id} + if monotonic() >= deadline: + pause_reason = "deadline" + break + if failure.code == "ingestion-watch-expired": + pause_reason = "deadline" + break + if retry_after is None: + retry_after = _retry_after(failure.retry_after) + if (failure.http_status not in RETRY_STATUS + and failure.code not in {"azure-response-ambiguous", "response-deadline-exceeded"}): + return failure_result("ingestion-inaccessible", failure.request_id, request_ids, + first_error, first_retry, failure.http_status) + if first_retry is None: + first_retry = {"code": failure.code, "status": failure.http_status, + "request_id": failure.request_id} + throttled = failure.http_status == 429 + delay = min(60, max(limits["interval_seconds"], delay * 2)) + else: + if monotonic() >= deadline: + pause_reason = "deadline" + break + body = response.body + if not isinstance(body, dict) or body.get("kind") != "azureBlob": + raise _failure("ingestion-status-invalid", "Expected azureBlob status.", request_id=response.request_id) + synchronization_status = body.get("synchronizationStatus", "not-reported") + if synchronization_status not in ("not-reported", "active", "creating", "deleting"): + raise _failure("ingestion-status-invalid", "Unknown knowledge-source synchronization availability.", + request_id=response.request_id) + current = _synchronization_state(body.get("currentSynchronizationState"), response.request_id) + last = _synchronization_state(body.get("lastSynchronizationState"), response.request_id) + for state in (current, last): + if state is not None: + _timestamp(state.get("startTime")) + _first_error(state) + state = current or last + start = _timestamp(state["startTime"]) if state else None + if start is not None and start >= not_before and (newest_run is None or start >= newest_run): + newest_run = start + error = _first_error(state) + if first_error is None and error: + first_error = {**error, "request_id": response.request_id} + latest = {"startTime": state["startTime"], + "state": "in-progress" if current else "completed-cycle-observed"} + for field in ("itemsUpdatesProcessed", "itemsUpdatesFailed", "itemsSkipped"): + if field in state: + if type(state[field]) is not int or state[field] < 0: + raise _failure("ingestion-status-invalid", "Observed synchronization counters are invalid.", + request_id=response.request_id) + latest[field] = state[field] + if current: + observed = "in-progress" + signature = digest(latest) + delay = (limits["interval_seconds"] if signature != previous + else min(60, delay * 2)) + previous = signature + else: + latest = {} + observed = "stale-success-or-unrelated-cycle" if state else "no-completed-cycle" + signature = digest({}) + delay = limits["interval_seconds"] if signature != previous else min(60, delay * 2) + previous = signature + if execution_guard is not None: + latest["indexer_execution"] = copy.deepcopy(execution_guard) + if first_response and require_new_cycle and last: + initial_cycle = (last.get("startTime"), last.get("endTime")) + first_response = False + if ( + last and last.get("endTime") is not None + and _timestamp(last.get("startTime")) >= max(not_before, newest_run or not_before) + and (last.get("startTime"), last.get("endTime")) != initial_cycle + and (excluded is None or (_timestamp(last.get("startTime")), _timestamp(last.get("endTime"))) != excluded) + ): + status = last.get("status") + if status is not None and not isinstance(status, str): + raise _failure("ingestion-status-invalid", "Synchronization status must be a string when present.") + if status in {"failure", "partialSuccess"}: + return failure_result(f"ingestion-{status}", response.request_id, + request_ids, first_error, first_retry) + if status in {None, "success"}: + start = _timestamp(last.get("startTime")) + end = _timestamp(last.get("endTime")) + failed = last.get("itemsUpdatesFailed") + processed = last.get("itemsUpdatesProcessed") + skipped = last.get("itemsSkipped") + if end < start or any(type(count) is not int or count < 0 for count in (failed, processed, skipped)): + raise _failure("ingestion-status-invalid", "Completed synchronization counters or interval are invalid.") + if failed or first_error: + return failure_result("ingestion-partialSuccess", response.request_id, + request_ids, first_error, first_retry) + if (not current and synchronization_status in ("active", "not-reported") + and (execution_guard is None + or _execution_completed_by(execution_guard, end, not_before))): + return { + "status": "verified", + "progress": report("completed"), + "synchronization": { + field: last[field] for field in + ("startTime", "endTime", "itemsUpdatesProcessed", "itemsUpdatesFailed", "itemsSkipped") + }, + "not_before": not_before.isoformat(), + "request_ids": request_ids, + "first_retry": first_retry, + "watch": {"schema_version": "1.0", "state": "completed", + "elapsed_seconds": max(0, monotonic() - started), + "status_checks": checks, "latest": latest}, + } + else: + raise _failure("ingestion-status-invalid", "Unknown synchronization completion status.") + elif last and not current: + observed = "stale-success-or-unrelated-cycle" + if indexer_name and checks < limits["max_requests"] and monotonic() < deadline and not cancelled(): + target = (f"{source['endpoint'].rstrip('/')}/indexers('{search_reconcile._odata_name(indexer_name)}')/status?" + + urlencode({"api-version": source["api_version"]})) + run_response = None + try: + run_response = read_status(target) + cutoff = max(not_before, newest_run or not_before) + if execution_guard is not None: + cutoff = min(cutoff, _timestamp(execution_guard["startTime"])) + execution = _execution_progress(run_response.body, cutoff) + if (execution is not None and execution_guard is not None + and _timestamp(execution["startTime"]) < _timestamp(execution_guard["startTime"])): + execution = None + if execution is not None: + if execution_guard is not None or execution["run_status"] == "inProgress": + execution_guard = copy.deepcopy(execution) + execution_signature = digest(execution) + if execution_signature != previous_execution: + delay = limits["interval_seconds"] + previous_execution = execution_signature + latest["indexer_execution"] = execution + if (execution.get("items_failed", 0) or execution["error_count"] + or execution["run_status"] in {"transientFailure", "persistentFailure"}): + first_error = first_error or { + "status": None, "message": "Latest relevant indexer execution reports failures; details withheld.", + "request_id": run_response.request_id, + } + except KeyboardInterrupt: + pause_reason = "cancelled" + break + except HelperFailure as failure: + if failure.request_id is None and run_response is not None: + failure.request_id = run_response.request_id + if failure.request_id and failure.request_id not in request_ids: + request_ids.append(failure.request_id) + if failure.code == "response-deadline-exceeded": + first_retry = first_retry or {"code": failure.code, "status": failure.http_status, + "request_id": failure.request_id} + if monotonic() >= deadline: + pause_reason = "deadline" + break + if failure.code == "ingestion-watch-expired": + pause_reason = "deadline" + break + if (failure.http_status not in RETRY_STATUS + and failure.code not in {"azure-response-ambiguous", "response-deadline-exceeded"}): + code = failure.code if failure.code in {"indexer-status-invalid", "indexer-status-error"} else "indexer-status-unverified" + return failure_result(code, failure.request_id, request_ids, + first_error, first_retry, failure.http_status) + first_retry = first_retry or {"code": failure.code, "status": failure.http_status, + "request_id": failure.request_id} + throttled = failure.http_status == 429 + if retry_after is None: + retry_after = _retry_after(failure.retry_after) + delay = min(60, delay * 2) + remaining = deadline - monotonic() + if cancelled(): + pause_reason = "cancelled" + break + if retry_after is not None and retry_after > min(60, remaining): + report("throttled" if throttled else "waiting") + pause_reason = "retry-after" + break + if remaining > 0 and checks < limits["max_requests"]: + try: + wait = min(max(delay, retry_after or 0), remaining) + report("throttled" if throttled else + "ingesting" if observed == "in-progress" else "waiting", wait) + sleep(wait) + except KeyboardInterrupt: + pause_reason = "cancelled" + break + if monotonic() >= deadline: + pause_reason = "deadline" + result = _readiness_failure( + "ingestion-timeout", request_ids[-1] if request_ids else None, + request_ids, first_error, first_retry, + ) + result["observed"] = observed + result["watch"] = {"schema_version": "1.0", "state": "paused", "reason": pause_reason, + "elapsed_seconds": max(0, monotonic() - started), + "status_checks": checks, "latest": latest, + "service_execution": "not-modified"} + if first_error or latest.get("itemsUpdatesFailed"): + result["code"] = "ingestion-partialSuccess" + result["progress"] = report("failed" if result["code"] == "ingestion-partialSuccess" else "paused") + result["safe_next_decision"] = ( + "Service reports item failures; retain the partial work and inspect exact private error evidence read-only. " + "A new watch does not repair failures; do not replay writes or default to cleanup." + ) if result["code"] == "ingestion-partialSuccess" else ( + "Client watch paused; this is not an ingestion failure. Resume GET-only with the unchanged input " + "and readiness receipt using blob_recheck.py --input --receipt . " + "Do not recreate, run/reset, delete, or infer corpus completion." + ) + return result + + +def _readiness_failure( + code: str, request_id: str | None, request_ids: list[str], + first_error: dict[str, Any] | None, first_retry: dict[str, Any] | None, + status: int | None = None, +) -> dict[str, Any]: + return { + "status": "unverified", "code": code, "request_id": request_id, + "http_status": status, "request_ids": request_ids, + "first_error": first_error, "first_retry": first_retry, + } + + +@reporting("blob-source") +def execute( + document: dict[str, Any], + *, + token_provider: TokenProvider = azure_cli_token, + transport: Transport = http_request, + storage_transport: Transport = http_request, + now: Callable[[], datetime] = lambda: datetime.now(timezone.utc), + monotonic: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, + checkpoint: Any = None, + progress: Progress | None = None, +) -> dict[str, Any]: + progress.update("validation") + reject_secrets(document) + require_allowed_fields(document, {"schema_version", "plan", "approval", "_computed_fingerprint"}, + label="input envelope") + plan = document.get("plan") + approval = document.get("approval") + if document.get("schema_version") != "1.0" or not isinstance(plan, dict) or not isinstance(approval, dict): + raise _failure("input-schema-invalid", "A typed user-approved envelope is required.") + require_allowed_fields(approval, {"confirmed", "fingerprint"}, label="approval") + fingerprint = digest(plan) + if approval.get("confirmed") is not True or approval.get("fingerprint") != fingerprint: + raise _failure("approval-mismatch", "The exact recomputed plan fingerprint must be approved.") + source, boundary = _validate_plan(plan) + progress.update("blob-inventory") + before = blob_inventory.discover( + boundary, plan["inventory_limits"], token_provider=token_provider, transport=storage_transport, + ) + if before["inventory_digest"] != plan["inventory_digest"]: + raise _failure("source-drift", "Current source evidence differs from approval; reconfirmation is required.") + not_before = now() + generated: list[dict[str, Any]] = [] + write_generated: list[dict[str, Any]] = [] + initial_read = True + creation_etag = None + embedding_transport = source_vector.guard_readback_transport(plan, transport) + + def reconcile_transport(method: str, url: str, token: str, **kwargs: Any) -> Any: + nonlocal initial_read + first_read = method == "GET" and initial_read + if first_read: + initial_read = False + kwargs.update(max_response_bytes=8 * 1024 * 1024, follow_redirects=False) + try: + response = embedding_transport(method, url, token, **kwargs) + except HelperFailure as failure: + if ( + first_read and failure.http_status == 404 and source["action"] == "create" + and source.get("expected_etag") is not None + ): + raise _failure( + "definition-drift", "The source bound by the approved ETag is absent; rebuild the plan.", + request_id=failure.request_id, + ) from failure + raise + if method == "GET" and response.status == 200 and isinstance(response.body, dict): + etag = search_reconcile.resolve_etag(search_reconcile.response_etags(response), response.request_id) + if etag is not None: + response = HttpResult(response.status, {**response.body, "@odata.etag": etag}, + response.headers, response.etag_values) + if creation_etag is not None and etag != creation_etag: + raise _failure("definition-drift", "Source version differs from the acknowledged create response.", + request_id=response.request_id) + if isinstance(response.body, dict): + if method == "GET" and response.status == 200: + parameters = response.body.get("azureBlobParameters") + if isinstance(parameters, dict): + try: + _connection_binding(parameters.get("connectionString"), boundary) + except HelperFailure as failure: + failure.request_id = response.request_id + raise + if first_read and response.status == 200: + if plan.get("expected_source_absent"): + raise _failure("definition-drift", "A source appeared after planning; rebuild the plan.", request_id=response.request_id) + if source.get("expected_etag") is not None and response.body.get("@odata.etag") != source["expected_etag"]: + raise _failure("definition-drift", "Source ETag differs from the approved readback.", request_id=response.request_id) + if method == "GET" and response.status == 200 and ( + plan.get("expected_source_absent") or "expected_generated" in plan + ): + if not isinstance(response.body.get("@odata.etag"), str) or not response.body["@odata.etag"]: + raise _failure("definition-evidence-missing", "Source readback must include its ETag.", request_id=response.request_id) + parameters = response.body.get("azureBlobParameters") + connection = parameters.get("connectionString") if isinstance(parameters, dict) else None + if isinstance(connection, str) and connection.startswith("ResourceId="): + if connection.removesuffix(";") != f"ResourceId={boundary['storage_id']}": + raise _failure("boundary-mismatch", "Readback targets a different Storage account.", request_id=response.request_id) + elif plan.get("expected_source_absent") and ( + not creation_etag or response.body.get("@odata.etag") != creation_etag + ): + raise _failure("source-binding-unverified", "Redacted creation readback lacks a matching acknowledged PUT ETag.", request_id=response.request_id) + generated.clear() + generated.extend(generated_resources(response.body)) + if method == "GET" and "expected_generated" in plan and generated != plan["expected_generated"]: + raise _failure("definition-drift", "Generated identities differ from the approved reuse plan.", request_id=response.request_id) + if method == "PUT" and response.status in {200, 201}: + write_generated[:] = generated + return response + + def created(*, response, url, body, headers): + nonlocal creation_etag + if (source["action"] != "create" or url != search_reconcile.resource_url(source) + or headers.get("If-None-Match") != "*" or "If-Match" in headers + or body != canonical_bytes(source["desired"])): + raise _failure("recheck-ownership-unproven", "Creation acknowledgement must bind the exact approved conditional wire.") + if checkpoint is not None: + checkpoint.acknowledge(plan, response, not_before, url=url, body=body, headers=headers) + creation_etag = search_reconcile.resolve_etag(search_reconcile.response_etags(response), response.request_id) + if creation_etag is None: + raise _failure("creation-version-unproven", "Successful create returned no ETag in body or HTTP headers; a later GET cannot prove its version.", + request_id=response.request_id) + + progress.update("source-reconciliation") + try: + result = search_reconcile.execute( + {"plan": source, "_computed_fingerprint": fingerprint}, + token_provider=token_provider, transport=reconcile_transport, on_created=created, + ) + except HelperFailure as failure: + if failure.writes: + failure.resources_remaining.extend(write_generated or generated) + elif failure.partial: + failure.resources_reused.extend(generated) + if checkpoint is not None and failure.writes: + result = blocked_result(failure, outcome="create-blob-knowledge-source", + fingerprint=fingerprint, owner=plan["owner"]) + result.update(readiness={"status": "unverified"}, knowledge_base="not-verified", retrieval="unverified") + return _checkpoint_result(result, checkpoint) + raise + writes = [ + {"action": "created", "type": item["type"], "name": item["name"]} + for item in result["resources"]["created"] + ] + readiness: dict[str, Any] = {"status": "unverified"} + after = None + owned_generated = list(generated or write_generated) if writes else [] + try: + expected_generated = list(generated) + if checkpoint is not None: + if source["action"] == "create" and ( + not creation_etag or creation_etag != result["verification"]["readback"]["etag"] + or write_generated and generated != write_generated + ): + raise _failure("recheck-ownership-unproven", "Checkpoint requires an acknowledged PUT ETag and unchanged generated identities.") + checkpoint.persist(plan, result, generated, not_before, + token_provider=token_provider, transport=transport) + readiness = monitor( + source, not_before=not_before, limits=plan["poll"], token_provider=token_provider, + transport=transport, require_new_cycle=not bool(writes) and checkpoint is None, + excluded_cycle=checkpoint.excluded_cycle if checkpoint is not None else None, + monotonic=monotonic, sleep=sleep, + progress=progress, + indexer_name=next((item["name"] for item in generated if item["type"] == "indexer"), None), + ) + if readiness["status"] != "verified": + raise HelperFailure( + readiness["code"], "Source reconciliation completed but ingestion readiness is unverified.", + blocked_at="verification", request_id=readiness["request_id"], + status=readiness["http_status"], + ) + progress.update("blob-readback") + after = blob_inventory.discover( + boundary, plan["inventory_limits"], token_provider=token_provider, transport=storage_transport, + ) + if after["inventory_digest"] != before["inventory_digest"]: + raise _failure("source-drift", "Source changed during ingestion; reconfirmation is required.") + progress.update("source-readback") + current, request_id = search_reconcile._get( + search_reconcile._resource_url(source), token_provider(SEARCH_AUDIENCE), transport=reconcile_transport, + ) + if ( + current is None + or search_reconcile._definition(current) != search_reconcile._definition(source["desired"]) + or current.get("@odata.etag") != result["verification"]["readback"]["etag"] + or generated != expected_generated + ): + raise _failure("definition-drift", "Source definition or ETag changed while monitoring.", request_id=request_id) + if request_id: + result["verification"]["request_ids"].append(request_id) + if len(generated) != 4 or {item["type"] for item in generated} != {"datasource", "indexer", "skillset", "index"}: + raise _failure("generated-resources-unverified", "Exact service-generated resource identities could not be read back.") + if checkpoint is not None: + checkpoint.verify(plan, token_provider=token_provider, transport=transport) + except HelperFailure as failure: + confirmed = copy.deepcopy(result) + result = blocked_result( + HelperFailure( + failure.code, failure.message, blocked_at=failure.blocked_at, writes=writes, + resources_remaining=result["ownership"]["run_owned"] + owned_generated, + request_id=failure.request_id, status=failure.http_status, partial=bool(writes), + ), + outcome="create-blob-knowledge-source", fingerprint=fingerprint, owner=plan["owner"], + ) + result["reconciliation"] = "completed" + result["writes_performed"] = writes + result["read_only_evidence"] = { + "inventory_request_ids": before["request_ids"] + (after["request_ids"] if after else []), + "configuration_request_ids": checkpoint.request_ids if checkpoint is not None else [], + } + result["confirmed_reconciliation"] = { + key: confirmed[key] for key in ("resources", "verification", "ownership") + } + result["readiness"] = {**readiness, "status": "unverified"} + if readiness.get("watch", {}).get("state") == "paused": + result["safe_next_decision"] = readiness["safe_next_decision"] + if not writes: + result["ownership"]["reused_not_owned"] = [ + {"type": "knowledge-source", "name": source["name"]}, *generated + ] + else: + result["outcome"] = "create-blob-knowledge-source" + result["readiness"] = readiness + result["verification"]["source_digest"] = after["inventory_digest"] + result["source"] = { + "type": "knowledge-source", "name": source["name"], + "generated": owned_generated if writes else generated, + "observed_generated": generated, + } + result["writes_performed"] = writes + result["source_evidence"] = { + "inventory_digest": before["inventory_digest"], "boundary": boundary, + "operator_reachability": "verified", "managed_ingestion_reachability": result["readiness"]["status"], + } + result.setdefault("warnings", []).append(SNAPSHOT_WARNING) + return _checkpoint_result(result, checkpoint) + + +def _checkpoint_result(result, checkpoint): + if "completed_writes" in result: + result["writes_performed"] = copy.deepcopy(result["completed_writes"]) + if checkpoint is not None: + result["warnings"].extend(checkpoint.recovery_warnings) + result["creation_acknowledgement"] = checkpoint.write_observation or { + "schema_version": "1.0", "state": "unavailable", + } + result["indexer_diagnostics"] = [ + item for item in checkpoint.diagnostics + if not item["field"].startswith(("datasource.", "indexer.", "skillset.", "index.")) + ] + result["generated_diagnostics"] = checkpoint.diagnostics + result["indexer_observations"] = checkpoint.observations + result["datasource_binding_observations"] = checkpoint.binding_observations + for item in checkpoint.diagnostics: + if item["severity"] == "warning" and item["message"] not in result.setdefault("warnings", []): + result["warnings"].append(item["message"]) + result["recheck_checkpoint"] = checkpoint.summary or {"status": "unavailable"} + result["recheck_write_acknowledgement"] = checkpoint.write_acknowledgement or {"status": "unavailable"} + result["recheck_acknowledgement"] = checkpoint.acknowledgement or checkpoint.write_acknowledgement or {"status": "unavailable"} + if checkpoint.summary is None: + result["safe_next_decision"] = "Preserve the first failure and resources. Use blob_recheck.py --recover with unchanged approved input and the retained acknowledgement, then --input/--receipt. Missing/conflicting write ETags or absent private evidence remain blockers; never borrow a GET version, replay creation or default to cleanup." + try: + checkpoint.finish(result) + except HelperFailure as failure: + if result["status"] == "completed": + original = result + result = blocked_result( + HelperFailure(failure.code, failure.message, blocked_at="evidence-retention", + writes=original["writes_performed"], + resources_remaining=original["ownership"]["run_owned"], + partial=bool(original["writes_performed"])), + outcome=original["outcome"], fingerprint=checkpoint.plan_digest, + ) + result["confirmed_reconciliation"] = original + result["writes_performed"] = original["writes_performed"] + result["result_evidence"] = {"status": "unavailable", "code": failure.code} + result.setdefault("warnings", []).append("Final private evidence retention failed; preserve the native result and first failure. No retry was performed.") + return result + + +def main(argv: list[str] | None = None) -> int: + try: + from .private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode + except ImportError: + from private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode + parser = argparse.ArgumentParser() + modes = parser.add_mutually_exclusive_group(required=True) + modes.add_argument("--discover", type=Path) + modes.add_argument("--plan", type=Path) + add_execution_output_argument(parser) + modes.add_argument("--input", type=Path) + parser.add_argument("--receipt-dir", type=Path) + parser.add_argument("--compact", action="store_true") + add_progress_argument(parser) + args = parser.parse_args(argv) + if args.compact and (not args.input or not args.receipt_dir): + parser.error("--compact requires --input and --receipt-dir to retain full private evidence.") + fingerprint = None + owner = None + progress = Progress("blob-source", enabled=args.progress) if args.input else None + execution_started = False + try: + validate_execution_output_mode(args) + if progress is not None: + progress.update("validation") + if args.receipt_dir and not args.input: + raise _failure("input-schema-invalid", "--receipt-dir requires --input; read-only reuse capture uses blob_recheck.py.") + if args.plan: + result = plan_source(_read_intent(args.plan)) + emit_plan_result(result, args.execution_output, preserve_unapproved_input=result["status"] == "planned") + return 0 if result["status"] == "planned" else 2 + elif args.discover: + try: + document = json.loads(args.discover.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise _failure("input-unreadable", "Discovery input must be readable UTF-8 JSON.") from exc + if not isinstance(document, dict): + raise _failure("input-schema-invalid", "Discovery input must be an object.") + reject_secrets(document) + require_allowed_fields(document, {"boundary", "inventory_limits"}, label="discovery input") + result = blob_inventory.discover(document.get("boundary"), document.get("inventory_limits")) + else: + document, plan, fingerprint = load_approved_input(args.input) + owner = plan.get("owner") + checkpoint = None + if args.receipt_dir: + try: + from . import blob_recheck + except ImportError: + import blob_recheck + checkpoint = blob_recheck.Checkpoint(args.receipt_dir, plan) + execution_started = True + if checkpoint is not None: + result = execute(document, checkpoint=checkpoint, progress=progress) + else: + result = execute(document, progress=progress) + except HelperFailure as failure: + if progress is not None and not execution_started: + progress.finish(failure=failure) + result = blocked_result(failure, outcome="blob-source-lifecycle", fingerprint=fingerprint, owner=owner) + if failure.partial and not failure.writes: + result["safe_next_decision"] = "Original create outcome is unproven. Preserve the mutation error and inspect observed resources read-only; no creation ownership, write replay or cleanup is authorized." + if args.compact: + try: + try: + from . import blob_recheck + except ImportError: + import blob_recheck + result = blob_recheck.compact_result(result, args.receipt_dir) + except HelperFailure as failure: + result.setdefault("warnings", []).append("compact-evidence-persistence-failed: full native result retained in output.") + result["presentation_failure"] = {"code": failure.code} + emit_result(result) + return 3 if result["status"] == "partial" else 2 + emit_result(result, preserve_unapproved_input=result["status"] == "planned") + return {"completed": 0, "discovered": 0, "planned": 0, "blocked": 2, "partial": 3}[result["status"]] + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/bootstrap-contracts.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/bootstrap-contracts.md index c4ca6041..0cc18d49 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/bootstrap-contracts.md +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/bootstrap-contracts.md @@ -1,150 +1,150 @@ -# Search bootstrap - -**New Basic/standard Search in an existing group**. -Other prerequisites/cleanup: [native bootstrap](../references/bootstrap-azure.md). - -Progress: stderr; `--no-progress` disables. - -## Choices - -Python 3.12+, signed-in CLI on Windows/POSIX; no login/install/context switch. -No preflight policy reads. -Prerequisites: **owner-verified references**, not helper-verified. -1–256 printable characters; punctuation allowed. -Errors omit values. - -Creation: `search`/`create`. Existing operations: [intake](search-intake-contracts.md); -legacy `reuse` stays strict, never hardens. -New locations fold ASCII case/whitespace (`East US` → `eastus`); -Not availability proof; bodies/hashes stay unchanged. -Basic: 1–3 replicas/one partition; standard: 1–12 replicas/1/2/3/4/6/12 partitions. -Fixed `SystemAssigned`, keyless auth and public networking; no role changes. -Non-secret tags; creation adds operation/owner tags. - -```json -{"schema_version":"2.0","resource_kind":"search","action":"create", -"subscription_id":"00000000-0000-0000-0000-000000000001","tenant_id":"00000000-0000-0000-0000-000000000002", -"resource_group":"rg1","name":"mysearchservice","location":"westus","sku":"basic","replicas":1,"partitions":1, -"public_network_access":"Enabled","tags":{"project":"knowledge"},"owner":"operator", -"prerequisites":{"quota":"checked; enough","pricing":"approved","network":"approved public","requirements":"settings","exclusive_name_authority":"exclusive name"}, -"limits":{"command_seconds":5,"wait_seconds":10,"wait_interval_seconds":1},"receipt_dir":""} -``` - -Replace examples/limits. Reuse: `exclusive_name_authority: null`; -required tags match; no ownership transfer. - -Region scope (closed): - -```json -{"schema_version":"2.0","subscription_id":"","tenant_id":"","receipt_dir":"","limits":{"command_seconds":5}} -``` - -No RG/name/SKU: account check plus one `az rest` GET: -`/subscriptions//providers/Microsoft.Search?api-version=2021-04-01`. -`searchServices.locations` supplies canonical `available_locations` (1–256). -Exit 0: `discovered`, no approval/artifact. -Create plan/apply: one fresh region GET each, including retained artifacts; no cache. -Unknown: `bootstrap-region-unsupported` plus choices; -denied/malformed: blocked. Reuse skips catalogs. Not SKU/quota/capacity/model/residency proof. - -```text -python "/helpers/bootstrap_azure.py" --regions scope.json -python "/helpers/bootstrap_azure.py" --plan choices.json -python "/helpers/bootstrap_azure.py" --apply "/.plan.json" --approve -``` - -`--plan` uses Azure reads/**local private writes**: `.plan.json` -body, before-state, choices, time, unapproved envelope. -Stdout: `artifact_id`/`approval_summary`, no paths/errors/hashes. -Labels: caller attestation, not helper verification; exact references stay private. -Show resource/network/capacity/tags/cost/limits; `--approve` consents once. -No cached consent or user checksums. - -15-minute expiry; submission markers prevent retries. -On drift/failure/expiry/changed choices, rerun `--plan`; discard consent. -Reconcile uncertainty; no suffix-create/body/hash edits. -Schema 1.0: regenerate (`bootstrap-contract-version`). Removed inputs -`max_polls`/`poll_seconds`/`poll_interval_seconds` are rejected. - -## Execution - -Argv: `az account show --subscription`, `az group show --name --subscription`, -Search ARM GET/PUT API `2025-05-01`. PUT uses `--body` then **one** -`"@" + str(body_path)` argument, Content-Type/client-request-ID headers. -POSIX runs `az` directly; Windows MSI/ZIP uses bundled `python.exe -IBm azure.cli`. -Other Windows layouts block. - -Only normally returned target `ResourceNotFound` proves absence. -Fresh principal/tenant/subscription/group/exact-name absence precede one PUT. -GET/PUT is **not atomic**: require exclusive name authority. -Mandatory atomicity: `bootstrap-concurrency-unresolved`, no invented conditionals. - -Reuse: two fresh material readbacks, required configuration/tags, -keyless identity/principal/tenant, endpoint, provisioning succeeded/status running. -`reused`: no mutation approval/executor or ownership adoption. -Wait uses the same identity; existence is not readiness. - -After PUT: `az resource wait --ids --api-version 2025-05-01 ---custom --interval --timeout `. -Fixed `properties`: succeeded/running or terminal states, lower/title/uppercase. -No caller expressions/custom polling. -Final raw GET establishes material match/ownership **before** readiness. -Wait failure remains primary even with ready GET; provider evidence is secondary. - -Limits: command 1–60 seconds, wait 1–900, interval 1–30. -Wait/final GET: separate `wait_seconds`/`command_seconds`; -sum is not an end-to-end deadline. -`subprocess.run`: argv, `shell=False`, best-effort timeout/cleanup. -CLI/OS/filesystems prevent hard time/process-tree/request-count guarantees. -No PID or confirmed-cleanup claim. Captured output is rejected **after capture** -above 1 MiB per stream: not a streaming cap or memory bound. Input/body/receipt -limits remain 1 MiB. Timeout/execution/receipt failure after attempted PUT is partial. - -Checks: target, principal/tenant, capacity/SKU, auth/network, identity, -required tags, ARM readiness. Metadata/ordering/unrelated tags, ID/enum casing, -equivalent optional defaults do not need another approval. -Never default missing material fields or return extra tags. -Material conflict, unknown ownership or failed readiness blocks/returns partial. - -## Evidence and failure - -Existing absolute private directory **outside the plugin**. -No symlinks/reparse paths. POSIX: directory-FD creation, identity/owner/private-mode -rechecks; exclusive safe leaves, mode 0600. -POSIX semantics required; later pathname uses are not pinned. -Windows checks owner/DACL (owner, SYSTEM, Administrators), not handle-relative -creation; `chmod` does not establish Windows privacy. No ACL changes. -Same-user/admin tampering is possible; unprovable privacy blocks. -Mocked/DrvFS is not native-filesystem or Azure evidence. - -Flushed receipts: operation/target/body/argv, client-request ID, readbacks/digests, -first error code/status/request ID, ownership. No raw errors/output/policy values/tokens/documents. -Message digest/`message_withheld`; secret-free labels/tags. -CLI file logging/dynamic installation: process-disabled. - -Creation-only policy diagnostics: -`RequestDisallowedByPolicy`, supported policy IDs or policy-violation details. -Provider error/statusDetails, never tags/configuration. -A bare 403/RBAC/network/quota failure triggers no policy calls. -Preserve first failure; diagnostic/receipt errors are secondary warnings. -At most eight referenced assignments/selected-subscription definitions -within 60 seconds: `az policy assignment show --name --scope` -and `az policy definition show --name --subscription`. No lists or global scans. -Built-ins/management groups/initiatives/exemptions/missing references need -the policy owner. No compliance/applicability proof, policy changes, write retries -or correction approval. - -Exit 0: `planned`/fresh `reused`/applied `completed` (ARM only). -Data-plane access/ingestion/retrieval stay **unverified**. -Exit 2: `blocked`, no confirmed/ambiguous Azure writes. -Exit 3: `partial`, `first_failure`, `attempted_writes`, `resources_remaining`; -unproven ownership is `unverified`, never run-owned. -Receipt failure: block before submission, partial handoff afterward. -No automatic rollback/deletion. Cleanup always needs separate consent and fresh -ownership/configuration/children/scoped-role evidence. - -Authorities: failure/conflict/uncertainty only. - -[Native wait](https://learn.microsoft.com/en-us/cli/azure/resource#az-resource-wait); -[location metadata](https://learn.microsoft.com/rest/api/resources/subscriptions/list-locations); -[provider GET](https://learn.microsoft.com/rest/api/resources/providers/get?view=rest-resources-2021-04-01). +# Search bootstrap + +**New Basic/standard Search in an existing group**. +Other prerequisites/cleanup: [native bootstrap](../references/bootstrap-azure.md). + +Progress: stderr; `--no-progress` disables. + +## Choices + +Python 3.12+, signed-in CLI on Windows/POSIX; no login/install/context switch. +No preflight policy reads. +Prerequisites: **owner-verified references**, not helper-verified. +1–256 printable characters; punctuation allowed. +Errors omit values. + +Creation: `search`/`create`. Existing operations: [intake](search-intake-contracts.md); +legacy `reuse` stays strict, never hardens. +New locations fold ASCII case/whitespace (`East US` → `eastus`); +Not availability proof; bodies/hashes stay unchanged. +Basic: 1–3 replicas/one partition; standard: 1–12 replicas/1/2/3/4/6/12 partitions. +Fixed `SystemAssigned`, keyless auth and public networking; no role changes. +Non-secret tags; creation adds operation/owner tags. + +```json +{"schema_version":"2.0","resource_kind":"search","action":"create", +"subscription_id":"00000000-0000-0000-0000-000000000001","tenant_id":"00000000-0000-0000-0000-000000000002", +"resource_group":"rg1","name":"mysearchservice","location":"westus","sku":"basic","replicas":1,"partitions":1, +"public_network_access":"Enabled","tags":{"project":"knowledge"},"owner":"operator", +"prerequisites":{"quota":"checked; enough","pricing":"approved","network":"approved public","requirements":"settings","exclusive_name_authority":"exclusive name"}, +"limits":{"command_seconds":5,"wait_seconds":10,"wait_interval_seconds":1},"receipt_dir":""} +``` + +Replace examples/limits. Reuse: `exclusive_name_authority: null`; +required tags match; no ownership transfer. + +Region scope (closed): + +```json +{"schema_version":"2.0","subscription_id":"","tenant_id":"","receipt_dir":"","limits":{"command_seconds":5}} +``` + +No RG/name/SKU: account check plus one `az rest` GET: +`/subscriptions//providers/Microsoft.Search?api-version=2021-04-01`. +`searchServices.locations` supplies canonical `available_locations` (1–256). +Exit 0: `discovered`, no approval/artifact. +Create plan/apply: one fresh region GET each, including retained artifacts; no cache. +Unknown: `bootstrap-region-unsupported` plus choices; +denied/malformed: blocked. Reuse skips catalogs. Not SKU/quota/capacity/model/residency proof. + +```text +python "/helpers/bootstrap_azure.py" --regions scope.json +python "/helpers/bootstrap_azure.py" --plan choices.json +python "/helpers/bootstrap_azure.py" --apply "/.plan.json" --approve +``` + +`--plan` uses Azure reads/**local private writes**: `.plan.json` +body, before-state, choices, time, unapproved envelope. +Stdout: `artifact_id`/`approval_summary`, no paths/errors/hashes. +Labels: caller attestation, not helper verification; exact references stay private. +Show resource/network/capacity/tags/cost/limits; `--approve` consents once. +No cached consent or user checksums. + +15-minute expiry; submission markers prevent retries. +On drift/failure/expiry/changed choices, rerun `--plan`; discard consent. +Reconcile uncertainty; no suffix-create/body/hash edits. +Schema 1.0: regenerate (`bootstrap-contract-version`). Removed inputs +`max_polls`/`poll_seconds`/`poll_interval_seconds` are rejected. + +## Execution + +Argv: `az account show --subscription`, `az group show --name --subscription`, +Search ARM GET/PUT API `2025-05-01`. PUT uses `--body` then **one** +`"@" + str(body_path)` argument, Content-Type/client-request-ID headers. +POSIX runs `az` directly; Windows MSI/ZIP uses bundled `python.exe -IBm azure.cli`. +Other Windows layouts block. + +Only normally returned target `ResourceNotFound` proves absence. +Fresh principal/tenant/subscription/group/exact-name absence precede one PUT. +GET/PUT is **not atomic**: require exclusive name authority. +Mandatory atomicity: `bootstrap-concurrency-unresolved`, no invented conditionals. + +Reuse: two fresh material readbacks, required configuration/tags, +keyless identity/principal/tenant, endpoint, provisioning succeeded/status running. +`reused`: no mutation approval/executor or ownership adoption. +Wait uses the same identity; existence is not readiness. + +After PUT: `az resource wait --ids --api-version 2025-05-01 +--custom --interval --timeout `. +Fixed `properties`: succeeded/running or terminal states, lower/title/uppercase. +No caller expressions/custom polling. +Final raw GET establishes material match/ownership **before** readiness. +Wait failure remains primary even with ready GET; provider evidence is secondary. + +Limits: command 1–60 seconds, wait 1–900, interval 1–30. +Wait/final GET: separate `wait_seconds`/`command_seconds`; +sum is not an end-to-end deadline. +`subprocess.run`: argv, `shell=False`, best-effort timeout/cleanup. +CLI/OS/filesystems prevent hard time/process-tree/request-count guarantees. +No PID or confirmed-cleanup claim. Captured output is rejected **after capture** +above 1 MiB per stream: not a streaming cap or memory bound. Input/body/receipt +limits remain 1 MiB. Timeout/execution/receipt failure after attempted PUT is partial. + +Checks: target, principal/tenant, capacity/SKU, auth/network, identity, +required tags, ARM readiness. Metadata/ordering/unrelated tags, ID/enum casing, +equivalent optional defaults do not need another approval. +Never default missing material fields or return extra tags. +Material conflict, unknown ownership or failed readiness blocks/returns partial. + +## Evidence and failure + +Existing absolute private directory **outside the plugin**. +No symlinks/reparse paths. POSIX: directory-FD creation, identity/owner/private-mode +rechecks; exclusive safe leaves, mode 0600. +POSIX semantics required; later pathname uses are not pinned. +Windows checks owner/DACL (owner, SYSTEM, Administrators), not handle-relative +creation; `chmod` does not establish Windows privacy. No ACL changes. +Same-user/admin tampering is possible; unprovable privacy blocks. +Mocked/DrvFS is not native-filesystem or Azure evidence. + +Flushed receipts: operation/target/body/argv, client-request ID, readbacks/digests, +first error code/status/request ID, ownership. No raw errors/output/policy values/tokens/documents. +Message digest/`message_withheld`; secret-free labels/tags. +CLI file logging/dynamic installation: process-disabled. + +Creation-only policy diagnostics: +`RequestDisallowedByPolicy`, supported policy IDs or policy-violation details. +Provider error/statusDetails, never tags/configuration. +A bare 403/RBAC/network/quota failure triggers no policy calls. +Preserve first failure; diagnostic/receipt errors are secondary warnings. +At most eight referenced assignments/selected-subscription definitions +within 60 seconds: `az policy assignment show --name --scope` +and `az policy definition show --name --subscription`. No lists or global scans. +Built-ins/management groups/initiatives/exemptions/missing references need +the policy owner. No compliance/applicability proof, policy changes, write retries +or correction approval. + +Exit 0: `planned`/fresh `reused`/applied `completed` (ARM only). +Data-plane access/ingestion/retrieval stay **unverified**. +Exit 2: `blocked`, no confirmed/ambiguous Azure writes. +Exit 3: `partial`, `first_failure`, `attempted_writes`, `resources_remaining`; +unproven ownership is `unverified`, never run-owned. +Receipt failure: block before submission, partial handoff afterward. +No automatic rollback/deletion. Cleanup always needs separate consent and fresh +ownership/configuration/children/scoped-role evidence. + +Authorities: failure/conflict/uncertainty only. + +[Native wait](https://learn.microsoft.com/en-us/cli/azure/resource#az-resource-wait); +[location metadata](https://learn.microsoft.com/rest/api/resources/subscriptions/list-locations); +[provider GET](https://learn.microsoft.com/rest/api/resources/providers/get?view=rest-resources-2021-04-01). diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/bootstrap_azure.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/bootstrap_azure.py index 4f75017a..0e68b77b 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/bootstrap_azure.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/bootstrap_azure.py @@ -1,716 +1,716 @@ -"""Closed Search bootstrap workflow. Never a caller-programmable command runner.""" -from __future__ import annotations - -import argparse -import copy -import json -import re -import sys -import time -import uuid -from pathlib import Path - -try: - from ._progress import Progress, add_progress_argument, reporting - from ._common import HelperFailure, blocked_result, digest, emit_result, normalize_azure_location, reject_secrets - from ._bootstrap_io import MAX_BYTES, failure, private_directory, private_file, read_json, run_cli -except ImportError: - from _progress import Progress, add_progress_argument, reporting - from _common import HelperFailure, blocked_result, digest, emit_result, normalize_azure_location, reject_secrets - from _bootstrap_io import MAX_BYTES, failure, private_directory, private_file, read_json, run_cli - -API = "2025-05-01" -PROVIDER_API = "2021-04-01" -SCHEMA = "2.0" -WAIT_CONDITION = ( - "(contains(['succeeded','Succeeded','SUCCEEDED'], properties.provisioningState) && " - "contains(['running','Running','RUNNING'], properties.status)) || " - "contains(['failed','Failed','FAILED','canceled','Canceled','CANCELED','cancelled','Cancelled','CANCELLED'," - "'deleting','Deleting','DELETING','deleted','Deleted','DELETED'], properties.provisioningState) || " - "contains(['error','Error','ERROR','degraded','Degraded','DEGRADED'], properties.status)" -) -GUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") -HASH = re.compile(r"sha256:[0-9a-f]{64}") -TEXT = re.compile(r"[A-Za-z0-9 ._@:/()-]{1,256}") -POLICY_ID = re.compile( - r"(?:/subscriptions/[0-9a-f-]{36}(?:/resourceGroups/[A-Za-z0-9_.()-]+" - r"(?:/providers/Microsoft.Search/searchServices/[a-z0-9-]+)?)?" - r"|/providers/Microsoft.Management/managementGroups/[A-Za-z0-9_.-]+)?" - r"/providers/Microsoft.Authorization/(?:policyAssignments|policyDefinitions|policySetDefinitions|policyExemptions)/[A-Za-z0-9_.-]+", - re.IGNORECASE, -) - - -def closed(value, keys, label): - if not isinstance(value, dict) or set(value) != set(keys): - raise failure("bootstrap-input-invalid", label + " needs exactly the documented fields.") - - -def text(value, pattern=TEXT): - return isinstance(value, str) and pattern.fullmatch(value) is not None - - -def validate(request): - closed(request, { - "schema_version", "resource_kind", "action", "subscription_id", "tenant_id", - "resource_group", "name", "location", "sku", "replicas", "partitions", - "public_network_access", "tags", "owner", "prerequisites", "limits", "receipt_dir", - }, "Bootstrap choices") - if request["schema_version"] != SCHEMA: - raise failure("bootstrap-contract-version", "Regenerate choices and artifacts using bootstrap schema 2.0 and native wait limits.") - reject_secrets(request) - if ( - request["resource_kind"] != "search" - or request["action"] not in ("create", "reuse") - or not text(request["subscription_id"], GUID) or not text(request["tenant_id"], GUID) - or not text(request["resource_group"], re.compile(r"[A-Za-z0-9_()-][A-Za-z0-9_.()-]{0,89}")) - or request["resource_group"].endswith(".") - or not text(request["name"], re.compile(r"(?=.{2,60}$)[a-z0-9][a-z0-9]+(?:-[a-z0-9]+)*")) - or normalize_azure_location(request["location"]) is None - or request["sku"] not in ("basic", "standard") - or type(request["replicas"]) is not int or not 1 <= request["replicas"] <= (3 if request["sku"] == "basic" else 12) - or type(request["partitions"]) is not int or request["partitions"] not in ( - (1,) if request["sku"] == "basic" else (1, 2, 3, 4, 6, 12)) - or request["public_network_access"] != "Enabled" or not text(request["owner"]) - ): - raise failure("bootstrap-input-invalid", "Select supported, explicit Search identity, keyless public networking and capacity.") - tags = request["tags"] - if not isinstance(tags, dict) or len(tags) > 20 or any( - not text(k, re.compile(r"[A-Za-z0-9_.-]{1,128}")) or not text(v) for k, v in tags.items() - ): - raise failure("bootstrap-input-invalid", "Tags must be bounded non-secret labels.") - if request["action"] == "create" and any(k in tags for k in ("foundry-iq-owner", "foundry-iq-operation")): - raise failure("bootstrap-input-invalid", "Creation ownership tags are generated, not caller overrides.") - closed(request["prerequisites"], {"quota", "pricing", "network", "requirements", "exclusive_name_authority"}, "Prerequisites") - for key, value in request["prerequisites"].items(): - if key == "exclusive_name_authority" and request["action"] == "reuse" and value is None: - continue - field = f"prerequisites.{key}" - if value is None or isinstance(value, str) and not value.strip(" "): - raise failure("bootstrap-prerequisite-missing", f"{field} requires nonempty owner-verified evidence.") - if not isinstance(value, str) or len(value) > 256: - raise failure("bootstrap-prerequisite-invalid", f"{field} must be a string of 1-256 printable characters.") - for position, character in enumerate(value, 1): - if not character.isprintable(): - raise failure( - "bootstrap-prerequisite-invalid", - f"{field} contains non-printable U+{ord(character):04X} at character {position}; evidence values are withheld.", - ) - closed(request["limits"], {"command_seconds", "wait_seconds", "wait_interval_seconds"}, "Limits") - for key, ceiling in (("command_seconds", 60), ("wait_seconds", 900), ("wait_interval_seconds", 30)): - if type(request["limits"][key]) is not int or not 1 <= request["limits"][key] <= ceiling: - raise failure("bootstrap-input-invalid", "Select explicit bounded command and readiness limits.") - private_directory(request["receipt_dir"]) - return request - - -def ids(request): - group = "/subscriptions/" + request["subscription_id"] + "/resourceGroups/" + request["resource_group"] - return group, group + "/providers/Microsoft.Search/searchServices/" + request["name"] - - -def provider_id(request): - return "/subscriptions/" + request["subscription_id"] + "/providers/Microsoft.Search" - - -def search_locations(value, expected_id): - if ( - not isinstance(value, dict) or folded(value.get("id")) != expected_id.casefold() - or folded(value.get("namespace")) != "microsoft.search" - or not isinstance(value.get("resourceTypes"), list) - or any(not isinstance(item, dict) or not isinstance(item.get("resourceType"), str) - for item in value["resourceTypes"]) - ): - raise failure("bootstrap-region-metadata-invalid", "Search provider metadata is incomplete or belongs to another subscription/provider.") - services = [item for item in value["resourceTypes"] if folded(item["resourceType"]) == "searchservices"] - if len(services) != 1: - raise failure("bootstrap-region-metadata-invalid", "Exactly one Search searchServices resource type is required.") - locations = services[0].get("locations") - if ( - not isinstance(locations, list) or not 1 <= len(locations) <= 256 - or any(normalize_azure_location(location) is None for location in locations) - ): - raise failure("bootstrap-region-metadata-invalid", "Search supported locations are missing, malformed or exceed the bounded list.") - return sorted({normalize_azure_location(location) for location in locations}) - - -def desired(request, operation_id): - # Only new planning normalizes choices; retained approved bodies keep their original representation. - tags = dict(request["tags"]) - if request["action"] == "create": - tags.update({"foundry-iq-owner": request["owner"], "foundry-iq-operation": operation_id}) - return { - "location": request["location"], "tags": tags, "sku": {"name": request["sku"]}, - "identity": {"type": "SystemAssigned"}, - "properties": {"replicaCount": request["replicas"], "partitionCount": request["partitions"], - "publicNetworkAccess": "Enabled", "disableLocalAuth": True}, - } - - -def _json(raw): - if len(raw) > MAX_BYTES: - raise failure("bootstrap-cli-output-limit", "CLI response exceeds its bound.") - try: - value = json.loads(raw.decode("utf-8")) - json.dumps(value, allow_nan=False) - if not isinstance(value, dict) or value.get("nextLink") or value.get("nextToken"): - raise ValueError("Not a complete object") - return value - except (ValueError, UnicodeError, RecursionError) as exc: - raise failure("bootstrap-response-invalid", "CLI must return one complete JSON object; empty or truncated output is not evidence.") from exc - - -def _error(raw): - """Only codes/UUIDs/reference IDs survive; arbitrary Azure messages are not safe receipts.""" - message = raw.decode("utf-8", errors="replace") - code_match = re.search(r'"code"\s*:\s*"([A-Za-z][A-Za-z0-9_.-]{0,100})"|(?:ERROR:\s*)?\(([A-Za-z][A-Za-z0-9_.-]{0,100})\)', message) - code = next((g for g in code_match.groups() if g), None) if code_match else "bootstrap-cli-failed" - request_match = re.search(r"request[\s-]?id[\"'\s:]+([0-9a-f-]{36})", message, re.I) - request_id = request_match.group(1).lower() if request_match and GUID.fullmatch(request_match.group(1).lower()) else None - status_match = re.search(r'"status(?:Code)?"\s*:\s*(4\d\d|5\d\d)', message) - status = int(status_match.group(1)) if status_match else None - reason = re.match(r"(?:ERROR:\s*)?([A-Za-z ]+)\(", message.strip()) - if status is None and reason: - status = {"Bad Request": 400, "Unauthorized": 401, "Forbidden": 403, "Not Found": 404, - "Conflict": 409, "Too Many Requests": 429, "Internal Server Error": 500, - "Service Unavailable": 503, "Gateway Timeout": 504}.get(reason.group(1)) - references = sorted(set(POLICY_ID.findall(message))) - implicated = code == "RequestDisallowedByPolicy" or bool(references) or '"PolicyViolation"' in message - result = failure(code, "Azure CLI failed; untrusted original message text is withheld from output and receipts.") - result.request_id, result.http_status = request_id, status - result.policy_ids = references[:8] - result.policy_implicated = implicated - result.message_digest = digest(message) - result.policy_overflow = len(references) > 8 - return result - - -def projection(resource, required_tags=()): - """Store selected configuration, not arbitrary provider fields/error text.""" - if resource is None: - return None - props, identity = resource.get("properties", {}), resource.get("identity", {}) - if not isinstance(props, dict) or not isinstance(identity, dict): - raise failure("bootstrap-response-invalid", "Search properties and identity must be objects.") - tags = resource.get("tags") - selected = {k.casefold() for k in required_tags} - return { - "id": resource.get("id"), "location": resource.get("location"), - "tags": {k: v for k, v in tags.items() if folded(k) in selected} if isinstance(tags, dict) else tags, - "sku": {"name": resource["sku"].get("name")} if isinstance(resource.get("sku"), dict) else resource.get("sku"), - "identity": {k: identity.get(k) for k in ("type", "principalId", "tenantId")}, - "properties": {k: props.get(k) for k in ( - "replicaCount", "partitionCount", "publicNetworkAccess", "disableLocalAuth", - "provisioningState", "status", "endpoint", - )}, - } - - -def matching(resource, request, body): - value = projection(resource, body["tags"]) - tags = resource.get("tags") if resource is not None else None - if tags is None and not body["tags"]: - tags = {} - group, target = ids(request) - location = normalize_azure_location(body["location"]) - if ( - value is None or not isinstance(value["id"], str) or value["id"].casefold() != target.casefold() - or location is None or normalize_azure_location(value["location"]) != location - or not isinstance(tags, dict) - or any([v for k, v in tags.items() if folded(k) == key.casefold()] != [expected] - for key, expected in body["tags"].items()) - or not isinstance(value["sku"], dict) or folded(value["sku"].get("name")) != body["sku"]["name"] - or folded(value["identity"]["type"]) != "systemassigned" - or any(type(value["properties"][k]) is not type(v) or ( - folded(value["properties"][k]) != v.casefold() if isinstance(v, str) else value["properties"][k] != v - ) for k, v in body["properties"].items()) - ): - raise failure("bootstrap-state-conflict", "Selected Search definition differs; shared or foreign state will not be modified.") - props = resource["properties"] - if props.get("authOptions") not in (None, {}) or resource["identity"].get("userAssignedIdentities") not in (None, {}): - raise failure("bootstrap-state-conflict", "Observed authentication configuration is not the selected keyless system identity.") - rules = props.get("networkRuleSet") - if rules is not None and (not isinstance(rules, dict) or rules.get("ipRules") not in (None, []) - or (rules.get("bypass") is not None and folded(rules["bypass"]) != "none")): - raise failure("bootstrap-state-conflict", "Additional network restrictions require the owning procedure.") - if props.get("privateEndpointConnections") not in (None, []) or props.get("sharedPrivateLinkResources") not in (None, []): - raise failure("bootstrap-state-conflict", "Private/shared network resources are outside this bootstrap slice.") - return value - - -def folded(value): - return value.casefold() if isinstance(value, str) else None - - -def reuse_binding(resource, request, body): - if resource is None: - return None - value = ready(resource, request, body) - if value is None: - raise failure("bootstrap-not-ready", "Fresh Search readback does not establish ARM readiness.") - return {"id": folded(value["id"]), "principal": folded(value["identity"]["principalId"]), - "tenant": folded(value["identity"]["tenantId"])} - - -def ready(resource, request, body): - return _readiness(resource, request, matching(resource, request, body)) - - -def _readiness(resource, request, value): - props = value["properties"] - if folded(props["provisioningState"]) in ("failed", "canceled", "cancelled", "deleting", "deleted") or folded(props["status"]) in ("error", "degraded"): - evidence = [resource.get("error"), resource["properties"].get("error"), - resource["properties"].get("statusDetails")] - raw = "\n".join(value if isinstance(value, str) else json.dumps(value) - for value in evidence if isinstance(value, (str, dict, list))) - error = _error(raw.encode("utf-8")) - if error.code == "bootstrap-cli-failed": - error.code = "bootstrap-provisioning-failed" - error.message = "Search entered a failed, deleting or degraded ARM state; provider message text is withheld." - error.args = (error.message,) - error.creation_failure = True - raise error - if ( - folded(props["provisioningState"]) != "succeeded" or folded(props["status"]) != "running" - or not text(folded(value["identity"]["principalId"]), GUID) - or folded(value["identity"]["tenantId"]) != request["tenant_id"] - or folded(props["endpoint"]) not in ( - "https://" + request["name"] + ".search.windows.net", - "https://" + request["name"] + ".search.windows.net/", - ) - ): - return None - return value - - -class Session: - def __init__(self, request, operation_id, cli=run_cli, clock=time.monotonic, *, target=None): - self.request, self.operation_id = request, operation_id - self.cli, self.clock = cli, clock - self.directory = private_directory(request["receipt_dir"]) - self.sequence = 0 - self.attempted = False - self.approved = False - self.owned = False - self.warnings = [] - self.diagnosed = False - self.policy_references = [] - self.target = target if target is not None else ids(request)[1] - - def record(self, event, value): - self.sequence += 1 - private_file(self.directory, f"{self.operation_id}.{self.sequence:03d}.{uuid.uuid4().hex}.receipt.json", { - "schema_version": "1.0", "operation_id": self.operation_id, "operation": "bootstrap-search", - "target": self.target, "event": event, "evidence": value, - }) - - def call(self, args, timeout=None, mutation=False, waiting=False): - self.record("command", {"argv": args}) - if mutation: - self.attempted = True - try: - response = self.cli( - args + ["--subscription", self.request["subscription_id"]], - self.request["limits"]["wait_seconds"] if waiting else ( - min(self.request["limits"]["command_seconds"], timeout) if timeout is not None else self.request["limits"]["command_seconds"]), - ) - except HelperFailure as error: - if mutation and error.code in ("bootstrap-tool-unavailable", "bootstrap-cli-start-failed"): - self.attempted = False - raise - rc, stdout, stderr = response - if len(stdout) > MAX_BYTES or len(stderr) > MAX_BYTES: - raise failure("bootstrap-cli-output-limit", "CLI output exceeded its bound.") - if rc: - error = _error(stderr or stdout) - try: - self.record("failure", {"code": error.code, "status": error.http_status, "request_id": error.request_id, - "message_digest": error.message_digest, "message_withheld": True, - "policy_ids": error.policy_ids, "policy_references_limited": error.policy_overflow, - "warnings": error.warnings}) - except HelperFailure: - self.warnings.append("Original CLI failure could not be persisted; retain the returned failure and ownership.") - raise error - if waiting: - if stdout.strip() not in (b"", b"null"): - raise failure("bootstrap-wait-result-invalid", "Native wait returned an unexpected result; readiness is unresolved.") - return None - return _json(stdout) - - def get(self, timeout=None): - try: - result = self.call(["rest", "--method", "get", "--url", - "https://management.azure.com" + self.target + "?api-version=" + API], timeout) - except HelperFailure as exc: - if (exc.code == "ResourceNotFound" and exc.http_status in (None, 404) - and not getattr(exc, "cleanup_unconfirmed", False)): - self.record("readback", {"id": self.target, "absence": "ResourceNotFound"}) - return None - raise - if not isinstance(result.get("id"), str) or result["id"].casefold() != self.target.casefold(): - raise failure("bootstrap-response-invalid", "ARM returned another identity.") - self.record("readback", {"id": self.target, "digest": digest(result)}) - return result - - def account_context(self): - account = self.call(["account", "show"]) - if (folded(account.get("id")) != self.request["subscription_id"] or folded(account.get("tenantId")) != self.request["tenant_id"] - or folded(account.get("state")) != "enabled" or account.get("environmentName") != "AzureCloud"): - raise failure("bootstrap-context-conflict", "Signed-in subscription/tenant is not the selected enabled context.") - user = account.get("user") - if not isinstance(user, dict) or not text(user.get("name")) or folded(user.get("type")) not in ("user", "serviceprincipal"): - raise failure("bootstrap-context-conflict", "Signed-in principal evidence is missing or unsupported.") - principal = folded(user["name"]) if text(folded(user["name"]), GUID) else user["name"] - return {"id": folded(account["id"]), "tenantId": folded(account["tenantId"]), "environmentName": account["environmentName"], - "user": {"name": principal, "type": folded(user["type"])}} - - def context(self): - account = self.account_context() - group = self.call(["group", "show", "--name", self.request["resource_group"]]) - props = group.get("properties") - if ( - not isinstance(group.get("id"), str) or group["id"].casefold() != ids(self.request)[0].casefold() - or not isinstance(props, dict) or folded(props.get("provisioningState")) != "succeeded" - ): - raise failure("bootstrap-group-not-ready", "The explicitly selected existing resource group must be ready.") - context = {"account": digest(account), "group": digest({"id": folded(group["id"]), "ready": True})} - self.record("context", {"account": account, "group_id": ids(self.request)[0], "group_digest": context["group"]}) - return context - - def regions(self, *, selected=None): - identity = provider_id(self.request) - value = self.call(["rest", "--method", "get", "--url", - "https://management.azure.com" + identity + "?api-version=" + PROVIDER_API]) - locations = search_locations(value, identity) - self.record("search-regions", {"provider_id": identity, "resource_type": "searchServices", - "api_version": PROVIDER_API, "digest": digest(value), "locations": locations}) - if selected is not None and normalize_azure_location(selected) not in locations: - error = failure("bootstrap-region-unsupported", "Select an advertised Search region; no typo correction or deployment-capacity inference.") - error.available_locations = locations - raise error - return locations - - def policy_evidence(self, original): - if self.diagnosed or not getattr(original, "policy_implicated", False): - return - self.diagnosed = True - end = self.clock() + 60 - queue = list(original.policy_ids) - seen = set() - self.warnings.append("Policy evidence is diagnostic only; cause/compliance remains unresolved for the policy owner.") - scopes = {self.target.casefold(), ids(self.request)[0].casefold(), ("/subscriptions/" + self.request["subscription_id"]).casefold()} - while queue and len(seen) < 8: - identity = queue.pop(0) - if identity.casefold() in seen: - continue - seen.add(identity.casefold()) - self.policy_references.append(identity) - remaining = end - self.clock() - if remaining <= 0: - break - scope, tail = re.split(r"/providers/Microsoft.Authorization/", identity, flags=re.I) - if scope.casefold() not in scopes or "/" not in tail: - self.warnings.append("Referenced policy scope is unsupported; hand off to the policy owner.") - continue - kind, name = tail.split("/", 1) - if kind.casefold() == "policyassignments": - args = ["policy", "assignment", "show", "--name", name, "--scope", scope] - elif kind.casefold() == "policydefinitions" and scope.casefold() == ("/subscriptions/" + self.request["subscription_id"]).casefold(): - args = ["policy", "definition", "show", "--name", name] - else: - self.warnings.append("Referenced policy kind requires owner investigation.") - continue - try: - value = self.call(args, remaining) - if self.clock() >= end: - raise failure("bootstrap-policy-timeout", "Policy evidence arrived after the diagnostic budget.") - if not isinstance(value.get("id"), str) or value["id"].casefold() != identity.casefold(): - raise failure("bootstrap-policy-evidence-mismatch", "Policy read returned another identity.") - # Values/rules can contain sensitive literals: retain identity, digest and - # classification only; the policy owner reads the referenced object. - properties = value.get("properties", value) - if not isinstance(properties, dict): - raise failure("bootstrap-response-invalid", "Policy properties are malformed.") - reference = properties.get("policyDefinitionId") - if isinstance(reference, str) and POLICY_ID.fullmatch(reference): - queue.append(reference) - self.record("policy-evidence", {"id": identity, "digest": digest(value), - "referenced_definition_id": reference if isinstance(reference, str) and POLICY_ID.fullmatch(reference) else None, - "enforcementMode": properties.get("enforcementMode") if properties.get("enforcementMode") in ("Default", "DoNotEnforce", "Enroll") else None, - "interpretation": "unresolved"}) - except HelperFailure as secondary: - self.warnings.append("Secondary policy diagnostic failed: " + secondary.code) - self.warnings.extend(secondary.warnings) - if queue or getattr(original, "policy_overflow", False) or not original.policy_ids: - self.warnings.append("Policy references are missing or exceed this bounded collector; no truncated completeness claim.") - - def blocked(self, error): - if self.attempted: - error.partial = True - error.writes = [{"operation": "PUT", "resource_id": self.target, "submission": "attempted-unverified"}] - error.resources_remaining = [{"resource_id": self.target, "run_owned": self.owned, - "cleanup": "separate consent and fresh ownership/children/roles required"}] - error.warnings.extend(self.warnings) - result = blocked_result(error, outcome="bootstrap-search", fingerprint=None, owner=self.request.get("owner")) - if error.code == "bootstrap-region-unsupported" and hasattr(error, "available_locations"): - result["available_locations"] = error.available_locations - if result["status"] == "partial": - result["approved_plan"] = {"confirmed": self.approved, "artifact_id": self.operation_id} - result["attempted_writes"] = result.pop("completed_writes") - result["completed_writes"] = [] - if not self.owned: - result["resources_remaining"]["unverified"] = result["resources_remaining"]["run_owned"] - result["resources_remaining"]["run_owned"] = [] - result["receipt_id"] = self.operation_id - if self.diagnosed: - result["policy_handoff"] = {"referenced_ids": self.policy_references, "complete": False, - "next_step": "Policy owner investigates exact references; no correction or retry is approved."} - return result - - -def discover_regions(request, *, cli=run_cli): - closed(request, {"schema_version", "subscription_id", "tenant_id", "receipt_dir", "limits"}, "Region discovery") - reject_secrets(request) - closed(request["limits"], {"command_seconds"}, "Region discovery limits") - if ( - request["schema_version"] != SCHEMA - or not text(request["subscription_id"], GUID) or not text(request["tenant_id"], GUID) - or type(request["limits"]["command_seconds"]) is not int - or not 1 <= request["limits"]["command_seconds"] <= 60 - ): - raise failure("bootstrap-input-invalid", "Select subscription/tenant and a 1-60 second region-discovery command limit.") - session = Session(request, str(uuid.uuid4()), cli, target=provider_id(request)) - try: - session.record("context", {"account": session.account_context()}) - locations = session.regions() - return {"status": "discovered", "resource_type": "Microsoft.Search/searchServices", - "available_locations": locations, "receipt_id": session.operation_id, - "mutation_approval_required": False, "execution_required": False, "writes_performed": [], - "verification": "Advertised region support only; not SKU, quota, capacity, models or residency approval."} - except HelperFailure as error: - return session.blocked(error) - - -def plan_request(request, *, cli=run_cli, clock=time.monotonic, execution_output=None): - validate(request) - request = copy.deepcopy(request) - request["location"] = normalize_azure_location(request["location"]) - operation_id = str(uuid.uuid4()) - session = Session(request, operation_id, cli, clock) - try: - context = session.context() - observed = session.get() - body = desired(request, operation_id) - if request["action"] == "create": - if observed is not None: - raise failure("bootstrap-state-conflict", "New intent requires exact-name absence; never overwrite or silently reuse.") - session.regions(selected=request["location"]) - refreshed = session.get() - if refreshed is not None: - raise failure("bootstrap-state-drift", "Exact-name absence changed; creation is blocked.") - else: - binding = reuse_binding(observed, request, body) - if binding is None: - raise failure("bootstrap-not-ready", "Reuse requires a running, keyless Search service with identity readback.") - refreshed = session.get() - if reuse_binding(refreshed, request, body) != binding: - raise failure("bootstrap-state-drift", "Selected Search identity changed; obtain fresh evidence.") - artifact = { - "operation": "bootstrap-search", "operation_id": operation_id, "created_at": int(time.time()), - "choices": copy.deepcopy(request), - "body": body, "before": {**context, "search": digest(observed)}, - } - envelope = {"schema_version": SCHEMA, "plan": artifact, - "approval": {"confirmed": False, "fingerprint": digest(artifact)}} - if execution_output is None: - private_file(session.directory, operation_id + ".plan.json", envelope) - else: - try: - from .private_artifacts import retain_execution_input - except ImportError: - from private_artifacts import retain_execution_input - execution_artifact = retain_execution_input(envelope, execution_output) - session.record("planned", {"body": body, "before": artifact["before"], - "readback": projection(refreshed, body["tags"]), "run_owned": False}) - create = request["action"] == "create" - if execution_output is not None: - return { - "status": "planned" if create else "reused", "artifact_id": operation_id, - "execution_artifact": execution_artifact, "execution_required": create, - "mutation_approval_required": create, "azure_mutation_performed": False, - "local_filesystem": {"artifact_created": True, "receipts_created": True, - "existing_files_changed": False}, - "summary": "Unapproved bootstrap artifact retained privately; review before separate approval.", - } - return { - "status": "planned" if create else "reused", "artifact_id": operation_id, - "execution_required": create, "mutation_approval_required": create, - "approval_summary": {"action": request["action"], "resource_id": session.target, - "location": request["location"], "sku": request["sku"], - "replicas": request["replicas"], "partitions": request["partitions"], - "network": "public; local authentication disabled; SystemAssigned", - "tags": body["tags"], "prerequisites": { - key: {"evidence_present": value is not None, - "verification": "caller-attested; not helper-verified" if value is not None else "not required for reuse"} - for key, value in request["prerequisites"].items() - }, - "limits": request["limits"], "cleanup": "separate approval"}, - "verification": {"arm_readiness": "verified" if not create else "unverified", - "data_plane_access": "unverified", "ingestion": "unverified", "retrieval": "unverified"}, - "writes_performed": [], "run_owned": False, - "observed_dependency": projection(refreshed, body["tags"]), - } - except HelperFailure as error: - return session.blocked(error) - - -def validate_artifact(envelope): - closed(envelope, {"schema_version", "plan", "approval"}, "Execution artifact") - if envelope["schema_version"] != SCHEMA: - raise failure("bootstrap-contract-version", "Regenerate the retained artifact with bootstrap schema 2.0; do not edit its checksum.") - plan, approval = envelope["plan"], envelope["approval"] - closed(plan, {"operation", "operation_id", "created_at", "choices", "body", "before"}, "Execution plan") - closed(approval, {"confirmed", "fingerprint"}, "Approval") - if ( - plan["operation"] != "bootstrap-search" - or not text(plan["operation_id"], GUID) or type(approval["confirmed"]) is not bool - or approval["fingerprint"] != digest(plan) - ): - raise failure("approval-mismatch", "Retain the unchanged planner artifact; integrity is checked internally.") - validate(plan["choices"]) - if type(plan["created_at"]) is not int or not plan["created_at"] <= time.time() <= plan["created_at"] + 900: - raise failure("bootstrap-artifact-expired", "Rerun planning; retained artifacts expire after fifteen minutes.") - closed(plan["before"], {"account", "group", "search"}, "Before-state") - if any(not text(v, HASH) for v in plan["before"].values()) or plan["body"] != desired(plan["choices"], plan["operation_id"]): - raise failure("bootstrap-artifact-invalid", "Artifact body or before-state differs from generated choices.") - return plan - - -@reporting("search-bootstrap") -def apply_artifact(envelope, *, approve=False, cli=run_cli, clock=time.monotonic, progress=None): - progress.update("validation") - plan = validate_artifact(envelope) - if approve is not True: - raise failure("approval-missing", "Explicit approval of the unchanged artifact is required before any execution reads or writes.") - request = plan["choices"] - if request["action"] != "create": - raise failure("bootstrap-execution-unnecessary", "Read-only reuse needs fresh planning, not mutation approval or execution.") - session = Session(request, plan["operation_id"], cli, clock) - session.approved = True - body = plan["body"] - try: - session.record("approved", {"plan": plan, "approval": {"confirmed": True, "fingerprint": digest(plan)}}) - progress.update("context-check") - context = session.context() - if context != {k: plan["before"][k] for k in ("account", "group")}: - raise failure("bootstrap-state-drift", "Account/group changed; rerun planning and discard old consent.") - progress.update("region-check") - session.regions(selected=request["location"]) - progress.update("absence-check") - if plan["before"]["search"] != digest(None) or session.get() is not None: - raise failure("bootstrap-state-drift", "Exact-name absence changed; never overwrite or retry through another name.") - body_path = private_file(session.directory, plan["operation_id"] + "." + uuid.uuid4().hex + ".body.json", body) - command = ["rest", "--method", "put", "--url", - "https://management.azure.com" + session.target + "?api-version=" + API, - "--headers", "Content-Type=application/json", "x-ms-client-request-id=" + str(uuid.uuid4()), - "--body", "@" + str(body_path)] - private_file(session.directory, plan["operation_id"] + ".submission.json", - {"operation_id": plan["operation_id"], "target": session.target, "body": body, "command": command}) - progress.update("search-submit") - try: - session.call(command, mutation=True) - except HelperFailure as original: - if not session.attempted: - raise - # Never retry PUT. Neither diagnostics nor a readback replaces the first error. - progress.update("arm-readback") - try: - observed = session.get() - if observed is not None: - matching(observed, request, body) - session.owned = True - session.record("failure-reconciliation", {"readback": projection(observed, body["tags"]), "run_owned": session.owned}) - if (original.code in ("RequestDisallowedByPolicy", "AuthorizationFailed", "InvalidSkuName") - and observed is None and not getattr(original, "cleanup_unconfirmed", False)): - session.attempted = False - except HelperFailure as secondary: - session.warnings.append("Read-only reconciliation failed: " + secondary.code) - session.warnings.extend(secondary.warnings) - session.policy_evidence(original) - raise original - progress.update("arm-wait") - try: - session.call(["resource", "wait", "--ids", session.target, "--api-version", API, - "--custom", WAIT_CONDITION, "--interval", str(request["limits"]["wait_interval_seconds"]), - "--timeout", str(request["limits"]["wait_seconds"])], waiting=True) - except HelperFailure as original: - try: - progress.update("arm-readback") - observed = session.get() - if observed is not None: - value = matching(observed, request, body) - session.owned = True - try: - _readiness(observed, request, value) - except HelperFailure as provider: - session.warnings.append("Terminal provider readback: " + provider.code) - session.record("terminal-provider-failure", { - "code": provider.code, "status": provider.http_status, "request_id": provider.request_id, - "message_digest": provider.message_digest, "message_withheld": True}) - session.policy_evidence(provider) - session.record("wait-failure-readback", {"readback": projection(observed, body["tags"]), "run_owned": session.owned}) - except HelperFailure as secondary: - session.warnings.append("Wait failure readback failed: " + secondary.code) - session.warnings.extend(secondary.warnings) - raise original - progress.update("arm-readback") - observed = session.get() - value = matching(observed, request, body) if observed is not None else None - if value is not None: - session.owned = True - value = _readiness(observed, request, value) - if value is None: - raise failure("bootstrap-readiness-timeout", "Final ARM readiness was not established; preserve the resource and operation receipts.") - session.record("arm-ready", {"readback": value, "run_owned": True}) - return {"status": "completed", "receipt_id": session.operation_id, "resource_id": session.target, - "run_owned": True, "writes_performed": [{"operation": "PUT", "resource_id": session.target}], - "observed_dependency": value, - "verification": {"arm_readiness": "verified", "data_plane_access": "unverified", - "ingestion": "unverified", "retrieval": "unverified"}, - "cleanup": "separate approval with fresh ownership/children/roles"} - except HelperFailure as error: - if session.attempted and getattr(error, "creation_failure", False): - session.policy_evidence(error) - return session.blocked(error) - - -def main(argv=None): - try: - from .private_artifacts import add_execution_output_argument, validate_execution_output_mode - except ImportError: - from private_artifacts import add_execution_output_argument, validate_execution_output_mode - parser = argparse.ArgumentParser(description="Discover Search regions or plan/apply one selected service; no models, roles or cleanup.") - mode = parser.add_mutually_exclusive_group(required=True) - mode.add_argument("--plan", type=Path) - mode.add_argument("--apply", type=Path) - mode.add_argument("--regions", type=Path) - parser.add_argument("--approve", action="store_true") - add_execution_output_argument(parser) - add_progress_argument(parser) - args = parser.parse_args(argv) - try: - validate_execution_output_mode(args) - if (args.plan or args.regions) and args.approve: - raise failure("bootstrap-input-invalid", "Planning cannot approve mutations.") - document = read_json(args.plan or args.apply or args.regions) - result = (discover_regions(document) if args.regions else - plan_request(document, **({"execution_output": args.execution_output} if args.execution_output else {})) if args.plan else apply_artifact( - document, approve=args.approve, progress=Progress("search-bootstrap", enabled=args.progress))) - except HelperFailure as error: - result = blocked_result(error, outcome="bootstrap-search", fingerprint=None) - emit_result(result) - return 3 if result["status"] == "partial" else 2 if result["status"] == "blocked" else 0 - - -if __name__ == "__main__": - sys.exit(main()) +"""Closed Search bootstrap workflow. Never a caller-programmable command runner.""" +from __future__ import annotations + +import argparse +import copy +import json +import re +import sys +import time +import uuid +from pathlib import Path + +try: + from ._progress import Progress, add_progress_argument, reporting + from ._common import HelperFailure, blocked_result, digest, emit_result, normalize_azure_location, reject_secrets + from ._bootstrap_io import MAX_BYTES, failure, private_directory, private_file, read_json, run_cli +except ImportError: + from _progress import Progress, add_progress_argument, reporting + from _common import HelperFailure, blocked_result, digest, emit_result, normalize_azure_location, reject_secrets + from _bootstrap_io import MAX_BYTES, failure, private_directory, private_file, read_json, run_cli + +API = "2025-05-01" +PROVIDER_API = "2021-04-01" +SCHEMA = "2.0" +WAIT_CONDITION = ( + "(contains(['succeeded','Succeeded','SUCCEEDED'], properties.provisioningState) && " + "contains(['running','Running','RUNNING'], properties.status)) || " + "contains(['failed','Failed','FAILED','canceled','Canceled','CANCELED','cancelled','Cancelled','CANCELLED'," + "'deleting','Deleting','DELETING','deleted','Deleted','DELETED'], properties.provisioningState) || " + "contains(['error','Error','ERROR','degraded','Degraded','DEGRADED'], properties.status)" +) +GUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") +HASH = re.compile(r"sha256:[0-9a-f]{64}") +TEXT = re.compile(r"[A-Za-z0-9 ._@:/()-]{1,256}") +POLICY_ID = re.compile( + r"(?:/subscriptions/[0-9a-f-]{36}(?:/resourceGroups/[A-Za-z0-9_.()-]+" + r"(?:/providers/Microsoft.Search/searchServices/[a-z0-9-]+)?)?" + r"|/providers/Microsoft.Management/managementGroups/[A-Za-z0-9_.-]+)?" + r"/providers/Microsoft.Authorization/(?:policyAssignments|policyDefinitions|policySetDefinitions|policyExemptions)/[A-Za-z0-9_.-]+", + re.IGNORECASE, +) + + +def closed(value, keys, label): + if not isinstance(value, dict) or set(value) != set(keys): + raise failure("bootstrap-input-invalid", label + " needs exactly the documented fields.") + + +def text(value, pattern=TEXT): + return isinstance(value, str) and pattern.fullmatch(value) is not None + + +def validate(request): + closed(request, { + "schema_version", "resource_kind", "action", "subscription_id", "tenant_id", + "resource_group", "name", "location", "sku", "replicas", "partitions", + "public_network_access", "tags", "owner", "prerequisites", "limits", "receipt_dir", + }, "Bootstrap choices") + if request["schema_version"] != SCHEMA: + raise failure("bootstrap-contract-version", "Regenerate choices and artifacts using bootstrap schema 2.0 and native wait limits.") + reject_secrets(request) + if ( + request["resource_kind"] != "search" + or request["action"] not in ("create", "reuse") + or not text(request["subscription_id"], GUID) or not text(request["tenant_id"], GUID) + or not text(request["resource_group"], re.compile(r"[A-Za-z0-9_()-][A-Za-z0-9_.()-]{0,89}")) + or request["resource_group"].endswith(".") + or not text(request["name"], re.compile(r"(?=.{2,60}$)[a-z0-9][a-z0-9]+(?:-[a-z0-9]+)*")) + or normalize_azure_location(request["location"]) is None + or request["sku"] not in ("basic", "standard") + or type(request["replicas"]) is not int or not 1 <= request["replicas"] <= (3 if request["sku"] == "basic" else 12) + or type(request["partitions"]) is not int or request["partitions"] not in ( + (1,) if request["sku"] == "basic" else (1, 2, 3, 4, 6, 12)) + or request["public_network_access"] != "Enabled" or not text(request["owner"]) + ): + raise failure("bootstrap-input-invalid", "Select supported, explicit Search identity, keyless public networking and capacity.") + tags = request["tags"] + if not isinstance(tags, dict) or len(tags) > 20 or any( + not text(k, re.compile(r"[A-Za-z0-9_.-]{1,128}")) or not text(v) for k, v in tags.items() + ): + raise failure("bootstrap-input-invalid", "Tags must be bounded non-secret labels.") + if request["action"] == "create" and any(k in tags for k in ("foundry-iq-owner", "foundry-iq-operation")): + raise failure("bootstrap-input-invalid", "Creation ownership tags are generated, not caller overrides.") + closed(request["prerequisites"], {"quota", "pricing", "network", "requirements", "exclusive_name_authority"}, "Prerequisites") + for key, value in request["prerequisites"].items(): + if key == "exclusive_name_authority" and request["action"] == "reuse" and value is None: + continue + field = f"prerequisites.{key}" + if value is None or isinstance(value, str) and not value.strip(" "): + raise failure("bootstrap-prerequisite-missing", f"{field} requires nonempty owner-verified evidence.") + if not isinstance(value, str) or len(value) > 256: + raise failure("bootstrap-prerequisite-invalid", f"{field} must be a string of 1-256 printable characters.") + for position, character in enumerate(value, 1): + if not character.isprintable(): + raise failure( + "bootstrap-prerequisite-invalid", + f"{field} contains non-printable U+{ord(character):04X} at character {position}; evidence values are withheld.", + ) + closed(request["limits"], {"command_seconds", "wait_seconds", "wait_interval_seconds"}, "Limits") + for key, ceiling in (("command_seconds", 60), ("wait_seconds", 900), ("wait_interval_seconds", 30)): + if type(request["limits"][key]) is not int or not 1 <= request["limits"][key] <= ceiling: + raise failure("bootstrap-input-invalid", "Select explicit bounded command and readiness limits.") + private_directory(request["receipt_dir"]) + return request + + +def ids(request): + group = "/subscriptions/" + request["subscription_id"] + "/resourceGroups/" + request["resource_group"] + return group, group + "/providers/Microsoft.Search/searchServices/" + request["name"] + + +def provider_id(request): + return "/subscriptions/" + request["subscription_id"] + "/providers/Microsoft.Search" + + +def search_locations(value, expected_id): + if ( + not isinstance(value, dict) or folded(value.get("id")) != expected_id.casefold() + or folded(value.get("namespace")) != "microsoft.search" + or not isinstance(value.get("resourceTypes"), list) + or any(not isinstance(item, dict) or not isinstance(item.get("resourceType"), str) + for item in value["resourceTypes"]) + ): + raise failure("bootstrap-region-metadata-invalid", "Search provider metadata is incomplete or belongs to another subscription/provider.") + services = [item for item in value["resourceTypes"] if folded(item["resourceType"]) == "searchservices"] + if len(services) != 1: + raise failure("bootstrap-region-metadata-invalid", "Exactly one Search searchServices resource type is required.") + locations = services[0].get("locations") + if ( + not isinstance(locations, list) or not 1 <= len(locations) <= 256 + or any(normalize_azure_location(location) is None for location in locations) + ): + raise failure("bootstrap-region-metadata-invalid", "Search supported locations are missing, malformed or exceed the bounded list.") + return sorted({normalize_azure_location(location) for location in locations}) + + +def desired(request, operation_id): + # Only new planning normalizes choices; retained approved bodies keep their original representation. + tags = dict(request["tags"]) + if request["action"] == "create": + tags.update({"foundry-iq-owner": request["owner"], "foundry-iq-operation": operation_id}) + return { + "location": request["location"], "tags": tags, "sku": {"name": request["sku"]}, + "identity": {"type": "SystemAssigned"}, + "properties": {"replicaCount": request["replicas"], "partitionCount": request["partitions"], + "publicNetworkAccess": "Enabled", "disableLocalAuth": True}, + } + + +def _json(raw): + if len(raw) > MAX_BYTES: + raise failure("bootstrap-cli-output-limit", "CLI response exceeds its bound.") + try: + value = json.loads(raw.decode("utf-8")) + json.dumps(value, allow_nan=False) + if not isinstance(value, dict) or value.get("nextLink") or value.get("nextToken"): + raise ValueError("Not a complete object") + return value + except (ValueError, UnicodeError, RecursionError) as exc: + raise failure("bootstrap-response-invalid", "CLI must return one complete JSON object; empty or truncated output is not evidence.") from exc + + +def _error(raw): + """Only codes/UUIDs/reference IDs survive; arbitrary Azure messages are not safe receipts.""" + message = raw.decode("utf-8", errors="replace") + code_match = re.search(r'"code"\s*:\s*"([A-Za-z][A-Za-z0-9_.-]{0,100})"|(?:ERROR:\s*)?\(([A-Za-z][A-Za-z0-9_.-]{0,100})\)', message) + code = next((g for g in code_match.groups() if g), None) if code_match else "bootstrap-cli-failed" + request_match = re.search(r"request[\s-]?id[\"'\s:]+([0-9a-f-]{36})", message, re.I) + request_id = request_match.group(1).lower() if request_match and GUID.fullmatch(request_match.group(1).lower()) else None + status_match = re.search(r'"status(?:Code)?"\s*:\s*(4\d\d|5\d\d)', message) + status = int(status_match.group(1)) if status_match else None + reason = re.match(r"(?:ERROR:\s*)?([A-Za-z ]+)\(", message.strip()) + if status is None and reason: + status = {"Bad Request": 400, "Unauthorized": 401, "Forbidden": 403, "Not Found": 404, + "Conflict": 409, "Too Many Requests": 429, "Internal Server Error": 500, + "Service Unavailable": 503, "Gateway Timeout": 504}.get(reason.group(1)) + references = sorted(set(POLICY_ID.findall(message))) + implicated = code == "RequestDisallowedByPolicy" or bool(references) or '"PolicyViolation"' in message + result = failure(code, "Azure CLI failed; untrusted original message text is withheld from output and receipts.") + result.request_id, result.http_status = request_id, status + result.policy_ids = references[:8] + result.policy_implicated = implicated + result.message_digest = digest(message) + result.policy_overflow = len(references) > 8 + return result + + +def projection(resource, required_tags=()): + """Store selected configuration, not arbitrary provider fields/error text.""" + if resource is None: + return None + props, identity = resource.get("properties", {}), resource.get("identity", {}) + if not isinstance(props, dict) or not isinstance(identity, dict): + raise failure("bootstrap-response-invalid", "Search properties and identity must be objects.") + tags = resource.get("tags") + selected = {k.casefold() for k in required_tags} + return { + "id": resource.get("id"), "location": resource.get("location"), + "tags": {k: v for k, v in tags.items() if folded(k) in selected} if isinstance(tags, dict) else tags, + "sku": {"name": resource["sku"].get("name")} if isinstance(resource.get("sku"), dict) else resource.get("sku"), + "identity": {k: identity.get(k) for k in ("type", "principalId", "tenantId")}, + "properties": {k: props.get(k) for k in ( + "replicaCount", "partitionCount", "publicNetworkAccess", "disableLocalAuth", + "provisioningState", "status", "endpoint", + )}, + } + + +def matching(resource, request, body): + value = projection(resource, body["tags"]) + tags = resource.get("tags") if resource is not None else None + if tags is None and not body["tags"]: + tags = {} + group, target = ids(request) + location = normalize_azure_location(body["location"]) + if ( + value is None or not isinstance(value["id"], str) or value["id"].casefold() != target.casefold() + or location is None or normalize_azure_location(value["location"]) != location + or not isinstance(tags, dict) + or any([v for k, v in tags.items() if folded(k) == key.casefold()] != [expected] + for key, expected in body["tags"].items()) + or not isinstance(value["sku"], dict) or folded(value["sku"].get("name")) != body["sku"]["name"] + or folded(value["identity"]["type"]) != "systemassigned" + or any(type(value["properties"][k]) is not type(v) or ( + folded(value["properties"][k]) != v.casefold() if isinstance(v, str) else value["properties"][k] != v + ) for k, v in body["properties"].items()) + ): + raise failure("bootstrap-state-conflict", "Selected Search definition differs; shared or foreign state will not be modified.") + props = resource["properties"] + if props.get("authOptions") not in (None, {}) or resource["identity"].get("userAssignedIdentities") not in (None, {}): + raise failure("bootstrap-state-conflict", "Observed authentication configuration is not the selected keyless system identity.") + rules = props.get("networkRuleSet") + if rules is not None and (not isinstance(rules, dict) or rules.get("ipRules") not in (None, []) + or (rules.get("bypass") is not None and folded(rules["bypass"]) != "none")): + raise failure("bootstrap-state-conflict", "Additional network restrictions require the owning procedure.") + if props.get("privateEndpointConnections") not in (None, []) or props.get("sharedPrivateLinkResources") not in (None, []): + raise failure("bootstrap-state-conflict", "Private/shared network resources are outside this bootstrap slice.") + return value + + +def folded(value): + return value.casefold() if isinstance(value, str) else None + + +def reuse_binding(resource, request, body): + if resource is None: + return None + value = ready(resource, request, body) + if value is None: + raise failure("bootstrap-not-ready", "Fresh Search readback does not establish ARM readiness.") + return {"id": folded(value["id"]), "principal": folded(value["identity"]["principalId"]), + "tenant": folded(value["identity"]["tenantId"])} + + +def ready(resource, request, body): + return _readiness(resource, request, matching(resource, request, body)) + + +def _readiness(resource, request, value): + props = value["properties"] + if folded(props["provisioningState"]) in ("failed", "canceled", "cancelled", "deleting", "deleted") or folded(props["status"]) in ("error", "degraded"): + evidence = [resource.get("error"), resource["properties"].get("error"), + resource["properties"].get("statusDetails")] + raw = "\n".join(value if isinstance(value, str) else json.dumps(value) + for value in evidence if isinstance(value, (str, dict, list))) + error = _error(raw.encode("utf-8")) + if error.code == "bootstrap-cli-failed": + error.code = "bootstrap-provisioning-failed" + error.message = "Search entered a failed, deleting or degraded ARM state; provider message text is withheld." + error.args = (error.message,) + error.creation_failure = True + raise error + if ( + folded(props["provisioningState"]) != "succeeded" or folded(props["status"]) != "running" + or not text(folded(value["identity"]["principalId"]), GUID) + or folded(value["identity"]["tenantId"]) != request["tenant_id"] + or folded(props["endpoint"]) not in ( + "https://" + request["name"] + ".search.windows.net", + "https://" + request["name"] + ".search.windows.net/", + ) + ): + return None + return value + + +class Session: + def __init__(self, request, operation_id, cli=run_cli, clock=time.monotonic, *, target=None): + self.request, self.operation_id = request, operation_id + self.cli, self.clock = cli, clock + self.directory = private_directory(request["receipt_dir"]) + self.sequence = 0 + self.attempted = False + self.approved = False + self.owned = False + self.warnings = [] + self.diagnosed = False + self.policy_references = [] + self.target = target if target is not None else ids(request)[1] + + def record(self, event, value): + self.sequence += 1 + private_file(self.directory, f"{self.operation_id}.{self.sequence:03d}.{uuid.uuid4().hex}.receipt.json", { + "schema_version": "1.0", "operation_id": self.operation_id, "operation": "bootstrap-search", + "target": self.target, "event": event, "evidence": value, + }) + + def call(self, args, timeout=None, mutation=False, waiting=False): + self.record("command", {"argv": args}) + if mutation: + self.attempted = True + try: + response = self.cli( + args + ["--subscription", self.request["subscription_id"]], + self.request["limits"]["wait_seconds"] if waiting else ( + min(self.request["limits"]["command_seconds"], timeout) if timeout is not None else self.request["limits"]["command_seconds"]), + ) + except HelperFailure as error: + if mutation and error.code in ("bootstrap-tool-unavailable", "bootstrap-cli-start-failed"): + self.attempted = False + raise + rc, stdout, stderr = response + if len(stdout) > MAX_BYTES or len(stderr) > MAX_BYTES: + raise failure("bootstrap-cli-output-limit", "CLI output exceeded its bound.") + if rc: + error = _error(stderr or stdout) + try: + self.record("failure", {"code": error.code, "status": error.http_status, "request_id": error.request_id, + "message_digest": error.message_digest, "message_withheld": True, + "policy_ids": error.policy_ids, "policy_references_limited": error.policy_overflow, + "warnings": error.warnings}) + except HelperFailure: + self.warnings.append("Original CLI failure could not be persisted; retain the returned failure and ownership.") + raise error + if waiting: + if stdout.strip() not in (b"", b"null"): + raise failure("bootstrap-wait-result-invalid", "Native wait returned an unexpected result; readiness is unresolved.") + return None + return _json(stdout) + + def get(self, timeout=None): + try: + result = self.call(["rest", "--method", "get", "--url", + "https://management.azure.com" + self.target + "?api-version=" + API], timeout) + except HelperFailure as exc: + if (exc.code == "ResourceNotFound" and exc.http_status in (None, 404) + and not getattr(exc, "cleanup_unconfirmed", False)): + self.record("readback", {"id": self.target, "absence": "ResourceNotFound"}) + return None + raise + if not isinstance(result.get("id"), str) or result["id"].casefold() != self.target.casefold(): + raise failure("bootstrap-response-invalid", "ARM returned another identity.") + self.record("readback", {"id": self.target, "digest": digest(result)}) + return result + + def account_context(self): + account = self.call(["account", "show"]) + if (folded(account.get("id")) != self.request["subscription_id"] or folded(account.get("tenantId")) != self.request["tenant_id"] + or folded(account.get("state")) != "enabled" or account.get("environmentName") != "AzureCloud"): + raise failure("bootstrap-context-conflict", "Signed-in subscription/tenant is not the selected enabled context.") + user = account.get("user") + if not isinstance(user, dict) or not text(user.get("name")) or folded(user.get("type")) not in ("user", "serviceprincipal"): + raise failure("bootstrap-context-conflict", "Signed-in principal evidence is missing or unsupported.") + principal = folded(user["name"]) if text(folded(user["name"]), GUID) else user["name"] + return {"id": folded(account["id"]), "tenantId": folded(account["tenantId"]), "environmentName": account["environmentName"], + "user": {"name": principal, "type": folded(user["type"])}} + + def context(self): + account = self.account_context() + group = self.call(["group", "show", "--name", self.request["resource_group"]]) + props = group.get("properties") + if ( + not isinstance(group.get("id"), str) or group["id"].casefold() != ids(self.request)[0].casefold() + or not isinstance(props, dict) or folded(props.get("provisioningState")) != "succeeded" + ): + raise failure("bootstrap-group-not-ready", "The explicitly selected existing resource group must be ready.") + context = {"account": digest(account), "group": digest({"id": folded(group["id"]), "ready": True})} + self.record("context", {"account": account, "group_id": ids(self.request)[0], "group_digest": context["group"]}) + return context + + def regions(self, *, selected=None): + identity = provider_id(self.request) + value = self.call(["rest", "--method", "get", "--url", + "https://management.azure.com" + identity + "?api-version=" + PROVIDER_API]) + locations = search_locations(value, identity) + self.record("search-regions", {"provider_id": identity, "resource_type": "searchServices", + "api_version": PROVIDER_API, "digest": digest(value), "locations": locations}) + if selected is not None and normalize_azure_location(selected) not in locations: + error = failure("bootstrap-region-unsupported", "Select an advertised Search region; no typo correction or deployment-capacity inference.") + error.available_locations = locations + raise error + return locations + + def policy_evidence(self, original): + if self.diagnosed or not getattr(original, "policy_implicated", False): + return + self.diagnosed = True + end = self.clock() + 60 + queue = list(original.policy_ids) + seen = set() + self.warnings.append("Policy evidence is diagnostic only; cause/compliance remains unresolved for the policy owner.") + scopes = {self.target.casefold(), ids(self.request)[0].casefold(), ("/subscriptions/" + self.request["subscription_id"]).casefold()} + while queue and len(seen) < 8: + identity = queue.pop(0) + if identity.casefold() in seen: + continue + seen.add(identity.casefold()) + self.policy_references.append(identity) + remaining = end - self.clock() + if remaining <= 0: + break + scope, tail = re.split(r"/providers/Microsoft.Authorization/", identity, flags=re.I) + if scope.casefold() not in scopes or "/" not in tail: + self.warnings.append("Referenced policy scope is unsupported; hand off to the policy owner.") + continue + kind, name = tail.split("/", 1) + if kind.casefold() == "policyassignments": + args = ["policy", "assignment", "show", "--name", name, "--scope", scope] + elif kind.casefold() == "policydefinitions" and scope.casefold() == ("/subscriptions/" + self.request["subscription_id"]).casefold(): + args = ["policy", "definition", "show", "--name", name] + else: + self.warnings.append("Referenced policy kind requires owner investigation.") + continue + try: + value = self.call(args, remaining) + if self.clock() >= end: + raise failure("bootstrap-policy-timeout", "Policy evidence arrived after the diagnostic budget.") + if not isinstance(value.get("id"), str) or value["id"].casefold() != identity.casefold(): + raise failure("bootstrap-policy-evidence-mismatch", "Policy read returned another identity.") + # Values/rules can contain sensitive literals: retain identity, digest and + # classification only; the policy owner reads the referenced object. + properties = value.get("properties", value) + if not isinstance(properties, dict): + raise failure("bootstrap-response-invalid", "Policy properties are malformed.") + reference = properties.get("policyDefinitionId") + if isinstance(reference, str) and POLICY_ID.fullmatch(reference): + queue.append(reference) + self.record("policy-evidence", {"id": identity, "digest": digest(value), + "referenced_definition_id": reference if isinstance(reference, str) and POLICY_ID.fullmatch(reference) else None, + "enforcementMode": properties.get("enforcementMode") if properties.get("enforcementMode") in ("Default", "DoNotEnforce", "Enroll") else None, + "interpretation": "unresolved"}) + except HelperFailure as secondary: + self.warnings.append("Secondary policy diagnostic failed: " + secondary.code) + self.warnings.extend(secondary.warnings) + if queue or getattr(original, "policy_overflow", False) or not original.policy_ids: + self.warnings.append("Policy references are missing or exceed this bounded collector; no truncated completeness claim.") + + def blocked(self, error): + if self.attempted: + error.partial = True + error.writes = [{"operation": "PUT", "resource_id": self.target, "submission": "attempted-unverified"}] + error.resources_remaining = [{"resource_id": self.target, "run_owned": self.owned, + "cleanup": "separate consent and fresh ownership/children/roles required"}] + error.warnings.extend(self.warnings) + result = blocked_result(error, outcome="bootstrap-search", fingerprint=None, owner=self.request.get("owner")) + if error.code == "bootstrap-region-unsupported" and hasattr(error, "available_locations"): + result["available_locations"] = error.available_locations + if result["status"] == "partial": + result["approved_plan"] = {"confirmed": self.approved, "artifact_id": self.operation_id} + result["attempted_writes"] = result.pop("completed_writes") + result["completed_writes"] = [] + if not self.owned: + result["resources_remaining"]["unverified"] = result["resources_remaining"]["run_owned"] + result["resources_remaining"]["run_owned"] = [] + result["receipt_id"] = self.operation_id + if self.diagnosed: + result["policy_handoff"] = {"referenced_ids": self.policy_references, "complete": False, + "next_step": "Policy owner investigates exact references; no correction or retry is approved."} + return result + + +def discover_regions(request, *, cli=run_cli): + closed(request, {"schema_version", "subscription_id", "tenant_id", "receipt_dir", "limits"}, "Region discovery") + reject_secrets(request) + closed(request["limits"], {"command_seconds"}, "Region discovery limits") + if ( + request["schema_version"] != SCHEMA + or not text(request["subscription_id"], GUID) or not text(request["tenant_id"], GUID) + or type(request["limits"]["command_seconds"]) is not int + or not 1 <= request["limits"]["command_seconds"] <= 60 + ): + raise failure("bootstrap-input-invalid", "Select subscription/tenant and a 1-60 second region-discovery command limit.") + session = Session(request, str(uuid.uuid4()), cli, target=provider_id(request)) + try: + session.record("context", {"account": session.account_context()}) + locations = session.regions() + return {"status": "discovered", "resource_type": "Microsoft.Search/searchServices", + "available_locations": locations, "receipt_id": session.operation_id, + "mutation_approval_required": False, "execution_required": False, "writes_performed": [], + "verification": "Advertised region support only; not SKU, quota, capacity, models or residency approval."} + except HelperFailure as error: + return session.blocked(error) + + +def plan_request(request, *, cli=run_cli, clock=time.monotonic, execution_output=None): + validate(request) + request = copy.deepcopy(request) + request["location"] = normalize_azure_location(request["location"]) + operation_id = str(uuid.uuid4()) + session = Session(request, operation_id, cli, clock) + try: + context = session.context() + observed = session.get() + body = desired(request, operation_id) + if request["action"] == "create": + if observed is not None: + raise failure("bootstrap-state-conflict", "New intent requires exact-name absence; never overwrite or silently reuse.") + session.regions(selected=request["location"]) + refreshed = session.get() + if refreshed is not None: + raise failure("bootstrap-state-drift", "Exact-name absence changed; creation is blocked.") + else: + binding = reuse_binding(observed, request, body) + if binding is None: + raise failure("bootstrap-not-ready", "Reuse requires a running, keyless Search service with identity readback.") + refreshed = session.get() + if reuse_binding(refreshed, request, body) != binding: + raise failure("bootstrap-state-drift", "Selected Search identity changed; obtain fresh evidence.") + artifact = { + "operation": "bootstrap-search", "operation_id": operation_id, "created_at": int(time.time()), + "choices": copy.deepcopy(request), + "body": body, "before": {**context, "search": digest(observed)}, + } + envelope = {"schema_version": SCHEMA, "plan": artifact, + "approval": {"confirmed": False, "fingerprint": digest(artifact)}} + if execution_output is None: + private_file(session.directory, operation_id + ".plan.json", envelope) + else: + try: + from .private_artifacts import retain_execution_input + except ImportError: + from private_artifacts import retain_execution_input + execution_artifact = retain_execution_input(envelope, execution_output) + session.record("planned", {"body": body, "before": artifact["before"], + "readback": projection(refreshed, body["tags"]), "run_owned": False}) + create = request["action"] == "create" + if execution_output is not None: + return { + "status": "planned" if create else "reused", "artifact_id": operation_id, + "execution_artifact": execution_artifact, "execution_required": create, + "mutation_approval_required": create, "azure_mutation_performed": False, + "local_filesystem": {"artifact_created": True, "receipts_created": True, + "existing_files_changed": False}, + "summary": "Unapproved bootstrap artifact retained privately; review before separate approval.", + } + return { + "status": "planned" if create else "reused", "artifact_id": operation_id, + "execution_required": create, "mutation_approval_required": create, + "approval_summary": {"action": request["action"], "resource_id": session.target, + "location": request["location"], "sku": request["sku"], + "replicas": request["replicas"], "partitions": request["partitions"], + "network": "public; local authentication disabled; SystemAssigned", + "tags": body["tags"], "prerequisites": { + key: {"evidence_present": value is not None, + "verification": "caller-attested; not helper-verified" if value is not None else "not required for reuse"} + for key, value in request["prerequisites"].items() + }, + "limits": request["limits"], "cleanup": "separate approval"}, + "verification": {"arm_readiness": "verified" if not create else "unverified", + "data_plane_access": "unverified", "ingestion": "unverified", "retrieval": "unverified"}, + "writes_performed": [], "run_owned": False, + "observed_dependency": projection(refreshed, body["tags"]), + } + except HelperFailure as error: + return session.blocked(error) + + +def validate_artifact(envelope): + closed(envelope, {"schema_version", "plan", "approval"}, "Execution artifact") + if envelope["schema_version"] != SCHEMA: + raise failure("bootstrap-contract-version", "Regenerate the retained artifact with bootstrap schema 2.0; do not edit its checksum.") + plan, approval = envelope["plan"], envelope["approval"] + closed(plan, {"operation", "operation_id", "created_at", "choices", "body", "before"}, "Execution plan") + closed(approval, {"confirmed", "fingerprint"}, "Approval") + if ( + plan["operation"] != "bootstrap-search" + or not text(plan["operation_id"], GUID) or type(approval["confirmed"]) is not bool + or approval["fingerprint"] != digest(plan) + ): + raise failure("approval-mismatch", "Retain the unchanged planner artifact; integrity is checked internally.") + validate(plan["choices"]) + if type(plan["created_at"]) is not int or not plan["created_at"] <= time.time() <= plan["created_at"] + 900: + raise failure("bootstrap-artifact-expired", "Rerun planning; retained artifacts expire after fifteen minutes.") + closed(plan["before"], {"account", "group", "search"}, "Before-state") + if any(not text(v, HASH) for v in plan["before"].values()) or plan["body"] != desired(plan["choices"], plan["operation_id"]): + raise failure("bootstrap-artifact-invalid", "Artifact body or before-state differs from generated choices.") + return plan + + +@reporting("search-bootstrap") +def apply_artifact(envelope, *, approve=False, cli=run_cli, clock=time.monotonic, progress=None): + progress.update("validation") + plan = validate_artifact(envelope) + if approve is not True: + raise failure("approval-missing", "Explicit approval of the unchanged artifact is required before any execution reads or writes.") + request = plan["choices"] + if request["action"] != "create": + raise failure("bootstrap-execution-unnecessary", "Read-only reuse needs fresh planning, not mutation approval or execution.") + session = Session(request, plan["operation_id"], cli, clock) + session.approved = True + body = plan["body"] + try: + session.record("approved", {"plan": plan, "approval": {"confirmed": True, "fingerprint": digest(plan)}}) + progress.update("context-check") + context = session.context() + if context != {k: plan["before"][k] for k in ("account", "group")}: + raise failure("bootstrap-state-drift", "Account/group changed; rerun planning and discard old consent.") + progress.update("region-check") + session.regions(selected=request["location"]) + progress.update("absence-check") + if plan["before"]["search"] != digest(None) or session.get() is not None: + raise failure("bootstrap-state-drift", "Exact-name absence changed; never overwrite or retry through another name.") + body_path = private_file(session.directory, plan["operation_id"] + "." + uuid.uuid4().hex + ".body.json", body) + command = ["rest", "--method", "put", "--url", + "https://management.azure.com" + session.target + "?api-version=" + API, + "--headers", "Content-Type=application/json", "x-ms-client-request-id=" + str(uuid.uuid4()), + "--body", "@" + str(body_path)] + private_file(session.directory, plan["operation_id"] + ".submission.json", + {"operation_id": plan["operation_id"], "target": session.target, "body": body, "command": command}) + progress.update("search-submit") + try: + session.call(command, mutation=True) + except HelperFailure as original: + if not session.attempted: + raise + # Never retry PUT. Neither diagnostics nor a readback replaces the first error. + progress.update("arm-readback") + try: + observed = session.get() + if observed is not None: + matching(observed, request, body) + session.owned = True + session.record("failure-reconciliation", {"readback": projection(observed, body["tags"]), "run_owned": session.owned}) + if (original.code in ("RequestDisallowedByPolicy", "AuthorizationFailed", "InvalidSkuName") + and observed is None and not getattr(original, "cleanup_unconfirmed", False)): + session.attempted = False + except HelperFailure as secondary: + session.warnings.append("Read-only reconciliation failed: " + secondary.code) + session.warnings.extend(secondary.warnings) + session.policy_evidence(original) + raise original + progress.update("arm-wait") + try: + session.call(["resource", "wait", "--ids", session.target, "--api-version", API, + "--custom", WAIT_CONDITION, "--interval", str(request["limits"]["wait_interval_seconds"]), + "--timeout", str(request["limits"]["wait_seconds"])], waiting=True) + except HelperFailure as original: + try: + progress.update("arm-readback") + observed = session.get() + if observed is not None: + value = matching(observed, request, body) + session.owned = True + try: + _readiness(observed, request, value) + except HelperFailure as provider: + session.warnings.append("Terminal provider readback: " + provider.code) + session.record("terminal-provider-failure", { + "code": provider.code, "status": provider.http_status, "request_id": provider.request_id, + "message_digest": provider.message_digest, "message_withheld": True}) + session.policy_evidence(provider) + session.record("wait-failure-readback", {"readback": projection(observed, body["tags"]), "run_owned": session.owned}) + except HelperFailure as secondary: + session.warnings.append("Wait failure readback failed: " + secondary.code) + session.warnings.extend(secondary.warnings) + raise original + progress.update("arm-readback") + observed = session.get() + value = matching(observed, request, body) if observed is not None else None + if value is not None: + session.owned = True + value = _readiness(observed, request, value) + if value is None: + raise failure("bootstrap-readiness-timeout", "Final ARM readiness was not established; preserve the resource and operation receipts.") + session.record("arm-ready", {"readback": value, "run_owned": True}) + return {"status": "completed", "receipt_id": session.operation_id, "resource_id": session.target, + "run_owned": True, "writes_performed": [{"operation": "PUT", "resource_id": session.target}], + "observed_dependency": value, + "verification": {"arm_readiness": "verified", "data_plane_access": "unverified", + "ingestion": "unverified", "retrieval": "unverified"}, + "cleanup": "separate approval with fresh ownership/children/roles"} + except HelperFailure as error: + if session.attempted and getattr(error, "creation_failure", False): + session.policy_evidence(error) + return session.blocked(error) + + +def main(argv=None): + try: + from .private_artifacts import add_execution_output_argument, validate_execution_output_mode + except ImportError: + from private_artifacts import add_execution_output_argument, validate_execution_output_mode + parser = argparse.ArgumentParser(description="Discover Search regions or plan/apply one selected service; no models, roles or cleanup.") + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--plan", type=Path) + mode.add_argument("--apply", type=Path) + mode.add_argument("--regions", type=Path) + parser.add_argument("--approve", action="store_true") + add_execution_output_argument(parser) + add_progress_argument(parser) + args = parser.parse_args(argv) + try: + validate_execution_output_mode(args) + if (args.plan or args.regions) and args.approve: + raise failure("bootstrap-input-invalid", "Planning cannot approve mutations.") + document = read_json(args.plan or args.apply or args.regions) + result = (discover_regions(document) if args.regions else + plan_request(document, **({"execution_output": args.execution_output} if args.execution_output else {})) if args.plan else apply_artifact( + document, approve=args.approve, progress=Progress("search-bootstrap", enabled=args.progress))) + except HelperFailure as error: + result = blocked_result(error, outcome="bootstrap-search", fingerprint=None) + emit_result(result) + return 3 if result["status"] == "partial" else 2 if result["status"] == "blocked" else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/cleanup_plan.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/cleanup_plan.py index 4c365787..fa7aa4f7 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/cleanup_plan.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/cleanup_plan.py @@ -1,735 +1,735 @@ -from __future__ import annotations - -import argparse -import copy -import json -import re -import sys -from pathlib import Path -from typing import Any, Callable - -try: - from . import prompt_cleanup, prompt_connect, search_reconcile, _cleanup_dependencies as dependencies, _cleanup_receipts as receipts - from ._common import ( - SEARCH_AUDIENCE, MANAGEMENT_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, - blocked_result, digest, emit_result, http_request, load_approved_input, - reject_secrets, require_allowed_fields, sdk_error_metadata, sdk_error_status, - validate_search_endpoint, - ) -except ImportError: - import prompt_cleanup - import prompt_connect - import search_reconcile - import _cleanup_dependencies as dependencies - import _cleanup_receipts as receipts - from _common import ( - SEARCH_AUDIENCE, MANAGEMENT_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, - blocked_result, digest, emit_result, http_request, load_approved_input, - reject_secrets, require_allowed_fields, sdk_error_metadata, sdk_error_status, - validate_search_endpoint, - ) - - -RETAIN = [ - "outside-plan resources and consumers", "original local and Storage documents", - "Search service", "accounts", "projects", "models", "role assignments", -] -REQUEST_FIELDS = { - "schema_version", "owner", "target", "creation_input_file", - "creation_result_file", "creation_response_file", - "creation_receipt_file", "inventory_limits", "agent_version", "agent_creation_response_file", "agent_creation_receipt_file", -} -SEARCH_FIELDS = {"type", "endpoint", "api_version", "name"} -PROMPT_FIELDS = {"type", "project_resource_id", "project_endpoint", "name", "version"} -RESPONSE_FIELDS = {"schema_version", "target", "operation", "status", "request_id", "body", "generated_resources"} - - -def _failure(code: str, message: str) -> HelperFailure: - return HelperFailure(code, message, blocked_at="cleanup-planning") - - -def _object(value: Any, label: str) -> dict[str, Any]: - if not isinstance(value, dict): - raise _failure("input-schema-invalid", f"{label} must be an object.") - return value - - -def _text(value: Any) -> bool: - return isinstance(value, str) and bool(value.strip()) - - -def _read_json(path: Path) -> dict[str, Any]: - try: - result = json.loads(path.read_text(encoding="utf-8")) - json.dumps(result, allow_nan=False) - except (OSError, UnicodeError, ValueError) as exc: - raise _failure("input-unreadable", "Select readable retained UTF-8 JSON records.") from exc - return _object(result, "Record") - - -def _target(value: Any) -> dict[str, Any]: - target = copy.deepcopy(_object(value, "Cleanup target")) - kind = target.get("type") - if not isinstance(kind, str): - raise _failure("target-selection-required", "Select one typed cleanup target.") - if kind == "hosted": - require_allowed_fields(target, {"type"}, label="Hosted cleanup target") - raise _failure("hosted-cleanup-unsupported", "Hosted teardown remains unsupported; retain the deployment and toolbox.") - if kind in {"knowledge-base", "knowledge-source"}: - require_allowed_fields(target, SEARCH_FIELDS, label="Search cleanup target") - search_reconcile.resource_url({**target, "resource_type": kind}) - target["endpoint"] = validate_search_endpoint(target["endpoint"]) - elif kind in {"prompt-agent-version", "project-connection"}: - fields = PROMPT_FIELDS if kind == "prompt-agent-version" else PROMPT_FIELDS - {"version"} - require_allowed_fields(target, fields, label="Prompt cleanup target") - project_id, endpoint = prompt_connect._project_identity(target) - target["project_resource_id"] = project_id.casefold() - target["project_endpoint"] = endpoint - if not _text(target.get("name")): - raise _failure("target-selection-required", "Select one exact agent or connection name.") - if kind == "prompt-agent-version" and ( - not isinstance(target.get("version"), str) - or re.fullmatch(r"[1-9][0-9]*", target["version"]) is None - ): - raise _failure("target-selection-required", "Select one exact numeric Prompt version, not latest or a list.") - else: - raise _failure("target-selection-required", "Select one supported exact cleanup target.") - return target - - -def _path(request: dict[str, Any], key: str, base_dir: Path) -> Path: - value = request.get(key) - if not _text(value): - raise _failure("ownership-unproven", "Retained approval, result and definitive create response files are required.") - path = Path(value) - return path if path.is_absolute() else base_dir / path - - -def _records( - request: dict[str, Any], target: dict[str, Any], base_dir: Path, -) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: - if "creation_receipt_file" in request: - if target["type"] != "knowledge-source" or "creation_response_file" in request: - raise _failure("input-schema-invalid", "Select either an original Blob checkpoint or native result/response records.") - try: - from . import blob_recheck - except ImportError: - import blob_recheck - original = _read_json(_path(request, "creation_result_file", base_dir)) - reject_secrets(original) - retained = original.get("recheck_checkpoint") - if (original.get("outcome") != "create-blob-knowledge-source" or original.get("status") not in ("completed", "partial") - or not isinstance(retained, dict) or retained.get("status") != "retained"): - raise _failure( - "generated-creation-evidence-unavailable", - "Require the original creation result retaining its generated checkpoint; ACK/recovery GETs cannot prove original child versions.", - ) - prior, receipt = blob_recheck._load( - _path(request, "creation_input_file", base_dir), _path(request, "creation_receipt_file", base_dir), - ) - if prior["source"]["action"] != "create" or prior["owner"] != request["owner"]: - raise _failure("ownership-unproven", "Only an original run-owned creation checkpoint is eligible, never captured reuse.") - original_ownership = _object( - original.get("resources_remaining" if original["status"] == "partial" else "ownership"), - "Original creation ownership", - ) - original_owner = original.get("owner") if original["status"] == "partial" else original_ownership.get("owner") - original_owned = original_ownership.get("run_owned") - if (retained.get("evidence_digest") != receipt["integrity"] - or retained.get("operation_id") != receipt["operation_id"] - or original.get("approved_plan") != {"confirmed": True, "fingerprint": receipt["plan_digest"]} - or original_owner != prior["owner"] or not isinstance(original_owned, list) - or receipt["creation"]["ownership"]["run_owned"][0] not in original_owned): - raise _failure("generated-ownership-unproven", "The original creation result must bind this exact checkpoint and approval; never substitute recovery snapshots.") - result = {"status": "completed", **copy.deepcopy(receipt["creation"])} - generated = receipt["creation"]["source"]["generated"] - body = copy.deepcopy(prior["source"]["desired"]) - body["@odata.etag"] = receipt["creation"]["verification"]["readback"]["etag"] - body["azureBlobParameters"]["createdResources"] = {item["type"]: item["name"] for item in generated} - snapshots = [ - {"type": item["type"], "name": item["name"], "etag": receipt["configuration"][item["type"]]["etag"], - "definition_digest": receipt["configuration"][item["type"]]["digest"]} - for item in generated - ] - return prior, result, {"body": body, "_checkpoint_validated": True, "_generated_snapshots": snapshots} - paths = [_path(request, field, base_dir) for field in ( - "creation_input_file", "creation_result_file", "creation_response_file", - )] - try: - _, prior, fingerprint = load_approved_input(paths[0]) - except UnicodeError as exc: - raise _failure("input-unreadable", "Retained approval must be UTF-8 JSON.") from exc - result, response = _read_json(paths[1]), _read_json(paths[2]) - reject_secrets(result) - reject_secrets(response) - require_allowed_fields(response, RESPONSE_FIELDS, label="Definitive create response record") - if ( - response.get("schema_version") != "1.0" - or not _text(response.get("request_id")) - or _target(response.get("target")) != target - ): - raise _failure("ownership-unproven", "Original create response must bind the exact scope and native request ID.") - if ( - result.get("status") != "completed" - or result.get("approved_plan") != {"confirmed": True, "fingerprint": fingerprint} - or _object(result.get("ownership"), "Creation ownership").get("owner") != prior.get("owner") - or prior.get("owner") != request["owner"] - ): - raise _failure("ownership-unproven", "Retained completed result, approval and accountable owner must agree.") - _object(response.get("body"), "Create response body") - return prior, result, response - - -def _owned_entry(result: dict[str, Any], target: dict[str, Any]) -> dict[str, Any]: - identity = {key: target[key] for key in ("type", "name", "version") if key in target} - - def matches(item: Any) -> bool: - return isinstance(item, dict) and all(item.get(key) == value for key, value in identity.items()) - - resources = _object(result.get("resources"), "Creation resources") - ownership = _object(result.get("ownership"), "Creation ownership") - for container, keys in ( - (resources, ("created", "reused", "updated", "skipped")), - (ownership, ("run_owned", "reused_not_owned")), - ): - for key in keys: - if not isinstance(container.get(key, []), list): - raise _failure("ownership-unproven", "Creation resource records must be complete lists.") - created = [item for item in resources.get("created", []) if matches(item)] - owned = [item for item in ownership.get("run_owned", []) if matches(item)] - if ( - len(created) != 1 or len(owned) != 1 or created[0] != owned[0] - or any(matches(item) for key in ("reused", "updated", "skipped") for item in resources.get(key, [])) - or any(matches(item) for item in ownership.get("reused_not_owned", [])) - ): - raise _failure("ownership-unproven", "Only one definitively created run-owned resource is eligible; never reused or updated.") - return created[0] - - -def _search_prior(prior: dict[str, Any], target: dict[str, Any]) -> dict[str, Any]: - source = prior - if prior.get("operation") in ("reconcile-and-ingest", "reconcile-and-monitor"): - source = _object(prior.get("source"), "Prior source") - if _object(source.get("desired"), "Prior source definition").get("kind") == "file": - try: - from . import file_source - except ImportError: - import file_source - file_source._validate_plan(prior) - else: - try: - from . import blob_source - except ImportError: - import blob_source - blob_source._validate_plan(prior) - search_reconcile._validate_plan(source) - if ( - source.get("operation") != "reconcile" or source.get("action") != "create" - or search_reconcile.resource_url(source) != search_reconcile.resource_url( - {**target, "resource_type": target["type"]} - ) - ): - raise _failure("ownership-unproven", "Creation approval must select this exact source/base with action create.") - return source - - -def _summary(target: dict[str, Any]) -> dict[str, Any]: - is_agent = target["type"] == "prompt-agent-version" - summary = { - "delete": [copy.deepcopy(target)], - "retain": RETAIN + ( - ["agent container", "prior and other agent versions", "project connection", "KB", "KS"] - if is_agent else ["all KS and generated indexes/pipelines", "agent versions", "project connections"] - ), - "blocked": [], - "order": ["Only this exact target; stop after failure. No dependent cleanup is chained."], - "impact": ( - "The selected version and its tool binding become unavailable; the project connection remains." - if is_agent else "The selected KB retrieval endpoint becomes unavailable; retained consumers are not detached." - ), - "ownership": "Original successful-create evidence required; owner metadata and equivalent GET are not proof or RBAC.", - "approval": "Separate cleanup consent: change only approval.confirmed after review.", - "hosted_cleanup": "unsupported", - } - if target["type"] == "knowledge-source": - summary.update( - retain=RETAIN + ["all KBs", "other KS/pipelines", "agent versions", "project connections"], - impact="The selected source, its uploaded File copies/indexed content and exact generated objects are deleted; originals remain.", - order=["Referencing KBs must already be absent or no longer reference this source; no detach is performed.", - "Delete only the source through Search; the service owns the exact approved cascade."], - ) - elif target["type"] == "project-connection": - summary.update( - retain=RETAIN + ["agent containers", "prior/other agent versions", "KBs", "KS/pipelines"], - impact="The selected project connection is removed; no retained agent version or tool is edited.", - order=["Verify all consumers; delete and verify the explicitly selected owned version first, if any.", - "Rescan protected consumers and conditionally delete only this connection; stop on failure."], - ) - return summary - - -def _original_generated(response): - names = dependencies.generated(response["body"]) - if response.get("_checkpoint_validated") is True: - snapshots = response["_generated_snapshots"] - else: - records = response.get("generated_resources") - if not isinstance(records, list): - raise _failure( - "generated-creation-evidence-unavailable", - "Original generated-object GET snapshots with ETags are required: source ETag and createdResources names " - "cannot distinguish a replaced child. Use the retained Blob checkpoint or original generated-object audit records.", - ) - snapshots = [] - for record in records: - if not isinstance(record, dict) or set(record) != {"type", "body"}: - raise _failure("input-schema-invalid", "Original generated records require type and complete native body.") - if not isinstance(record["type"], str) or record["type"] not in names: - raise _failure("generated-ownership-unproven", "Unexpected original generated resource type.") - body = _object(record["body"], "Original generated body") - snapshots.append({ - "type": record["type"], "name": body.get("name"), "etag": body.get("@odata.etag"), - "definition_digest": digest(body), - }) - if ( - len(snapshots) != len(names) - or any(item.get("type") not in names or item.get("name") != names[item["type"]] - or not _text(item.get("etag")) for item in snapshots) - or len({item["type"] for item in snapshots}) != len(names) - ): - raise _failure("generated-ownership-unproven", "Original child snapshots must match the exact complete acknowledged cascade.") - return sorted(snapshots, key=lambda item: item["type"]) - - -def _absent(target: dict[str, Any], request_id: str | None = None) -> dict[str, Any]: - summary = _summary(target) - summary["delete"] = [] - if target["type"] == "knowledge-source": - summary["impact"] = "Source already absent; generated-object absence is unverified. No independent child deletion is authorized." - summary["retain"].append("unverified generated objects") - return { - "status": "already-absent", "outcome": "plan-cleanup", - "execution_required": False, "mutation_approval_required": False, - "approval_summary": summary, - "verification": {"absence": True, "request_ids": [request_id] if request_id else []}, - } - - -def _planned( - target: dict[str, Any], plan: dict[str, Any], executor: str, - retained_targets: list[dict[str, Any]], -) -> dict[str, Any]: - fingerprint = digest(plan) - summary = _summary(target) - summary["retained_targets"] = retained_targets - return { - "status": "planned", "outcome": "plan-cleanup", "executor": executor, - "execution_required": True, "mutation_approval_required": True, - "plan_fingerprint": fingerprint, - "execution_input": { - "schema_version": "1.0", "plan": plan, - "approval": {"confirmed": False, "fingerprint": fingerprint}, - }, - "approval_summary": summary, - } - - -def _plan_search( - request: dict[str, Any], target: dict[str, Any], prior: dict[str, Any], - result: dict[str, Any], response: dict[str, Any], *, - token_provider: TokenProvider, transport: Transport, -) -> dict[str, Any]: - source = _search_prior(prior, target) - owned = _owned_entry(result, target) - body = response["body"] - owned_digest = digest(search_reconcile._definition(body)) - etag = body.get("@odata.etag") - if ( - (response.get("_checkpoint_validated") is not True and ( - response.get("operation") != "search-create" or type(response.get("status")) is not int or response["status"] != 201 - )) or not _text(etag) - or body.get("name") != target["name"] - or not search_reconcile.definitions_match(source["desired"], body) - or owned.get("definition_digest") != owned_digest or owned.get("etag") != etag - ): - raise _failure("ownership-unproven", "Require the original HTTP 201 create body and matching retained owned readback, not GET recovery.") - plan = { - "operation": "delete", "outcome": "cleanup-search-resource", "plan_kind": "cleanup", - "cleanup_approved": True, "owner": request["owner"], - "resource_type": target["type"], "endpoint": target["endpoint"], - "name": target["name"], "api_version": target["api_version"], - "owned_definition_digest": owned_digest, "expected_etag": etag, - } - token = token_provider(SEARCH_AUDIENCE) - current, request_id = search_reconcile.read_resource(search_reconcile.resource_url(plan), token, transport=transport) - if current is None: - return _absent(target, request_id) - if ( - digest(search_reconcile._definition(current)) != owned_digest - or current.get("@odata.etag") != etag - ): - raise _failure("definition-drift", "Current definition or creation ETag changed; same-name replacement is not owned.") - if target["type"] == "knowledge-source": - original = _original_generated(response) - if dependencies.generated(current) != dependencies.generated(body): - raise _failure("generated-ownership-unproven", "Generated identities differ from the acknowledged source creation.") - snapshot = dependencies.search_snapshot( - plan, current, token, transport=transport, bounds=dependencies.limits(request.get("inventory_limits")), - ) - if snapshot["generated"] != original: - raise _failure("generated-incarnation-drift", "Generated definitions/ETags differ from original retained ownership evidence.") - plan["dependency_guard"] = snapshot - search_reconcile._validate_plan(plan) - planned = _planned(target, plan, "helpers/search_reconcile.py", []) - planned["approval_summary"]["delete"].extend([ - {"type": item["type"], "name": item["name"], "endpoint": target["endpoint"], "service_managed": True} - for item in original - ]) - return planned - reject_secrets(current) - plan["desired"] = copy.deepcopy(current) - search_reconcile._validate_plan(plan) - retained = [ - {**target, "type": "knowledge-source", "name": source["name"]} - for source in current["knowledgeSources"] - ] - return _planned(target, plan, "helpers/search_reconcile.py", retained) - - -def _plan_prompt( - request: dict[str, Any], target: dict[str, Any], prior: dict[str, Any], - result: dict[str, Any], response: dict[str, Any], *, - sdk_loader: Callable[[], tuple[Any, Any, Any, Any, Any]], -) -> dict[str, Any]: - prompt_connect._validate_plan(prior) - owned = _owned_entry(result, target) - prior_project, prior_endpoint = prompt_connect._project_identity(prior) - body = response["body"] - require_allowed_fields(body, {"name", "version", "definition", "id", "created_at"}, label="SDK create-version response") - definition = _object(body.get("definition"), "Created Prompt definition") - if ( - response.get("operation") != "agents.create_version" or response.get("status") != "succeeded" - or prior_project.casefold() != target["project_resource_id"] - or prior_endpoint != target["project_endpoint"] - or prior["agent"]["name"] != target["name"] or prior["agent"]["version"] == target["version"] - or body.get("name") != target["name"] or body.get("version") != target["version"] - or definition.get("kind") != "prompt" - or owned.get("definition_digest") != digest(definition) - or receipts.version_identity(body) is None - ): - raise _failure("ownership-unproven", "Require the original successful SDK new-version return, never the baseline or recovered equivalent.") - AIProjectClient, _, _, _, extras = sdk_loader() - AzureCliCredential, AzureError = extras - client = AIProjectClient(endpoint=target["project_endpoint"], credential=AzureCliCredential()) - try: - try: - current = client.agents.get_version(agent_name=target["name"], agent_version=target["version"]) - except AzureError as exc: - if sdk_error_status(exc) == 404: - return _absent(target, sdk_error_metadata(exc).get("request_id")) - raise HelperFailure( - message="Exact Prompt version readback failed; no inventory or deletion attempted.", - blocked_at="cleanup-planning", **sdk_error_metadata(exc, "agent-readback-failed"), - ) from exc - try: - actual = current.definition.as_dict() - identity_matches = current.name == target["name"] and str(current.version) == target["version"] - except (AttributeError, TypeError, ValueError) as exc: - raise _failure("agent-readback-invalid", "Exact version readback is incomplete.") from exc - if (not identity_matches or not isinstance(actual, dict) or digest(actual) != owned["definition_digest"] - or receipts.version_identity(current) != receipts.version_identity(body)): - raise _failure("definition-drift", "Exact Prompt version identity or definition changed.") - finally: - client.close() - plan = { - "operation": "delete", "outcome": "cleanup-prompt-version", "plan_kind": "cleanup", - "cleanup_approved": True, "sdk_major": 2, "owner": request["owner"], - "project_resource_id": target["project_resource_id"], "project_endpoint": target["project_endpoint"], - "agent": { - "name": target["name"], "version": target["version"], "run_owned": True, - "owned_definition_digest": owned["definition_digest"], - "owned_version_identity": receipts.version_identity(body), - }, - } - prompt_cleanup._validate_plan(plan) - return _planned(target, plan, "helpers/prompt_cleanup.py", [ - {**target, "version": prior["agent"]["version"]}, - { - "type": "project-connection", "project_resource_id": target["project_resource_id"], - "project_endpoint": target["project_endpoint"], "name": prior["connection"]["name"], - }, - ]) - - -def _plan_connection(request, target, prior, result, response, *, base_dir, token_provider, transport, sdk_loader): - prompt_connect._validate_plan(prior) - project, endpoint = prompt_connect._project_identity(prior) - body = response["body"] - created = {"type": "project-connection", "name": target["name"]} - owned_write = {"action": "created", "connection": target["name"]} - resources = _object(result.get("resources"), "Creation resources") - ownership = _object(result.get("ownership"), "Creation ownership") - verification = _object(result.get("verification"), "Creation verification") - readback = _object(verification.get("connection_readback"), "Creation connection readback") - def matches(item): - return isinstance(item, dict) and ( - (item.get("type") == "project-connection" and item.get("name") == target["name"]) - or item.get("connection") == target["name"] - ) - - if any(not isinstance(container.get(key, []), list) for container, keys in ( - (resources, ("created", "reused", "updated", "skipped")), (ownership, ("run_owned", "reused_not_owned")), - ) for key in keys): - raise _failure("ownership-unproven", "Creation resource and ownership lists must be complete.") - etag = body.get("etag") or body.get("@odata.etag") - if ( - project.casefold() != target["project_resource_id"] or endpoint != target["project_endpoint"] - or prior["connection"]["name"] != target["name"] or prior["connection"]["action"] != "create" - or response.get("operation") != "project-connection-create" or type(response.get("status")) is not int - or response["status"] != 201 or not _text(etag) - or [item for item in resources.get("created", []) if matches(item)] != [created] - or any(matches(item) for key in ("reused", "updated", "skipped") for item in resources.get(key, [])) - or [item for item in ownership.get("run_owned", []) if matches(item)] != [owned_write] - or any(matches(item) for item in ownership.get("reused_not_owned", [])) - or readback.get("name") != target["name"] or readback.get("definition_digest") != digest(body) - ): - raise _failure("ownership-unproven", "Retain the exact acknowledged created project connection; updated/reused/recovered GETs are not create evidence.") - if not prompt_connect._connection_readback(prior, body)[0]: - raise _failure("ownership-unproven", "The original connection body must match its approved project and configuration.") - plan = { - "operation": "delete", "outcome": "cleanup-prompt-connection", "plan_kind": "cleanup", - "cleanup_approved": True, "sdk_major": 2, "owner": request["owner"], - "project_resource_id": target["project_resource_id"], "project_endpoint": target["project_endpoint"], - "connection": {"name": target["name"], "run_owned": True, "expected_etag": etag, "owned_definition_digest": digest(body)}, - } - url = prompt_connect._connection_url(plan) - token = token_provider(MANAGEMENT_AUDIENCE) - current, request_id = prompt_cleanup._get_connection(url, token, transport=transport) - if current is not None and current != body: - raise _failure("definition-drift", "The project connection changed since acknowledged creation.") - selected = None - if "agent_version" in request: - selected_target = { - **target, "type": "prompt-agent-version", "name": prior["agent"]["name"], "version": request["agent_version"], - } - selected_request = { - "schema_version": "1.0", "owner": request["owner"], "target": selected_target, - "creation_input_file": request["creation_input_file"], "creation_result_file": request["creation_result_file"], - "creation_response_file": request["agent_creation_response_file"], - } - selected = plan_cleanup(selected_request, base_dir=base_dir, token_provider=token_provider, transport=transport, sdk_loader=sdk_loader) - if selected["status"] == "planned": - plan["agent"] = selected["execution_input"]["plan"]["agent"] - if current is None: - if selected is not None and selected["status"] == "planned": - selected["approval_summary"]["already_absent"] = [target] - return selected - return _absent(target, request_id) - plan["dependency_guard"] = dependencies.prompt_snapshot( - plan, current, sdk_loader=sdk_loader, bounds=dependencies.limits(request.get("inventory_limits")), - ) - refreshed, _ = prompt_cleanup._get_connection(url, token, transport=transport) - if refreshed != current: - raise _failure("definition-drift", "Connection changed during project consumer discovery.") - prompt_cleanup._validate_plan(plan) - retained = [ - {**target, "type": "prompt-agent-version", "name": item["name"], "version": item["version"]} - for item in plan["dependency_guard"]["versions"] - if not plan.get("agent") or (item["name"], item["version"]) != (plan["agent"]["name"], plan["agent"]["version"]) - ] - planned = _planned(target, plan, "helpers/prompt_cleanup.py", retained) - if selected is not None and selected["status"] == "planned": - planned["approval_summary"]["delete"].insert(0, selected["approval_summary"]["delete"][0]) - return planned - - -def _plan_protected(request, target, prior, record, *, base_dir, token_provider, transport, sdk_loader): - snapshot = record["snapshot"] - outcome = ("cleanup-search-resource" if target["type"] in ("knowledge-base", "knowledge-source") - else "cleanup-prompt-version" if target["type"] == "prompt-agent-version" else "cleanup-prompt-connection") - plan = {"operation": "delete", "outcome": outcome, "plan_kind": "cleanup", - "cleanup_approved": True, "owner": request["owner"]} - if record["owner"] != request["owner"]: - raise _failure("ownership-unproven", "Original producer owner and cleanup accountable owner differ.") - if target["type"] in ("knowledge-base", "knowledge-source"): - source = _search_prior(prior, target) - if snapshot["definition_digest"] != digest(search_reconcile._definition(source["desired"])): - raise _failure("ownership-unproven", "Receipt does not bind the original approved Search definition.") - plan.update(resource_type=target["type"], **{k: target[k] for k in ("endpoint", "name", "api_version")}, - owned_definition_digest=snapshot["definition_digest"], expected_etag=snapshot["etag"]) - token = token_provider(SEARCH_AUDIENCE) - current, request_id = search_reconcile.read_resource(search_reconcile.resource_url(plan), token, transport=transport) - if current is None: - return _absent(target, request_id) - if digest(search_reconcile._definition(current)) != snapshot["definition_digest"] or current.get("@odata.etag") != snapshot["etag"]: - raise _failure("definition-drift", "Current Search state differs from the original producer snapshot.") - if target["type"] == "knowledge-source": - if dependencies.generated(current) != record["acknowledgement"]["generated"]: - raise _failure("generated-ownership-unproven", "Current generated identities differ from the native create acknowledgement.") - plan["dependency_guard"] = dependencies.search_snapshot( - plan, current, token, transport=transport, bounds=dependencies.limits(request.get("inventory_limits"))) - if plan["dependency_guard"]["generated"] != snapshot["generated"]: - raise _failure("generated-incarnation-drift", "Generated resources differ from original producer snapshots.") - else: - reject_secrets(current) - plan["desired"] = current - search_reconcile._validate_plan(plan) - planned = _planned(target, plan, "helpers/search_reconcile.py", []) - if target["type"] == "knowledge-source": - planned["approval_summary"]["delete"].extend( - {"type": child["type"], "name": child["name"], "endpoint": target["endpoint"], "service_managed": True} - for child in snapshot["generated"]) - else: - planned["approval_summary"]["retained_targets"] = [ - {**target, "type": "knowledge-source", "name": item["name"]} for item in current["knowledgeSources"]] - return planned - if prior.get("operation") == "create-initial-prompt-agent": - try: - from . import _initial_prompt - except ImportError: - import _initial_prompt - _initial_prompt.validate(prior) - if (target["type"] != "prompt-agent-version" or record["acknowledgement"]["operation"] != "agents.create" - or snapshot["definition_digest"] != digest(prior["agent"]["definition"])): - raise _failure("ownership-unproven", "Initial receipts authorize only the acknowledged version, never a connection/container.") - else: - prompt_connect._validate_plan(prior) - if target["type"] == "prompt-agent-version" and record["acknowledgement"]["operation"] != "agents.create_version": - raise _failure("ownership-unproven", "Connect receipts must come from the native new-version operation.") - project, endpoint = prompt_connect._project_identity(prior) - if project.casefold() != target["project_resource_id"] or endpoint != target["project_endpoint"]: - raise _failure("ownership-unproven", "Receipt project differs from original creation approval.") - plan.update(sdk_major=2, project_resource_id=target["project_resource_id"], project_endpoint=target["project_endpoint"]) - if target["type"] == "prompt-agent-version": - if prior["agent"]["name"] != target["name"] or prior["agent"].get("version") == target["version"]: - raise _failure("ownership-unproven", "Only the acknowledged new version is eligible, never the baseline.") - Client, _, _, _, extras = sdk_loader() - Credential, AzureError = extras - client = Client(endpoint=endpoint, credential=Credential()) - try: - current = client.agents.get_version(agent_name=target["name"], agent_version=target["version"]) - if (current.name != target["name"] or str(current.version) != target["version"] - or digest(current.definition.as_dict()) != snapshot["definition_digest"] - or receipts.version_identity(current) != snapshot["version_identity"]): - raise _failure("definition-drift", "Selected version differs from the original SDK return.") - except AzureError as exc: - if sdk_error_status(exc) == 404: - return _absent(target) - raise HelperFailure(message="Exact version readback failed.", blocked_at="cleanup-planning", - **sdk_error_metadata(exc, "agent-readback-failed")) from exc - finally: - client.close() - plan["agent"] = {"name": target["name"], "version": target["version"], "run_owned": True, - "owned_definition_digest": snapshot["definition_digest"], "owned_version_identity": snapshot["version_identity"]} - prompt_cleanup._validate_plan(plan) - return _planned(target, plan, "helpers/prompt_cleanup.py", []) - if prior["connection"]["name"] != target["name"] or prior["connection"]["action"] != "create": - raise _failure("ownership-unproven", "Connection receipt must bind original create intent.") - plan["connection"] = {"name": target["name"], "run_owned": True, "expected_etag": snapshot["etag"], - "owned_definition_digest": snapshot["definition_digest"]} - token = token_provider(MANAGEMENT_AUDIENCE) - url = prompt_connect._connection_url(plan) - current, request_id = prompt_cleanup._get_connection(url, token, transport=transport) - if current is not None and (digest(current) != snapshot["definition_digest"] or not prompt_connect._connection_readback(prior, current)[0]): - raise _failure("definition-drift", "Connection differs from its original acknowledged producer state.") - selected = None - if "agent_version" in request: - selected = plan_cleanup({ - "schema_version": "1.0", "owner": request["owner"], - "target": {**target, "type": "prompt-agent-version", "name": prior["agent"]["name"], "version": request["agent_version"]}, - "creation_input_file": request["creation_input_file"], - "creation_receipt_file": request["agent_creation_receipt_file"], - }, base_dir=base_dir, token_provider=token_provider, transport=transport, sdk_loader=sdk_loader) - if selected["status"] == "planned": - plan["agent"] = selected["execution_input"]["plan"]["agent"] - if current is None: - if selected and selected["status"] == "planned": - selected["approval_summary"]["already_absent"] = [target] - return selected - return _absent(target, request_id) - plan["dependency_guard"] = dependencies.prompt_snapshot( - plan, current, sdk_loader=sdk_loader, bounds=dependencies.limits(request.get("inventory_limits"))) - if prompt_cleanup._get_connection(url, token, transport=transport)[0] != current: - raise _failure("definition-drift", "Connection changed during consumer discovery.") - prompt_cleanup._validate_plan(plan) - planned = _planned(target, plan, "helpers/prompt_cleanup.py", []) - if selected and selected["status"] == "planned": - planned["approval_summary"]["delete"].insert(0, selected["approval_summary"]["delete"][0]) - return planned - - -def plan_cleanup( - request: dict[str, Any], *, base_dir: Path = Path("."), - token_provider: TokenProvider = azure_cli_token, transport: Transport = http_request, - sdk_loader: Callable[[], tuple[Any, Any, Any, Any, Any]] = prompt_cleanup.load_cleanup_sdk, -) -> dict[str, Any]: - _object(request, "Cleanup planning request") - reject_secrets(request) - require_allowed_fields(request, REQUEST_FIELDS, label="Cleanup planning request") - if request.get("schema_version") != "1.0" or not _text(request.get("owner")): - raise _failure("input-schema-invalid", "schema_version 1.0 and accountable owner are required.") - target = _target(request.get("target")) - dependencies.limits(request.get("inventory_limits")) - selection_fields = {"agent_version", "agent_creation_response_file", "agent_creation_receipt_file"} & set(request) - evidence_field = "agent_creation_receipt_file" if "agent_creation_receipt_file" in request else "agent_creation_response_file" - if selection_fields and (selection_fields != {"agent_version", evidence_field} or target["type"] != "project-connection"): - raise _failure("input-schema-invalid", "Only connection cleanup can explicitly select both agent_version and its original create response.") - if selection_fields and ( - not isinstance(request["agent_version"], str) or re.fullmatch(r"[1-9][0-9]*", request["agent_version"]) is None - or not _text(request[evidence_field]) - ): - raise _failure("target-selection-required", "Select an exact numeric created agent version and original SDK response file.") - if "creation_receipt_file" in request: - try: - from .blob_recheck import read_private - except ImportError: - from blob_recheck import read_private - path = _path(request, "creation_receipt_file", base_dir) - if read_private(path).get("kind") == "cleanup-creation-receipt": - if ("creation_response_file" in request or "creation_result_file" in request - or (selection_fields and evidence_field != "agent_creation_receipt_file")): - raise _failure("input-schema-invalid", "Protected receipts cannot be mixed with manual response adapters.") - prior, record = receipts.load(_path(request, "creation_input_file", base_dir), path, target) - return _plan_protected(request, target, prior, record, base_dir=base_dir, - token_provider=token_provider, transport=transport, sdk_loader=sdk_loader) - if evidence_field == "agent_creation_receipt_file": - raise _failure("input-schema-invalid", "Selected producer receipts require a protected connection receipt.") - prior, result, response = _records(request, target, base_dir) - if target["type"] in {"knowledge-base", "knowledge-source"}: - return _plan_search( - request, target, prior, result, response, token_provider=token_provider, transport=transport, - ) - if target["type"] == "project-connection": - return _plan_connection( - request, target, prior, result, response, base_dir=base_dir, - token_provider=token_provider, transport=transport, sdk_loader=sdk_loader, - ) - return _plan_prompt(request, target, prior, result, response, sdk_loader=sdk_loader) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--plan", type=Path, required=True) - args = parser.parse_args(argv) - request: dict[str, Any] = {} - try: - request = _read_json(args.plan) - result = plan_cleanup(request, base_dir=args.plan.resolve().parent) - emit_result(result, preserve_unapproved_input=result["status"] == "planned") - except HelperFailure as failure: - result = blocked_result(failure, outcome="plan-cleanup", fingerprint=None, owner=request.get("owner")) - result["approval_summary"] = { - "delete": [], "retain": RETAIN + ["selected target and all its dependencies"], - "blocked": [failure.code], "order": [], "hosted_cleanup": "unsupported", - } - try: - result["approval_summary"]["retained_targets"] = [_target(request.get("target"))] - except HelperFailure: - pass - emit_result(result) - return 2 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) +from __future__ import annotations + +import argparse +import copy +import json +import re +import sys +from pathlib import Path +from typing import Any, Callable + +try: + from . import prompt_cleanup, prompt_connect, search_reconcile, _cleanup_dependencies as dependencies, _cleanup_receipts as receipts + from ._common import ( + SEARCH_AUDIENCE, MANAGEMENT_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, + blocked_result, digest, emit_result, http_request, load_approved_input, + reject_secrets, require_allowed_fields, sdk_error_metadata, sdk_error_status, + validate_search_endpoint, + ) +except ImportError: + import prompt_cleanup + import prompt_connect + import search_reconcile + import _cleanup_dependencies as dependencies + import _cleanup_receipts as receipts + from _common import ( + SEARCH_AUDIENCE, MANAGEMENT_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, + blocked_result, digest, emit_result, http_request, load_approved_input, + reject_secrets, require_allowed_fields, sdk_error_metadata, sdk_error_status, + validate_search_endpoint, + ) + + +RETAIN = [ + "outside-plan resources and consumers", "original local and Storage documents", + "Search service", "accounts", "projects", "models", "role assignments", +] +REQUEST_FIELDS = { + "schema_version", "owner", "target", "creation_input_file", + "creation_result_file", "creation_response_file", + "creation_receipt_file", "inventory_limits", "agent_version", "agent_creation_response_file", "agent_creation_receipt_file", +} +SEARCH_FIELDS = {"type", "endpoint", "api_version", "name"} +PROMPT_FIELDS = {"type", "project_resource_id", "project_endpoint", "name", "version"} +RESPONSE_FIELDS = {"schema_version", "target", "operation", "status", "request_id", "body", "generated_resources"} + + +def _failure(code: str, message: str) -> HelperFailure: + return HelperFailure(code, message, blocked_at="cleanup-planning") + + +def _object(value: Any, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise _failure("input-schema-invalid", f"{label} must be an object.") + return value + + +def _text(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _read_json(path: Path) -> dict[str, Any]: + try: + result = json.loads(path.read_text(encoding="utf-8")) + json.dumps(result, allow_nan=False) + except (OSError, UnicodeError, ValueError) as exc: + raise _failure("input-unreadable", "Select readable retained UTF-8 JSON records.") from exc + return _object(result, "Record") + + +def _target(value: Any) -> dict[str, Any]: + target = copy.deepcopy(_object(value, "Cleanup target")) + kind = target.get("type") + if not isinstance(kind, str): + raise _failure("target-selection-required", "Select one typed cleanup target.") + if kind == "hosted": + require_allowed_fields(target, {"type"}, label="Hosted cleanup target") + raise _failure("hosted-cleanup-unsupported", "Hosted teardown remains unsupported; retain the deployment and toolbox.") + if kind in {"knowledge-base", "knowledge-source"}: + require_allowed_fields(target, SEARCH_FIELDS, label="Search cleanup target") + search_reconcile.resource_url({**target, "resource_type": kind}) + target["endpoint"] = validate_search_endpoint(target["endpoint"]) + elif kind in {"prompt-agent-version", "project-connection"}: + fields = PROMPT_FIELDS if kind == "prompt-agent-version" else PROMPT_FIELDS - {"version"} + require_allowed_fields(target, fields, label="Prompt cleanup target") + project_id, endpoint = prompt_connect._project_identity(target) + target["project_resource_id"] = project_id.casefold() + target["project_endpoint"] = endpoint + if not _text(target.get("name")): + raise _failure("target-selection-required", "Select one exact agent or connection name.") + if kind == "prompt-agent-version" and ( + not isinstance(target.get("version"), str) + or re.fullmatch(r"[1-9][0-9]*", target["version"]) is None + ): + raise _failure("target-selection-required", "Select one exact numeric Prompt version, not latest or a list.") + else: + raise _failure("target-selection-required", "Select one supported exact cleanup target.") + return target + + +def _path(request: dict[str, Any], key: str, base_dir: Path) -> Path: + value = request.get(key) + if not _text(value): + raise _failure("ownership-unproven", "Retained approval, result and definitive create response files are required.") + path = Path(value) + return path if path.is_absolute() else base_dir / path + + +def _records( + request: dict[str, Any], target: dict[str, Any], base_dir: Path, +) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: + if "creation_receipt_file" in request: + if target["type"] != "knowledge-source" or "creation_response_file" in request: + raise _failure("input-schema-invalid", "Select either an original Blob checkpoint or native result/response records.") + try: + from . import blob_recheck + except ImportError: + import blob_recheck + original = _read_json(_path(request, "creation_result_file", base_dir)) + reject_secrets(original) + retained = original.get("recheck_checkpoint") + if (original.get("outcome") != "create-blob-knowledge-source" or original.get("status") not in ("completed", "partial") + or not isinstance(retained, dict) or retained.get("status") != "retained"): + raise _failure( + "generated-creation-evidence-unavailable", + "Require the original creation result retaining its generated checkpoint; ACK/recovery GETs cannot prove original child versions.", + ) + prior, receipt = blob_recheck._load( + _path(request, "creation_input_file", base_dir), _path(request, "creation_receipt_file", base_dir), + ) + if prior["source"]["action"] != "create" or prior["owner"] != request["owner"]: + raise _failure("ownership-unproven", "Only an original run-owned creation checkpoint is eligible, never captured reuse.") + original_ownership = _object( + original.get("resources_remaining" if original["status"] == "partial" else "ownership"), + "Original creation ownership", + ) + original_owner = original.get("owner") if original["status"] == "partial" else original_ownership.get("owner") + original_owned = original_ownership.get("run_owned") + if (retained.get("evidence_digest") != receipt["integrity"] + or retained.get("operation_id") != receipt["operation_id"] + or original.get("approved_plan") != {"confirmed": True, "fingerprint": receipt["plan_digest"]} + or original_owner != prior["owner"] or not isinstance(original_owned, list) + or receipt["creation"]["ownership"]["run_owned"][0] not in original_owned): + raise _failure("generated-ownership-unproven", "The original creation result must bind this exact checkpoint and approval; never substitute recovery snapshots.") + result = {"status": "completed", **copy.deepcopy(receipt["creation"])} + generated = receipt["creation"]["source"]["generated"] + body = copy.deepcopy(prior["source"]["desired"]) + body["@odata.etag"] = receipt["creation"]["verification"]["readback"]["etag"] + body["azureBlobParameters"]["createdResources"] = {item["type"]: item["name"] for item in generated} + snapshots = [ + {"type": item["type"], "name": item["name"], "etag": receipt["configuration"][item["type"]]["etag"], + "definition_digest": receipt["configuration"][item["type"]]["digest"]} + for item in generated + ] + return prior, result, {"body": body, "_checkpoint_validated": True, "_generated_snapshots": snapshots} + paths = [_path(request, field, base_dir) for field in ( + "creation_input_file", "creation_result_file", "creation_response_file", + )] + try: + _, prior, fingerprint = load_approved_input(paths[0]) + except UnicodeError as exc: + raise _failure("input-unreadable", "Retained approval must be UTF-8 JSON.") from exc + result, response = _read_json(paths[1]), _read_json(paths[2]) + reject_secrets(result) + reject_secrets(response) + require_allowed_fields(response, RESPONSE_FIELDS, label="Definitive create response record") + if ( + response.get("schema_version") != "1.0" + or not _text(response.get("request_id")) + or _target(response.get("target")) != target + ): + raise _failure("ownership-unproven", "Original create response must bind the exact scope and native request ID.") + if ( + result.get("status") != "completed" + or result.get("approved_plan") != {"confirmed": True, "fingerprint": fingerprint} + or _object(result.get("ownership"), "Creation ownership").get("owner") != prior.get("owner") + or prior.get("owner") != request["owner"] + ): + raise _failure("ownership-unproven", "Retained completed result, approval and accountable owner must agree.") + _object(response.get("body"), "Create response body") + return prior, result, response + + +def _owned_entry(result: dict[str, Any], target: dict[str, Any]) -> dict[str, Any]: + identity = {key: target[key] for key in ("type", "name", "version") if key in target} + + def matches(item: Any) -> bool: + return isinstance(item, dict) and all(item.get(key) == value for key, value in identity.items()) + + resources = _object(result.get("resources"), "Creation resources") + ownership = _object(result.get("ownership"), "Creation ownership") + for container, keys in ( + (resources, ("created", "reused", "updated", "skipped")), + (ownership, ("run_owned", "reused_not_owned")), + ): + for key in keys: + if not isinstance(container.get(key, []), list): + raise _failure("ownership-unproven", "Creation resource records must be complete lists.") + created = [item for item in resources.get("created", []) if matches(item)] + owned = [item for item in ownership.get("run_owned", []) if matches(item)] + if ( + len(created) != 1 or len(owned) != 1 or created[0] != owned[0] + or any(matches(item) for key in ("reused", "updated", "skipped") for item in resources.get(key, [])) + or any(matches(item) for item in ownership.get("reused_not_owned", [])) + ): + raise _failure("ownership-unproven", "Only one definitively created run-owned resource is eligible; never reused or updated.") + return created[0] + + +def _search_prior(prior: dict[str, Any], target: dict[str, Any]) -> dict[str, Any]: + source = prior + if prior.get("operation") in ("reconcile-and-ingest", "reconcile-and-monitor"): + source = _object(prior.get("source"), "Prior source") + if _object(source.get("desired"), "Prior source definition").get("kind") == "file": + try: + from . import file_source + except ImportError: + import file_source + file_source._validate_plan(prior) + else: + try: + from . import blob_source + except ImportError: + import blob_source + blob_source._validate_plan(prior) + search_reconcile._validate_plan(source) + if ( + source.get("operation") != "reconcile" or source.get("action") != "create" + or search_reconcile.resource_url(source) != search_reconcile.resource_url( + {**target, "resource_type": target["type"]} + ) + ): + raise _failure("ownership-unproven", "Creation approval must select this exact source/base with action create.") + return source + + +def _summary(target: dict[str, Any]) -> dict[str, Any]: + is_agent = target["type"] == "prompt-agent-version" + summary = { + "delete": [copy.deepcopy(target)], + "retain": RETAIN + ( + ["agent container", "prior and other agent versions", "project connection", "KB", "KS"] + if is_agent else ["all KS and generated indexes/pipelines", "agent versions", "project connections"] + ), + "blocked": [], + "order": ["Only this exact target; stop after failure. No dependent cleanup is chained."], + "impact": ( + "The selected version and its tool binding become unavailable; the project connection remains." + if is_agent else "The selected KB retrieval endpoint becomes unavailable; retained consumers are not detached." + ), + "ownership": "Original successful-create evidence required; owner metadata and equivalent GET are not proof or RBAC.", + "approval": "Separate cleanup consent: change only approval.confirmed after review.", + "hosted_cleanup": "unsupported", + } + if target["type"] == "knowledge-source": + summary.update( + retain=RETAIN + ["all KBs", "other KS/pipelines", "agent versions", "project connections"], + impact="The selected source, its uploaded File copies/indexed content and exact generated objects are deleted; originals remain.", + order=["Referencing KBs must already be absent or no longer reference this source; no detach is performed.", + "Delete only the source through Search; the service owns the exact approved cascade."], + ) + elif target["type"] == "project-connection": + summary.update( + retain=RETAIN + ["agent containers", "prior/other agent versions", "KBs", "KS/pipelines"], + impact="The selected project connection is removed; no retained agent version or tool is edited.", + order=["Verify all consumers; delete and verify the explicitly selected owned version first, if any.", + "Rescan protected consumers and conditionally delete only this connection; stop on failure."], + ) + return summary + + +def _original_generated(response): + names = dependencies.generated(response["body"]) + if response.get("_checkpoint_validated") is True: + snapshots = response["_generated_snapshots"] + else: + records = response.get("generated_resources") + if not isinstance(records, list): + raise _failure( + "generated-creation-evidence-unavailable", + "Original generated-object GET snapshots with ETags are required: source ETag and createdResources names " + "cannot distinguish a replaced child. Use the retained Blob checkpoint or original generated-object audit records.", + ) + snapshots = [] + for record in records: + if not isinstance(record, dict) or set(record) != {"type", "body"}: + raise _failure("input-schema-invalid", "Original generated records require type and complete native body.") + if not isinstance(record["type"], str) or record["type"] not in names: + raise _failure("generated-ownership-unproven", "Unexpected original generated resource type.") + body = _object(record["body"], "Original generated body") + snapshots.append({ + "type": record["type"], "name": body.get("name"), "etag": body.get("@odata.etag"), + "definition_digest": digest(body), + }) + if ( + len(snapshots) != len(names) + or any(item.get("type") not in names or item.get("name") != names[item["type"]] + or not _text(item.get("etag")) for item in snapshots) + or len({item["type"] for item in snapshots}) != len(names) + ): + raise _failure("generated-ownership-unproven", "Original child snapshots must match the exact complete acknowledged cascade.") + return sorted(snapshots, key=lambda item: item["type"]) + + +def _absent(target: dict[str, Any], request_id: str | None = None) -> dict[str, Any]: + summary = _summary(target) + summary["delete"] = [] + if target["type"] == "knowledge-source": + summary["impact"] = "Source already absent; generated-object absence is unverified. No independent child deletion is authorized." + summary["retain"].append("unverified generated objects") + return { + "status": "already-absent", "outcome": "plan-cleanup", + "execution_required": False, "mutation_approval_required": False, + "approval_summary": summary, + "verification": {"absence": True, "request_ids": [request_id] if request_id else []}, + } + + +def _planned( + target: dict[str, Any], plan: dict[str, Any], executor: str, + retained_targets: list[dict[str, Any]], +) -> dict[str, Any]: + fingerprint = digest(plan) + summary = _summary(target) + summary["retained_targets"] = retained_targets + return { + "status": "planned", "outcome": "plan-cleanup", "executor": executor, + "execution_required": True, "mutation_approval_required": True, + "plan_fingerprint": fingerprint, + "execution_input": { + "schema_version": "1.0", "plan": plan, + "approval": {"confirmed": False, "fingerprint": fingerprint}, + }, + "approval_summary": summary, + } + + +def _plan_search( + request: dict[str, Any], target: dict[str, Any], prior: dict[str, Any], + result: dict[str, Any], response: dict[str, Any], *, + token_provider: TokenProvider, transport: Transport, +) -> dict[str, Any]: + source = _search_prior(prior, target) + owned = _owned_entry(result, target) + body = response["body"] + owned_digest = digest(search_reconcile._definition(body)) + etag = body.get("@odata.etag") + if ( + (response.get("_checkpoint_validated") is not True and ( + response.get("operation") != "search-create" or type(response.get("status")) is not int or response["status"] != 201 + )) or not _text(etag) + or body.get("name") != target["name"] + or not search_reconcile.definitions_match(source["desired"], body) + or owned.get("definition_digest") != owned_digest or owned.get("etag") != etag + ): + raise _failure("ownership-unproven", "Require the original HTTP 201 create body and matching retained owned readback, not GET recovery.") + plan = { + "operation": "delete", "outcome": "cleanup-search-resource", "plan_kind": "cleanup", + "cleanup_approved": True, "owner": request["owner"], + "resource_type": target["type"], "endpoint": target["endpoint"], + "name": target["name"], "api_version": target["api_version"], + "owned_definition_digest": owned_digest, "expected_etag": etag, + } + token = token_provider(SEARCH_AUDIENCE) + current, request_id = search_reconcile.read_resource(search_reconcile.resource_url(plan), token, transport=transport) + if current is None: + return _absent(target, request_id) + if ( + digest(search_reconcile._definition(current)) != owned_digest + or current.get("@odata.etag") != etag + ): + raise _failure("definition-drift", "Current definition or creation ETag changed; same-name replacement is not owned.") + if target["type"] == "knowledge-source": + original = _original_generated(response) + if dependencies.generated(current) != dependencies.generated(body): + raise _failure("generated-ownership-unproven", "Generated identities differ from the acknowledged source creation.") + snapshot = dependencies.search_snapshot( + plan, current, token, transport=transport, bounds=dependencies.limits(request.get("inventory_limits")), + ) + if snapshot["generated"] != original: + raise _failure("generated-incarnation-drift", "Generated definitions/ETags differ from original retained ownership evidence.") + plan["dependency_guard"] = snapshot + search_reconcile._validate_plan(plan) + planned = _planned(target, plan, "helpers/search_reconcile.py", []) + planned["approval_summary"]["delete"].extend([ + {"type": item["type"], "name": item["name"], "endpoint": target["endpoint"], "service_managed": True} + for item in original + ]) + return planned + reject_secrets(current) + plan["desired"] = copy.deepcopy(current) + search_reconcile._validate_plan(plan) + retained = [ + {**target, "type": "knowledge-source", "name": source["name"]} + for source in current["knowledgeSources"] + ] + return _planned(target, plan, "helpers/search_reconcile.py", retained) + + +def _plan_prompt( + request: dict[str, Any], target: dict[str, Any], prior: dict[str, Any], + result: dict[str, Any], response: dict[str, Any], *, + sdk_loader: Callable[[], tuple[Any, Any, Any, Any, Any]], +) -> dict[str, Any]: + prompt_connect._validate_plan(prior) + owned = _owned_entry(result, target) + prior_project, prior_endpoint = prompt_connect._project_identity(prior) + body = response["body"] + require_allowed_fields(body, {"name", "version", "definition", "id", "created_at"}, label="SDK create-version response") + definition = _object(body.get("definition"), "Created Prompt definition") + if ( + response.get("operation") != "agents.create_version" or response.get("status") != "succeeded" + or prior_project.casefold() != target["project_resource_id"] + or prior_endpoint != target["project_endpoint"] + or prior["agent"]["name"] != target["name"] or prior["agent"]["version"] == target["version"] + or body.get("name") != target["name"] or body.get("version") != target["version"] + or definition.get("kind") != "prompt" + or owned.get("definition_digest") != digest(definition) + or receipts.version_identity(body) is None + ): + raise _failure("ownership-unproven", "Require the original successful SDK new-version return, never the baseline or recovered equivalent.") + AIProjectClient, _, _, _, extras = sdk_loader() + AzureCliCredential, AzureError = extras + client = AIProjectClient(endpoint=target["project_endpoint"], credential=AzureCliCredential()) + try: + try: + current = client.agents.get_version(agent_name=target["name"], agent_version=target["version"]) + except AzureError as exc: + if sdk_error_status(exc) == 404: + return _absent(target, sdk_error_metadata(exc).get("request_id")) + raise HelperFailure( + message="Exact Prompt version readback failed; no inventory or deletion attempted.", + blocked_at="cleanup-planning", **sdk_error_metadata(exc, "agent-readback-failed"), + ) from exc + try: + actual = current.definition.as_dict() + identity_matches = current.name == target["name"] and str(current.version) == target["version"] + except (AttributeError, TypeError, ValueError) as exc: + raise _failure("agent-readback-invalid", "Exact version readback is incomplete.") from exc + if (not identity_matches or not isinstance(actual, dict) or digest(actual) != owned["definition_digest"] + or receipts.version_identity(current) != receipts.version_identity(body)): + raise _failure("definition-drift", "Exact Prompt version identity or definition changed.") + finally: + client.close() + plan = { + "operation": "delete", "outcome": "cleanup-prompt-version", "plan_kind": "cleanup", + "cleanup_approved": True, "sdk_major": 2, "owner": request["owner"], + "project_resource_id": target["project_resource_id"], "project_endpoint": target["project_endpoint"], + "agent": { + "name": target["name"], "version": target["version"], "run_owned": True, + "owned_definition_digest": owned["definition_digest"], + "owned_version_identity": receipts.version_identity(body), + }, + } + prompt_cleanup._validate_plan(plan) + return _planned(target, plan, "helpers/prompt_cleanup.py", [ + {**target, "version": prior["agent"]["version"]}, + { + "type": "project-connection", "project_resource_id": target["project_resource_id"], + "project_endpoint": target["project_endpoint"], "name": prior["connection"]["name"], + }, + ]) + + +def _plan_connection(request, target, prior, result, response, *, base_dir, token_provider, transport, sdk_loader): + prompt_connect._validate_plan(prior) + project, endpoint = prompt_connect._project_identity(prior) + body = response["body"] + created = {"type": "project-connection", "name": target["name"]} + owned_write = {"action": "created", "connection": target["name"]} + resources = _object(result.get("resources"), "Creation resources") + ownership = _object(result.get("ownership"), "Creation ownership") + verification = _object(result.get("verification"), "Creation verification") + readback = _object(verification.get("connection_readback"), "Creation connection readback") + def matches(item): + return isinstance(item, dict) and ( + (item.get("type") == "project-connection" and item.get("name") == target["name"]) + or item.get("connection") == target["name"] + ) + + if any(not isinstance(container.get(key, []), list) for container, keys in ( + (resources, ("created", "reused", "updated", "skipped")), (ownership, ("run_owned", "reused_not_owned")), + ) for key in keys): + raise _failure("ownership-unproven", "Creation resource and ownership lists must be complete.") + etag = body.get("etag") or body.get("@odata.etag") + if ( + project.casefold() != target["project_resource_id"] or endpoint != target["project_endpoint"] + or prior["connection"]["name"] != target["name"] or prior["connection"]["action"] != "create" + or response.get("operation") != "project-connection-create" or type(response.get("status")) is not int + or response["status"] != 201 or not _text(etag) + or [item for item in resources.get("created", []) if matches(item)] != [created] + or any(matches(item) for key in ("reused", "updated", "skipped") for item in resources.get(key, [])) + or [item for item in ownership.get("run_owned", []) if matches(item)] != [owned_write] + or any(matches(item) for item in ownership.get("reused_not_owned", [])) + or readback.get("name") != target["name"] or readback.get("definition_digest") != digest(body) + ): + raise _failure("ownership-unproven", "Retain the exact acknowledged created project connection; updated/reused/recovered GETs are not create evidence.") + if not prompt_connect._connection_readback(prior, body)[0]: + raise _failure("ownership-unproven", "The original connection body must match its approved project and configuration.") + plan = { + "operation": "delete", "outcome": "cleanup-prompt-connection", "plan_kind": "cleanup", + "cleanup_approved": True, "sdk_major": 2, "owner": request["owner"], + "project_resource_id": target["project_resource_id"], "project_endpoint": target["project_endpoint"], + "connection": {"name": target["name"], "run_owned": True, "expected_etag": etag, "owned_definition_digest": digest(body)}, + } + url = prompt_connect._connection_url(plan) + token = token_provider(MANAGEMENT_AUDIENCE) + current, request_id = prompt_cleanup._get_connection(url, token, transport=transport) + if current is not None and current != body: + raise _failure("definition-drift", "The project connection changed since acknowledged creation.") + selected = None + if "agent_version" in request: + selected_target = { + **target, "type": "prompt-agent-version", "name": prior["agent"]["name"], "version": request["agent_version"], + } + selected_request = { + "schema_version": "1.0", "owner": request["owner"], "target": selected_target, + "creation_input_file": request["creation_input_file"], "creation_result_file": request["creation_result_file"], + "creation_response_file": request["agent_creation_response_file"], + } + selected = plan_cleanup(selected_request, base_dir=base_dir, token_provider=token_provider, transport=transport, sdk_loader=sdk_loader) + if selected["status"] == "planned": + plan["agent"] = selected["execution_input"]["plan"]["agent"] + if current is None: + if selected is not None and selected["status"] == "planned": + selected["approval_summary"]["already_absent"] = [target] + return selected + return _absent(target, request_id) + plan["dependency_guard"] = dependencies.prompt_snapshot( + plan, current, sdk_loader=sdk_loader, bounds=dependencies.limits(request.get("inventory_limits")), + ) + refreshed, _ = prompt_cleanup._get_connection(url, token, transport=transport) + if refreshed != current: + raise _failure("definition-drift", "Connection changed during project consumer discovery.") + prompt_cleanup._validate_plan(plan) + retained = [ + {**target, "type": "prompt-agent-version", "name": item["name"], "version": item["version"]} + for item in plan["dependency_guard"]["versions"] + if not plan.get("agent") or (item["name"], item["version"]) != (plan["agent"]["name"], plan["agent"]["version"]) + ] + planned = _planned(target, plan, "helpers/prompt_cleanup.py", retained) + if selected is not None and selected["status"] == "planned": + planned["approval_summary"]["delete"].insert(0, selected["approval_summary"]["delete"][0]) + return planned + + +def _plan_protected(request, target, prior, record, *, base_dir, token_provider, transport, sdk_loader): + snapshot = record["snapshot"] + outcome = ("cleanup-search-resource" if target["type"] in ("knowledge-base", "knowledge-source") + else "cleanup-prompt-version" if target["type"] == "prompt-agent-version" else "cleanup-prompt-connection") + plan = {"operation": "delete", "outcome": outcome, "plan_kind": "cleanup", + "cleanup_approved": True, "owner": request["owner"]} + if record["owner"] != request["owner"]: + raise _failure("ownership-unproven", "Original producer owner and cleanup accountable owner differ.") + if target["type"] in ("knowledge-base", "knowledge-source"): + source = _search_prior(prior, target) + if snapshot["definition_digest"] != digest(search_reconcile._definition(source["desired"])): + raise _failure("ownership-unproven", "Receipt does not bind the original approved Search definition.") + plan.update(resource_type=target["type"], **{k: target[k] for k in ("endpoint", "name", "api_version")}, + owned_definition_digest=snapshot["definition_digest"], expected_etag=snapshot["etag"]) + token = token_provider(SEARCH_AUDIENCE) + current, request_id = search_reconcile.read_resource(search_reconcile.resource_url(plan), token, transport=transport) + if current is None: + return _absent(target, request_id) + if digest(search_reconcile._definition(current)) != snapshot["definition_digest"] or current.get("@odata.etag") != snapshot["etag"]: + raise _failure("definition-drift", "Current Search state differs from the original producer snapshot.") + if target["type"] == "knowledge-source": + if dependencies.generated(current) != record["acknowledgement"]["generated"]: + raise _failure("generated-ownership-unproven", "Current generated identities differ from the native create acknowledgement.") + plan["dependency_guard"] = dependencies.search_snapshot( + plan, current, token, transport=transport, bounds=dependencies.limits(request.get("inventory_limits"))) + if plan["dependency_guard"]["generated"] != snapshot["generated"]: + raise _failure("generated-incarnation-drift", "Generated resources differ from original producer snapshots.") + else: + reject_secrets(current) + plan["desired"] = current + search_reconcile._validate_plan(plan) + planned = _planned(target, plan, "helpers/search_reconcile.py", []) + if target["type"] == "knowledge-source": + planned["approval_summary"]["delete"].extend( + {"type": child["type"], "name": child["name"], "endpoint": target["endpoint"], "service_managed": True} + for child in snapshot["generated"]) + else: + planned["approval_summary"]["retained_targets"] = [ + {**target, "type": "knowledge-source", "name": item["name"]} for item in current["knowledgeSources"]] + return planned + if prior.get("operation") == "create-initial-prompt-agent": + try: + from . import _initial_prompt + except ImportError: + import _initial_prompt + _initial_prompt.validate(prior) + if (target["type"] != "prompt-agent-version" or record["acknowledgement"]["operation"] != "agents.create" + or snapshot["definition_digest"] != digest(prior["agent"]["definition"])): + raise _failure("ownership-unproven", "Initial receipts authorize only the acknowledged version, never a connection/container.") + else: + prompt_connect._validate_plan(prior) + if target["type"] == "prompt-agent-version" and record["acknowledgement"]["operation"] != "agents.create_version": + raise _failure("ownership-unproven", "Connect receipts must come from the native new-version operation.") + project, endpoint = prompt_connect._project_identity(prior) + if project.casefold() != target["project_resource_id"] or endpoint != target["project_endpoint"]: + raise _failure("ownership-unproven", "Receipt project differs from original creation approval.") + plan.update(sdk_major=2, project_resource_id=target["project_resource_id"], project_endpoint=target["project_endpoint"]) + if target["type"] == "prompt-agent-version": + if prior["agent"]["name"] != target["name"] or prior["agent"].get("version") == target["version"]: + raise _failure("ownership-unproven", "Only the acknowledged new version is eligible, never the baseline.") + Client, _, _, _, extras = sdk_loader() + Credential, AzureError = extras + client = Client(endpoint=endpoint, credential=Credential()) + try: + current = client.agents.get_version(agent_name=target["name"], agent_version=target["version"]) + if (current.name != target["name"] or str(current.version) != target["version"] + or digest(current.definition.as_dict()) != snapshot["definition_digest"] + or receipts.version_identity(current) != snapshot["version_identity"]): + raise _failure("definition-drift", "Selected version differs from the original SDK return.") + except AzureError as exc: + if sdk_error_status(exc) == 404: + return _absent(target) + raise HelperFailure(message="Exact version readback failed.", blocked_at="cleanup-planning", + **sdk_error_metadata(exc, "agent-readback-failed")) from exc + finally: + client.close() + plan["agent"] = {"name": target["name"], "version": target["version"], "run_owned": True, + "owned_definition_digest": snapshot["definition_digest"], "owned_version_identity": snapshot["version_identity"]} + prompt_cleanup._validate_plan(plan) + return _planned(target, plan, "helpers/prompt_cleanup.py", []) + if prior["connection"]["name"] != target["name"] or prior["connection"]["action"] != "create": + raise _failure("ownership-unproven", "Connection receipt must bind original create intent.") + plan["connection"] = {"name": target["name"], "run_owned": True, "expected_etag": snapshot["etag"], + "owned_definition_digest": snapshot["definition_digest"]} + token = token_provider(MANAGEMENT_AUDIENCE) + url = prompt_connect._connection_url(plan) + current, request_id = prompt_cleanup._get_connection(url, token, transport=transport) + if current is not None and (digest(current) != snapshot["definition_digest"] or not prompt_connect._connection_readback(prior, current)[0]): + raise _failure("definition-drift", "Connection differs from its original acknowledged producer state.") + selected = None + if "agent_version" in request: + selected = plan_cleanup({ + "schema_version": "1.0", "owner": request["owner"], + "target": {**target, "type": "prompt-agent-version", "name": prior["agent"]["name"], "version": request["agent_version"]}, + "creation_input_file": request["creation_input_file"], + "creation_receipt_file": request["agent_creation_receipt_file"], + }, base_dir=base_dir, token_provider=token_provider, transport=transport, sdk_loader=sdk_loader) + if selected["status"] == "planned": + plan["agent"] = selected["execution_input"]["plan"]["agent"] + if current is None: + if selected and selected["status"] == "planned": + selected["approval_summary"]["already_absent"] = [target] + return selected + return _absent(target, request_id) + plan["dependency_guard"] = dependencies.prompt_snapshot( + plan, current, sdk_loader=sdk_loader, bounds=dependencies.limits(request.get("inventory_limits"))) + if prompt_cleanup._get_connection(url, token, transport=transport)[0] != current: + raise _failure("definition-drift", "Connection changed during consumer discovery.") + prompt_cleanup._validate_plan(plan) + planned = _planned(target, plan, "helpers/prompt_cleanup.py", []) + if selected and selected["status"] == "planned": + planned["approval_summary"]["delete"].insert(0, selected["approval_summary"]["delete"][0]) + return planned + + +def plan_cleanup( + request: dict[str, Any], *, base_dir: Path = Path("."), + token_provider: TokenProvider = azure_cli_token, transport: Transport = http_request, + sdk_loader: Callable[[], tuple[Any, Any, Any, Any, Any]] = prompt_cleanup.load_cleanup_sdk, +) -> dict[str, Any]: + _object(request, "Cleanup planning request") + reject_secrets(request) + require_allowed_fields(request, REQUEST_FIELDS, label="Cleanup planning request") + if request.get("schema_version") != "1.0" or not _text(request.get("owner")): + raise _failure("input-schema-invalid", "schema_version 1.0 and accountable owner are required.") + target = _target(request.get("target")) + dependencies.limits(request.get("inventory_limits")) + selection_fields = {"agent_version", "agent_creation_response_file", "agent_creation_receipt_file"} & set(request) + evidence_field = "agent_creation_receipt_file" if "agent_creation_receipt_file" in request else "agent_creation_response_file" + if selection_fields and (selection_fields != {"agent_version", evidence_field} or target["type"] != "project-connection"): + raise _failure("input-schema-invalid", "Only connection cleanup can explicitly select both agent_version and its original create response.") + if selection_fields and ( + not isinstance(request["agent_version"], str) or re.fullmatch(r"[1-9][0-9]*", request["agent_version"]) is None + or not _text(request[evidence_field]) + ): + raise _failure("target-selection-required", "Select an exact numeric created agent version and original SDK response file.") + if "creation_receipt_file" in request: + try: + from .blob_recheck import read_private + except ImportError: + from blob_recheck import read_private + path = _path(request, "creation_receipt_file", base_dir) + if read_private(path).get("kind") == "cleanup-creation-receipt": + if ("creation_response_file" in request or "creation_result_file" in request + or (selection_fields and evidence_field != "agent_creation_receipt_file")): + raise _failure("input-schema-invalid", "Protected receipts cannot be mixed with manual response adapters.") + prior, record = receipts.load(_path(request, "creation_input_file", base_dir), path, target) + return _plan_protected(request, target, prior, record, base_dir=base_dir, + token_provider=token_provider, transport=transport, sdk_loader=sdk_loader) + if evidence_field == "agent_creation_receipt_file": + raise _failure("input-schema-invalid", "Selected producer receipts require a protected connection receipt.") + prior, result, response = _records(request, target, base_dir) + if target["type"] in {"knowledge-base", "knowledge-source"}: + return _plan_search( + request, target, prior, result, response, token_provider=token_provider, transport=transport, + ) + if target["type"] == "project-connection": + return _plan_connection( + request, target, prior, result, response, base_dir=base_dir, + token_provider=token_provider, transport=transport, sdk_loader=sdk_loader, + ) + return _plan_prompt(request, target, prior, result, response, sdk_loader=sdk_loader) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--plan", type=Path, required=True) + args = parser.parse_args(argv) + request: dict[str, Any] = {} + try: + request = _read_json(args.plan) + result = plan_cleanup(request, base_dir=args.plan.resolve().parent) + emit_result(result, preserve_unapproved_input=result["status"] == "planned") + except HelperFailure as failure: + result = blocked_result(failure, outcome="plan-cleanup", fingerprint=None, owner=request.get("owner")) + result["approval_summary"] = { + "delete": [], "retain": RETAIN + ["selected target and all its dependencies"], + "blocked": [failure.code], "order": [], "hosted_cleanup": "unsupported", + } + try: + result["approval_summary"]["retained_targets"] = [_target(request.get("target"))] + except HelperFailure: + pass + emit_result(result) + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/contracts.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/contracts.md index 48d63fa2..f0a8fb03 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/contracts.md +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/contracts.md @@ -1,154 +1,154 @@ -# Deterministic helper contracts - -Read before planning/approval; scope discovery. -Owners assess content/ambiguity/roles/consent. [Private output](private-artifacts.md). -Inspect source only on `blocked`/`partial`. - -## Common envelope and process contract - -UTF-8 JSON: - -```json -{"schema_version":"1.0","plan":{},"approval":{"confirmed":true,"fingerprint":"sha256:"}} -``` - -Canonical JSON: UTF-8, key-sorted, ASCII-escaped, compact. Approve the unchanged -plan; helpers verify fingerprints. Exclude credentials, tokens, keys, SAS, -passwords, content and secret-bearing connection strings. -Use CLI tokens. File CU: MI or approved ARM/ENV; never retain secrets. -Envelope, approval, plan, nested controls and file records are -closed schemas: undeclared fields block before authentication. -`desired` remains a complete service body. Results recursively redact secret fields. -Azure Policy evaluation belongs to the creation owner, not these helpers. -[Bootstrap](bootstrap-contracts.md): no preflight policy reads. -Bind known requirements and child fingerprint in the parent plan; no policy precheck. -Never add undeclared policy/tag fields; if required settings cannot reach the supported request body, -return `azure-policy-setting-unsupported` before creation. - -Mutation commands emit compact JSON to stdout: exit `0` is `completed`, `2` no-write `blocked`, -`3` written/ambiguous `partial`. Preserve JSON/exit status. -Read same identity after ambiguous writes. Unacknowledged Search creates stay -`partial`: `resources_remaining.reused`, not owned. Otherwise prove approved state/absence. - -429: [read recovery](throttle-recovery.md), plus File queue retry below. - -File/Blob/ADLS and Search apply: [_progress.py](_progress.py): -stderr JSONL; `--no-progress` disables. Planning/discovery stays quiet. -Libraries: `progress=Progress("")`; children share one terminal event. -Bounded activity/remaining checks, elapsed seconds and counts. -Same stage: once/second; transitions/terminal: immediate. -File POST: five-second heartbeat, no polling/ETA; metadata proves File ingestion; -cycle updates aren't unique documents; ARM readiness isn't KB/retrieval/data-plane proof. -I/O failure: `progress-output-failed` warning; primary failures/execution unchanged. -Failed native stderr drains to null at shutdown; custom sinks untouched. Final JSON is authoritative. - -## File source application - -`python helpers/file_source.py --plan request.json`: read-only; -unapproved `execution_input`/`approval_summary`; exit `0` is `planned`, not readiness. -See [request/example](../knowledge-sources/create-file.md#proposed-plan). -Require `extraction_mode`; [optional embeddings](vector-contracts.md) are independent. -Planning: no bootstrap/roles/uploads/cleanup/approval; owner-owned requirements. -Fresh exact reuse sets `execution_required`/`mutation_approval_required` false; -no approval/execution. Create: approve the unchanged envelope: - -```text -python helpers/file_source.py --input approved.json --cleanup-receipt-dir --upload-receipt-dir -``` - -Require `operation: reconcile-and-ingest`, `cleanup_approved: false`, -`owner`, `source` and `ingestion`. Children target the same endpoint/name/API/owner. -Validate both before mutation; reconcile before upload/readback. -ACKs/journal: [resume](file-upload-recovery.md). -Reuse requires exact markers, never new files or per-file cleanup. -Parent/approved upload-only routes. `source` uses the -[reconciliation shape](#search-resource-reconciliation) with `kind: "file"`; -`ingestion`: - -```json -{"operation":"ingest","endpoint":"https://svc.search.windows.net","name":"src","api_version":"2026-08-01-preview","local_root":"","service_tier":"basic","extraction_mode":"minimal","owner":"o@x","cleanup_approved":false,"files":[{"path":"guide.txt","size":123,"mtime_ns":1700000000000000000,"sha256":"sha256:<64-hex>","media_type":"text/plain"}],"inventory_digest":"sha256:<64-hex>","expected_server_inventory_digest":"sha256:<64-hex>"} -``` - -## Search resource reconciliation - -```text -python helpers/search_reconcile.py --input -``` - -Require `operation: reconcile`, `outcome`, `resource_type`, `endpoint`, `name`, -`api_version`, `action`, complete approved `desired`, `owner`, and -`cleanup_approved: false`. Use `action: create`, or approved `action: update` -with `expected_etag`. Exact existing state is zero-write. - -Blob/ADLS plans also require `source_evidence` with `verified: true` and an -`inventory_digest`. ADLS requires `path_verified: true`, `acl_verified: true`. -These attest reconciliation, not evidence collection/readiness: apply through -the Blob parent. File `standard`: `ai_services_managed_identity` or -`ai_services_key_acquisition` via File parent, or existing `ai_services_api_key_environment`. - -KB `--plan`: [intent/wire/approval](kb-contracts.md); save unapproved `execution_input`, -never hand-build bodies/hashes. `verified_source` binds `verified: true`, `name`, -normalized `definition_digest`; fresh same-API GET must match before create/update/reuse. -GA omits preview fields/models. Low/medium needs a separate chat model; -model-free agent retrieval requires preview minimal/extractive. - -For cleanup, [plan](../lifecycle/cleanup.md), then use the same executor. -Bind `operation: delete`, `plan_kind: cleanup`, `cleanup_approved: true`, -`expected_etag`, and `owned_definition_digest`. Recheck ownership; verify absence. - -## Blob/ADLS source application - -Blob/ADLS alone loads [folder scope/bounds/drift/cleanup](blob-contracts.md). -`blob_source.py --discover` inventories; `--plan` builds unapproved artifacts -or verifies approval-free reuse; -`--input` applies the approved parent and monitors ingestion. Search -reconciliation alone never proves Blob readiness. - -## File ingestion internals - -`file_source.py` invokes this child after validating both plans. -Standalone CLI never authorizes new uploads. - -Reject path/inventory/extraction/server-state/marker drift. Sequential uploads; -one final inventory. Only per-file 415 permits continuation; systemic failures stop. -[File 429](file-upload-recovery.md): one bounded retry. POST: 180s; reads: 60s. -Timeout/409/5xx: read proof, never POST replay. Metadata proves ingestion, not -retrieval; status-only ACKs remain pending. -Never return bytes/local root. - -## Existing Prompt Agent fallback - -Only for unavailable MCP/missing connection/version operations; -never authorization denial/conflicting state: - -```text -python helpers/prompt_connect.py --input -``` - -The plan requires `operation: connect`, `sdk_major: 2`, exact project resource -ID and matching endpoint identity, `connection`, `agent`, `rbac_verified`, `allowed_tools: -["knowledge_base_retrieve"]`, `require_approval: "never"`, -`permission_forwarding`, `owner`, and `cleanup_approved: false`. -The [fallback](../agents/connect-prompt-sdk-fallback.md) defines `--plan`: -intent yields unapproved `execution_input` + summary; hashes stay internal. -Legacy inputs omit dependency/version snapshots. -Approve grounding changes. Forwarding -is either `{"mode":"not-applicable"}` or -`{"mode":"structured-input","name":"search_auth_token"}`. -Reuse one complete match; multiple matches block. -ARM uses `Microsoft.CognitiveServices/accounts/projects/connections`. -Optional boolean `connection.is_shared_to_all` defaults to `true`. -Fallback defines normalization/bounded warnings for creation, -ambiguous-write recovery/reuse. `agent_invocation: not-run` isn't E2E proof. - -Prompt cleanup (planner above blocks unproven connection consumers): - -```text -python helpers/prompt_cleanup.py --input -``` - -Bind `operation: delete`, `plan_kind: cleanup`, `cleanup_approved: true`, -`sdk_major: 2`, exact project identity and `agent`/`connection` with -`run_owned: true`, exact identity and `owned_definition_digest`. -Connections also bind `expected_etag`. Delete version before connection; -verify absence; preserve prior versions/project/model/KB/roles. +# Deterministic helper contracts + +Read before planning/approval; scope discovery. +Owners assess content/ambiguity/roles/consent. [Private output](private-artifacts.md). +Inspect source only on `blocked`/`partial`. + +## Common envelope and process contract + +UTF-8 JSON: + +```json +{"schema_version":"1.0","plan":{},"approval":{"confirmed":true,"fingerprint":"sha256:"}} +``` + +Canonical JSON: UTF-8, key-sorted, ASCII-escaped, compact. Approve the unchanged +plan; helpers verify fingerprints. Exclude credentials, tokens, keys, SAS, +passwords, content and secret-bearing connection strings. +Use CLI tokens. File CU: MI or approved ARM/ENV; never retain secrets. +Envelope, approval, plan, nested controls and file records are +closed schemas: undeclared fields block before authentication. +`desired` remains a complete service body. Results recursively redact secret fields. +Azure Policy evaluation belongs to the creation owner, not these helpers. +[Bootstrap](bootstrap-contracts.md): no preflight policy reads. +Bind known requirements and child fingerprint in the parent plan; no policy precheck. +Never add undeclared policy/tag fields; if required settings cannot reach the supported request body, +return `azure-policy-setting-unsupported` before creation. + +Mutation commands emit compact JSON to stdout: exit `0` is `completed`, `2` no-write `blocked`, +`3` written/ambiguous `partial`. Preserve JSON/exit status. +Read same identity after ambiguous writes. Unacknowledged Search creates stay +`partial`: `resources_remaining.reused`, not owned. Otherwise prove approved state/absence. + +429: [read recovery](throttle-recovery.md), plus File queue retry below. + +File/Blob/ADLS and Search apply: [_progress.py](_progress.py): +stderr JSONL; `--no-progress` disables. Planning/discovery stays quiet. +Libraries: `progress=Progress("")`; children share one terminal event. +Bounded activity/remaining checks, elapsed seconds and counts. +Same stage: once/second; transitions/terminal: immediate. +File POST: five-second heartbeat, no polling/ETA; metadata proves File ingestion; +cycle updates aren't unique documents; ARM readiness isn't KB/retrieval/data-plane proof. +I/O failure: `progress-output-failed` warning; primary failures/execution unchanged. +Failed native stderr drains to null at shutdown; custom sinks untouched. Final JSON is authoritative. + +## File source application + +`python helpers/file_source.py --plan request.json`: read-only; +unapproved `execution_input`/`approval_summary`; exit `0` is `planned`, not readiness. +See [request/example](../knowledge-sources/create-file.md#proposed-plan). +Require `extraction_mode`; [optional embeddings](vector-contracts.md) are independent. +Planning: no bootstrap/roles/uploads/cleanup/approval; owner-owned requirements. +Fresh exact reuse sets `execution_required`/`mutation_approval_required` false; +no approval/execution. Create: approve the unchanged envelope: + +```text +python helpers/file_source.py --input approved.json --cleanup-receipt-dir --upload-receipt-dir +``` + +Require `operation: reconcile-and-ingest`, `cleanup_approved: false`, +`owner`, `source` and `ingestion`. Children target the same endpoint/name/API/owner. +Validate both before mutation; reconcile before upload/readback. +ACKs/journal: [resume](file-upload-recovery.md). +Reuse requires exact markers, never new files or per-file cleanup. +Parent/approved upload-only routes. `source` uses the +[reconciliation shape](#search-resource-reconciliation) with `kind: "file"`; +`ingestion`: + +```json +{"operation":"ingest","endpoint":"https://svc.search.windows.net","name":"src","api_version":"2026-08-01-preview","local_root":"","service_tier":"basic","extraction_mode":"minimal","owner":"o@x","cleanup_approved":false,"files":[{"path":"guide.txt","size":123,"mtime_ns":1700000000000000000,"sha256":"sha256:<64-hex>","media_type":"text/plain"}],"inventory_digest":"sha256:<64-hex>","expected_server_inventory_digest":"sha256:<64-hex>"} +``` + +## Search resource reconciliation + +```text +python helpers/search_reconcile.py --input +``` + +Require `operation: reconcile`, `outcome`, `resource_type`, `endpoint`, `name`, +`api_version`, `action`, complete approved `desired`, `owner`, and +`cleanup_approved: false`. Use `action: create`, or approved `action: update` +with `expected_etag`. Exact existing state is zero-write. + +Blob/ADLS plans also require `source_evidence` with `verified: true` and an +`inventory_digest`. ADLS requires `path_verified: true`, `acl_verified: true`. +These attest reconciliation, not evidence collection/readiness: apply through +the Blob parent. File `standard`: `ai_services_managed_identity` or +`ai_services_key_acquisition` via File parent, or existing `ai_services_api_key_environment`. + +KB `--plan`: [intent/wire/approval](kb-contracts.md); save unapproved `execution_input`, +never hand-build bodies/hashes. `verified_source` binds `verified: true`, `name`, +normalized `definition_digest`; fresh same-API GET must match before create/update/reuse. +GA omits preview fields/models. Low/medium needs a separate chat model; +model-free agent retrieval requires preview minimal/extractive. + +For cleanup, [plan](../lifecycle/cleanup.md), then use the same executor. +Bind `operation: delete`, `plan_kind: cleanup`, `cleanup_approved: true`, +`expected_etag`, and `owned_definition_digest`. Recheck ownership; verify absence. + +## Blob/ADLS source application + +Blob/ADLS alone loads [folder scope/bounds/drift/cleanup](blob-contracts.md). +`blob_source.py --discover` inventories; `--plan` builds unapproved artifacts +or verifies approval-free reuse; +`--input` applies the approved parent and monitors ingestion. Search +reconciliation alone never proves Blob readiness. + +## File ingestion internals + +`file_source.py` invokes this child after validating both plans. +Standalone CLI never authorizes new uploads. + +Reject path/inventory/extraction/server-state/marker drift. Sequential uploads; +one final inventory. Only per-file 415 permits continuation; systemic failures stop. +[File 429](file-upload-recovery.md): one bounded retry. POST: 180s; reads: 60s. +Timeout/409/5xx: read proof, never POST replay. Metadata proves ingestion, not +retrieval; status-only ACKs remain pending. +Never return bytes/local root. + +## Existing Prompt Agent fallback + +Only for unavailable MCP/missing connection/version operations; +never authorization denial/conflicting state: + +```text +python helpers/prompt_connect.py --input +``` + +The plan requires `operation: connect`, `sdk_major: 2`, exact project resource +ID and matching endpoint identity, `connection`, `agent`, `rbac_verified`, `allowed_tools: +["knowledge_base_retrieve"]`, `require_approval: "never"`, +`permission_forwarding`, `owner`, and `cleanup_approved: false`. +The [fallback](../agents/connect-prompt-sdk-fallback.md) defines `--plan`: +intent yields unapproved `execution_input` + summary; hashes stay internal. +Legacy inputs omit dependency/version snapshots. +Approve grounding changes. Forwarding +is either `{"mode":"not-applicable"}` or +`{"mode":"structured-input","name":"search_auth_token"}`. +Reuse one complete match; multiple matches block. +ARM uses `Microsoft.CognitiveServices/accounts/projects/connections`. +Optional boolean `connection.is_shared_to_all` defaults to `true`. +Fallback defines normalization/bounded warnings for creation, +ambiguous-write recovery/reuse. `agent_invocation: not-run` isn't E2E proof. + +Prompt cleanup (planner above blocks unproven connection consumers): + +```text +python helpers/prompt_cleanup.py --input +``` + +Bind `operation: delete`, `plan_kind: cleanup`, `cleanup_approved: true`, +`sdk_major: 2`, exact project identity and `agent`/`connection` with +`run_owned: true`, exact identity and `owned_definition_digest`. +Connections also bind `expected_etag`. Delete version before connection; +verify absence; preserve prior versions/project/model/KB/roles. diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/cu_ingestion_auth.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/cu_ingestion_auth.py index 06c6d1bd..89afa2f0 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/cu_ingestion_auth.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/cu_ingestion_auth.py @@ -1,245 +1,245 @@ -"""Shared CU auth disclosure; private acquisition is restricted to File source PUTs.""" -from __future__ import annotations - -import base64 -import copy -import json -import re - -try: - from . import _bootstrap_io - from ._common import HelperFailure, HttpResult, MANAGEMENT_AUDIENCE, azure_cli_token, digest, require_allowed_fields -except ImportError: - import _bootstrap_io - from _common import HelperFailure, HttpResult, MANAGEMENT_AUDIENCE, azure_cli_token, digest, require_allowed_fields - - -API_VERSION = "2024-10-01" -PURPOSE = "file-standard-cu-source-put" -RESOURCE = re.compile( - r"/subscriptions/([0-9a-fA-F-]{36})/resourceGroups/[A-Za-z0-9_.()-]{1,90}" - r"/providers/Microsoft\.CognitiveServices/accounts/[A-Za-z0-9][A-Za-z0-9_.-]{1,63}", - re.IGNORECASE, -) -ENDPOINT = re.compile(r"https://[a-z0-9][a-z0-9-]{0,62}\.services\.ai\.azure\.com/?") -UUID = re.compile(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}") - - -def approval_summary(adapter, auth, *, creating): - modes = { - "file": {"system-assigned", "api-key-arm", "api-key-environment"}, - "azureBlob": {"system-assigned"}, "adlsGen2": {"system-assigned"}, - } - if adapter not in modes or auth not in modes[adapter]: - raise failure("cu-adapter-auth-unsupported", "Select the supported adapter's verified CU auth contract; no cross-adapter auth fallback.") - return { - "adapter": adapter, "mode": auth, - "contract": ( - "File system MI is implemented in inspected service code; live compatibility is unverified. Explicit legacy key modes remain supported, without fallback." - if adapter == "file" else - "Documented Search system-assigned CU identity; Cognitive Services User on the selected CU account. No key fallback." - ), - "credential_read": ( - "none: exact source reuse" if not creating else - "private ARM listKeys/key1 for the exact source PUT only" if auth == "api-key-arm" else - "explicit existing ENV only" if auth == "api-key-environment" else - "none: managed identity" - ), - "setup": ( - "Reuse verified dependencies. Resolve only missing changes through packaged native bootstrap with exact identity/role/scope and explicit permission/local-auth approval; source execution does not alter them and unsupported updates block." - if creating else "No CU setup or reingestion for exact source reuse." - ), - "consent": "Disclose auth and cost/data/access in the concrete source approval; no separate manual credential/MI/ENV confirmation. Material changes require refreshed approval.", - } - - -def failure(code, message, *, status=None, partial=False): - return HelperFailure(code, message, blocked_at="cu-authentication", status=status, partial=partial) - - -def token_principal(token): - # This binds CLI token identity, not token validity; ARM validates the credential. - try: - payload = token.split(".")[1] - claims = json.loads(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4))) - if any(not isinstance(claims.get(k), str) or UUID.fullmatch(claims[k]) is None for k in ("tid", "oid")): - raise ValueError() - return claims["tid"], digest({"object_id": claims["oid"]}) - except Exception: - raise failure("cu-context-unavailable", "The signed-in ARM token lacks a usable tenant/object identity binding; token details withheld.") from None - - -def account_context(): - """Read non-secret CLI identity metadata; never run a key-returning subprocess.""" - try: - code, stdout, _ = _bootstrap_io.run_cli(["account", "show"], 30) - value = json.loads(stdout) if code == 0 else None - user = value.get("user") if isinstance(value, dict) else None - if ( - not isinstance(user, dict) or user.get("type") not in ("user", "servicePrincipal") - or not isinstance(user.get("name"), str) or not user["name"] - or value.get("environmentName") != "AzureCloud" or value.get("state") != "Enabled" - ): - raise ValueError() - tenant, principal = token_principal(azure_cli_token(MANAGEMENT_AUDIENCE)) - if tenant != value["tenantId"]: - raise ValueError() - context = { - "subscription_id": value["id"], "tenant_id": value["tenantId"], - "principal_digest": principal, - } - validate_context(context) - return context - except Exception: - raise failure("cu-context-unavailable", "Use an existing signed-in AzureCloud CLI context; identity details withheld.") from None - - -def validate_context(context): - if not isinstance(context, dict): - raise failure("cu-context-invalid", "Retain the planner's exact signed-in context.") - require_allowed_fields(context, {"subscription_id", "tenant_id", "principal_digest"}, label="File CU context") - if ( - any(not isinstance(context.get(k), str) or UUID.fullmatch(context[k]) is None - for k in ("subscription_id", "tenant_id")) - or not isinstance(context.get("principal_digest"), str) - or re.fullmatch(r"sha256:[0-9a-f]{64}", context["principal_digest"]) is None - ): - raise failure("cu-context-invalid", "Retain complete tenant, subscription and principal binding.") - - -def acquisition(choice, context): - result = { - "resource_id": choice["resource_id"], "endpoint": choice["endpoint"].rstrip("/"), - "context": copy.deepcopy(context), "api_version": API_VERSION, "key_name": "key1", "purpose": PURPOSE, - } - validate_acquisition(result, choice["endpoint"]) - return result - - -def validate_acquisition(value, endpoint): - if not isinstance(value, dict): - raise failure("cu-acquisition-invalid", "Retain the exact approved File CU acquisition contract.") - require_allowed_fields(value, {"resource_id", "endpoint", "context", "api_version", "key_name", "purpose"}, - label="File CU acquisition") - resource = RESOURCE.fullmatch(value.get("resource_id", "")) if isinstance(value.get("resource_id"), str) else None - if ( - resource is None or not isinstance(value.get("endpoint"), str) - or ENDPOINT.fullmatch(value["endpoint"]) is None - or value["endpoint"] != endpoint.rstrip("/") - or value.get("api_version") != API_VERSION or value.get("key_name") != "key1" - or value.get("purpose") != PURPOSE - ): - raise failure("cu-acquisition-invalid", "Only the exact selected AIServices account, endpoint and key1 source-PUT purpose are supported.") - validate_context(value.get("context")) - if resource.group(1).casefold() != value["context"]["subscription_id"].casefold(): - raise failure("cu-subscription-mismatch", "Select the approved CU account's existing CLI subscription, then refresh the plan; no context switching was performed.") - - -def check_context(expected, provider): - current = provider() - validate_context(current) - if current != expected: - raise failure("cu-context-drift", "Signed-in tenant, subscription or principal changed; restore the approved context or refresh the plan and approval.") - - -class PrivateKey: - def __init__(self, contract, *, token_provider, transport, context_provider, recheck): - self.contract = contract - self.token_provider = token_provider - self.raw_transport = transport - self.context_provider = context_provider - self.recheck = recheck - self._secrets = () - self._attempted = False - - def acquire(self): - if self._attempted: - raise failure("cu-acquisition-repeated", "Credential acquisition is single-attempt; no alternate key, account or auth channel was tried.") - self._attempted = True - expected = self.contract["context"] - check_context(expected, self.context_provider) - self.recheck() - try: - token = self.token_provider(MANAGEMENT_AUDIENCE) - if token_principal(token) != (expected["tenant_id"], expected["principal_digest"]): - raise failure("cu-context-drift", "ARM token identity differs from the approved caller.") - check_context(expected, self.context_provider) - result = self.raw_transport( - "POST", f"{MANAGEMENT_AUDIENCE}{self.contract['resource_id']}/listKeys?api-version={API_VERSION}", - token, follow_redirects=False, max_response_bytes=16384, - ) - except HelperFailure as error: - if error.code in ("cu-context-drift", "cu-context-unavailable"): - raise failure(error.code, "Signed-in ARM identity no longer matches the usable approved context; refresh approval.") from None - raise self._acquisition_failure(error.http_status) from None - except Exception: - raise self._acquisition_failure(None) from None - if result.status != 200: - raise self._acquisition_failure(result.status) - body = result.body - if ( - not isinstance(body, dict) - or not isinstance(body.get("key1"), str) - or not 1 <= len(body["key1"]) <= 4096 - or not body["key1"].isascii() or any(c.isspace() or ord(c) < 33 for c in body["key1"]) - or (body.get("key2") is not None and ( - not isinstance(body["key2"], str) or len(body["key2"]) > 4096 - )) - ): - raise failure("cu-key-response-invalid", "ARM listKeys did not return a usable key1; no alternate key was selected.") - self._secrets = tuple(v for k in ("key1", "key2") if isinstance(v := body.get(k), str) and v) - check_context(expected, self.context_provider) - return body["key1"] - - @staticmethod - def _acquisition_failure(status): - status = status if type(status) is int and 100 <= status <= 599 else None - if status in (401, 403): - return failure( - "cu-key-access-denied", - "The signed-in caller needs Microsoft.CognitiveServices/accounts/listKeys/action on the exact approved CU account. Have its owner resolve access under separate concrete approval; no roles or policies were changed.", - status=status, - ) - return failure("cu-key-acquisition-failed", "ARM listKeys failed for the approved CU account; details withheld and no retry or auth fallback performed.", status=status) - - def _redact(self, value): - if isinstance(value, str): - for secret in self._secrets: - value = value.replace(secret, "[REDACTED]") - return value - if isinstance(value, dict): - return {self._redact(k): self._redact(v) for k, v in value.items()} - if isinstance(value, list): - return [self._redact(v) for v in value] - if isinstance(value, tuple): - return tuple(self._redact(v) for v in value) - return value - - def transport(self, method, url, token, **kwargs): - if not self._secrets: - return self.raw_transport(method, url, token, **kwargs) - kwargs["follow_redirects"] = False - try: - result = self.raw_transport(method, url, token, **kwargs) - except HelperFailure as error: - raise HelperFailure( - self._redact(error.code), "Azure request failed after private CU credential acquisition; response details withheld.", - blocked_at=self._redact(error.blocked_at), - status=error.http_status if type(error.http_status) is int and 100 <= error.http_status <= 599 else None, - partial=error.partial, request_id=self._redact(error.request_id), - writes=self._redact(error.writes), resources_remaining=self._redact(error.resources_remaining), - resources_reused=self._redact(error.resources_reused), warnings=self._redact(error.warnings), - ) from None - except Exception: - raise failure("cu-private-request-failed", "Azure request failed; private request/response details withheld.", - partial=method not in ("GET", "HEAD")) from None - etags = self._redact(result.etag_values) - versions = [ - *(result.etag_values or ()), - *(value for name, value in result.headers.items() if name.lower() == "etag"), - result.body.get("@odata.etag") if isinstance(result.body, dict) else None, - ] - if any(self._redact(value) != value for value in versions): - # Redaction must not turn conflicting secret-bearing versions into equal evidence. - etags = ("",) - return HttpResult(result.status, self._redact(result.body), self._redact(result.headers), etags) +"""Shared CU auth disclosure; private acquisition is restricted to File source PUTs.""" +from __future__ import annotations + +import base64 +import copy +import json +import re + +try: + from . import _bootstrap_io + from ._common import HelperFailure, HttpResult, MANAGEMENT_AUDIENCE, azure_cli_token, digest, require_allowed_fields +except ImportError: + import _bootstrap_io + from _common import HelperFailure, HttpResult, MANAGEMENT_AUDIENCE, azure_cli_token, digest, require_allowed_fields + + +API_VERSION = "2024-10-01" +PURPOSE = "file-standard-cu-source-put" +RESOURCE = re.compile( + r"/subscriptions/([0-9a-fA-F-]{36})/resourceGroups/[A-Za-z0-9_.()-]{1,90}" + r"/providers/Microsoft\.CognitiveServices/accounts/[A-Za-z0-9][A-Za-z0-9_.-]{1,63}", + re.IGNORECASE, +) +ENDPOINT = re.compile(r"https://[a-z0-9][a-z0-9-]{0,62}\.services\.ai\.azure\.com/?") +UUID = re.compile(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}") + + +def approval_summary(adapter, auth, *, creating): + modes = { + "file": {"system-assigned", "api-key-arm", "api-key-environment"}, + "azureBlob": {"system-assigned"}, "adlsGen2": {"system-assigned"}, + } + if adapter not in modes or auth not in modes[adapter]: + raise failure("cu-adapter-auth-unsupported", "Select the supported adapter's verified CU auth contract; no cross-adapter auth fallback.") + return { + "adapter": adapter, "mode": auth, + "contract": ( + "File system MI is implemented in inspected service code; live compatibility is unverified. Explicit legacy key modes remain supported, without fallback." + if adapter == "file" else + "Documented Search system-assigned CU identity; Cognitive Services User on the selected CU account. No key fallback." + ), + "credential_read": ( + "none: exact source reuse" if not creating else + "private ARM listKeys/key1 for the exact source PUT only" if auth == "api-key-arm" else + "explicit existing ENV only" if auth == "api-key-environment" else + "none: managed identity" + ), + "setup": ( + "Reuse verified dependencies. Resolve only missing changes through packaged native bootstrap with exact identity/role/scope and explicit permission/local-auth approval; source execution does not alter them and unsupported updates block." + if creating else "No CU setup or reingestion for exact source reuse." + ), + "consent": "Disclose auth and cost/data/access in the concrete source approval; no separate manual credential/MI/ENV confirmation. Material changes require refreshed approval.", + } + + +def failure(code, message, *, status=None, partial=False): + return HelperFailure(code, message, blocked_at="cu-authentication", status=status, partial=partial) + + +def token_principal(token): + # This binds CLI token identity, not token validity; ARM validates the credential. + try: + payload = token.split(".")[1] + claims = json.loads(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4))) + if any(not isinstance(claims.get(k), str) or UUID.fullmatch(claims[k]) is None for k in ("tid", "oid")): + raise ValueError() + return claims["tid"], digest({"object_id": claims["oid"]}) + except Exception: + raise failure("cu-context-unavailable", "The signed-in ARM token lacks a usable tenant/object identity binding; token details withheld.") from None + + +def account_context(): + """Read non-secret CLI identity metadata; never run a key-returning subprocess.""" + try: + code, stdout, _ = _bootstrap_io.run_cli(["account", "show"], 30) + value = json.loads(stdout) if code == 0 else None + user = value.get("user") if isinstance(value, dict) else None + if ( + not isinstance(user, dict) or user.get("type") not in ("user", "servicePrincipal") + or not isinstance(user.get("name"), str) or not user["name"] + or value.get("environmentName") != "AzureCloud" or value.get("state") != "Enabled" + ): + raise ValueError() + tenant, principal = token_principal(azure_cli_token(MANAGEMENT_AUDIENCE)) + if tenant != value["tenantId"]: + raise ValueError() + context = { + "subscription_id": value["id"], "tenant_id": value["tenantId"], + "principal_digest": principal, + } + validate_context(context) + return context + except Exception: + raise failure("cu-context-unavailable", "Use an existing signed-in AzureCloud CLI context; identity details withheld.") from None + + +def validate_context(context): + if not isinstance(context, dict): + raise failure("cu-context-invalid", "Retain the planner's exact signed-in context.") + require_allowed_fields(context, {"subscription_id", "tenant_id", "principal_digest"}, label="File CU context") + if ( + any(not isinstance(context.get(k), str) or UUID.fullmatch(context[k]) is None + for k in ("subscription_id", "tenant_id")) + or not isinstance(context.get("principal_digest"), str) + or re.fullmatch(r"sha256:[0-9a-f]{64}", context["principal_digest"]) is None + ): + raise failure("cu-context-invalid", "Retain complete tenant, subscription and principal binding.") + + +def acquisition(choice, context): + result = { + "resource_id": choice["resource_id"], "endpoint": choice["endpoint"].rstrip("/"), + "context": copy.deepcopy(context), "api_version": API_VERSION, "key_name": "key1", "purpose": PURPOSE, + } + validate_acquisition(result, choice["endpoint"]) + return result + + +def validate_acquisition(value, endpoint): + if not isinstance(value, dict): + raise failure("cu-acquisition-invalid", "Retain the exact approved File CU acquisition contract.") + require_allowed_fields(value, {"resource_id", "endpoint", "context", "api_version", "key_name", "purpose"}, + label="File CU acquisition") + resource = RESOURCE.fullmatch(value.get("resource_id", "")) if isinstance(value.get("resource_id"), str) else None + if ( + resource is None or not isinstance(value.get("endpoint"), str) + or ENDPOINT.fullmatch(value["endpoint"]) is None + or value["endpoint"] != endpoint.rstrip("/") + or value.get("api_version") != API_VERSION or value.get("key_name") != "key1" + or value.get("purpose") != PURPOSE + ): + raise failure("cu-acquisition-invalid", "Only the exact selected AIServices account, endpoint and key1 source-PUT purpose are supported.") + validate_context(value.get("context")) + if resource.group(1).casefold() != value["context"]["subscription_id"].casefold(): + raise failure("cu-subscription-mismatch", "Select the approved CU account's existing CLI subscription, then refresh the plan; no context switching was performed.") + + +def check_context(expected, provider): + current = provider() + validate_context(current) + if current != expected: + raise failure("cu-context-drift", "Signed-in tenant, subscription or principal changed; restore the approved context or refresh the plan and approval.") + + +class PrivateKey: + def __init__(self, contract, *, token_provider, transport, context_provider, recheck): + self.contract = contract + self.token_provider = token_provider + self.raw_transport = transport + self.context_provider = context_provider + self.recheck = recheck + self._secrets = () + self._attempted = False + + def acquire(self): + if self._attempted: + raise failure("cu-acquisition-repeated", "Credential acquisition is single-attempt; no alternate key, account or auth channel was tried.") + self._attempted = True + expected = self.contract["context"] + check_context(expected, self.context_provider) + self.recheck() + try: + token = self.token_provider(MANAGEMENT_AUDIENCE) + if token_principal(token) != (expected["tenant_id"], expected["principal_digest"]): + raise failure("cu-context-drift", "ARM token identity differs from the approved caller.") + check_context(expected, self.context_provider) + result = self.raw_transport( + "POST", f"{MANAGEMENT_AUDIENCE}{self.contract['resource_id']}/listKeys?api-version={API_VERSION}", + token, follow_redirects=False, max_response_bytes=16384, + ) + except HelperFailure as error: + if error.code in ("cu-context-drift", "cu-context-unavailable"): + raise failure(error.code, "Signed-in ARM identity no longer matches the usable approved context; refresh approval.") from None + raise self._acquisition_failure(error.http_status) from None + except Exception: + raise self._acquisition_failure(None) from None + if result.status != 200: + raise self._acquisition_failure(result.status) + body = result.body + if ( + not isinstance(body, dict) + or not isinstance(body.get("key1"), str) + or not 1 <= len(body["key1"]) <= 4096 + or not body["key1"].isascii() or any(c.isspace() or ord(c) < 33 for c in body["key1"]) + or (body.get("key2") is not None and ( + not isinstance(body["key2"], str) or len(body["key2"]) > 4096 + )) + ): + raise failure("cu-key-response-invalid", "ARM listKeys did not return a usable key1; no alternate key was selected.") + self._secrets = tuple(v for k in ("key1", "key2") if isinstance(v := body.get(k), str) and v) + check_context(expected, self.context_provider) + return body["key1"] + + @staticmethod + def _acquisition_failure(status): + status = status if type(status) is int and 100 <= status <= 599 else None + if status in (401, 403): + return failure( + "cu-key-access-denied", + "The signed-in caller needs Microsoft.CognitiveServices/accounts/listKeys/action on the exact approved CU account. Have its owner resolve access under separate concrete approval; no roles or policies were changed.", + status=status, + ) + return failure("cu-key-acquisition-failed", "ARM listKeys failed for the approved CU account; details withheld and no retry or auth fallback performed.", status=status) + + def _redact(self, value): + if isinstance(value, str): + for secret in self._secrets: + value = value.replace(secret, "[REDACTED]") + return value + if isinstance(value, dict): + return {self._redact(k): self._redact(v) for k, v in value.items()} + if isinstance(value, list): + return [self._redact(v) for v in value] + if isinstance(value, tuple): + return tuple(self._redact(v) for v in value) + return value + + def transport(self, method, url, token, **kwargs): + if not self._secrets: + return self.raw_transport(method, url, token, **kwargs) + kwargs["follow_redirects"] = False + try: + result = self.raw_transport(method, url, token, **kwargs) + except HelperFailure as error: + raise HelperFailure( + self._redact(error.code), "Azure request failed after private CU credential acquisition; response details withheld.", + blocked_at=self._redact(error.blocked_at), + status=error.http_status if type(error.http_status) is int and 100 <= error.http_status <= 599 else None, + partial=error.partial, request_id=self._redact(error.request_id), + writes=self._redact(error.writes), resources_remaining=self._redact(error.resources_remaining), + resources_reused=self._redact(error.resources_reused), warnings=self._redact(error.warnings), + ) from None + except Exception: + raise failure("cu-private-request-failed", "Azure request failed; private request/response details withheld.", + partial=method not in ("GET", "HEAD")) from None + etags = self._redact(result.etag_values) + versions = [ + *(result.etag_values or ()), + *(value for name, value in result.headers.items() if name.lower() == "etag"), + result.body.get("@odata.etag") if isinstance(result.body, dict) else None, + ] + if any(self._redact(value) != value for value in versions): + # Redaction must not turn conflicting secret-bearing versions into equal evidence. + etags = ("",) + return HttpResult(result.status, self._redact(result.body), self._redact(result.headers), etags) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/document-assessment.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/document-assessment.md index 9e288e63..a75f5046 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/document-assessment.md +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/document-assessment.md @@ -1,124 +1,124 @@ -# Bounded document assessment - -Read **only when document sample inspection is selected** from content-fit intake. -Structural facts only; no conversion or admission proof. -Refusal remains valid. No upload, OCR, rasterization, model call, -macro/script/formula execution, external URI/object resolution or file extraction -to disk. Never inspect binary documents using text viewers. - -## Exact approval and read contract - -Before any read, approve exact local file/SHA-256/scope/ceilings. Separately approve -whole-file hashing if identity is unknown. No inspection/hash/worker on decline. -One file/invocation; no automatic batches, retries, wider scope or larger limits. - -PDF: explicit 1-based pages, maximum eight selected pages/200 total pages. -Others require `--whole-document`, not guessed page subsets; narrower non-PDF -scope is unsupported. Pagination/rendering is not assessed. -Whole-file hashing and PDF trailer/xref/object-stream/page-tree parsing remain -necessary; compressed containers may include unselected objects. Only selected -PDF streams are assessed, never other pages to guess modality. Decline if this -whole-file structural metadata parsing is unwanted. - -Use a customer-controlled private local scope. No URLs, device/UNC/mapped-network -paths, symlinks/reparse points or untrusted writable directories. Linux mount -locality and concurrent file replacement remain operator responsibilities. -Blob/ADLS: reuse customer-provided, separately authorized local sample copies. -This tool implements **no remote read/download adapter**. Remote requests return -`remote-sample-access-not-supported`, not a container/subscription crawl. -Separate sample-access approval names exact account/container/blob/version/ETag, -byte range/full-file read, destination, retention and limits. Local path/hash -approval remains required; no remote provenance verification or wider prefix. - -```text -python "\helpers\document_assess.py" --approve-inspection --file "C:\approved\sample.pdf" --sha256 <64-lowercase-hex> --pages 1,3 -python "\helpers\document_assess.py" --approve-inspection --file "C:\approved\sample.docx" --sha256 <64-lowercase-hex> --whole-document -``` - -POSIX: native paths/quoting. Optional `--format`: -`auto`, `pdf`, `text`, `markdown`, `html`, `json`, `docx`, `pptx`, `xlsx`, `png`, -`jpeg`. Byte signatures/OOXML content types determine binary adapters, never -extensions/MIME. Text auto mode reports UTF-8 text; HTML/JSON need explicit hints for structure, -Markdown for lexical counts. Hints cannot override binary detection, ingestion -mode or Search server admission. Auto-detected PDF still needs exact pages. - -## Declared installation and limits - -Python 3.11+, Windows/Linux. Separately approve installation outside inspection; -no automatic install or library guessing: - -```text -python -m pip install -r "\helpers\requirements-assessment.txt" -``` - -`pypdf==6.8.0` (BSD-3-Clause): PDF; `defusedxml==0.7.1` (PSF license): Office XML; -others use stdlib. Missing/wrong selected-adapter versions block, without -affecting other adapters. Nothing vendored; MIT-compatible distribution requires -retaining dependency notices/non-endorsement terms when bundling. -Repository development: `uv sync --locked` (`pyproject.toml`/`uv.lock`). - -| Bound | Fixed ceiling | -|---|---| -| File / worker / output | 16 MiB input; 256 MiB worker; 10 CPU seconds; 15 seconds wall including startup/read; 8 KiB JSON | -| Text / HTML / JSON | 2 MiB UTF-8 input; JSON depth 64/100,000 nodes; HTML 100,000 start tags | -| Office ZIP/XML | 256 entries; 2 MiB per inflated entry; 16 MiB aggregate declared inflation; compression ratio 100; 100,000 inspected XML nodes/depth 64 | -| Images | PNG IHDR/JPEG SOF0/1/2 headers only; 100 million declared pixels; 4,096 JPEG markers; no pixel decompression | - -ZIP: stored/deflated only; encrypted/macro packages, duplicate/unsafe paths and -inflation excess block. Capped in-memory reads, no disk extraction. XML forbids -DTDs/entities/external references. OS limits cover forged sizes, decompression, -cycles and allocations before post-parse node checks. - -Windows Job Object: process commit-memory/CPU/active-process limits. Linux: -`RLIMIT_AS`/`RLIMIT_CPU`, zero core/file-output limits; not RSS quotas. -Install before source reads/parser imports or block. Parent timeout/output guards -kill only their owned PID. Audit denies writes/network/processes; after trusted -parser/codec imports, file/directory reads too. Unsupported lazy features block. -This is **not an OS security sandbox or proof against every hostile document**. -OS paging/crash dumps remain outside the helper's privacy boundary. - -## Fact semantics and coverage - -| Adapter | Reported structural facts; exclusions | -|---|---| -| PDF | Page-tree count; selected-page Unicode length/nonempty flag; direct image/path paint and Form calls. Not unique/visible images, lines or tables. Forms untraversed/their text unassessed; total images/drawings unknown. | -| Text/Markdown | UTF-8 character count including whitespace/markup, not an AST or visual classification. Other encodings unassessed. | -| HTML | Data-event characters excluding script/style; recognized img/table/tr/td/th tag counts. Tolerant syntax, not rendering, table relationships or fetched images. | -| JSON | String-value characters, object/array/key counts, not keys' contents, schema semantics or numeric text interpretation. | -| DOCX | Main-document text/paragraph/table/blip elements; headers/footers/notes/embedded objects unassessed. | -| PPTX | Standard numbered slide parts' text/paragraph/table/blip elements and part count; not order, notes, diagrams or linked objects. | -| XLSX | Standard worksheet/shared-string/table parts: stored text, formula/cell/table counts, declared sheets. Shared strings counted once, not cell occurrences; no evaluation/numeric/chart interpretation. | -| PNG/JPEG | Dimensions/numeric headers; PNG IHDR CRC checked. Pixels, orientation, further frames and completeness unassessed. | - -Office: conventional transitional namespaces/parts only. Unreferenced standard -slide/sheet parts may be counted; relationships unresolved. Image references/ -table elements prove neither visibility, preserved relationships nor semantics. -Every report keeps input profile `unknown`, OCR need/answerability/layout -relationships `not-assessed`. `not-applicable` distinguishes non-page formats or -irrelevant typed counters; unknown facts never become false zero counts. -Text output/Markdown representation cannot prove text-only input or choose CU. - -## Return and fallback - -Exit `0`/`completed`, `assessment: assessed`: adapter facts only. -Exit `2`/`blocked`, `assessment: not-assessed`, `facts: null`: approval/scope/identity/ -limits, malformed/encrypted/unsupported content, missing adapter, warning or -worker failure. Do not invent worker-death causes. Legacy Office, other formats -and unsupported profiles are unassessed—not rejected for service ingestion. -All reports say `service_admission: not-assessed`; preserve the blocker and use -description/answer questions, never assume text-only or silently downgrade. - -After identity verification, bind source SHA-256 and canonical path SHA-256 -(UTF-8 absolute path, Windows case-normalized). Changed bytes block before parsing; -original File bytes remain unchanged. -No raw text/keys/snippets/images/drawing operands/archive names in stdout/stderr/ -artifacts. Only aggregate JSON leaves worker memory; no temporary files/deletion. -Return coverage/uncertainty to content-fit, then source-owner plan/ingestion -approvals—not fabricated evidence. - -Authorities: failure/conflict/uncertainty only. -- [pypdf contract/license](https://pypi.org/project/pypdf/6.8.0/) -- [XML parser/license](https://github.com/tiran/defusedxml) -- [OOXML](https://ecma-international.org/publications-and-standards/standards/ecma-376/) -- [PNG specification](https://www.w3.org/TR/png-3/) -- [JPEG T.81](https://www.itu.int/rec/T-REC-T.81) +# Bounded document assessment + +Read **only when document sample inspection is selected** from content-fit intake. +Structural facts only; no conversion or admission proof. +Refusal remains valid. No upload, OCR, rasterization, model call, +macro/script/formula execution, external URI/object resolution or file extraction +to disk. Never inspect binary documents using text viewers. + +## Exact approval and read contract + +Before any read, approve exact local file/SHA-256/scope/ceilings. Separately approve +whole-file hashing if identity is unknown. No inspection/hash/worker on decline. +One file/invocation; no automatic batches, retries, wider scope or larger limits. + +PDF: explicit 1-based pages, maximum eight selected pages/200 total pages. +Others require `--whole-document`, not guessed page subsets; narrower non-PDF +scope is unsupported. Pagination/rendering is not assessed. +Whole-file hashing and PDF trailer/xref/object-stream/page-tree parsing remain +necessary; compressed containers may include unselected objects. Only selected +PDF streams are assessed, never other pages to guess modality. Decline if this +whole-file structural metadata parsing is unwanted. + +Use a customer-controlled private local scope. No URLs, device/UNC/mapped-network +paths, symlinks/reparse points or untrusted writable directories. Linux mount +locality and concurrent file replacement remain operator responsibilities. +Blob/ADLS: reuse customer-provided, separately authorized local sample copies. +This tool implements **no remote read/download adapter**. Remote requests return +`remote-sample-access-not-supported`, not a container/subscription crawl. +Separate sample-access approval names exact account/container/blob/version/ETag, +byte range/full-file read, destination, retention and limits. Local path/hash +approval remains required; no remote provenance verification or wider prefix. + +```text +python "\helpers\document_assess.py" --approve-inspection --file "C:\approved\sample.pdf" --sha256 <64-lowercase-hex> --pages 1,3 +python "\helpers\document_assess.py" --approve-inspection --file "C:\approved\sample.docx" --sha256 <64-lowercase-hex> --whole-document +``` + +POSIX: native paths/quoting. Optional `--format`: +`auto`, `pdf`, `text`, `markdown`, `html`, `json`, `docx`, `pptx`, `xlsx`, `png`, +`jpeg`. Byte signatures/OOXML content types determine binary adapters, never +extensions/MIME. Text auto mode reports UTF-8 text; HTML/JSON need explicit hints for structure, +Markdown for lexical counts. Hints cannot override binary detection, ingestion +mode or Search server admission. Auto-detected PDF still needs exact pages. + +## Declared installation and limits + +Python 3.11+, Windows/Linux. Separately approve installation outside inspection; +no automatic install or library guessing: + +```text +python -m pip install -r "\helpers\requirements-assessment.txt" +``` + +`pypdf==6.8.0` (BSD-3-Clause): PDF; `defusedxml==0.7.1` (PSF license): Office XML; +others use stdlib. Missing/wrong selected-adapter versions block, without +affecting other adapters. Nothing vendored; MIT-compatible distribution requires +retaining dependency notices/non-endorsement terms when bundling. +Repository development: `uv sync --locked` (`pyproject.toml`/`uv.lock`). + +| Bound | Fixed ceiling | +|---|---| +| File / worker / output | 16 MiB input; 256 MiB worker; 10 CPU seconds; 15 seconds wall including startup/read; 8 KiB JSON | +| Text / HTML / JSON | 2 MiB UTF-8 input; JSON depth 64/100,000 nodes; HTML 100,000 start tags | +| Office ZIP/XML | 256 entries; 2 MiB per inflated entry; 16 MiB aggregate declared inflation; compression ratio 100; 100,000 inspected XML nodes/depth 64 | +| Images | PNG IHDR/JPEG SOF0/1/2 headers only; 100 million declared pixels; 4,096 JPEG markers; no pixel decompression | + +ZIP: stored/deflated only; encrypted/macro packages, duplicate/unsafe paths and +inflation excess block. Capped in-memory reads, no disk extraction. XML forbids +DTDs/entities/external references. OS limits cover forged sizes, decompression, +cycles and allocations before post-parse node checks. + +Windows Job Object: process commit-memory/CPU/active-process limits. Linux: +`RLIMIT_AS`/`RLIMIT_CPU`, zero core/file-output limits; not RSS quotas. +Install before source reads/parser imports or block. Parent timeout/output guards +kill only their owned PID. Audit denies writes/network/processes; after trusted +parser/codec imports, file/directory reads too. Unsupported lazy features block. +This is **not an OS security sandbox or proof against every hostile document**. +OS paging/crash dumps remain outside the helper's privacy boundary. + +## Fact semantics and coverage + +| Adapter | Reported structural facts; exclusions | +|---|---| +| PDF | Page-tree count; selected-page Unicode length/nonempty flag; direct image/path paint and Form calls. Not unique/visible images, lines or tables. Forms untraversed/their text unassessed; total images/drawings unknown. | +| Text/Markdown | UTF-8 character count including whitespace/markup, not an AST or visual classification. Other encodings unassessed. | +| HTML | Data-event characters excluding script/style; recognized img/table/tr/td/th tag counts. Tolerant syntax, not rendering, table relationships or fetched images. | +| JSON | String-value characters, object/array/key counts, not keys' contents, schema semantics or numeric text interpretation. | +| DOCX | Main-document text/paragraph/table/blip elements; headers/footers/notes/embedded objects unassessed. | +| PPTX | Standard numbered slide parts' text/paragraph/table/blip elements and part count; not order, notes, diagrams or linked objects. | +| XLSX | Standard worksheet/shared-string/table parts: stored text, formula/cell/table counts, declared sheets. Shared strings counted once, not cell occurrences; no evaluation/numeric/chart interpretation. | +| PNG/JPEG | Dimensions/numeric headers; PNG IHDR CRC checked. Pixels, orientation, further frames and completeness unassessed. | + +Office: conventional transitional namespaces/parts only. Unreferenced standard +slide/sheet parts may be counted; relationships unresolved. Image references/ +table elements prove neither visibility, preserved relationships nor semantics. +Every report keeps input profile `unknown`, OCR need/answerability/layout +relationships `not-assessed`. `not-applicable` distinguishes non-page formats or +irrelevant typed counters; unknown facts never become false zero counts. +Text output/Markdown representation cannot prove text-only input or choose CU. + +## Return and fallback + +Exit `0`/`completed`, `assessment: assessed`: adapter facts only. +Exit `2`/`blocked`, `assessment: not-assessed`, `facts: null`: approval/scope/identity/ +limits, malformed/encrypted/unsupported content, missing adapter, warning or +worker failure. Do not invent worker-death causes. Legacy Office, other formats +and unsupported profiles are unassessed—not rejected for service ingestion. +All reports say `service_admission: not-assessed`; preserve the blocker and use +description/answer questions, never assume text-only or silently downgrade. + +After identity verification, bind source SHA-256 and canonical path SHA-256 +(UTF-8 absolute path, Windows case-normalized). Changed bytes block before parsing; +original File bytes remain unchanged. +No raw text/keys/snippets/images/drawing operands/archive names in stdout/stderr/ +artifacts. Only aggregate JSON leaves worker memory; no temporary files/deletion. +Return coverage/uncertainty to content-fit, then source-owner plan/ingestion +approvals—not fabricated evidence. + +Authorities: failure/conflict/uncertainty only. +- [pypdf contract/license](https://pypi.org/project/pypdf/6.8.0/) +- [XML parser/license](https://github.com/tiran/defusedxml) +- [OOXML](https://ecma-international.org/publications-and-standards/standards/ecma-376/) +- [PNG specification](https://www.w3.org/TR/png-3/) +- [JPEG T.81](https://www.itu.int/rec/T-REC-T.81) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/document_assess.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/document_assess.py index 6f8833f6..4c597b9e 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/document_assess.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/document_assess.py @@ -1,353 +1,353 @@ -"""Consent-gated input-document structural facts; never emit document content.""" -from __future__ import annotations - -import argparse -import hashlib -import io -import json -import os -from pathlib import Path -import re -import stat -import subprocess -import sys -import threading - -PARSER_VERSION = "6.8.0" -MAX_INPUT = 16 * 1024 * 1024 -MAX_PAGES = 200 -MAX_SELECTED = 8 -MAX_OUTPUT = 8192 -MAX_REQUEST = 16384 -WALL_SECONDS = 15 -FORMATS = {"auto", "pdf", "text", "markdown", "html", "json", "docx", "pptx", - "xlsx", "png", "jpeg"} - - -class Blocked(ValueError): - """A public, content-free blocker code.""" - - -def blocked(code): - return {"schema_version": "1.0", "status": "blocked", "assessment": "not-assessed", "code": code, - "service_admission": "not-assessed", "facts": None} - - -def validate_request(request): - if not isinstance(request, dict) or request.get("approved") is not True: - raise Blocked("inspection-not-approved") - if not {"approved", "path", "sha256"} <= set(request) or set(request) - { - "approved", "path", "sha256", "pages", "whole_document", "format" - }: - raise Blocked("invalid-request") - path, digest, pages = request["path"], request["sha256"], request.get("pages") - if isinstance(path, str) and path.lower().startswith(("http:", "https:", "abfs:", "abfss:")): - raise Blocked("remote-sample-access-not-supported") - if (not isinstance(path, str) or len(path) > 4096 or "\0" in path - or not Path(path).is_absolute()): - raise Blocked("invalid-source-path") - if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest): - raise Blocked("invalid-source-identity") - if (pages is not None and (not isinstance(pages, list) or not 1 <= len(pages) <= MAX_SELECTED - or any(type(page) is not int or not 1 <= page <= MAX_PAGES for page in pages) - or len(set(pages)) != len(pages))): - raise Blocked("invalid-page-selection") - if type(request.get("whole_document", False)) is not bool or ( - (pages is not None) == request.get("whole_document", False) - ): - raise Blocked("explicit-document-scope-required") - if not isinstance(request.get("format", "auto"), str) or request.get("format", "auto") not in FORMATS: - raise Blocked("unsupported-format-hint") - - -def read_source(request): - path = Path(request["path"]) - if sys.platform == "win32": - import ctypes - - # Reject device namespaces, UNC, ADS and mapped network drives. - if (str(path).startswith("\\\\") or ":" in str(path)[2:] - or ctypes.windll.kernel32.GetDriveTypeW(str(path.anchor)) not in (3, 6)): - raise Blocked("nonlocal-source") - if any(item.is_symlink() or ( - getattr(item.lstat(), "st_file_attributes", 0) & 0x400 - ) for item in (path, *path.parents)): - raise Blocked("source-link-unsupported") - canonical = path.resolve(strict=True) - with canonical.open("rb") as source: - before = os.fstat(source.fileno()) - if not stat.S_ISREG(before.st_mode): - raise Blocked("nonregular-source") - if before.st_size > MAX_INPUT: - raise Blocked("input-limit") - data = source.read(MAX_INPUT + 1) - after = os.fstat(source.fileno()) - if len(data) > MAX_INPUT: - raise Blocked("input-limit") - if (before.st_size, before.st_mtime_ns, before.st_ino) != ( - after.st_size, after.st_mtime_ns, after.st_ino - ) or len(data) != before.st_size: - raise Blocked("source-changed") - if hashlib.sha256(data).hexdigest() != request["sha256"]: - raise Blocked("source-identity-mismatch") - binding = hashlib.sha256(os.path.normcase(str(canonical)).encode("utf-8")).hexdigest() - return data, binding - - -def deny_document_reads(event, args): - if event in {"open", "os.listdir", "os.scandir"}: - raise PermissionError("Document external read denied") - - -def assess_bytes(data, selected): - import codecs - import logging - import warnings - - try: - import pypdf - except ImportError: - raise Blocked("parser-dependency-missing") from None - if pypdf.__version__ != PARSER_VERSION: - raise Blocked("parser-version-unsupported") - from pypdf.errors import PdfReadError, PdfStreamError - from pypdf.generic import ContentStream - - class RejectWarning(logging.Handler): - def emit(self, record): - raise Blocked("pdf-parser-warning") - - logger = logging.getLogger("pypdf") - logger.handlers = [RejectWarning()] - logger.propagate = False - logger.setLevel(logging.WARNING) - warnings.simplefilter("error") - for encoding in ("charmap", "utf-16-be", "utf-16-le", "utf-8", "latin-1", "ascii"): - codecs.lookup(encoding) - # Parser imports are trusted installation reads; once bytes are supplied, - # no PDF-directed file read (even local) is permitted. - sys.addaudithook(deny_document_reads) - if not data.startswith(b"%PDF-") or not data.rstrip().endswith(b"%%EOF"): - raise Blocked("invalid-pdf") - try: - reader = pypdf.PdfReader(io.BytesIO(data), strict=True) - if reader.is_encrypted: - raise Blocked("encrypted-pdf") - page_count = len(reader.pages) - if page_count > MAX_PAGES: - raise Blocked("page-limit") - if max(selected) > page_count: - raise Blocked("page-out-of-range") - pages = [] - for number in selected: - page = reader.pages[number - 1] - # Direct stream operators, not object inventory or rendered entities. - content = page.get_contents() - operations = ContentStream(content, reader).operations if content is not None else [] - images, drawings, forms = 0, 0, 0 - for operands, operator in operations: - if operator == b"INLINE IMAGE": - images += 1 - elif operator in {b"S", b"s", b"f", b"F", b"f*", b"B", b"B*", b"b", b"b*"}: - drawings += 1 - elif operator == b"Do": - obj = page["/Resources"]["/XObject"][operands[0]].get_object() - subtype = obj["/Subtype"] - if subtype == "/Image": - images += 1 - elif subtype == "/Form": - forms += 1 - else: - raise Blocked("unsupported-paint-object") - # Forms can reference other pages/resources. Do not traverse them to - # estimate modality; report text unknown instead of widening scope. - characters = None if forms else len(page.extract_text()) - pages.append({ - "page": number, - "text_characters": characters, - "text_available": None if characters is None else characters > 0, - "text_status": "not-assessed-form-content" if forms else "assessed", - "direct_image_paints": images, - "direct_path_paints": drawings, - "form_invocations": forms, - "total_images": None, - "total_drawings": None, - "table_structure": "not-assessed", - "layout_relationships": "not-assessed", - }) - return {"page_count": page_count, "pages": pages, "input_profile": "unknown", - "answerability": "not-assessed", "ocr_need": "not-assessed"} - except (PdfReadError, PdfStreamError, KeyError, IndexError, TypeError, - UnicodeError, NotImplementedError): - raise Blocked("pdf-parse-unsupported") from None - except Warning: - raise Blocked("pdf-parser-warning") from None - except PermissionError: - raise Blocked("pdf-external-read-blocked") from None - - -def dispatch(data, request): - hint = request.get("format", "auto") - if data.startswith(b"%PDF-"): - if hint not in {"auto", "pdf"}: - raise Blocked("format-hint-mismatch") - if request.get("pages") is None: - raise Blocked("pdf-page-selection-required") - return "pdf", f"pypdf=={PARSER_VERSION}", assess_bytes(data, request["pages"]) - if request.get("pages") is not None: - raise Blocked("non-pdf-requires-whole-document-scope") - from _document_adapters import AdapterBlocked, assess_document - - try: - return assess_document(data, hint, deny_document_reads) - except AdapterBlocked as error: - raise Blocked(str(error)) from None - - -def worker(): - # Isolated mode excludes ambient PYTHONPATH; add only this packaged helper. - sys.path.insert(0, str(Path(__file__).resolve().parent)) - from _document_limits import deny_side_effects, install_limits - - try: - job_handle = install_limits() - except (OSError, ValueError): - return blocked("worker-limits-unavailable") - sys.addaudithook(deny_side_effects) - binding = {} - try: - raw = sys.stdin.buffer.read(MAX_REQUEST + 1) - if len(raw) > MAX_REQUEST: - raise Blocked("request-limit") - request = json.loads(raw) - validate_request(request) - data, path_digest = read_source(request) - binding = {"source_sha256": request["sha256"], "path_sha256": path_digest} - kind, parser, facts = dispatch(data, request) - return {"schema_version": "1.0", "status": "completed", "assessment": "assessed", - "service_admission": "not-assessed", "format": kind, "parser": parser, - **binding, "facts": facts} - except Blocked as error: - return dict(blocked(str(error)), **binding) - except MemoryError: - return dict(blocked("worker-memory-limit"), **binding) - except RecursionError: - return dict(blocked("document-recursion-limit"), **binding) - except (OSError, ValueError, OverflowError): - return dict(blocked("document-input-or-parse-failed"), **binding) - finally: - # This local deliberately remains live through parsing on Windows. - _ = job_handle - - -def run_worker(command, request, timeout=WALL_SECONDS): - """Pipe-only IPC; bounded output reader, no sample or stderr artifacts.""" - encoded = json.dumps(request, separators=(",", ":")).encode() - if len(encoded) > MAX_REQUEST: - return blocked("request-limit") - output = bytearray() - exceeded = threading.Event() - io_failed = threading.Event() - with subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL) as process: - def drain(): - try: - chunk = process.stdout.read(MAX_OUTPUT + 1) - if len(chunk) > MAX_OUTPUT: - exceeded.set() - process.kill() - else: - output.extend(chunk) - except OSError: - io_failed.set() - - def send(): - try: - process.stdin.write(encoded) - process.stdin.close() - except OSError: - io_failed.set() - - thread = threading.Thread(target=drain, daemon=True) - writer = threading.Thread(target=send, daemon=True) - thread.start() - writer.start() - try: - process.wait(timeout=timeout) - except subprocess.TimeoutExpired: - process.kill() - process.wait() - return blocked("worker-timeout") - finally: - if process.poll() is None: - process.kill() - process.wait() - thread.join() - writer.join() - if exceeded.is_set(): - return blocked("worker-output-limit") - if process.returncode not in (0, 2): - return blocked("worker-failed-or-resource-limit") - if io_failed.is_set(): - return blocked("worker-io-failed") - try: - result = json.loads(output) - except (ValueError, UnicodeError): - return blocked("worker-invalid-output") - if not isinstance(result, dict) or result.get("status") not in {"completed", "blocked"}: - return blocked("worker-invalid-output") - return result - - -def assess(request): - try: - validate_request(request) - return run_worker([sys.executable, "-I", "-B", str(Path(__file__).resolve()), - "--worker"], request) - except Blocked as error: - return blocked(str(error)) - except OSError: - return blocked("worker-start-failed") - - -class SafeArgumentParser(argparse.ArgumentParser): - def error(self, message): - raise Blocked("invalid-cli-arguments") - - -def main(): - def private_failure(exc_type, exc_value, traceback): - print(json.dumps(blocked("worker-failed-or-resource-limit"))) - - # Unexpected parser/runtime failures must not print attacker-controlled - # exception strings or stack traces, including when --worker is invoked. - sys.excepthook = private_failure - if sys.argv[1:] == ["--worker"]: - result = worker() - else: - parser = SafeArgumentParser(description=__doc__) - parser.add_argument("--approve-inspection", action="store_true") - parser.add_argument("--file") - parser.add_argument("--sha256") - parser.add_argument("--pages", help="Explicit 1-based pages, e.g. 1,3; no ranges") - parser.add_argument("--whole-document", action="store_true", - help="Approve whole bounded non-PDF document, not rendered pages") - parser.add_argument("--format", choices=sorted(FORMATS), default="auto") - try: - args = parser.parse_args() - if not args.approve_inspection: - raise Blocked("inspection-not-approved") - pages = None if args.pages is None else [int(value) for value in args.pages.split(",")] - result = assess({"approved": True, "path": args.file, - "sha256": args.sha256, "pages": pages, - "whole_document": args.whole_document, "format": args.format}) - except Blocked as error: - result = blocked(str(error)) - except ValueError: - result = blocked("invalid-page-selection") - print(json.dumps(result, separators=(",", ":"), ensure_ascii=True)) - return 0 if result["status"] == "completed" else 2 - - -if __name__ == "__main__": - raise SystemExit(main()) +"""Consent-gated input-document structural facts; never emit document content.""" +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import os +from pathlib import Path +import re +import stat +import subprocess +import sys +import threading + +PARSER_VERSION = "6.8.0" +MAX_INPUT = 16 * 1024 * 1024 +MAX_PAGES = 200 +MAX_SELECTED = 8 +MAX_OUTPUT = 8192 +MAX_REQUEST = 16384 +WALL_SECONDS = 15 +FORMATS = {"auto", "pdf", "text", "markdown", "html", "json", "docx", "pptx", + "xlsx", "png", "jpeg"} + + +class Blocked(ValueError): + """A public, content-free blocker code.""" + + +def blocked(code): + return {"schema_version": "1.0", "status": "blocked", "assessment": "not-assessed", "code": code, + "service_admission": "not-assessed", "facts": None} + + +def validate_request(request): + if not isinstance(request, dict) or request.get("approved") is not True: + raise Blocked("inspection-not-approved") + if not {"approved", "path", "sha256"} <= set(request) or set(request) - { + "approved", "path", "sha256", "pages", "whole_document", "format" + }: + raise Blocked("invalid-request") + path, digest, pages = request["path"], request["sha256"], request.get("pages") + if isinstance(path, str) and path.lower().startswith(("http:", "https:", "abfs:", "abfss:")): + raise Blocked("remote-sample-access-not-supported") + if (not isinstance(path, str) or len(path) > 4096 or "\0" in path + or not Path(path).is_absolute()): + raise Blocked("invalid-source-path") + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise Blocked("invalid-source-identity") + if (pages is not None and (not isinstance(pages, list) or not 1 <= len(pages) <= MAX_SELECTED + or any(type(page) is not int or not 1 <= page <= MAX_PAGES for page in pages) + or len(set(pages)) != len(pages))): + raise Blocked("invalid-page-selection") + if type(request.get("whole_document", False)) is not bool or ( + (pages is not None) == request.get("whole_document", False) + ): + raise Blocked("explicit-document-scope-required") + if not isinstance(request.get("format", "auto"), str) or request.get("format", "auto") not in FORMATS: + raise Blocked("unsupported-format-hint") + + +def read_source(request): + path = Path(request["path"]) + if sys.platform == "win32": + import ctypes + + # Reject device namespaces, UNC, ADS and mapped network drives. + if (str(path).startswith("\\\\") or ":" in str(path)[2:] + or ctypes.windll.kernel32.GetDriveTypeW(str(path.anchor)) not in (3, 6)): + raise Blocked("nonlocal-source") + if any(item.is_symlink() or ( + getattr(item.lstat(), "st_file_attributes", 0) & 0x400 + ) for item in (path, *path.parents)): + raise Blocked("source-link-unsupported") + canonical = path.resolve(strict=True) + with canonical.open("rb") as source: + before = os.fstat(source.fileno()) + if not stat.S_ISREG(before.st_mode): + raise Blocked("nonregular-source") + if before.st_size > MAX_INPUT: + raise Blocked("input-limit") + data = source.read(MAX_INPUT + 1) + after = os.fstat(source.fileno()) + if len(data) > MAX_INPUT: + raise Blocked("input-limit") + if (before.st_size, before.st_mtime_ns, before.st_ino) != ( + after.st_size, after.st_mtime_ns, after.st_ino + ) or len(data) != before.st_size: + raise Blocked("source-changed") + if hashlib.sha256(data).hexdigest() != request["sha256"]: + raise Blocked("source-identity-mismatch") + binding = hashlib.sha256(os.path.normcase(str(canonical)).encode("utf-8")).hexdigest() + return data, binding + + +def deny_document_reads(event, args): + if event in {"open", "os.listdir", "os.scandir"}: + raise PermissionError("Document external read denied") + + +def assess_bytes(data, selected): + import codecs + import logging + import warnings + + try: + import pypdf + except ImportError: + raise Blocked("parser-dependency-missing") from None + if pypdf.__version__ != PARSER_VERSION: + raise Blocked("parser-version-unsupported") + from pypdf.errors import PdfReadError, PdfStreamError + from pypdf.generic import ContentStream + + class RejectWarning(logging.Handler): + def emit(self, record): + raise Blocked("pdf-parser-warning") + + logger = logging.getLogger("pypdf") + logger.handlers = [RejectWarning()] + logger.propagate = False + logger.setLevel(logging.WARNING) + warnings.simplefilter("error") + for encoding in ("charmap", "utf-16-be", "utf-16-le", "utf-8", "latin-1", "ascii"): + codecs.lookup(encoding) + # Parser imports are trusted installation reads; once bytes are supplied, + # no PDF-directed file read (even local) is permitted. + sys.addaudithook(deny_document_reads) + if not data.startswith(b"%PDF-") or not data.rstrip().endswith(b"%%EOF"): + raise Blocked("invalid-pdf") + try: + reader = pypdf.PdfReader(io.BytesIO(data), strict=True) + if reader.is_encrypted: + raise Blocked("encrypted-pdf") + page_count = len(reader.pages) + if page_count > MAX_PAGES: + raise Blocked("page-limit") + if max(selected) > page_count: + raise Blocked("page-out-of-range") + pages = [] + for number in selected: + page = reader.pages[number - 1] + # Direct stream operators, not object inventory or rendered entities. + content = page.get_contents() + operations = ContentStream(content, reader).operations if content is not None else [] + images, drawings, forms = 0, 0, 0 + for operands, operator in operations: + if operator == b"INLINE IMAGE": + images += 1 + elif operator in {b"S", b"s", b"f", b"F", b"f*", b"B", b"B*", b"b", b"b*"}: + drawings += 1 + elif operator == b"Do": + obj = page["/Resources"]["/XObject"][operands[0]].get_object() + subtype = obj["/Subtype"] + if subtype == "/Image": + images += 1 + elif subtype == "/Form": + forms += 1 + else: + raise Blocked("unsupported-paint-object") + # Forms can reference other pages/resources. Do not traverse them to + # estimate modality; report text unknown instead of widening scope. + characters = None if forms else len(page.extract_text()) + pages.append({ + "page": number, + "text_characters": characters, + "text_available": None if characters is None else characters > 0, + "text_status": "not-assessed-form-content" if forms else "assessed", + "direct_image_paints": images, + "direct_path_paints": drawings, + "form_invocations": forms, + "total_images": None, + "total_drawings": None, + "table_structure": "not-assessed", + "layout_relationships": "not-assessed", + }) + return {"page_count": page_count, "pages": pages, "input_profile": "unknown", + "answerability": "not-assessed", "ocr_need": "not-assessed"} + except (PdfReadError, PdfStreamError, KeyError, IndexError, TypeError, + UnicodeError, NotImplementedError): + raise Blocked("pdf-parse-unsupported") from None + except Warning: + raise Blocked("pdf-parser-warning") from None + except PermissionError: + raise Blocked("pdf-external-read-blocked") from None + + +def dispatch(data, request): + hint = request.get("format", "auto") + if data.startswith(b"%PDF-"): + if hint not in {"auto", "pdf"}: + raise Blocked("format-hint-mismatch") + if request.get("pages") is None: + raise Blocked("pdf-page-selection-required") + return "pdf", f"pypdf=={PARSER_VERSION}", assess_bytes(data, request["pages"]) + if request.get("pages") is not None: + raise Blocked("non-pdf-requires-whole-document-scope") + from _document_adapters import AdapterBlocked, assess_document + + try: + return assess_document(data, hint, deny_document_reads) + except AdapterBlocked as error: + raise Blocked(str(error)) from None + + +def worker(): + # Isolated mode excludes ambient PYTHONPATH; add only this packaged helper. + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from _document_limits import deny_side_effects, install_limits + + try: + job_handle = install_limits() + except (OSError, ValueError): + return blocked("worker-limits-unavailable") + sys.addaudithook(deny_side_effects) + binding = {} + try: + raw = sys.stdin.buffer.read(MAX_REQUEST + 1) + if len(raw) > MAX_REQUEST: + raise Blocked("request-limit") + request = json.loads(raw) + validate_request(request) + data, path_digest = read_source(request) + binding = {"source_sha256": request["sha256"], "path_sha256": path_digest} + kind, parser, facts = dispatch(data, request) + return {"schema_version": "1.0", "status": "completed", "assessment": "assessed", + "service_admission": "not-assessed", "format": kind, "parser": parser, + **binding, "facts": facts} + except Blocked as error: + return dict(blocked(str(error)), **binding) + except MemoryError: + return dict(blocked("worker-memory-limit"), **binding) + except RecursionError: + return dict(blocked("document-recursion-limit"), **binding) + except (OSError, ValueError, OverflowError): + return dict(blocked("document-input-or-parse-failed"), **binding) + finally: + # This local deliberately remains live through parsing on Windows. + _ = job_handle + + +def run_worker(command, request, timeout=WALL_SECONDS): + """Pipe-only IPC; bounded output reader, no sample or stderr artifacts.""" + encoded = json.dumps(request, separators=(",", ":")).encode() + if len(encoded) > MAX_REQUEST: + return blocked("request-limit") + output = bytearray() + exceeded = threading.Event() + io_failed = threading.Event() + with subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL) as process: + def drain(): + try: + chunk = process.stdout.read(MAX_OUTPUT + 1) + if len(chunk) > MAX_OUTPUT: + exceeded.set() + process.kill() + else: + output.extend(chunk) + except OSError: + io_failed.set() + + def send(): + try: + process.stdin.write(encoded) + process.stdin.close() + except OSError: + io_failed.set() + + thread = threading.Thread(target=drain, daemon=True) + writer = threading.Thread(target=send, daemon=True) + thread.start() + writer.start() + try: + process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + return blocked("worker-timeout") + finally: + if process.poll() is None: + process.kill() + process.wait() + thread.join() + writer.join() + if exceeded.is_set(): + return blocked("worker-output-limit") + if process.returncode not in (0, 2): + return blocked("worker-failed-or-resource-limit") + if io_failed.is_set(): + return blocked("worker-io-failed") + try: + result = json.loads(output) + except (ValueError, UnicodeError): + return blocked("worker-invalid-output") + if not isinstance(result, dict) or result.get("status") not in {"completed", "blocked"}: + return blocked("worker-invalid-output") + return result + + +def assess(request): + try: + validate_request(request) + return run_worker([sys.executable, "-I", "-B", str(Path(__file__).resolve()), + "--worker"], request) + except Blocked as error: + return blocked(str(error)) + except OSError: + return blocked("worker-start-failed") + + +class SafeArgumentParser(argparse.ArgumentParser): + def error(self, message): + raise Blocked("invalid-cli-arguments") + + +def main(): + def private_failure(exc_type, exc_value, traceback): + print(json.dumps(blocked("worker-failed-or-resource-limit"))) + + # Unexpected parser/runtime failures must not print attacker-controlled + # exception strings or stack traces, including when --worker is invoked. + sys.excepthook = private_failure + if sys.argv[1:] == ["--worker"]: + result = worker() + else: + parser = SafeArgumentParser(description=__doc__) + parser.add_argument("--approve-inspection", action="store_true") + parser.add_argument("--file") + parser.add_argument("--sha256") + parser.add_argument("--pages", help="Explicit 1-based pages, e.g. 1,3; no ranges") + parser.add_argument("--whole-document", action="store_true", + help="Approve whole bounded non-PDF document, not rendered pages") + parser.add_argument("--format", choices=sorted(FORMATS), default="auto") + try: + args = parser.parse_args() + if not args.approve_inspection: + raise Blocked("inspection-not-approved") + pages = None if args.pages is None else [int(value) for value in args.pages.split(",")] + result = assess({"approved": True, "path": args.file, + "sha256": args.sha256, "pages": pages, + "whole_document": args.whole_document, "format": args.format}) + except Blocked as error: + result = blocked(str(error)) + except ValueError: + result = blocked("invalid-page-selection") + print(json.dumps(result, separators=(",", ":"), ensure_ascii=True)) + return 0 if result["status"] == "completed" else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file-upload-recovery.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file-upload-recovery.md index 67e82a91..6beec370 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file-upload-recovery.md +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file-upload-recovery.md @@ -1,135 +1,135 @@ -# File upload continuation and partial batches - -After partial File creation, retain the source. Never rerun creation, infer MI -from redacted GET, or recover by recreation/deletion/reset/new roles. -Generic File MI reuse/provenance guards remain unchanged. - -## Retain original evidence before creation - -```text -python helpers/file_source.py --input --cleanup-receipt-dir --upload-receipt-dir -``` - -Use the [private directory helper](private-artifacts.md); no automatic directory/ -ACL changes. The required journal retains original approval/context, actual -conditional source PUT ACK/request ID/ETag, exclusive **pre-POST** attempts and -separate upload responses. Never edit/remove/reconstruct entries. Integrity isn't -a service signature: retain exclusive control, no concurrent writers. - -File CLI creation requires protected `--cleanup-receipt-dir` and -`--upload-receipt-dir` pre-write; separate directories, no guessed paths. Plan with -`--execution-output `; show reference/counts, not -hash-bearing envelopes. Plan JSON/library and non-File policies are unchanged. - -Persistence failure stops requests. An attempt without result stays uncertain, -even if POST wasn't sent. Missing ACK/ETag cannot come from GET. Journals neither -replace [cleanup provenance](../lifecycle/cleanup.md) nor authorize deletion. -Without originals: retain source, `file-upload-provenance-missing`, not -"recreate required". - -## Supported upload-only plan - -Closed request: - -```json -{"schema_version":"1.0","receipt_directory":""} -``` - -```text -python helpers/file_upload.py --plan resume-request.json --execution-output -python helpers/file_upload.py --input -``` - -Planning is read-only: an **unapproved**, closed version `1.0` `resume-file-uploads` -envelope binds original plan/creation ACK/journal digests, owner, exact -never-attempted ordinals and receipt directory; -`cleanup_approved` stays false. Review and newly approve these uploads. An empty -eligible list needs no mutation or execution; use the returned observation. -Never edit ordinals/hashes to authorize attempted files. - -Execution validates the original unchanged approved corpus (paths, full hashes, -size, timestamps, mode, metadata), tenant/subscription/principal, source URL/API/ -definition/ACK ETag, and applicable CU account/Search MI/role/network snapshots. -It neither acquires CU keys nor changes authentication. It invokes only existing -metadata GETs and direct File upload POSTs using the original Search identity; -no source PUT, source update, new role, indexer operation, deletion, KB or model call. -POST targets the original validated -`/knowledgesources('')/files?api-version=2026-08-01-preview&pageSize=200` -multipart contract; approval/privacy gates apply. - -Continuation excludes previously attempted files, including exhausted 429 retries, -timeout/409/5xx and interrupted attempts. Only the original operation may retry its -received File-upload 429 once, as below. An immediate or delayed **empty** inventory -never proves in-flight termination for unknown outcomes. Positive exact file-ID/ -marker/hash/size readback can prove completed ingestion, but never invents an -upload ACK or source ownership. -Drift blocks; never repair by write. - -## Bounded batching and truthful progress - -File upload is synchronous: extraction, chunking, embedding, indexing and metadata -persistence finish before success. No File indexer, schedule or asynchronous status -polling. Each sequential POST has a 180-second processing/response allowance, -separate from the 60-second read budget; stricter approved outer deadlines win. -Timeout is not remote cancellation. Retain 200/201 ACKs without relisting source/ -full inventory after every healthy file. One final bounded inventory checks the -batch; follow validated `@odata.nextLink` exactly, without rebuilding parameters. -A per-file 415 permits independent continuation. Auth/access, deadlines, -persistence, drift and unknown outcomes stop new uploads. Repeated 429, or later -429 after the operation's one retry opportunity, stops without more reads/sleeps. -Preserve prior confirmed files; never hammer the remaining corpus. - -[429 recovery](throttle-recovery.md) supplies one server-respecting delay/read -opportunity, at most 30 seconds waiting/60 seconds per complete read sequence, -including all pages and response reads. Overlong delays stop, not shorten. -Journal-retained UTC backoff blocks resume planning/execution before any network; -fresh approval never waives it. `file_batch.backoff` reports waiting/unresolved/ -elapsed. Legacy 429 without timing stays unresolved; no invented deadline. -`--no-progress` disables observations. Otherwise stderr emits counts, a `waiting` -event with the bounded HTTP-429 delay, and terminal state. During POST, an immediate -and then five-second content-free heartbeat reports ordinal/total/attempt, elapsed -time and already ingested count: "1/25 ingested; processing file 2/25". This observer -never makes requests. No within-file percentage, ETA, filenames or private content. -Increment completion only after ACK/metadata proof and required receipt persistence. - -`file_batch.files` records bounded relative names/full hashes, upload state, -verification state and sanitized request IDs. Counts distinguish: -`accepted` (original 200/201 ACK), `confirmed` (exact file-ID/markers without -reported error), `failed`, `pending` (ACK but not confirmed), -`unverified` (uncertain attempt), and `not_attempted`. -Accepted overlaps verification counts; do not sum it with them. -`ingested` counts synchronous completion proved by valid ACK metadata or exact -completed-file readback. A status-only ACK is insufficient. Original ACK file IDs -remain binding even if its remaining metadata is incomplete. Failed readback stays -partial while preserving known prior ingestion; `retrieval` stays `unverified`. -File completion does not prove OCR quality or KB retrieval. - -Report "24 ingested, one failed/unverified" only with this proof; otherwise -distinguish accepted/pending. Preserve outcomes, not a blanket blocker. A subset -handoff must disclose pending files may appear and subset isolation is not proven; -any KB use still needs its existing separate approval. No automatic KB writes. - -## One documented queue-rejection retry - -The official File guide identifies upload HTTP 429 as a full processing queue and -recommends bounded parallelism/exponential backoff; Retry-After is not guaranteed. -Only this File upload route permits one same-operation retry across the batch, -not per file. No private throttle marker/error text is required. Honor valid -Retry-After; missing/invalid metadata uses the diagnosed first backoff step of one -second. Shared 30-second wait/60-second complete preflight limits apply. -After waiting, revalidate unchanged source/corpus and one fresh bounded inventory. -Positive proof avoids another POST; absence is not the retry authority: the -received queue-rejection 429 is. Persist exclusive original/retry attempt and -result records when journaling. Retry the original URL, token, multipart bytes/ -boundary and approved payload; source creation happens once. - -The MI canary's stricter one-upload approval forbids this retry. No PUT/DELETE/ -update retry, new roles or auth substitution. Each upload creates a new file ID, -even for the same filename; no filename-idempotency key exists. Timeout/504/409/ -5xx remain unknown, never replayable from empty inventory or elapsed time. -`file-upload-retry-safety-unproven` retains that blocker. Exhausted files stay -failed; continuation uploads only never-attempted files under fresh approval, -not another attempt at an exhausted file. - -Authorities: failure/conflict/uncertainty only. -[File upload/troubleshooting](https://learn.microsoft.com/azure/search/agentic-knowledge-source-how-to-file). +# File upload continuation and partial batches + +After partial File creation, retain the source. Never rerun creation, infer MI +from redacted GET, or recover by recreation/deletion/reset/new roles. +Generic File MI reuse/provenance guards remain unchanged. + +## Retain original evidence before creation + +```text +python helpers/file_source.py --input --cleanup-receipt-dir --upload-receipt-dir +``` + +Use the [private directory helper](private-artifacts.md); no automatic directory/ +ACL changes. The required journal retains original approval/context, actual +conditional source PUT ACK/request ID/ETag, exclusive **pre-POST** attempts and +separate upload responses. Never edit/remove/reconstruct entries. Integrity isn't +a service signature: retain exclusive control, no concurrent writers. + +File CLI creation requires protected `--cleanup-receipt-dir` and +`--upload-receipt-dir` pre-write; separate directories, no guessed paths. Plan with +`--execution-output `; show reference/counts, not +hash-bearing envelopes. Plan JSON/library and non-File policies are unchanged. + +Persistence failure stops requests. An attempt without result stays uncertain, +even if POST wasn't sent. Missing ACK/ETag cannot come from GET. Journals neither +replace [cleanup provenance](../lifecycle/cleanup.md) nor authorize deletion. +Without originals: retain source, `file-upload-provenance-missing`, not +"recreate required". + +## Supported upload-only plan + +Closed request: + +```json +{"schema_version":"1.0","receipt_directory":""} +``` + +```text +python helpers/file_upload.py --plan resume-request.json --execution-output +python helpers/file_upload.py --input +``` + +Planning is read-only: an **unapproved**, closed version `1.0` `resume-file-uploads` +envelope binds original plan/creation ACK/journal digests, owner, exact +never-attempted ordinals and receipt directory; +`cleanup_approved` stays false. Review and newly approve these uploads. An empty +eligible list needs no mutation or execution; use the returned observation. +Never edit ordinals/hashes to authorize attempted files. + +Execution validates the original unchanged approved corpus (paths, full hashes, +size, timestamps, mode, metadata), tenant/subscription/principal, source URL/API/ +definition/ACK ETag, and applicable CU account/Search MI/role/network snapshots. +It neither acquires CU keys nor changes authentication. It invokes only existing +metadata GETs and direct File upload POSTs using the original Search identity; +no source PUT, source update, new role, indexer operation, deletion, KB or model call. +POST targets the original validated +`/knowledgesources('')/files?api-version=2026-08-01-preview&pageSize=200` +multipart contract; approval/privacy gates apply. + +Continuation excludes previously attempted files, including exhausted 429 retries, +timeout/409/5xx and interrupted attempts. Only the original operation may retry its +received File-upload 429 once, as below. An immediate or delayed **empty** inventory +never proves in-flight termination for unknown outcomes. Positive exact file-ID/ +marker/hash/size readback can prove completed ingestion, but never invents an +upload ACK or source ownership. +Drift blocks; never repair by write. + +## Bounded batching and truthful progress + +File upload is synchronous: extraction, chunking, embedding, indexing and metadata +persistence finish before success. No File indexer, schedule or asynchronous status +polling. Each sequential POST has a 180-second processing/response allowance, +separate from the 60-second read budget; stricter approved outer deadlines win. +Timeout is not remote cancellation. Retain 200/201 ACKs without relisting source/ +full inventory after every healthy file. One final bounded inventory checks the +batch; follow validated `@odata.nextLink` exactly, without rebuilding parameters. +A per-file 415 permits independent continuation. Auth/access, deadlines, +persistence, drift and unknown outcomes stop new uploads. Repeated 429, or later +429 after the operation's one retry opportunity, stops without more reads/sleeps. +Preserve prior confirmed files; never hammer the remaining corpus. + +[429 recovery](throttle-recovery.md) supplies one server-respecting delay/read +opportunity, at most 30 seconds waiting/60 seconds per complete read sequence, +including all pages and response reads. Overlong delays stop, not shorten. +Journal-retained UTC backoff blocks resume planning/execution before any network; +fresh approval never waives it. `file_batch.backoff` reports waiting/unresolved/ +elapsed. Legacy 429 without timing stays unresolved; no invented deadline. +`--no-progress` disables observations. Otherwise stderr emits counts, a `waiting` +event with the bounded HTTP-429 delay, and terminal state. During POST, an immediate +and then five-second content-free heartbeat reports ordinal/total/attempt, elapsed +time and already ingested count: "1/25 ingested; processing file 2/25". This observer +never makes requests. No within-file percentage, ETA, filenames or private content. +Increment completion only after ACK/metadata proof and required receipt persistence. + +`file_batch.files` records bounded relative names/full hashes, upload state, +verification state and sanitized request IDs. Counts distinguish: +`accepted` (original 200/201 ACK), `confirmed` (exact file-ID/markers without +reported error), `failed`, `pending` (ACK but not confirmed), +`unverified` (uncertain attempt), and `not_attempted`. +Accepted overlaps verification counts; do not sum it with them. +`ingested` counts synchronous completion proved by valid ACK metadata or exact +completed-file readback. A status-only ACK is insufficient. Original ACK file IDs +remain binding even if its remaining metadata is incomplete. Failed readback stays +partial while preserving known prior ingestion; `retrieval` stays `unverified`. +File completion does not prove OCR quality or KB retrieval. + +Report "24 ingested, one failed/unverified" only with this proof; otherwise +distinguish accepted/pending. Preserve outcomes, not a blanket blocker. A subset +handoff must disclose pending files may appear and subset isolation is not proven; +any KB use still needs its existing separate approval. No automatic KB writes. + +## One documented queue-rejection retry + +The official File guide identifies upload HTTP 429 as a full processing queue and +recommends bounded parallelism/exponential backoff; Retry-After is not guaranteed. +Only this File upload route permits one same-operation retry across the batch, +not per file. No private throttle marker/error text is required. Honor valid +Retry-After; missing/invalid metadata uses the diagnosed first backoff step of one +second. Shared 30-second wait/60-second complete preflight limits apply. +After waiting, revalidate unchanged source/corpus and one fresh bounded inventory. +Positive proof avoids another POST; absence is not the retry authority: the +received queue-rejection 429 is. Persist exclusive original/retry attempt and +result records when journaling. Retry the original URL, token, multipart bytes/ +boundary and approved payload; source creation happens once. + +The MI canary's stricter one-upload approval forbids this retry. No PUT/DELETE/ +update retry, new roles or auth substitution. Each upload creates a new file ID, +even for the same filename; no filename-idempotency key exists. Timeout/504/409/ +5xx remain unknown, never replayable from empty inventory or elapsed time. +`file-upload-retry-safety-unproven` retains that blocker. Exhausted files stay +failed; continuation uploads only never-attempted files under fresh approval, +not another attempt at an exhausted file. + +Authorities: failure/conflict/uncertainty only. +[File upload/troubleshooting](https://learn.microsoft.com/azure/search/agentic-knowledge-source-how-to-file). diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_cu_canary.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_cu_canary.py index 92d65254..106ac898 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_cu_canary.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_cu_canary.py @@ -1,540 +1,540 @@ -"""Approval-gated one-file OCR canary; never invoked automatically by ingestion.""" -from __future__ import annotations - -import argparse -import copy -import hashlib -import json -import re -import secrets -import sys -import time -import zlib -from email import policy -from email.parser import BytesParser -from pathlib import Path -from urllib.parse import urlencode - -try: - from . import _bootstrap_io, file_source, file_ingest, file_cu_mi, search_reconcile - from ._common import (HelperFailure, MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, azure_cli_token, - blocked_result, digest, emit_result, http_request, load_approved_input, - require_allowed_fields, reject_secrets) -except ImportError: - import _bootstrap_io, file_source, file_ingest, file_cu_mi, search_reconcile - from _common import (HelperFailure, MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, azure_cli_token, - blocked_result, digest, emit_result, http_request, load_approved_input, - require_allowed_fields, reject_secrets) - -PDF_NAME = "cu-mi-probe.pdf" -MARKER = re.compile(r"CUOCR[2-9]{8}") -FIELD = re.compile(r"[A-Za-z][A-Za-z0-9_]{0,127}") -# Pinned implementation, not deployed proof; see references/file-cu-canary.md. -# Never populate this from caller attestations or an arbitrary readback field. -OCR_CONTENT_MAPPINGS = { - "2026-08-01-preview": { - "field": "snippet", "parent_field": "snippet_parent_id", "path_field": "metadata_storage_path", - "authority": { - "repository": "AzureSearch", "commit": "2e241af8939a0891836b78278df113dab31a5964", - "contract": "file-standard-cu-index-v1", - }, - }, -} -DISCLOSURE = ( - "One synthetic image-only PDF is uploaded to Search and processed by billable CU OCR; " - "CU may create an analyzer, content may cross regions, and Search retains generated data. " - "No embeddings, KB/chat or direct CU probes. Existing resources/roles/network/local-auth stay unchanged. " - "Client time/attempt caps are not a monetary cap or cancellation of remote processing. " - "Cleanup needs separate approval; retain the private source/file/index inventory." -) -GLYPHS = { - "C": ("01111", "10000", "10000", "10000", "10000", "10000", "01111"), - "U": ("10001", "10001", "10001", "10001", "10001", "10001", "01110"), - "O": ("01110", "10001", "10001", "10001", "10001", "10001", "01110"), - "R": ("11110", "10001", "10001", "11110", "10100", "10010", "10001"), - "2": ("01110", "10001", "00001", "00010", "00100", "01000", "11111"), - "3": ("11110", "00001", "00001", "01110", "00001", "00001", "11110"), - "4": ("00010", "00110", "01010", "10010", "11111", "00010", "00010"), - "5": ("11111", "10000", "10000", "11110", "00001", "00001", "11110"), - "6": ("01110", "10000", "10000", "11110", "10001", "10001", "01110"), - "7": ("11111", "00001", "00010", "00100", "01000", "01000", "01000"), - "8": ("01110", "10001", "10001", "01110", "10001", "10001", "01110"), - "9": ("01110", "10001", "10001", "01111", "00001", "00001", "01110"), -} - - -def fail(code, message, *, partial=False, request_id=None, status=None): - return HelperFailure(code, message, blocked_at="file-cu-canary", partial=partial, request_id=request_id, status=status) - - -def pdf_bytes(marker): - if not isinstance(marker, str) or MARKER.fullmatch(marker) is None: - raise fail("canary-marker-invalid", "Use CUOCR followed by eight digits 2–9.") - scale, margin = 12, 24 - width, height = len(marker) * 6 * scale + 2 * margin, 7 * scale + 2 * margin - image = bytearray(b"\xff" * width * height) - for n, letter in enumerate(marker): - for y, row in enumerate(GLYPHS[letter]): - for x, pixel in enumerate(row): - if pixel == "1": - for yy in range(scale): - start = (margin + y * scale + yy) * width + margin + (n * 6 + x) * scale - image[start:start + scale] = b"\x00" * scale - compressed = zlib.compress(bytes(image), 9) - draw = b"q 600 0 0 84 20 20 cm /Image0 Do Q\n" - objects = [ - b"<< /Type /Catalog /Pages 2 0 R >>", - b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", - b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 640 124] /Resources << /XObject << /Image0 4 0 R >> >> /Contents 5 0 R >>", - f"<< /Type /XObject /Subtype /Image /Width {width} /Height {height} /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /FlateDecode /Length {len(compressed)} >>\nstream\n".encode() - + compressed + b"\nendstream", - f"<< /Length {len(draw)} >>\nstream\n".encode() + draw + b"endstream", - ] - out = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n") - offsets = [0] - for n, obj in enumerate(objects, 1): - offsets.append(len(out)) - out.extend(f"{n} 0 obj\n".encode() + obj + b"\nendobj\n") - xref = len(out) - out.extend(b"xref\n0 6\n0000000000 65535 f \n") - for offset in offsets[1:]: - out.extend(f"{offset:010d} 00000 n \n".encode()) - out.extend(f"trailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode()) - if len(out) > 65536 or marker.encode() in out: - raise fail("canary-pdf-invalid", "Probe must be bounded image-only bytes without a text marker.") - return bytes(out) - - -def prepare(directory, marker=None): - directory = _bootstrap_io.private_directory(directory) - marker = marker or "CUOCR" + "".join(secrets.choice("23456789") for _ in range(8)) - data = pdf_bytes(marker) - _bootstrap_io.private_bytes(directory, PDF_NAME, data) - manifest = {"file": PDF_NAME, "marker": marker, "sha256": hashlib.sha256(data).hexdigest(), - "bytes": len(data), "pages": 1, "text_layer": False} - path = _bootstrap_io.private_file(directory, "cu-mi-probe-manifest.json", manifest) - return {"status": "prepared", "manifest": str(path), "writes_performed": [], - "azure_operations": 0, "note": "Local synthetic artifacts only; no live approval or resource selection."} - - -def _bounds(value): - if not isinstance(value, dict): - raise fail("canary-bounds-missing", "Select explicit timeout, HTTP request, polling attempt and interval caps.") - ranges = {"timeout_seconds": (30, 600), "max_requests": (16, 100), - "max_poll_attempts": (1, 10), "poll_interval_seconds": (1, 30)} - require_allowed_fields(value, set(ranges), label="canary bounds") - if any(type(value.get(k)) is not int or not low <= value[k] <= high for k, (low, high) in ranges.items()): - raise fail("canary-bounds-invalid", "Use timeout 30–600s, HTTP requests 16–100, polls 1–10 and interval 1–30s.") - - -def _ocr_mapping(api_version, field): - mapping = OCR_CONTENT_MAPPINGS.get(api_version) - if mapping is None: - raise fail("canary-ocr-mapping-unverified", - "No pinned File Standard OCR-to-index field mapping is verified for this API. " - "Do not create/upload; verify the first-party mapping and update the supported contract before planning.") - if field != mapping["field"]: - raise fail("canary-field-unverified", "The selected field does not match the pinned File Standard OCR-content mapping.") - return copy.deepcopy(mapping) - - -def _verified_file(source_result, ingestion): - files = source_result["verification"]["readback"].get("files") - expected = ingestion["files"][0] - if ( - not isinstance(files, list) or len(files) != 1 or not isinstance(files[0], dict) - or not isinstance(files[0].get("fileId"), str) or not files[0]["fileId"].strip() - or files[0].get("fileName") != expected["path"] - or files[0].get("sha256") != expected["sha256"] or files[0].get("size") != expected["size"] - ): - raise fail("canary-file-identity-unverified", "Require the uploaded synthetic file's exact ID, path, hash and size readback.") - return files[0] - - -def _verify_index_fields(index, mapping): - fields = index.get("fields") - if not isinstance(fields, list): - raise fail("canary-field-unverified", "Generated index fields must match the pinned File OCR mapping.") - for name in (mapping["field"], mapping["parent_field"], mapping["path_field"]): - selected = [field for field in fields if isinstance(field, dict) and field.get("name") == name] - if len(selected) != 1 or selected[0].get("type") != "Edm.String" or selected[0].get("retrievable") is not True: - raise fail("canary-field-unverified", "Require unique retrievable string fields for pinned OCR content and file identity.") - if name == mapping["field"] and ( - selected[0].get("searchable") is not True - or any(selected[0].get(flag) is not False for flag in ("filterable", "sortable", "facetable")) - ): - raise fail("canary-field-unverified", "The canonical snippet field does not match the pinned searchable OCR-content schema.") - - -def _contains_marker(value, marker): - if isinstance(value, str): - return marker in re.sub(r"[^A-Z0-9]", "", value.upper()) - if isinstance(value, dict): - return any(_contains_marker(k, marker) or _contains_marker(v, marker) for k, v in value.items()) - if isinstance(value, (list, tuple)): - return any(_contains_marker(item, marker) for item in value) - return False - - -def _check_non_image_channels(plan, marker): - ingestion = plan["ingestion"] - # Produce the actual multipart envelope with only the image bytes omitted. - try: - body, boundary = file_ingest._multipart(ingestion, ingestion["files"][0], b"", digest(plan)) - message = BytesParser(policy=policy.default).parsebytes( - f"Content-Type: multipart/form-data; boundary={boundary}\r\n\r\n".encode("ascii") + body - ) - except (ValueError, UnicodeError, TypeError): - raise fail("canary-upload-envelope-unverified", "The non-image upload envelope could not be produced safely.") from None - parts = list(message.iter_parts()) - if ( - message.defects or len(parts) != 2 or any(part.defects for part in parts) - or [part.get_param("name", header="content-disposition") for part in parts] != ["metadata", "content"] - or parts[0].get_content_type() != "application/json" or parts[1].get_payload(decode=True) != b"" - ): - raise fail("canary-upload-envelope-unverified", "The produced non-image upload envelope is not the supported multipart contract.") - try: - metadata = json.loads(parts[0].get_payload(decode=True)) - channels = [plan["source"]["desired"], file_ingest._list_url(ingestion), - body.decode("utf-8"), metadata, [list(part.items()) for part in parts]] - except (ValueError, UnicodeError, TypeError): - raise fail("canary-upload-envelope-unverified", "The produced non-image upload envelope could not be inspected.") from None - if _contains_marker(channels, marker): - raise fail("canary-marker-in-metadata", "The OCR marker must not occur in source/upload names, owner, metadata or other non-image upload channels.") - - -def _file_contract(plan, marker): - expected = pdf_bytes(marker) - if not isinstance(plan, dict) or not isinstance(plan.get("ingestion"), dict): - raise fail("canary-file-contract-invalid", "Retain the complete File plan.") - root = file_ingest.resolve_local_root(plan["ingestion"].get("local_root")) - actual = file_ingest._resolve_inventory_path(root, PDF_NAME) - try: - if actual.stat().st_size != len(expected): - raise fail("canary-content-drift", "Probe size differs from the bounded synthetic image-only PDF.") - except OSError: - raise fail("canary-content-drift", "Selected synthetic PDF is unreadable.") from None - file_source._validate_plan(plan) - source, ingestion = plan["source"], plan["ingestion"] - settings = source["desired"]["fileParameters"]["ingestionParameters"] - if ( - plan.get("file_cu_plan_version") != "1.2" or source.get("action") != "create" - or source["api_version"] != "2026-08-01-preview" - or settings.get("contentExtractionMode") != "standard" or "embedding" in plan - or settings.get("embeddingModel") is not None or settings.get("chatCompletionModel") is not None - or settings.get("disableImageVerbalization") is not True - or len(ingestion["files"]) != 1 or ingestion["files"][0]["path"] != PDF_NAME - ): - raise fail("canary-file-contract-invalid", "Canary requires fresh conditional File Standard MI creation, one synthetic PDF, no embedding/chat and unchanged August API.") - _check_non_image_channels(plan, marker) - records = file_ingest.snapshot_inventory(root, [PDF_NAME], service_tier=ingestion["service_tier"]) - if records != ingestion["files"] or records[0]["sha256"] != "sha256:" + hashlib.sha256(expected).hexdigest(): - raise fail("canary-content-drift", "Synthetic inventory changed; regenerate the concrete plan before approval.") - - -def plan_canary(request, *, token_provider=azure_cli_token, transport=http_request, now=time.time): - if not isinstance(request, dict): - raise fail("canary-input-invalid", "Canary intent must be an object.") - require_allowed_fields(request, {"schema_version", "file_request", "marker", "content_field", "bounds", "receipt_directory"}, - label="File CU canary intent") - reject_secrets(request) - _bounds(request.get("bounds")) - if request.get("schema_version") != "1.0" or not isinstance(request.get("content_field"), str) or not FIELD.fullmatch(request["content_field"]): - raise fail("canary-input-invalid", "Use schema 1.0 and an exact simple generated content field name.") - pdf_bytes(request.get("marker")) - directory = _bootstrap_io.private_directory(request.get("receipt_directory")) - fr = request.get("file_request") - if not isinstance(fr, dict): - raise fail("canary-input-invalid", "Supply an explicit existing Search/CU File request; no resource defaults or provisioning.") - # Reject a key mode before even read-only resource discovery. - cu = fr.get("content_understanding") - if not isinstance(cu, dict) or cu.get("auth", "system-assigned") != "system-assigned": - raise fail("canary-auth-invalid", "This canary only tests File managed identity; no keys or fallback.") - if fr.get("paths") != [PDF_NAME] or fr.get("extraction_mode") != "standard" or fr.get("vectorization") != "none": - raise fail("canary-file-contract-invalid", "Select exactly the generated probe PDF, Standard extraction and no vectors.") - version = fr.get("api_version", file_ingest.API_VERSION) - file_ingest.validate_api_version(version) - mapping = _ocr_mapping(version, request["content_field"]) - root = file_ingest.resolve_local_root(fr.get("local_root")) - selected = file_ingest._resolve_inventory_path(root, PDF_NAME) - try: - if selected.stat().st_size != len(pdf_bytes(request["marker"])): - raise fail("canary-content-drift", "Selected PDF is not the bounded synthetic probe.") - except OSError: - raise fail("canary-content-drift", "Selected probe is unreadable.") from None - result = file_source.plan_source(fr, token_provider=token_provider, transport=transport) - fp = result["execution_input"]["plan"] - _file_contract(fp, request["marker"]) - created = int(now()) - plan = { - "operation": "validate-file-cu-mi", "version": "1.1", "file_plan": fp, "ocr_mapping": mapping, - "marker": request["marker"], "content_field": request["content_field"], - "bounds": copy.deepcopy(request["bounds"]), "receipt_directory": str(directory), - "created_at": created, "expires_at": created + 900, "disclosure": DISCLOSURE, - } - fingerprint = digest(plan) - return { - "status": "planned", "plan_fingerprint": fingerprint, - "execution_input": {"schema_version": "1.0", "plan": plan, - "approval": {"confirmed": False, "fingerprint": fingerprint}}, - "approval_summary": {"source": result["approval_summary"], "bounds": plan["bounds"], - "expires_at": plan["expires_at"], "disclosure": DISCLOSURE, - "expected_outcome": "Indexed OCR marker without client keys; principal attribution remains separate."}, - "writes_performed": [], - } - - -class BoundedTransport: - def __init__(self, plan, raw, clock, record): - self.plan, self.raw, self.clock, self.record = plan, raw, clock, record - self.deadline = clock() + plan["bounds"]["timeout_seconds"] - self.calls, self.puts, self.uploads = 0, 0, 0 - self.upload_ack_failure = None - self.index_url = None - fp = plan["file_plan"] - self.source_url = search_reconcile.resource_url(fp["source"]) - self.files_prefix = self.source_url.split("?")[0] + "/files" - cu = fp["content_understanding"] - mi = cu["managed_identity"] - self.arm_urls = { - f"{MANAGEMENT_AUDIENCE}{cu['resource_id']}?api-version=2024-10-01", - f"{MANAGEMENT_AUDIENCE}{mi['search_resource_id']}?api-version={file_cu_mi.SEARCH_API}", - f"{MANAGEMENT_AUDIENCE}{mi['role_assignment_id']}?api-version={file_cu_mi.ROLE_API}", - } - - def __call__(self, method, url, token, **kwargs): - if self.upload_ack_failure is not None: - raise self.upload_ack_failure - remaining = self.deadline - self.clock() - if remaining <= 0 or self.calls >= self.plan["bounds"]["max_requests"]: - raise fail("canary-bound-reached", "Client request/deadline cap reached; remote processing may continue.", partial=bool(self.puts)) - file_url = url.startswith(self.files_prefix + "?") - allowed = method == "GET" and ( - url in self.arm_urls or url == self.source_url or file_url - or self.index_url is not None and (url == self.index_url or url.startswith(self.index_url.split("?")[0] + "/docs?")) - ) - if method == "PUT" and url == self.source_url and self.puts == 0 and kwargs.get("headers", {}).get("If-None-Match") == "*": - allowed = True - self.puts += 1 - elif method == "POST" and file_url and self.uploads == 0: - allowed = True - self.uploads += 1 - if not allowed: - raise fail("canary-operation-forbidden", "Only exact resource GETs, one conditional source PUT and one selected upload are approved.", partial=bool(self.puts)) - self.calls += 1 - kwargs.update(follow_redirects=False, max_response_bytes=min(524288, kwargs.get("max_response_bytes", 524288)), - response_deadline=min(self.deadline, kwargs.get("response_deadline", self.deadline)), - timeout=max(0.01, min(float(kwargs.get("timeout", 60)), remaining))) - try: - response = self.raw(method, url, token, **kwargs) - except HelperFailure as failure: - failure.recovery_deadline = min(self.deadline, failure.recovery_deadline or self.deadline) - raise - if method == "POST" and response.status in (200, 201, 202): - try: - self.record("upload-http-ack", {"status": response.status, "request_id": response.request_id, - "ingestion": "not yet verified"}) - except HelperFailure as error: - # Return the real ACK: persistence is not an ambiguous upload. - self.upload_ack_failure = error - return search_reconcile.HttpResult( - response.status, response.body, response.headers, response.etag_values, - min(self.deadline, response.recovery_deadline or self.deadline), - self.upload_ack_failure, - ) - - -def execute(document, *, token_provider=azure_cli_token, transport=http_request, now=time.time, - clock=time.monotonic, sleep=time.sleep): - if not isinstance(document, dict): - raise fail("canary-input-invalid", "Canary execution requires an object envelope.") - require_allowed_fields(document, {"schema_version", "plan", "approval", "_computed_fingerprint"}, label="canary envelope") - if document.get("schema_version") != "1.0": - raise fail("canary-input-invalid", "Use canary envelope schema_version 1.0.") - plan = document.get("plan") - if not isinstance(plan, dict): - raise fail("canary-input-invalid", "Use the unchanged canary execution envelope.") - require_allowed_fields(plan, {"operation", "version", "file_plan", "marker", "content_field", "ocr_mapping", "bounds", - "receipt_directory", "created_at", "expires_at", "disclosure"}, label="canary plan") - fingerprint = digest(plan) - if document.get("approval") != {"confirmed": True, "fingerprint": fingerprint} or document.get("_computed_fingerprint") != fingerprint: - raise fail("approval-missing", "Exact canary scope, synthetic content/cost and bounded verification need unchanged fingerprinted approval.") - if ( - plan.get("operation") != "validate-file-cu-mi" or plan.get("version") != "1.1" - or plan.get("disclosure") != DISCLOSURE or not isinstance(plan.get("content_field"), str) - or not FIELD.fullmatch(plan["content_field"]) - or type(plan.get("created_at")) is not int or type(plan.get("expires_at")) is not int - or plan["expires_at"] - plan["created_at"] != 900 or not plan["created_at"] <= now() < plan["expires_at"] - ): - raise fail("canary-plan-stale", "Canary plan is invalid or outside its 15-minute approval window; refresh discovery.") - _bounds(plan.get("bounds")) - _file_contract(plan["file_plan"], plan["marker"]) - if plan.get("ocr_mapping") != _ocr_mapping(plan["file_plan"]["source"]["api_version"], plan["content_field"]): - raise fail("canary-ocr-mapping-stale", "The approved OCR mapping differs from the pinned helper contract; a fresh plan and approval are required.") - directory = _bootstrap_io.private_directory(plan["receipt_directory"]) - prefix = "cu-mi-" + fingerprint.split(":")[1][:16] - refs = [] - - def record(stage, value): - path = _bootstrap_io.private_file(directory, prefix + "-" + stage + ".json", { - "fingerprint": fingerprint, "stage": stage, "value": value, - }) - refs.append(str(path)) - - record("started", {"approved_input": {k: v for k, v in document.items() if k != "_computed_fingerprint"}, - "state": "No Azure mutation yet; later completion is not atomic."}) - bounded = BoundedTransport(plan, transport, clock, record) - fp = plan["file_plan"] - child_digest = digest(fp) - source_result = None - def source_ack(**evidence): - response = evidence["response"] - record("source-http-ack", {"status": response.status, "request_id": response.request_id, - "etag_evidence": search_reconcile.response_etags(response)}) - - try: - source_result = file_source.execute( - {"schema_version": "1.0", "plan": fp, "_computed_fingerprint": child_digest, - "approval": {"confirmed": True, "fingerprint": child_digest}}, - token_provider=token_provider, transport=bounded, mi_on_created=source_ack, - allow_upload_retry=False, - ) - if bounded.upload_ack_failure is not None: - raise bounded.upload_ack_failure - record("file-completed", source_result) - original_file = _verified_file(source_result, fp["ingestion"]) - mapping = plan["ocr_mapping"] - token = token_provider(SEARCH_AUDIENCE) - current, _ = search_reconcile.read_resource(bounded.source_url, token, transport=bounded) - file_source.verify_content_understanding_readback(fp["content_understanding"], current) - created = current.get("fileParameters", {}).get("createdResources") - if not isinstance(created, dict) or set(created) != {"index"} or not isinstance(created["index"], str) or not re.fullmatch(r"[a-z0-9][a-z0-9_-]{1,127}", created["index"]): - raise fail("canary-index-unverified", "Fresh File readback must identify exactly one generated index.") - retained = source_result["verification"]["readback"]["source"] - if current.get("@odata.etag") != retained["etag"] or not search_reconcile.definitions_match(fp["source"]["desired"], current): - raise fail("canary-source-drift", "Source version changed before indexed OCR verification.") - version = fp["source"]["api_version"] - bounded.index_url = f"{fp['source']['endpoint'].rstrip('/')}/indexes('{created['index']}')?api-version={version}" - index, _ = search_reconcile.read_resource(bounded.index_url, token, transport=bounded) - if not isinstance(index, dict) or index.get("name") != created["index"] or not isinstance(index.get("@odata.etag"), str) or not index["@odata.etag"]: - raise fail("canary-index-unverified", "Require the generated index's matching name and fresh ETag.") - _verify_index_fields(index, mapping) - query = bounded.index_url.split("?")[0] + "/docs?" + urlencode({ - "api-version": version, "search": "*", - "$select": ",".join((mapping["field"], mapping["parent_field"], mapping["path_field"])), "$top": 3, - }) - matched = False - for attempt in range(plan["bounds"]["max_poll_attempts"]): - response = bounded("GET", query, token) - if response.status != 200 or not isinstance(response.body, dict) or not isinstance(response.body.get("value"), list): - raise fail("canary-index-query-failed", "Generated-index query did not return bounded document rows.", - request_id=response.request_id, status=response.status) - rows = response.body["value"] - if len(rows) > 3: - raise fail("canary-index-query-failed", "Generated-index response exceeded the approved row cap.") - if any( - not isinstance(row, dict) or not isinstance(row.get(mapping["field"]), str) - or row.get(mapping["parent_field"]) != original_file["fileId"] - or row.get(mapping["path_field"]) != original_file["fileName"] - for row in rows - ): - raise fail("canary-document-binding-unverified", "Indexed rows must contain canonical OCR text and the uploaded file's exact parent ID/path; chunk IDs or metadata alone are not proof.") - matched = any(_contains_marker(row[mapping["field"]], plan["marker"]) for row in rows) - if matched: - break - if attempt + 1 < plan["bounds"]["max_poll_attempts"]: - sleep(min(plan["bounds"]["poll_interval_seconds"], max(0, bounded.deadline - clock()))) - if not matched: - raise fail("canary-ocr-unverified", "Upload/source creation is not extraction proof: indexed OCR marker was not observed within the caps.") - after, _ = search_reconcile.read_resource(bounded.source_url, token, transport=bounded) - if not isinstance(after, dict) or after.get("@odata.etag") != current.get("@odata.etag") or after.get("fileParameters", {}).get("createdResources") != created: - raise fail("canary-source-drift", "Generated source/index binding changed during OCR verification.") - final_index, _ = search_reconcile.read_resource(bounded.index_url, token, transport=bounded) - if final_index != index: - raise fail("canary-index-drift", "Generated index changed during OCR verification.") - binding, _ = file_cu_mi.read_binding(fp["content_understanding"], fp["source"]["endpoint"], - token_provider=token_provider, transport=bounded) - if binding != fp["cu_identity_state"]: - raise fail("canary-identity-drift", "Search/CU identity-role binding changed during validation.") - result = {**copy.deepcopy(source_result), "outcome": "validate-file-cu-mi", - "approved_plan": {"fingerprint": fingerprint, "confirmed": True}, - "status": "completed", "verdict": "keyless-functional-pass", "indexed_ocr_marker": True, - "principal_attribution": "unverified", "backend_rollout": "unverified", - "identity_evidence": fp["cu_identity_state"], "source_result": source_result, - "ocr_evidence": {"mapping": mapping, "file": original_file}, - "index": created["index"], "request_count": bounded.calls, - "cleanup": {"status": "separate-plan-and-approval-required", - "instructions": "Retain inventory; separately approved guarded source cleanup only."}, - "warnings": ["Functional evidence is for this run only, not an atomic deployment or release claim.", - "No backend CU principal/telemetry was observed."]} - except HelperFailure as error: - if bounded.upload_ack_failure is not None and error is not bounded.upload_ack_failure: - checkpoint_error = bounded.upload_ack_failure - checkpoint_error.writes = error.writes - checkpoint_error.resources_remaining = error.resources_remaining - checkpoint_error.resources_reused = error.resources_reused - checkpoint_error.resources_unverified = error.resources_unverified - checkpoint_error.warnings.extend(error.warnings) - checkpoint_error.partial = error.partial - error = checkpoint_error - if source_result is not None: - error.partial = True - error.writes = [ - {**resource, "action": "created", "type": resource.get("type", "knowledge-source-file")} - for resource in source_result["resources"]["created"] - ] + error.writes - error.resources_remaining = source_result["ownership"]["run_owned"] + error.resources_remaining - result = blocked_result(error, outcome="validate-file-cu-mi", fingerprint=fingerprint, owner=fp["owner"]) - result.update(verdict="unverified", indexed_ocr_marker=False, principal_attribution="unverified", - backend_rollout="unverified", request_count=bounded.calls) - result.setdefault("warnings", []).append("Remote processing/costs may continue; do not replay mutations or infer ownership from an ambiguous create.") - if bounded.upload_ack_failure is not None: - result["warnings"].append("Upload HTTP ACK could not be retained; validation stopped. Only listed receipt_refs are confirmed; do not replay the upload.") - try: - record("result", result) - except HelperFailure: - if result["status"] == "completed": - error = fail("canary-receipt-failed", "Validation finished but its final private receipt could not be persisted.", partial=True) - error.writes = [ - {**resource, "action": "created", "type": resource.get("type", "knowledge-source-file")} - for resource in source_result["resources"]["created"] - ] - error.resources_remaining = source_result["ownership"]["run_owned"] - result = {**blocked_result(error, outcome="validate-file-cu-mi", fingerprint=fingerprint, owner=fp["owner"]), - "indexed_ocr_marker": True, "principal_attribution": "unverified", "backend_rollout": "unverified", - "source_result": source_result, "request_count": bounded.calls} - result["verdict"] = "unverified" - result.setdefault("warnings", []).append("Final private receipt persistence failed; retain this ownership handoff and existing checkpoints. Do not replay.") - return {**result, "receipt_refs": refs} - - -def main(argv=None): - parser = argparse.ArgumentParser() - modes = parser.add_mutually_exclusive_group(required=True) - modes.add_argument("--prepare", metavar="EXISTING_PRIVATE_DIRECTORY") - modes.add_argument("--plan", type=Path) - modes.add_argument("--input", type=Path) - args = parser.parse_args(argv) - try: - if args.prepare: - result = prepare(args.prepare) - elif args.plan: - result = plan_canary(_bootstrap_io.read_json(args.plan)) - directory = result["execution_input"]["plan"]["receipt_directory"] - path = _bootstrap_io.private_file(directory, "cu-mi-plan-" + result["plan_fingerprint"].split(":")[1][:16] + ".json", - result["execution_input"]) - result = {k: v for k, v in result.items() if k != "execution_input"} - result["execution_input_ref"] = str(path) - else: - document, _, fingerprint = load_approved_input(args.input) - document["_computed_fingerprint"] = fingerprint - result = execute(document) - emit_result(result) - return 0 if result["status"] in ("prepared", "planned", "completed") else 3 if result["status"] == "partial" else 2 - except HelperFailure as error: - emit_result(blocked_result(error, outcome="validate-file-cu-mi", fingerprint=None, owner=None)) - return 3 if error.partial else 2 - - -if __name__ == "__main__": - sys.exit(main()) +"""Approval-gated one-file OCR canary; never invoked automatically by ingestion.""" +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import re +import secrets +import sys +import time +import zlib +from email import policy +from email.parser import BytesParser +from pathlib import Path +from urllib.parse import urlencode + +try: + from . import _bootstrap_io, file_source, file_ingest, file_cu_mi, search_reconcile + from ._common import (HelperFailure, MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, azure_cli_token, + blocked_result, digest, emit_result, http_request, load_approved_input, + require_allowed_fields, reject_secrets) +except ImportError: + import _bootstrap_io, file_source, file_ingest, file_cu_mi, search_reconcile + from _common import (HelperFailure, MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, azure_cli_token, + blocked_result, digest, emit_result, http_request, load_approved_input, + require_allowed_fields, reject_secrets) + +PDF_NAME = "cu-mi-probe.pdf" +MARKER = re.compile(r"CUOCR[2-9]{8}") +FIELD = re.compile(r"[A-Za-z][A-Za-z0-9_]{0,127}") +# Pinned implementation, not deployed proof; see references/file-cu-canary.md. +# Never populate this from caller attestations or an arbitrary readback field. +OCR_CONTENT_MAPPINGS = { + "2026-08-01-preview": { + "field": "snippet", "parent_field": "snippet_parent_id", "path_field": "metadata_storage_path", + "authority": { + "repository": "AzureSearch", "commit": "2e241af8939a0891836b78278df113dab31a5964", + "contract": "file-standard-cu-index-v1", + }, + }, +} +DISCLOSURE = ( + "One synthetic image-only PDF is uploaded to Search and processed by billable CU OCR; " + "CU may create an analyzer, content may cross regions, and Search retains generated data. " + "No embeddings, KB/chat or direct CU probes. Existing resources/roles/network/local-auth stay unchanged. " + "Client time/attempt caps are not a monetary cap or cancellation of remote processing. " + "Cleanup needs separate approval; retain the private source/file/index inventory." +) +GLYPHS = { + "C": ("01111", "10000", "10000", "10000", "10000", "10000", "01111"), + "U": ("10001", "10001", "10001", "10001", "10001", "10001", "01110"), + "O": ("01110", "10001", "10001", "10001", "10001", "10001", "01110"), + "R": ("11110", "10001", "10001", "11110", "10100", "10010", "10001"), + "2": ("01110", "10001", "00001", "00010", "00100", "01000", "11111"), + "3": ("11110", "00001", "00001", "01110", "00001", "00001", "11110"), + "4": ("00010", "00110", "01010", "10010", "11111", "00010", "00010"), + "5": ("11111", "10000", "10000", "11110", "00001", "00001", "11110"), + "6": ("01110", "10000", "10000", "11110", "10001", "10001", "01110"), + "7": ("11111", "00001", "00010", "00100", "01000", "01000", "01000"), + "8": ("01110", "10001", "10001", "01110", "10001", "10001", "01110"), + "9": ("01110", "10001", "10001", "01111", "00001", "00001", "01110"), +} + + +def fail(code, message, *, partial=False, request_id=None, status=None): + return HelperFailure(code, message, blocked_at="file-cu-canary", partial=partial, request_id=request_id, status=status) + + +def pdf_bytes(marker): + if not isinstance(marker, str) or MARKER.fullmatch(marker) is None: + raise fail("canary-marker-invalid", "Use CUOCR followed by eight digits 2–9.") + scale, margin = 12, 24 + width, height = len(marker) * 6 * scale + 2 * margin, 7 * scale + 2 * margin + image = bytearray(b"\xff" * width * height) + for n, letter in enumerate(marker): + for y, row in enumerate(GLYPHS[letter]): + for x, pixel in enumerate(row): + if pixel == "1": + for yy in range(scale): + start = (margin + y * scale + yy) * width + margin + (n * 6 + x) * scale + image[start:start + scale] = b"\x00" * scale + compressed = zlib.compress(bytes(image), 9) + draw = b"q 600 0 0 84 20 20 cm /Image0 Do Q\n" + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 640 124] /Resources << /XObject << /Image0 4 0 R >> >> /Contents 5 0 R >>", + f"<< /Type /XObject /Subtype /Image /Width {width} /Height {height} /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /FlateDecode /Length {len(compressed)} >>\nstream\n".encode() + + compressed + b"\nendstream", + f"<< /Length {len(draw)} >>\nstream\n".encode() + draw + b"endstream", + ] + out = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n") + offsets = [0] + for n, obj in enumerate(objects, 1): + offsets.append(len(out)) + out.extend(f"{n} 0 obj\n".encode() + obj + b"\nendobj\n") + xref = len(out) + out.extend(b"xref\n0 6\n0000000000 65535 f \n") + for offset in offsets[1:]: + out.extend(f"{offset:010d} 00000 n \n".encode()) + out.extend(f"trailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode()) + if len(out) > 65536 or marker.encode() in out: + raise fail("canary-pdf-invalid", "Probe must be bounded image-only bytes without a text marker.") + return bytes(out) + + +def prepare(directory, marker=None): + directory = _bootstrap_io.private_directory(directory) + marker = marker or "CUOCR" + "".join(secrets.choice("23456789") for _ in range(8)) + data = pdf_bytes(marker) + _bootstrap_io.private_bytes(directory, PDF_NAME, data) + manifest = {"file": PDF_NAME, "marker": marker, "sha256": hashlib.sha256(data).hexdigest(), + "bytes": len(data), "pages": 1, "text_layer": False} + path = _bootstrap_io.private_file(directory, "cu-mi-probe-manifest.json", manifest) + return {"status": "prepared", "manifest": str(path), "writes_performed": [], + "azure_operations": 0, "note": "Local synthetic artifacts only; no live approval or resource selection."} + + +def _bounds(value): + if not isinstance(value, dict): + raise fail("canary-bounds-missing", "Select explicit timeout, HTTP request, polling attempt and interval caps.") + ranges = {"timeout_seconds": (30, 600), "max_requests": (16, 100), + "max_poll_attempts": (1, 10), "poll_interval_seconds": (1, 30)} + require_allowed_fields(value, set(ranges), label="canary bounds") + if any(type(value.get(k)) is not int or not low <= value[k] <= high for k, (low, high) in ranges.items()): + raise fail("canary-bounds-invalid", "Use timeout 30–600s, HTTP requests 16–100, polls 1–10 and interval 1–30s.") + + +def _ocr_mapping(api_version, field): + mapping = OCR_CONTENT_MAPPINGS.get(api_version) + if mapping is None: + raise fail("canary-ocr-mapping-unverified", + "No pinned File Standard OCR-to-index field mapping is verified for this API. " + "Do not create/upload; verify the first-party mapping and update the supported contract before planning.") + if field != mapping["field"]: + raise fail("canary-field-unverified", "The selected field does not match the pinned File Standard OCR-content mapping.") + return copy.deepcopy(mapping) + + +def _verified_file(source_result, ingestion): + files = source_result["verification"]["readback"].get("files") + expected = ingestion["files"][0] + if ( + not isinstance(files, list) or len(files) != 1 or not isinstance(files[0], dict) + or not isinstance(files[0].get("fileId"), str) or not files[0]["fileId"].strip() + or files[0].get("fileName") != expected["path"] + or files[0].get("sha256") != expected["sha256"] or files[0].get("size") != expected["size"] + ): + raise fail("canary-file-identity-unverified", "Require the uploaded synthetic file's exact ID, path, hash and size readback.") + return files[0] + + +def _verify_index_fields(index, mapping): + fields = index.get("fields") + if not isinstance(fields, list): + raise fail("canary-field-unverified", "Generated index fields must match the pinned File OCR mapping.") + for name in (mapping["field"], mapping["parent_field"], mapping["path_field"]): + selected = [field for field in fields if isinstance(field, dict) and field.get("name") == name] + if len(selected) != 1 or selected[0].get("type") != "Edm.String" or selected[0].get("retrievable") is not True: + raise fail("canary-field-unverified", "Require unique retrievable string fields for pinned OCR content and file identity.") + if name == mapping["field"] and ( + selected[0].get("searchable") is not True + or any(selected[0].get(flag) is not False for flag in ("filterable", "sortable", "facetable")) + ): + raise fail("canary-field-unverified", "The canonical snippet field does not match the pinned searchable OCR-content schema.") + + +def _contains_marker(value, marker): + if isinstance(value, str): + return marker in re.sub(r"[^A-Z0-9]", "", value.upper()) + if isinstance(value, dict): + return any(_contains_marker(k, marker) or _contains_marker(v, marker) for k, v in value.items()) + if isinstance(value, (list, tuple)): + return any(_contains_marker(item, marker) for item in value) + return False + + +def _check_non_image_channels(plan, marker): + ingestion = plan["ingestion"] + # Produce the actual multipart envelope with only the image bytes omitted. + try: + body, boundary = file_ingest._multipart(ingestion, ingestion["files"][0], b"", digest(plan)) + message = BytesParser(policy=policy.default).parsebytes( + f"Content-Type: multipart/form-data; boundary={boundary}\r\n\r\n".encode("ascii") + body + ) + except (ValueError, UnicodeError, TypeError): + raise fail("canary-upload-envelope-unverified", "The non-image upload envelope could not be produced safely.") from None + parts = list(message.iter_parts()) + if ( + message.defects or len(parts) != 2 or any(part.defects for part in parts) + or [part.get_param("name", header="content-disposition") for part in parts] != ["metadata", "content"] + or parts[0].get_content_type() != "application/json" or parts[1].get_payload(decode=True) != b"" + ): + raise fail("canary-upload-envelope-unverified", "The produced non-image upload envelope is not the supported multipart contract.") + try: + metadata = json.loads(parts[0].get_payload(decode=True)) + channels = [plan["source"]["desired"], file_ingest._list_url(ingestion), + body.decode("utf-8"), metadata, [list(part.items()) for part in parts]] + except (ValueError, UnicodeError, TypeError): + raise fail("canary-upload-envelope-unverified", "The produced non-image upload envelope could not be inspected.") from None + if _contains_marker(channels, marker): + raise fail("canary-marker-in-metadata", "The OCR marker must not occur in source/upload names, owner, metadata or other non-image upload channels.") + + +def _file_contract(plan, marker): + expected = pdf_bytes(marker) + if not isinstance(plan, dict) or not isinstance(plan.get("ingestion"), dict): + raise fail("canary-file-contract-invalid", "Retain the complete File plan.") + root = file_ingest.resolve_local_root(plan["ingestion"].get("local_root")) + actual = file_ingest._resolve_inventory_path(root, PDF_NAME) + try: + if actual.stat().st_size != len(expected): + raise fail("canary-content-drift", "Probe size differs from the bounded synthetic image-only PDF.") + except OSError: + raise fail("canary-content-drift", "Selected synthetic PDF is unreadable.") from None + file_source._validate_plan(plan) + source, ingestion = plan["source"], plan["ingestion"] + settings = source["desired"]["fileParameters"]["ingestionParameters"] + if ( + plan.get("file_cu_plan_version") != "1.2" or source.get("action") != "create" + or source["api_version"] != "2026-08-01-preview" + or settings.get("contentExtractionMode") != "standard" or "embedding" in plan + or settings.get("embeddingModel") is not None or settings.get("chatCompletionModel") is not None + or settings.get("disableImageVerbalization") is not True + or len(ingestion["files"]) != 1 or ingestion["files"][0]["path"] != PDF_NAME + ): + raise fail("canary-file-contract-invalid", "Canary requires fresh conditional File Standard MI creation, one synthetic PDF, no embedding/chat and unchanged August API.") + _check_non_image_channels(plan, marker) + records = file_ingest.snapshot_inventory(root, [PDF_NAME], service_tier=ingestion["service_tier"]) + if records != ingestion["files"] or records[0]["sha256"] != "sha256:" + hashlib.sha256(expected).hexdigest(): + raise fail("canary-content-drift", "Synthetic inventory changed; regenerate the concrete plan before approval.") + + +def plan_canary(request, *, token_provider=azure_cli_token, transport=http_request, now=time.time): + if not isinstance(request, dict): + raise fail("canary-input-invalid", "Canary intent must be an object.") + require_allowed_fields(request, {"schema_version", "file_request", "marker", "content_field", "bounds", "receipt_directory"}, + label="File CU canary intent") + reject_secrets(request) + _bounds(request.get("bounds")) + if request.get("schema_version") != "1.0" or not isinstance(request.get("content_field"), str) or not FIELD.fullmatch(request["content_field"]): + raise fail("canary-input-invalid", "Use schema 1.0 and an exact simple generated content field name.") + pdf_bytes(request.get("marker")) + directory = _bootstrap_io.private_directory(request.get("receipt_directory")) + fr = request.get("file_request") + if not isinstance(fr, dict): + raise fail("canary-input-invalid", "Supply an explicit existing Search/CU File request; no resource defaults or provisioning.") + # Reject a key mode before even read-only resource discovery. + cu = fr.get("content_understanding") + if not isinstance(cu, dict) or cu.get("auth", "system-assigned") != "system-assigned": + raise fail("canary-auth-invalid", "This canary only tests File managed identity; no keys or fallback.") + if fr.get("paths") != [PDF_NAME] or fr.get("extraction_mode") != "standard" or fr.get("vectorization") != "none": + raise fail("canary-file-contract-invalid", "Select exactly the generated probe PDF, Standard extraction and no vectors.") + version = fr.get("api_version", file_ingest.API_VERSION) + file_ingest.validate_api_version(version) + mapping = _ocr_mapping(version, request["content_field"]) + root = file_ingest.resolve_local_root(fr.get("local_root")) + selected = file_ingest._resolve_inventory_path(root, PDF_NAME) + try: + if selected.stat().st_size != len(pdf_bytes(request["marker"])): + raise fail("canary-content-drift", "Selected PDF is not the bounded synthetic probe.") + except OSError: + raise fail("canary-content-drift", "Selected probe is unreadable.") from None + result = file_source.plan_source(fr, token_provider=token_provider, transport=transport) + fp = result["execution_input"]["plan"] + _file_contract(fp, request["marker"]) + created = int(now()) + plan = { + "operation": "validate-file-cu-mi", "version": "1.1", "file_plan": fp, "ocr_mapping": mapping, + "marker": request["marker"], "content_field": request["content_field"], + "bounds": copy.deepcopy(request["bounds"]), "receipt_directory": str(directory), + "created_at": created, "expires_at": created + 900, "disclosure": DISCLOSURE, + } + fingerprint = digest(plan) + return { + "status": "planned", "plan_fingerprint": fingerprint, + "execution_input": {"schema_version": "1.0", "plan": plan, + "approval": {"confirmed": False, "fingerprint": fingerprint}}, + "approval_summary": {"source": result["approval_summary"], "bounds": plan["bounds"], + "expires_at": plan["expires_at"], "disclosure": DISCLOSURE, + "expected_outcome": "Indexed OCR marker without client keys; principal attribution remains separate."}, + "writes_performed": [], + } + + +class BoundedTransport: + def __init__(self, plan, raw, clock, record): + self.plan, self.raw, self.clock, self.record = plan, raw, clock, record + self.deadline = clock() + plan["bounds"]["timeout_seconds"] + self.calls, self.puts, self.uploads = 0, 0, 0 + self.upload_ack_failure = None + self.index_url = None + fp = plan["file_plan"] + self.source_url = search_reconcile.resource_url(fp["source"]) + self.files_prefix = self.source_url.split("?")[0] + "/files" + cu = fp["content_understanding"] + mi = cu["managed_identity"] + self.arm_urls = { + f"{MANAGEMENT_AUDIENCE}{cu['resource_id']}?api-version=2024-10-01", + f"{MANAGEMENT_AUDIENCE}{mi['search_resource_id']}?api-version={file_cu_mi.SEARCH_API}", + f"{MANAGEMENT_AUDIENCE}{mi['role_assignment_id']}?api-version={file_cu_mi.ROLE_API}", + } + + def __call__(self, method, url, token, **kwargs): + if self.upload_ack_failure is not None: + raise self.upload_ack_failure + remaining = self.deadline - self.clock() + if remaining <= 0 or self.calls >= self.plan["bounds"]["max_requests"]: + raise fail("canary-bound-reached", "Client request/deadline cap reached; remote processing may continue.", partial=bool(self.puts)) + file_url = url.startswith(self.files_prefix + "?") + allowed = method == "GET" and ( + url in self.arm_urls or url == self.source_url or file_url + or self.index_url is not None and (url == self.index_url or url.startswith(self.index_url.split("?")[0] + "/docs?")) + ) + if method == "PUT" and url == self.source_url and self.puts == 0 and kwargs.get("headers", {}).get("If-None-Match") == "*": + allowed = True + self.puts += 1 + elif method == "POST" and file_url and self.uploads == 0: + allowed = True + self.uploads += 1 + if not allowed: + raise fail("canary-operation-forbidden", "Only exact resource GETs, one conditional source PUT and one selected upload are approved.", partial=bool(self.puts)) + self.calls += 1 + kwargs.update(follow_redirects=False, max_response_bytes=min(524288, kwargs.get("max_response_bytes", 524288)), + response_deadline=min(self.deadline, kwargs.get("response_deadline", self.deadline)), + timeout=max(0.01, min(float(kwargs.get("timeout", 60)), remaining))) + try: + response = self.raw(method, url, token, **kwargs) + except HelperFailure as failure: + failure.recovery_deadline = min(self.deadline, failure.recovery_deadline or self.deadline) + raise + if method == "POST" and response.status in (200, 201, 202): + try: + self.record("upload-http-ack", {"status": response.status, "request_id": response.request_id, + "ingestion": "not yet verified"}) + except HelperFailure as error: + # Return the real ACK: persistence is not an ambiguous upload. + self.upload_ack_failure = error + return search_reconcile.HttpResult( + response.status, response.body, response.headers, response.etag_values, + min(self.deadline, response.recovery_deadline or self.deadline), + self.upload_ack_failure, + ) + + +def execute(document, *, token_provider=azure_cli_token, transport=http_request, now=time.time, + clock=time.monotonic, sleep=time.sleep): + if not isinstance(document, dict): + raise fail("canary-input-invalid", "Canary execution requires an object envelope.") + require_allowed_fields(document, {"schema_version", "plan", "approval", "_computed_fingerprint"}, label="canary envelope") + if document.get("schema_version") != "1.0": + raise fail("canary-input-invalid", "Use canary envelope schema_version 1.0.") + plan = document.get("plan") + if not isinstance(plan, dict): + raise fail("canary-input-invalid", "Use the unchanged canary execution envelope.") + require_allowed_fields(plan, {"operation", "version", "file_plan", "marker", "content_field", "ocr_mapping", "bounds", + "receipt_directory", "created_at", "expires_at", "disclosure"}, label="canary plan") + fingerprint = digest(plan) + if document.get("approval") != {"confirmed": True, "fingerprint": fingerprint} or document.get("_computed_fingerprint") != fingerprint: + raise fail("approval-missing", "Exact canary scope, synthetic content/cost and bounded verification need unchanged fingerprinted approval.") + if ( + plan.get("operation") != "validate-file-cu-mi" or plan.get("version") != "1.1" + or plan.get("disclosure") != DISCLOSURE or not isinstance(plan.get("content_field"), str) + or not FIELD.fullmatch(plan["content_field"]) + or type(plan.get("created_at")) is not int or type(plan.get("expires_at")) is not int + or plan["expires_at"] - plan["created_at"] != 900 or not plan["created_at"] <= now() < plan["expires_at"] + ): + raise fail("canary-plan-stale", "Canary plan is invalid or outside its 15-minute approval window; refresh discovery.") + _bounds(plan.get("bounds")) + _file_contract(plan["file_plan"], plan["marker"]) + if plan.get("ocr_mapping") != _ocr_mapping(plan["file_plan"]["source"]["api_version"], plan["content_field"]): + raise fail("canary-ocr-mapping-stale", "The approved OCR mapping differs from the pinned helper contract; a fresh plan and approval are required.") + directory = _bootstrap_io.private_directory(plan["receipt_directory"]) + prefix = "cu-mi-" + fingerprint.split(":")[1][:16] + refs = [] + + def record(stage, value): + path = _bootstrap_io.private_file(directory, prefix + "-" + stage + ".json", { + "fingerprint": fingerprint, "stage": stage, "value": value, + }) + refs.append(str(path)) + + record("started", {"approved_input": {k: v for k, v in document.items() if k != "_computed_fingerprint"}, + "state": "No Azure mutation yet; later completion is not atomic."}) + bounded = BoundedTransport(plan, transport, clock, record) + fp = plan["file_plan"] + child_digest = digest(fp) + source_result = None + def source_ack(**evidence): + response = evidence["response"] + record("source-http-ack", {"status": response.status, "request_id": response.request_id, + "etag_evidence": search_reconcile.response_etags(response)}) + + try: + source_result = file_source.execute( + {"schema_version": "1.0", "plan": fp, "_computed_fingerprint": child_digest, + "approval": {"confirmed": True, "fingerprint": child_digest}}, + token_provider=token_provider, transport=bounded, mi_on_created=source_ack, + allow_upload_retry=False, + ) + if bounded.upload_ack_failure is not None: + raise bounded.upload_ack_failure + record("file-completed", source_result) + original_file = _verified_file(source_result, fp["ingestion"]) + mapping = plan["ocr_mapping"] + token = token_provider(SEARCH_AUDIENCE) + current, _ = search_reconcile.read_resource(bounded.source_url, token, transport=bounded) + file_source.verify_content_understanding_readback(fp["content_understanding"], current) + created = current.get("fileParameters", {}).get("createdResources") + if not isinstance(created, dict) or set(created) != {"index"} or not isinstance(created["index"], str) or not re.fullmatch(r"[a-z0-9][a-z0-9_-]{1,127}", created["index"]): + raise fail("canary-index-unverified", "Fresh File readback must identify exactly one generated index.") + retained = source_result["verification"]["readback"]["source"] + if current.get("@odata.etag") != retained["etag"] or not search_reconcile.definitions_match(fp["source"]["desired"], current): + raise fail("canary-source-drift", "Source version changed before indexed OCR verification.") + version = fp["source"]["api_version"] + bounded.index_url = f"{fp['source']['endpoint'].rstrip('/')}/indexes('{created['index']}')?api-version={version}" + index, _ = search_reconcile.read_resource(bounded.index_url, token, transport=bounded) + if not isinstance(index, dict) or index.get("name") != created["index"] or not isinstance(index.get("@odata.etag"), str) or not index["@odata.etag"]: + raise fail("canary-index-unverified", "Require the generated index's matching name and fresh ETag.") + _verify_index_fields(index, mapping) + query = bounded.index_url.split("?")[0] + "/docs?" + urlencode({ + "api-version": version, "search": "*", + "$select": ",".join((mapping["field"], mapping["parent_field"], mapping["path_field"])), "$top": 3, + }) + matched = False + for attempt in range(plan["bounds"]["max_poll_attempts"]): + response = bounded("GET", query, token) + if response.status != 200 or not isinstance(response.body, dict) or not isinstance(response.body.get("value"), list): + raise fail("canary-index-query-failed", "Generated-index query did not return bounded document rows.", + request_id=response.request_id, status=response.status) + rows = response.body["value"] + if len(rows) > 3: + raise fail("canary-index-query-failed", "Generated-index response exceeded the approved row cap.") + if any( + not isinstance(row, dict) or not isinstance(row.get(mapping["field"]), str) + or row.get(mapping["parent_field"]) != original_file["fileId"] + or row.get(mapping["path_field"]) != original_file["fileName"] + for row in rows + ): + raise fail("canary-document-binding-unverified", "Indexed rows must contain canonical OCR text and the uploaded file's exact parent ID/path; chunk IDs or metadata alone are not proof.") + matched = any(_contains_marker(row[mapping["field"]], plan["marker"]) for row in rows) + if matched: + break + if attempt + 1 < plan["bounds"]["max_poll_attempts"]: + sleep(min(plan["bounds"]["poll_interval_seconds"], max(0, bounded.deadline - clock()))) + if not matched: + raise fail("canary-ocr-unverified", "Upload/source creation is not extraction proof: indexed OCR marker was not observed within the caps.") + after, _ = search_reconcile.read_resource(bounded.source_url, token, transport=bounded) + if not isinstance(after, dict) or after.get("@odata.etag") != current.get("@odata.etag") or after.get("fileParameters", {}).get("createdResources") != created: + raise fail("canary-source-drift", "Generated source/index binding changed during OCR verification.") + final_index, _ = search_reconcile.read_resource(bounded.index_url, token, transport=bounded) + if final_index != index: + raise fail("canary-index-drift", "Generated index changed during OCR verification.") + binding, _ = file_cu_mi.read_binding(fp["content_understanding"], fp["source"]["endpoint"], + token_provider=token_provider, transport=bounded) + if binding != fp["cu_identity_state"]: + raise fail("canary-identity-drift", "Search/CU identity-role binding changed during validation.") + result = {**copy.deepcopy(source_result), "outcome": "validate-file-cu-mi", + "approved_plan": {"fingerprint": fingerprint, "confirmed": True}, + "status": "completed", "verdict": "keyless-functional-pass", "indexed_ocr_marker": True, + "principal_attribution": "unverified", "backend_rollout": "unverified", + "identity_evidence": fp["cu_identity_state"], "source_result": source_result, + "ocr_evidence": {"mapping": mapping, "file": original_file}, + "index": created["index"], "request_count": bounded.calls, + "cleanup": {"status": "separate-plan-and-approval-required", + "instructions": "Retain inventory; separately approved guarded source cleanup only."}, + "warnings": ["Functional evidence is for this run only, not an atomic deployment or release claim.", + "No backend CU principal/telemetry was observed."]} + except HelperFailure as error: + if bounded.upload_ack_failure is not None and error is not bounded.upload_ack_failure: + checkpoint_error = bounded.upload_ack_failure + checkpoint_error.writes = error.writes + checkpoint_error.resources_remaining = error.resources_remaining + checkpoint_error.resources_reused = error.resources_reused + checkpoint_error.resources_unverified = error.resources_unverified + checkpoint_error.warnings.extend(error.warnings) + checkpoint_error.partial = error.partial + error = checkpoint_error + if source_result is not None: + error.partial = True + error.writes = [ + {**resource, "action": "created", "type": resource.get("type", "knowledge-source-file")} + for resource in source_result["resources"]["created"] + ] + error.writes + error.resources_remaining = source_result["ownership"]["run_owned"] + error.resources_remaining + result = blocked_result(error, outcome="validate-file-cu-mi", fingerprint=fingerprint, owner=fp["owner"]) + result.update(verdict="unverified", indexed_ocr_marker=False, principal_attribution="unverified", + backend_rollout="unverified", request_count=bounded.calls) + result.setdefault("warnings", []).append("Remote processing/costs may continue; do not replay mutations or infer ownership from an ambiguous create.") + if bounded.upload_ack_failure is not None: + result["warnings"].append("Upload HTTP ACK could not be retained; validation stopped. Only listed receipt_refs are confirmed; do not replay the upload.") + try: + record("result", result) + except HelperFailure: + if result["status"] == "completed": + error = fail("canary-receipt-failed", "Validation finished but its final private receipt could not be persisted.", partial=True) + error.writes = [ + {**resource, "action": "created", "type": resource.get("type", "knowledge-source-file")} + for resource in source_result["resources"]["created"] + ] + error.resources_remaining = source_result["ownership"]["run_owned"] + result = {**blocked_result(error, outcome="validate-file-cu-mi", fingerprint=fingerprint, owner=fp["owner"]), + "indexed_ocr_marker": True, "principal_attribution": "unverified", "backend_rollout": "unverified", + "source_result": source_result, "request_count": bounded.calls} + result["verdict"] = "unverified" + result.setdefault("warnings", []).append("Final private receipt persistence failed; retain this ownership handoff and existing checkpoints. Do not replay.") + return {**result, "receipt_refs": refs} + + +def main(argv=None): + parser = argparse.ArgumentParser() + modes = parser.add_mutually_exclusive_group(required=True) + modes.add_argument("--prepare", metavar="EXISTING_PRIVATE_DIRECTORY") + modes.add_argument("--plan", type=Path) + modes.add_argument("--input", type=Path) + args = parser.parse_args(argv) + try: + if args.prepare: + result = prepare(args.prepare) + elif args.plan: + result = plan_canary(_bootstrap_io.read_json(args.plan)) + directory = result["execution_input"]["plan"]["receipt_directory"] + path = _bootstrap_io.private_file(directory, "cu-mi-plan-" + result["plan_fingerprint"].split(":")[1][:16] + ".json", + result["execution_input"]) + result = {k: v for k, v in result.items() if k != "execution_input"} + result["execution_input_ref"] = str(path) + else: + document, _, fingerprint = load_approved_input(args.input) + document["_computed_fingerprint"] = fingerprint + result = execute(document) + emit_result(result) + return 0 if result["status"] in ("prepared", "planned", "completed") else 3 if result["status"] == "partial" else 2 + except HelperFailure as error: + emit_result(blocked_result(error, outcome="validate-file-cu-mi", fingerprint=None, owner=None)) + return 3 if error.partial else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_cu_mi.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_cu_mi.py index 768e0662..67f0a191 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_cu_mi.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_cu_mi.py @@ -1,206 +1,206 @@ -"""File Standard system-MI binding; implementation evidence is not rollout proof.""" -from __future__ import annotations - -import copy -import re - -try: - from . import _bootstrap_io, cu_ingestion_auth, search_reconcile - from ._common import MANAGEMENT_AUDIENCE, HelperFailure, digest, require_allowed_fields -except ImportError: - import _bootstrap_io, cu_ingestion_auth, search_reconcile - from _common import MANAGEMENT_AUDIENCE, HelperFailure, digest, require_allowed_fields - -SEARCH_API = "2025-05-01" -ROLE_API = "2022-04-01" -CU_ROLE = "a97b65f3-24c7-4388-baec-2e87135dc908" -SEARCH_ID = re.compile( - r"/subscriptions/[0-9a-fA-F-]{36}/resourceGroups/[A-Za-z0-9_.()-]{1,90}" - r"/providers/Microsoft\.Search/searchServices/(?=.{2,60}$)((?a:[a-z0-9][a-z0-9]+(?:-[a-z0-9]+)*))", re.I, -) -REDACTED = (None, "", "", "***") - - -def fail(code, message, *, request_id=None, status=None): - return HelperFailure(code, message, blocked_at="cu-managed-identity", request_id=request_id, status=status) - - -def validate_choice(value, resource_id): - if not isinstance(value, dict): - raise fail("cu-mi-prerequisite-missing", "Resolve existing Search resource ID and exact CU-scoped Cognitive Services User assignment. Missing identity/role changes need packaged bootstrap plans and explicit approval.") - require_allowed_fields(value, {"search_resource_id", "role_assignment_id"}, label="File CU managed identity") - search = value.get("search_resource_id") - role = value.get("role_assignment_id") - prefix = resource_id + "/providers/Microsoft.Authorization/roleAssignments/" - if ( - not isinstance(search, str) or SEARCH_ID.fullmatch(search) is None - or not isinstance(role, str) or not role.casefold().startswith(prefix.casefold()) - or cu_ingestion_auth.UUID.fullmatch(role[len(prefix):]) is None - ): - raise fail("cu-mi-binding-invalid", "Select exact Search and CU-account-scoped role assignment IDs; no inferred account, inherited/custom role or user-assigned identity.") - return copy.deepcopy(value) - - -def read_binding(choice, endpoint, *, token_provider, transport): - selected = validate_choice(choice.get("managed_identity"), choice["resource_id"]) - match = SEARCH_ID.fullmatch(selected["search_resource_id"]) - if endpoint.rstrip("/").casefold() != f"https://{match.group(1)}.search.windows.net".casefold(): - raise fail("cu-mi-search-mismatch", "Search resource ID must match the exact approved Search endpoint.") - ids = [] - - def get(resource, version): - response = transport( - "GET", f"{MANAGEMENT_AUDIENCE}{resource}?api-version={version}", - token_provider(MANAGEMENT_AUDIENCE), follow_redirects=False, max_response_bytes=65536, - ) - if response.status != 200 or not isinstance(response.body, dict): - raise fail("cu-mi-prerequisite-unavailable", "Cannot GET the exact Search identity or scoped CU role. Have its owner resolve read access or plan missing setup; no key fallback or permission changes.", - request_id=response.request_id, status=response.status) - if str(response.body.get("id", "")).casefold() != resource.casefold(): - raise fail("cu-mi-resource-mismatch", "ARM returned another resource or scope; refresh the selected bindings.", request_id=response.request_id) - if response.request_id: - ids.append(response.request_id) - return response.body - - search = get(selected["search_resource_id"], SEARCH_API) - identity, props = search.get("identity"), search.get("properties") - if ( - not isinstance(identity, dict) or not isinstance(props, dict) - or "SystemAssigned" not in str(identity.get("type", "")).replace(" ", "").split(",") - or any(not isinstance(identity.get(k), str) or cu_ingestion_auth.UUID.fullmatch(identity[k]) is None - for k in ("principalId", "tenantId")) - or str(props.get("provisioningState", "")).casefold() != "succeeded" - or str(props.get("status", "")).casefold() != "running" - or str(props.get("endpoint", "")).rstrip("/").casefold() not in ( - endpoint.rstrip("/").casefold(), endpoint.rstrip("/").casefold().removeprefix("https://"), - ) - ): - raise fail("cu-mi-identity-unverified", "Require ready Search system-assigned principal/tenant and endpoint readback; resolve missing identity with a separately approved bootstrap plan.") - role = get(selected["role_assignment_id"], ROLE_API) - rp = role.get("properties") - expected_role = re.compile(r"/subscriptions/[0-9a-fA-F-]{36}/providers/Microsoft.Authorization/roleDefinitions/" + CU_ROLE, re.I) - if ( - not isinstance(rp, dict) - or str(rp.get("principalId", "")).casefold() != identity["principalId"].casefold() - or rp.get("principalType") != "ServicePrincipal" - or str(rp.get("scope", "")).casefold() != choice["resource_id"].casefold() - or expected_role.fullmatch(str(rp.get("roleDefinitionId", ""))) is None - or rp.get("condition") not in (None, "") - ): - raise fail("cu-mi-role-unverified", "Require the selected Search principal's unconditional Cognitive Services User assignment on this CU account. Plan a missing scoped role explicitly; no self-grant or key fallback.") - return { - **selected, "search_endpoint": endpoint.rstrip("/"), - "principal_id": identity["principalId"], "tenant_id": identity["tenantId"], - "identity_type": identity["type"], "search_api_version": SEARCH_API, "role_api_version": ROLE_API, - "role_definition_id": rp["roleDefinitionId"], "cu_resource_id": choice["resource_id"], - "search_network": {k: copy.deepcopy(props.get(k)) for k in ("publicNetworkAccess", "networkRuleSet")}, - }, ids - - -def validate_state(state, choice, endpoint): - if not isinstance(state, dict): - raise fail("cu-mi-prerequisite-missing", "Retain the planner's verified Search identity/role binding.") - fields = {"search_resource_id", "role_assignment_id", "search_endpoint", "principal_id", "tenant_id", - "identity_type", "search_api_version", "role_api_version", "role_definition_id", - "cu_resource_id", "search_network"} - require_allowed_fields(state, fields, label="File CU identity state") - selected = validate_choice(choice["managed_identity"], choice["resource_id"]) - if ( - set(state) != fields or any(state.get(k) != v for k, v in selected.items()) - or state["search_endpoint"] != endpoint.rstrip("/") - or state["cu_resource_id"] != choice["resource_id"] - or state["search_api_version"] != SEARCH_API or state["role_api_version"] != ROLE_API - or not isinstance(state["search_network"], dict) - or any(not isinstance(state[k], str) or cu_ingestion_auth.UUID.fullmatch(state[k]) is None - for k in ("principal_id", "tenant_id")) - or "SystemAssigned" not in str(state["identity_type"]).replace(" ", "").split(",") - or not str(state["role_definition_id"]).casefold().endswith("/roledefinitions/" + CU_ROLE) - ): - raise fail("cu-mi-binding-invalid", "Retain complete, unchanged identity, role, API and account bindings.") - - -def verify_reuse(request, plan, current): - paths = (request.get("reuse_input_file"), request.get("reuse_result_file")) - if not all(isinstance(p, str) and p for p in paths): - raise fail("cu-mi-provenance-required", "A redacted source GET cannot establish key versus MI auth. Retain the original approved version 1.2 File input and completed creation result for exact reuse.") - prior, result = (_bootstrap_io.read_json(path) for path in paths) - if ( - not isinstance(prior, dict) or not isinstance(result, dict) - or not isinstance(prior.get("plan"), dict) or not isinstance(prior.get("approval"), dict) - or not isinstance(result.get("resources"), dict) - or not isinstance(result["resources"].get("created"), list) - ): - raise fail("cu-mi-provenance-mismatch", "Retain complete approved input and completed File result objects.") - require_allowed_fields(prior, {"schema_version", "plan", "approval"}, label="MI creation input") - try: - from . import file_source - except ImportError: - import file_source - approved = prior.get("approval", {}) - pp = prior.get("plan", {}) - file_source._validate_plan(pp) - fingerprint = digest(pp) - created = result.get("resources", {}).get("created", []) - source = pp.get("source", {}) - if ( - prior.get("schema_version") != "1.0" or pp.get("file_cu_plan_version") != "1.2" - or approved != {"confirmed": True, "fingerprint": fingerprint} - or source.get("action") != "create" or source.get("endpoint") != plan["source"]["endpoint"] - or source.get("name") != plan["source"]["name"] or pp.get("owner") != plan["owner"] - or pp.get("cu_identity_state") != plan["cu_identity_state"] - or pp.get("cu_resource_state") != plan["cu_resource_state"] - or result.get("status") != "completed" - or result.get("approved_plan") != {"confirmed": True, "fingerprint": fingerprint} - or not search_reconcile.definitions_match(source.get("desired"), current) - or not any(isinstance(item, dict) and item.get("type") == "knowledge-source" - and item.get("name") == source["name"] and item.get("etag") == current.get("@odata.etag") - and item.get("definition_digest") == digest(search_reconcile._definition(current)) - for item in created) - ): - raise fail("cu-mi-provenance-mismatch", "Retained MI creation evidence does not bind the fresh source/ETag/account/identity. No auth inference, reingestion or ownership claim.") - - -def guard_create(plan, transport, recheck, checkpoint=None): - """An expected-absent MI plan must not adopt another actor's redacted source.""" - url = search_reconcile.resource_url(plan["source"]) - acknowledged = False - created_etag = None - - def on_created(**evidence): - nonlocal created_etag - # The reconciler calls this after ACK, outside ambiguous transport recovery. - if checkpoint is not None: - try: - checkpoint(**evidence) - except HelperFailure: - raise - except Exception: - raise fail("cu-mi-checkpoint-failed", "Acknowledged MI creation checkpoint failed; private details withheld.") from None - try: - response = evidence["response"] - created_etag = search_reconcile.resolve_etag(search_reconcile.response_etags(response), response.request_id) - except HelperFailure: - raise fail("cu-mi-ack-version-unverified", "MI creation was acknowledged, but its version evidence is invalid. Retain ownership; do not replay.") from None - if created_etag is None: - raise fail("cu-mi-ack-version-unverified", "MI creation was acknowledged without an ETag; redacted auth cannot be bound to a created version. Retain ownership; do not replay.") - - def guarded(method, target, token, **kwargs): - nonlocal acknowledged, created_etag - if method == "PUT" and target == url: - recheck() - result = transport(method, target, token, **kwargs) - if target == url: - if method == "PUT" and result.status in (200, 201): - acknowledged = True - if method == "GET" and result.status == 200 and plan["source"]["action"] == "create" and not acknowledged: - raise fail("cu-mi-source-drift", "Expected source absence changed before creation; do not infer MI from redacted readback. Refresh discovery and provenance.") - if method == "GET" and result.status == 200 and acknowledged: - try: - observed_etag = search_reconcile.resolve_etag(search_reconcile.response_etags(result), result.request_id) - except HelperFailure: - raise fail("cu-mi-readback-version-unverified", "Created MI source readback has invalid version evidence.") from None - if observed_etag != created_etag: - raise fail("cu-mi-source-drift", "Source version changed after acknowledged MI creation; redacted auth is unproven. Inspect ownership before cleanup.") - return result - - return guarded, on_created +"""File Standard system-MI binding; implementation evidence is not rollout proof.""" +from __future__ import annotations + +import copy +import re + +try: + from . import _bootstrap_io, cu_ingestion_auth, search_reconcile + from ._common import MANAGEMENT_AUDIENCE, HelperFailure, digest, require_allowed_fields +except ImportError: + import _bootstrap_io, cu_ingestion_auth, search_reconcile + from _common import MANAGEMENT_AUDIENCE, HelperFailure, digest, require_allowed_fields + +SEARCH_API = "2025-05-01" +ROLE_API = "2022-04-01" +CU_ROLE = "a97b65f3-24c7-4388-baec-2e87135dc908" +SEARCH_ID = re.compile( + r"/subscriptions/[0-9a-fA-F-]{36}/resourceGroups/[A-Za-z0-9_.()-]{1,90}" + r"/providers/Microsoft\.Search/searchServices/(?=.{2,60}$)((?a:[a-z0-9][a-z0-9]+(?:-[a-z0-9]+)*))", re.I, +) +REDACTED = (None, "", "", "***") + + +def fail(code, message, *, request_id=None, status=None): + return HelperFailure(code, message, blocked_at="cu-managed-identity", request_id=request_id, status=status) + + +def validate_choice(value, resource_id): + if not isinstance(value, dict): + raise fail("cu-mi-prerequisite-missing", "Resolve existing Search resource ID and exact CU-scoped Cognitive Services User assignment. Missing identity/role changes need packaged bootstrap plans and explicit approval.") + require_allowed_fields(value, {"search_resource_id", "role_assignment_id"}, label="File CU managed identity") + search = value.get("search_resource_id") + role = value.get("role_assignment_id") + prefix = resource_id + "/providers/Microsoft.Authorization/roleAssignments/" + if ( + not isinstance(search, str) or SEARCH_ID.fullmatch(search) is None + or not isinstance(role, str) or not role.casefold().startswith(prefix.casefold()) + or cu_ingestion_auth.UUID.fullmatch(role[len(prefix):]) is None + ): + raise fail("cu-mi-binding-invalid", "Select exact Search and CU-account-scoped role assignment IDs; no inferred account, inherited/custom role or user-assigned identity.") + return copy.deepcopy(value) + + +def read_binding(choice, endpoint, *, token_provider, transport): + selected = validate_choice(choice.get("managed_identity"), choice["resource_id"]) + match = SEARCH_ID.fullmatch(selected["search_resource_id"]) + if endpoint.rstrip("/").casefold() != f"https://{match.group(1)}.search.windows.net".casefold(): + raise fail("cu-mi-search-mismatch", "Search resource ID must match the exact approved Search endpoint.") + ids = [] + + def get(resource, version): + response = transport( + "GET", f"{MANAGEMENT_AUDIENCE}{resource}?api-version={version}", + token_provider(MANAGEMENT_AUDIENCE), follow_redirects=False, max_response_bytes=65536, + ) + if response.status != 200 or not isinstance(response.body, dict): + raise fail("cu-mi-prerequisite-unavailable", "Cannot GET the exact Search identity or scoped CU role. Have its owner resolve read access or plan missing setup; no key fallback or permission changes.", + request_id=response.request_id, status=response.status) + if str(response.body.get("id", "")).casefold() != resource.casefold(): + raise fail("cu-mi-resource-mismatch", "ARM returned another resource or scope; refresh the selected bindings.", request_id=response.request_id) + if response.request_id: + ids.append(response.request_id) + return response.body + + search = get(selected["search_resource_id"], SEARCH_API) + identity, props = search.get("identity"), search.get("properties") + if ( + not isinstance(identity, dict) or not isinstance(props, dict) + or "SystemAssigned" not in str(identity.get("type", "")).replace(" ", "").split(",") + or any(not isinstance(identity.get(k), str) or cu_ingestion_auth.UUID.fullmatch(identity[k]) is None + for k in ("principalId", "tenantId")) + or str(props.get("provisioningState", "")).casefold() != "succeeded" + or str(props.get("status", "")).casefold() != "running" + or str(props.get("endpoint", "")).rstrip("/").casefold() not in ( + endpoint.rstrip("/").casefold(), endpoint.rstrip("/").casefold().removeprefix("https://"), + ) + ): + raise fail("cu-mi-identity-unverified", "Require ready Search system-assigned principal/tenant and endpoint readback; resolve missing identity with a separately approved bootstrap plan.") + role = get(selected["role_assignment_id"], ROLE_API) + rp = role.get("properties") + expected_role = re.compile(r"/subscriptions/[0-9a-fA-F-]{36}/providers/Microsoft.Authorization/roleDefinitions/" + CU_ROLE, re.I) + if ( + not isinstance(rp, dict) + or str(rp.get("principalId", "")).casefold() != identity["principalId"].casefold() + or rp.get("principalType") != "ServicePrincipal" + or str(rp.get("scope", "")).casefold() != choice["resource_id"].casefold() + or expected_role.fullmatch(str(rp.get("roleDefinitionId", ""))) is None + or rp.get("condition") not in (None, "") + ): + raise fail("cu-mi-role-unverified", "Require the selected Search principal's unconditional Cognitive Services User assignment on this CU account. Plan a missing scoped role explicitly; no self-grant or key fallback.") + return { + **selected, "search_endpoint": endpoint.rstrip("/"), + "principal_id": identity["principalId"], "tenant_id": identity["tenantId"], + "identity_type": identity["type"], "search_api_version": SEARCH_API, "role_api_version": ROLE_API, + "role_definition_id": rp["roleDefinitionId"], "cu_resource_id": choice["resource_id"], + "search_network": {k: copy.deepcopy(props.get(k)) for k in ("publicNetworkAccess", "networkRuleSet")}, + }, ids + + +def validate_state(state, choice, endpoint): + if not isinstance(state, dict): + raise fail("cu-mi-prerequisite-missing", "Retain the planner's verified Search identity/role binding.") + fields = {"search_resource_id", "role_assignment_id", "search_endpoint", "principal_id", "tenant_id", + "identity_type", "search_api_version", "role_api_version", "role_definition_id", + "cu_resource_id", "search_network"} + require_allowed_fields(state, fields, label="File CU identity state") + selected = validate_choice(choice["managed_identity"], choice["resource_id"]) + if ( + set(state) != fields or any(state.get(k) != v for k, v in selected.items()) + or state["search_endpoint"] != endpoint.rstrip("/") + or state["cu_resource_id"] != choice["resource_id"] + or state["search_api_version"] != SEARCH_API or state["role_api_version"] != ROLE_API + or not isinstance(state["search_network"], dict) + or any(not isinstance(state[k], str) or cu_ingestion_auth.UUID.fullmatch(state[k]) is None + for k in ("principal_id", "tenant_id")) + or "SystemAssigned" not in str(state["identity_type"]).replace(" ", "").split(",") + or not str(state["role_definition_id"]).casefold().endswith("/roledefinitions/" + CU_ROLE) + ): + raise fail("cu-mi-binding-invalid", "Retain complete, unchanged identity, role, API and account bindings.") + + +def verify_reuse(request, plan, current): + paths = (request.get("reuse_input_file"), request.get("reuse_result_file")) + if not all(isinstance(p, str) and p for p in paths): + raise fail("cu-mi-provenance-required", "A redacted source GET cannot establish key versus MI auth. Retain the original approved version 1.2 File input and completed creation result for exact reuse.") + prior, result = (_bootstrap_io.read_json(path) for path in paths) + if ( + not isinstance(prior, dict) or not isinstance(result, dict) + or not isinstance(prior.get("plan"), dict) or not isinstance(prior.get("approval"), dict) + or not isinstance(result.get("resources"), dict) + or not isinstance(result["resources"].get("created"), list) + ): + raise fail("cu-mi-provenance-mismatch", "Retain complete approved input and completed File result objects.") + require_allowed_fields(prior, {"schema_version", "plan", "approval"}, label="MI creation input") + try: + from . import file_source + except ImportError: + import file_source + approved = prior.get("approval", {}) + pp = prior.get("plan", {}) + file_source._validate_plan(pp) + fingerprint = digest(pp) + created = result.get("resources", {}).get("created", []) + source = pp.get("source", {}) + if ( + prior.get("schema_version") != "1.0" or pp.get("file_cu_plan_version") != "1.2" + or approved != {"confirmed": True, "fingerprint": fingerprint} + or source.get("action") != "create" or source.get("endpoint") != plan["source"]["endpoint"] + or source.get("name") != plan["source"]["name"] or pp.get("owner") != plan["owner"] + or pp.get("cu_identity_state") != plan["cu_identity_state"] + or pp.get("cu_resource_state") != plan["cu_resource_state"] + or result.get("status") != "completed" + or result.get("approved_plan") != {"confirmed": True, "fingerprint": fingerprint} + or not search_reconcile.definitions_match(source.get("desired"), current) + or not any(isinstance(item, dict) and item.get("type") == "knowledge-source" + and item.get("name") == source["name"] and item.get("etag") == current.get("@odata.etag") + and item.get("definition_digest") == digest(search_reconcile._definition(current)) + for item in created) + ): + raise fail("cu-mi-provenance-mismatch", "Retained MI creation evidence does not bind the fresh source/ETag/account/identity. No auth inference, reingestion or ownership claim.") + + +def guard_create(plan, transport, recheck, checkpoint=None): + """An expected-absent MI plan must not adopt another actor's redacted source.""" + url = search_reconcile.resource_url(plan["source"]) + acknowledged = False + created_etag = None + + def on_created(**evidence): + nonlocal created_etag + # The reconciler calls this after ACK, outside ambiguous transport recovery. + if checkpoint is not None: + try: + checkpoint(**evidence) + except HelperFailure: + raise + except Exception: + raise fail("cu-mi-checkpoint-failed", "Acknowledged MI creation checkpoint failed; private details withheld.") from None + try: + response = evidence["response"] + created_etag = search_reconcile.resolve_etag(search_reconcile.response_etags(response), response.request_id) + except HelperFailure: + raise fail("cu-mi-ack-version-unverified", "MI creation was acknowledged, but its version evidence is invalid. Retain ownership; do not replay.") from None + if created_etag is None: + raise fail("cu-mi-ack-version-unverified", "MI creation was acknowledged without an ETag; redacted auth cannot be bound to a created version. Retain ownership; do not replay.") + + def guarded(method, target, token, **kwargs): + nonlocal acknowledged, created_etag + if method == "PUT" and target == url: + recheck() + result = transport(method, target, token, **kwargs) + if target == url: + if method == "PUT" and result.status in (200, 201): + acknowledged = True + if method == "GET" and result.status == 200 and plan["source"]["action"] == "create" and not acknowledged: + raise fail("cu-mi-source-drift", "Expected source absence changed before creation; do not infer MI from redacted readback. Refresh discovery and provenance.") + if method == "GET" and result.status == 200 and acknowledged: + try: + observed_etag = search_reconcile.resolve_etag(search_reconcile.response_etags(result), result.request_id) + except HelperFailure: + raise fail("cu-mi-readback-version-unverified", "Created MI source readback has invalid version evidence.") from None + if observed_etag != created_etag: + raise fail("cu-mi-source-drift", "Source version changed after acknowledged MI creation; redacted auth is unproven. Inspect ownership before cleanup.") + return result + + return guarded, on_created diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_ingest.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_ingest.py index 810d663b..1635e8ae 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_ingest.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_ingest.py @@ -1,1040 +1,1040 @@ -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import re -import stat -import sys -import time -from pathlib import Path, PurePosixPath -from typing import Any -from urllib.parse import parse_qs, urlencode, urlsplit - -try: - from ._progress import Progress, add_progress_argument, reporting -except ImportError: - from _progress import Progress, add_progress_argument, reporting - -try: - from ._common import ( - SEARCH_AUDIENCE, - HelperFailure, - ReadRecovery, - TokenProvider, - Transport, - azure_cli_token, - blocked_result, - digest, - emit_result, - file_digest, - http_request, - load_approved_input, - odata_name, - reject_secrets, - require_allowed_fields, - validate_search_endpoint, - ) -except ImportError: - from _common import ( # type: ignore[no-redef] - SEARCH_AUDIENCE, - HelperFailure, - ReadRecovery, - TokenProvider, - Transport, - azure_cli_token, - blocked_result, - digest, - emit_result, - file_digest, - http_request, - load_approved_input, - odata_name, - reject_secrets, - require_allowed_fields, - validate_search_endpoint, - ) - - -API_VERSION = "2026-08-01-preview" -MAX_INVENTORY_PAGES = 200 -MAX_SERVER_FILES = 200 -INVENTORY_READ_TIMEOUT_SECONDS = 60 -MAX_INVENTORY_RESPONSE_BYTES = 1024 * 1024 -MAX_FILE_BYTES = { - "free": 50 * 1024 * 1024, - "basic": 50 * 1024 * 1024, - "dedicated": 100 * 1024 * 1024, - "serverless": 100 * 1024 * 1024, -} -MEDIA_TYPE = re.compile( - r"^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]*/[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]*$" -) -FILE_MEDIA_HINTS = { - ".txt": "text/plain", ".md": "text/markdown", - ".pdf": "application/pdf", ".html": "text/html", ".htm": "text/html", - ".csv": "text/csv", ".json": "application/json", ".sh": "application/x-sh", - ".doc": "application/msword", - ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - ".ppt": "application/vnd.ms-powerpoint", - ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", - ".xls": "application/vnd.ms-excel", - ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - ".jpeg": "image/jpeg", ".jpg": "image/jpeg", ".png": "image/png", - ".bmp": "image/bmp", ".heif": "image/heif", ".heic": "image/heic", - ".tiff": "image/tiff", ".tif": "image/tiff", ".gif": "image/gif", - ".webp": "image/webp", ".svg": "image/svg+xml", ".avif": "image/avif", - ".ico": "image/vnd.microsoft.icon", -} - - -def validate_api_version(value: Any) -> None: - if value != API_VERSION: - raise HelperFailure( - "api-version-invalid", - f"This File helper requires {API_VERSION} for its metadata upload contract. " - "The service also supports 2026-05-01-preview minimal extraction; use a compatible client " - "or explicitly approve August, never silently change the requested API.", - blocked_at="input-resolution", - ) - - -def media_type_hint(path: str) -> str: - """Return a portable filename hint, never a claim of server content detection.""" - return FILE_MEDIA_HINTS.get(PurePosixPath(path).suffix.lower(), "application/octet-stream") - - -def _reject_credential_path(path: Path) -> None: - lowered = [part.lower() for part in path.parts] - if ( - any(part in {".ssh", ".aws", ".azure", ".git", "credentials"} or part == ".env" or part.startswith(".env.") - for part in lowered) - or lowered[-1] in {"id_rsa", "id_dsa", "id_ecdsa", "id_ed25519"} - or path.suffix.lower() in {".pem", ".key", ".pfx", ".p12", ".kdbx"} - ): - raise HelperFailure( - "credential-file-forbidden", "Credential files are outside document ingestion.", - blocked_at="input-resolution", - ) - - -def _odata_name(name: Any) -> str: - return odata_name(name) - - -def _list_url(plan: dict[str, Any]) -> str: - endpoint = validate_search_endpoint(plan.get("endpoint")) - validate_api_version(plan.get("api_version")) - name = _odata_name(plan.get("name")) - return ( - f"{endpoint}/knowledgesources('{name}')/files?" - + urlencode({"api-version": API_VERSION, "pageSize": 200}) - ) - - -def _normalize_inventory(files: list[dict[str, Any]]) -> list[dict[str, Any]]: - fields = ( - "fileId", - "fileName", - "prefix", - "metadata", - "parsingMode", - "extractionMode", - "fileSizeBytes", - "errorMessage", - ) - normalized = [ - {field: item.get(field) for field in fields if field in item} - for item in files - ] - return sorted( - normalized, - key=lambda item: (str(item.get("fileName")), str(item.get("fileId"))), - ) - - -def _list_files( - url: str, - token: str, - *, - transport: Transport, - recovery: ReadRecovery | None = None, -) -> tuple[list[dict[str, Any]], list[str]]: - try: - origin = urlsplit(url) - origin_port = origin.port - except ValueError as exc: - raise HelperFailure( - "continuation-url-invalid", "File inventory URL is malformed.", - blocked_at="verification", - ) from exc - current_url: str | None = url - seen: set[str] = set() - files: list[dict[str, Any]] = [] - request_ids: list[str] = [] - requests = 0 - deadline = time.monotonic() + INVENTORY_READ_TIMEOUT_SECONDS - if recovery is not None: - deadline = min(deadline, recovery.deadline) - - def inventory_transport(method, target, credential, **options): - nonlocal requests - if requests >= MAX_INVENTORY_PAGES: - raise HelperFailure( - "file-list-limit-exceeded", "Complete file inventory exceeds 200 pages/requests.", - blocked_at="verification", - ) - requests += 1 - return transport(method, target, credential, **options) - - while current_url: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise HelperFailure( - "file-list-timeout", "Complete file inventory read exceeded its 60-second deadline.", - blocked_at="verification", - request_id=request_ids[-1] if request_ids else None, - ) - if len(seen) >= MAX_INVENTORY_PAGES: - raise HelperFailure( - "file-list-limit-exceeded", "Complete file inventory exceeds 200 pages/requests.", - blocked_at="verification", - request_id=request_ids[-1] if request_ids else None, - ) - try: - current_url.encode("ascii") - current = urlsplit(current_url) - current_port = current.port - except (ValueError, UnicodeEncodeError) as exc: - raise HelperFailure( - "continuation-url-invalid", "Search returned a malformed continuation URL.", - blocked_at="verification", - ) from exc - query = parse_qs(current.query) - if ( - current_url in seen - or current.scheme != "https" - or current.hostname != origin.hostname - or current_port != origin_port - or current.path != origin.path - or query.get("api-version") != [API_VERSION] - or current.username - or current.password - ): - raise HelperFailure( - "continuation-url-invalid", - "Search returned a continuation URL outside the approved service.", - blocked_at="verification", - ) - seen.add(current_url) - try: - options = dict(timeout=remaining, response_deadline=deadline, - max_response_bytes=MAX_INVENTORY_RESPONSE_BYTES, follow_redirects=False) - result = (recovery.get(current_url, token, transport=inventory_transport, - max_requests=MAX_INVENTORY_PAGES - requests, **options) - if recovery is not None else inventory_transport("GET", current_url, token, **options)) - except HelperFailure as failure: - if failure.code not in {"response-deadline-exceeded", "read-recovery-budget-exhausted"}: - raise - raise HelperFailure( - "file-list-timeout", "Complete file inventory read exceeded its effective deadline (at most 60 seconds).", - blocked_at="verification", request_id=failure.request_id, - status=failure.http_status, warnings=failure.warnings, - ) from failure - if time.monotonic() >= deadline: - raise HelperFailure( - "file-list-timeout", "Complete file inventory read exceeded its 60-second deadline.", - blocked_at="verification", - request_id=result.request_id, - ) - if result.status != 200 or not isinstance(result.body, dict): - raise HelperFailure( - "file-list-invalid", - "File-list readback did not return a JSON object.", - blocked_at="reconciliation", - request_id=result.request_id, - status=result.status, - ) - values = result.body.get("value") - if isinstance(values, list) and len(files) + len(values) > MAX_SERVER_FILES: - raise HelperFailure( - "file-list-limit-exceeded", "Complete file inventory exceeds 200 records.", - blocked_at="verification", - request_id=result.request_id, - ) - if not isinstance(values, list) or not all( - isinstance(item, dict) for item in values - ): - raise HelperFailure( - "file-list-invalid", - "File-list readback did not contain an object array.", - blocked_at="reconciliation", - request_id=result.request_id, - ) - files.extend(values) - if result.request_id: - request_ids.append(result.request_id) - next_link = result.body.get("@odata.nextLink") - if next_link is not None and not isinstance(next_link, str): - raise HelperFailure( - "continuation-url-invalid", - "Search returned an invalid continuation URL.", - blocked_at="verification", - ) - current_url = next_link - return files, recovery.request_ids if recovery is not None else request_ids - - -def _validate_relative_path(value: Any) -> str: - if not isinstance(value, str) or not value or "\\" in value: - raise HelperFailure( - "inventory-path-invalid", - "Every inventory path must be a non-empty normalized POSIX path.", - blocked_at="input-resolution", - ) - path = PurePosixPath(value) - if ( - path.is_absolute() - or path.as_posix() != value - or any(part in {"", ".", ".."} for part in path.parts) - ): - raise HelperFailure( - "inventory-path-invalid", - f"Inventory path is not safely relative: {value!r}.", - blocked_at="input-resolution", - ) - if any("\r" in part or "\n" in part or ":" in part for part in path.parts): - raise HelperFailure( - "inventory-path-invalid", - f"Inventory path contains a forbidden segment: {value!r}.", - blocked_at="input-resolution", - ) - return path.as_posix() - - -def _reject_links(path: Path) -> None: - for probe in reversed((path, *path.parents)): - attributes = getattr(probe.lstat(), "st_file_attributes", 0) - if probe.is_symlink() or attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT: - raise HelperFailure( - "inventory-path-invalid", - "Local inventory cannot traverse links or reparse points.", - blocked_at="input-resolution", - ) - - -def resolve_local_root(value: Any) -> Path: - """Resolve the explicit data boundary without following links or reparse points.""" - if not isinstance(value, str) or not os.path.isabs(value): - raise HelperFailure( - "local-root-invalid", - "local_root must be an explicit absolute path.", - blocked_at="input-resolution", - ) - try: - path = Path(value) - _reject_links(path) - root = path.resolve(strict=True) - if not root.is_dir(): - raise OSError("not a directory") - except OSError as exc: - raise HelperFailure( - "local-root-invalid", - "local_root must be an existing readable real directory.", - blocked_at="input-resolution", - ) from exc - return root - - -def _resolve_inventory_path(root: Path, value: Any) -> Path: - relative = _validate_relative_path(value) - path = root.joinpath(*PurePosixPath(relative).parts) - try: - _reject_links(path) - resolved = path.resolve(strict=True) - resolved.relative_to(root) - except (OSError, ValueError) as exc: - raise HelperFailure( - "inventory-path-invalid", - f"Inventory path escapes or is unreadable: {relative}.", - blocked_at="input-resolution", - ) from exc - if not resolved.is_file() or resolved.is_symlink(): - raise HelperFailure( - "inventory-path-invalid", - f"Inventory entry is not a regular file: {relative}.", - blocked_at="input-resolution", - ) - _reject_credential_path(resolved) - return resolved - - -def _resolve_file(root: Path, record: dict[str, Any]) -> Path: - relative = _validate_relative_path(record.get("path")) - resolved = _resolve_inventory_path(root, relative) - try: - observed = resolved.stat() - matches = ( - record.get("size") == observed.st_size - and record.get("mtime_ns") == observed.st_mtime_ns - and record.get("sha256") == file_digest(resolved) - ) - except OSError as exc: - raise HelperFailure( - "inventory-unreadable", f"Selected file cannot be read: {relative}.", - blocked_at="input-resolution", - ) from exc - if not matches: - raise HelperFailure( - "inventory-drift", - f"Size, modification time, or SHA-256 changed for {relative}.", - blocked_at="confirmation", - ) - return resolved - - -def snapshot_inventory( - root: Path, paths: list[str], *, service_tier: str -) -> list[dict[str, Any]]: - """Freeze selected files with MIME hints; Search determines actual support.""" - if not isinstance(service_tier, str) or service_tier not in MAX_FILE_BYTES: - raise HelperFailure( - "service-tier-invalid", "Select a supported service_tier.", - blocked_at="input-resolution", - ) - if not isinstance(paths, list) or not 1 <= len(paths) <= 200: - raise HelperFailure( - "inventory-invalid", "Select between 1 and 200 explicit file paths.", - blocked_at="input-resolution", - ) - relative_paths = [_validate_relative_path(path) for path in paths] - if len(set(relative_paths)) != len(relative_paths): - raise HelperFailure( - "inventory-invalid", "Selected file paths must be unique.", - blocked_at="input-resolution", - ) - records = [] - for relative in sorted(relative_paths): - path = _resolve_inventory_path(root, relative) - media_type = media_type_hint(relative) - try: - observed = path.stat() - if not 0 < observed.st_size <= MAX_FILE_BYTES[service_tier]: - raise HelperFailure( - "inventory-invalid", "File size is empty or exceeds the selected tier limit.", - blocked_at="input-resolution", - ) - record = { - "path": relative, "size": observed.st_size, - "mtime_ns": observed.st_mtime_ns, "sha256": file_digest(path), - "media_type": media_type, - } - _resolve_file(root, record) - except OSError as exc: - raise HelperFailure( - "inventory-unreadable", "Selected file cannot be read.", - blocked_at="input-resolution", - ) from exc - records.append(record) - return records - - -def read_inventory( - plan: dict[str, Any], token: str, *, transport: Transport -) -> tuple[list[dict[str, Any]], list[str]]: - """Read all pages for the exact File source, using guarded continuations.""" - return _list_files(_list_url(plan), token, transport=transport) - - -def inventory_digest(files: list[dict[str, Any]]) -> str: - return digest(_normalize_inventory(files)) - - -def reconcile_inventory( - plan: dict[str, Any], - before: list[dict[str, Any]], - *, - allow_new_uploads: bool = False, -) -> dict[str, dict[str, Any]]: - """Validate all existing markers before planning or performing any upload.""" - expected_names = {record["path"] for record in plan["files"]} - if any(item.get("fileName") not in expected_names for item in before): - raise HelperFailure( - "server-inventory-conflict", - "The server inventory contains files outside the approved inventory.", - blocked_at="reconciliation", - ) - matched = {} - for record in plan["files"]: - matches = [item for item in before if item.get("fileName") == record["path"]] - if len(matches) > 1: - raise HelperFailure( - "duplicate-file-record", f"Multiple server records exist for {record['path']}.", - blocked_at="reconciliation", - ) - if matches: - if not _matches(matches[0], plan, record): - raise HelperFailure( - "file-record-conflict", f"Server record conflicts with approved file {record['path']}.", - blocked_at="reconciliation", - ) - matched[record["path"]] = matches[0] - elif not allow_new_uploads: - raise HelperFailure( - "reused-source-upload-forbidden", - "New files cannot be uploaded into a reused source because individual-file cleanup is unsupported.", - blocked_at="reconciliation", - ) - return matched - - -def _metadata(plan: dict[str, Any], record: dict[str, Any]) -> dict[str, str]: - supplied = record.get("metadata") or {} - if not isinstance(supplied, dict) or not all( - isinstance(key, str) and isinstance(value, str) - for key, value in supplied.items() - ): - raise HelperFailure( - "file-metadata-invalid", - "File metadata must contain only string keys and values.", - blocked_at="input-resolution", - ) - metadata = dict(supplied) - metadata.update( - { - "foundryIqSha256": record["sha256"], - "foundryIqSizeBytes": str(record["size"]), - "foundryIqInventory": plan["inventory_digest"], - "foundryIqOwner": str(plan["owner"]), - "foundryIqSource": str(plan["name"]), - } - ) - return metadata - - -def _matches( - server: dict[str, Any], - plan: dict[str, Any], - record: dict[str, Any], -) -> bool: - metadata = server.get("metadata") or {} - expected = _metadata(plan, record) - return ( - server.get("fileName") == record["path"] - and server.get("fileSizeBytes") == record["size"] - and server.get("errorMessage") is None - and isinstance(metadata, dict) - and all(metadata.get(key) == value for key, value in expected.items()) - ) - - -def _multipart( - plan: dict[str, Any], - record: dict[str, Any], - content: bytes, - fingerprint: str, -) -> tuple[bytes, str]: - boundary = "foundry-iq-" + fingerprint.removeprefix("sha256:")[:24] - metadata = { - "fileName": record["path"], - "metadata": _metadata(plan, record), - } - media_type = record.get("media_type") - if not isinstance(media_type, str) or not media_type: - media_type = media_type_hint(record["path"]) - pieces = [ - f"--{boundary}\r\n".encode("ascii"), - b'Content-Disposition: form-data; name="metadata"\r\n', - b"Content-Type: application/json\r\n\r\n", - json.dumps(metadata, sort_keys=True, separators=(",", ":")).encode("utf-8"), - b"\r\n", - f"--{boundary}\r\n".encode("ascii"), - b'Content-Disposition: form-data; name="content"; filename="upload"\r\n', - f"Content-Type: {media_type}\r\n\r\n".encode("ascii"), - content, - b"\r\n", - f"--{boundary}--\r\n".encode("ascii"), - ] - return b"".join(pieces), boundary - - -def _validate_plan(plan: dict[str, Any]) -> tuple[Path, list[dict[str, Any]]]: - reject_secrets(plan) - require_allowed_fields( - plan, - { - "operation", - "outcome", - "endpoint", - "name", - "api_version", - "local_root", - "files", - "inventory_digest", - "expected_server_inventory_digest", - "service_tier", - "extraction_mode", - "rbac", - "network", - "owner", - "cleanup_approved", - }, - label="File ingestion plan", - ) - for field, allowed in ( - ("rbac", {"assignments"}), - ("network", {"posture", "evidence"}), - ): - section = plan.get(field) - if section is not None: - if not isinstance(section, dict): - raise HelperFailure( - "input-schema-invalid", - f"{field} must be an object.", - blocked_at="input-resolution", - ) - require_allowed_fields(section, allowed, label=field) - if plan.get("operation") != "ingest" or plan.get("cleanup_approved") is not False: - raise HelperFailure( - "operation-invalid", - "File ingestion requires operation ingest and cleanup_approved false.", - blocked_at="input-resolution", - ) - if not isinstance(plan.get("name"), str) or not plan["name"]: - raise HelperFailure( - "name-invalid", - "Knowledge source name is required.", - blocked_at="input-resolution", - ) - if not isinstance(plan.get("owner"), str) or not plan["owner"]: - raise HelperFailure( - "owner-invalid", - "File ingestion owner is required.", - blocked_at="input-resolution", - ) - root = resolve_local_root(plan.get("local_root")) - records = plan.get("files") - if not isinstance(records, list) or not records or len(records) > 200: - raise HelperFailure( - "inventory-invalid", - "files must contain between 1 and 200 entries.", - blocked_at="input-resolution", - ) - if not all(isinstance(record, dict) for record in records): - raise HelperFailure( - "inventory-invalid", - "Every file inventory entry must be an object.", - blocked_at="input-resolution", - ) - for record in records: - require_allowed_fields( - record, - {"path", "size", "mtime_ns", "sha256", "media_type", "metadata"}, - label="File inventory record", - ) - service_tier = plan.get("service_tier") - if service_tier not in MAX_FILE_BYTES: - raise HelperFailure( - "service-tier-invalid", - "service_tier must be free, basic, dedicated, or serverless.", - blocked_at="input-resolution", - ) - extraction_mode = plan.get("extraction_mode") - if extraction_mode not in {"minimal", "standard"}: - raise HelperFailure( - "extraction-mode-invalid", - "extraction_mode must be minimal or standard.", - blocked_at="input-resolution", - ) - paths = [_validate_relative_path(record.get("path")) for record in records] - if paths != sorted(paths) or len(paths) != len(set(paths)): - raise HelperFailure( - "inventory-invalid", - "File inventory paths must be unique and byte-sorted.", - blocked_at="input-resolution", - ) - if digest(records) != plan.get("inventory_digest"): - raise HelperFailure( - "inventory-drift", - "inventory_digest does not match the approved file records.", - blocked_at="confirmation", - ) - for record in records: - size = record.get("size") - mtime_ns = record.get("mtime_ns") - media_type = record.get("media_type") - if ( - not isinstance(size, int) - or size <= 0 - or size > MAX_FILE_BYTES[service_tier] - or not isinstance(mtime_ns, int) - or mtime_ns <= 0 - or not isinstance(media_type, str) - or MEDIA_TYPE.fullmatch(media_type) is None - ): - raise HelperFailure( - "inventory-invalid", - "Every file requires positive size/mtime, media type, and a tier-valid size.", - blocked_at="input-resolution", - ) - _resolve_file(root, record) - _metadata(plan, record) - return root, records - - -def _confirm_ambiguous_upload( - list_url: str, - token: str, - transport: Transport, - record: dict[str, Any], - plan: dict[str, Any], - request_ids: list[str], - failure: HelperFailure, - warnings: list[str], -) -> dict[str, Any] | None: - """Resolve an ambiguous upload with bounded readback, never upload replay. - - Only ambiguous outcomes (transport failure, timeout, 409/429/5xx) reach - this helper. A definitive 200/201 response never calls it. Returns the - matching server record when the readback proves the approved file - exists, otherwise ``None`` so the caller reports ``partial``. - """ - recovery = ReadRecovery() - try: - recovery.delay(failure) - observed, _ = _list_files( - list_url, token, transport=transport, recovery=recovery, - ) - except HelperFailure as read_failure: - warnings.append(f"Upload readback failed ({read_failure.code}); original upload failure retained.") - return None - finally: - request_ids.extend(recovery.request_ids) - warnings.extend(recovery.diagnostics()) - matches = [item for item in observed if item.get("fileName") == record["path"]] - if len(matches) != 1 or not _matches(matches[0], plan, record): - return None - return matches[0] - - -def _remaining_files(resources: list[dict[str, Any]]) -> list[dict[str, Any]]: - return [ - { - "type": "knowledge-source-file", - "fileName": resource["fileName"], - "sha256": resource["sha256"], - } - for resource in resources - ] - - -@reporting("file-upload") -def execute( - document: dict[str, Any], - *, - token_provider: TokenProvider = azure_cli_token, - transport: Transport = http_request, - allow_new_uploads: bool = False, - progress: Progress | None = None, - upload_session=None, - source_check=None, - allow_upload_retry=True, -) -> dict[str, Any]: - if allow_new_uploads or upload_session is not None or len(document.get("plan", {}).get("files", [])) > 1: - try: - from .file_upload import run_batch - except ImportError: - from file_upload import run_batch - return run_batch(document, token_provider=token_provider, transport=transport, progress=progress, - allow_new_uploads=allow_new_uploads, session=upload_session, - source_check=source_check, allow_upload_retry=allow_upload_retry) - progress.update("file-inventory") - plan = document["plan"] - fingerprint = document["_computed_fingerprint"] - root, records = _validate_plan(plan) - list_url = _list_url(plan) - token = token_provider(SEARCH_AUDIENCE) - readonly = ReadRecovery() if not allow_new_uploads else None - before, request_ids = _list_files(list_url, token, transport=transport, recovery=readonly) - warnings: list[str] = list(readonly.warnings) if readonly is not None else [] - if inventory_digest(before) != plan.get("expected_server_inventory_digest"): - raise HelperFailure( - "server-inventory-drift", - "Server file inventory changed after approval.", - blocked_at="reconciliation", - warnings=readonly.diagnostics() if readonly is not None else [], - ) - matched = reconcile_inventory(plan, before, allow_new_uploads=allow_new_uploads) - - created: list[dict[str, Any]] = [] - reused: list[dict[str, Any]] = [] - acknowledged_ids: list[str] = [] - progress.update("file-upload", uploads_acknowledged=0, files_reused=0) - for record in records: - progress.update("file-upload", uploads_acknowledged=len(created), files_reused=len(reused)) - if record["path"] in matched: - reused.append( - { - "fileId": matched[record["path"]].get("fileId"), - "fileName": record["path"], - "sha256": record["sha256"], - } - ) - continue - - path = _resolve_file(root, record) - try: - content = path.read_bytes() - except OSError as exc: - raise HelperFailure( - "inventory-unreadable", - f"Approved file became unreadable: {record['path']}.", - blocked_at="execution", - writes=created, - resources_remaining=_remaining_files(created), - partial=bool(created), - ) from exc - try: - post_read_stat = path.stat() - except OSError as exc: - raise HelperFailure( - "inventory-unreadable", - f"Approved file became unreadable: {record['path']}.", - blocked_at="execution", - writes=created, - resources_remaining=_remaining_files(created), - partial=bool(created), - ) from exc - content_digest = "sha256:" + hashlib.sha256(content).hexdigest() - if ( - content_digest != record["sha256"] - or len(content) != record["size"] - or post_read_stat.st_mtime_ns != record["mtime_ns"] - ): - raise HelperFailure( - "inventory-drift", - f"Approved file changed before upload: {record['path']}.", - blocked_at="confirmation", - writes=created, - resources_remaining=_remaining_files(created), - partial=bool(created), - ) - body, boundary = _multipart(plan, record, content, fingerprint) - try: - result = transport( - "POST", - list_url, - token, - body=body, - headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, - ) - except HelperFailure as failure: - if failure.http_status is not None and 400 <= failure.http_status < 500 and failure.http_status not in {408, 409, 429}: - raise HelperFailure( - failure.code, failure.message, blocked_at=failure.blocked_at, - writes=created, resources_remaining=_remaining_files(created), - request_id=failure.request_id, status=failure.http_status, partial=bool(created), - ) from failure - # The transport outcome is ambiguous (we do not know whether the - # server received the write): attempt one readback before - # concluding partial, per the ambiguous-write contract. - confirmed = _confirm_ambiguous_upload( - list_url, token, transport, record, plan, request_ids, failure, warnings - ) - if confirmed is not None: - created.append({"fileName": record["path"], "sha256": record["sha256"]}) - if failure.request_id: - request_ids.append(failure.request_id) - continue - uncertain = { - "action": "upload-unverified", - "type": "knowledge-source-file", - "fileName": record["path"], - "sha256": record["sha256"], - } - raise HelperFailure( - failure.code, - failure.message, - blocked_at=failure.blocked_at, - writes=created + [uncertain], - resources_remaining=_remaining_files(created) + [uncertain], - request_id=failure.request_id, - status=failure.http_status, - partial=True, - warnings=warnings, - ) from failure - if result.status not in {200, 201}: - ambiguous = result.status in {408, 409, 429} or result.status >= 500 - if ambiguous: - confirmed = _confirm_ambiguous_upload( - list_url, token, transport, record, plan, request_ids, - HelperFailure("upload-failed", "Upload response is ambiguous.", - blocked_at="execution", status=result.status, - request_id=result.request_id, retry_after=result.retry_after, - recovery_deadline=result.recovery_deadline), - warnings, - ) - if confirmed is not None: - created.append( - {"fileName": record["path"], "sha256": record["sha256"]} - ) - if result.request_id: - request_ids.append(result.request_id) - continue - uncertain = { - "action": "upload-unverified", - "type": "knowledge-source-file", - "fileName": record["path"], - "sha256": record["sha256"], - } - raise HelperFailure( - "upload-failed", - f"Upload returned unexpected HTTP {result.status}.", - blocked_at="execution", - writes=created + ([uncertain] if ambiguous else []), - resources_remaining=( - _remaining_files(created) + ([uncertain] if ambiguous else []) - ), - request_id=result.request_id, - status=result.status, - partial=bool(created) or ambiguous, - warnings=warnings, - ) - # A definitive 200/201 response is not ambiguous: trust it rather than - # re-listing the whole source after every single file. The complete - # inventory is verified once, in bulk, after the loop. - created.append({"fileName": record["path"], "sha256": record["sha256"]}) - if result.request_id: - request_ids.append(result.request_id) - acknowledged_ids.append(ReadRecovery.safe_id(result.request_id)) - - progress.update("file-readback", uploads_acknowledged=len(created), files_reused=len(reused)) - ack_warnings = (["Acknowledged upload request IDs: " + ", ".join(acknowledged_ids)] - if acknowledged_ids else []) - recovery = ReadRecovery() - try: - after, after_request_ids = _list_files( - list_url, token, transport=transport, recovery=recovery, - ) - except HelperFailure as failure: - raise HelperFailure( - failure.code, - failure.message, - blocked_at=failure.blocked_at, - writes=created + failure.writes, - resources_remaining=( - _remaining_files(created) + failure.resources_remaining - ), - resources_reused=failure.resources_reused, - resources_unverified=failure.resources_unverified, - warnings=[*warnings, *ack_warnings, *failure.warnings, *recovery.diagnostics()], - request_id=failure.request_id, - status=failure.http_status, - partial=bool(created or failure.partial), - ) from failure - request_ids.extend(after_request_ids) - warnings.extend(recovery.warnings) - if len(after) != len(records): - raise HelperFailure( - "readback-mismatch", - "Final server inventory count differs from the approved inventory.", - blocked_at="verification", - writes=created, - resources_remaining=_remaining_files(created), - request_id=request_ids[-1] if request_ids else None, - partial=bool(created), - warnings=[*warnings, *ack_warnings, *recovery.diagnostics()], - ) - verified: list[dict[str, Any]] = [] - for record in records: - matches = [item for item in after if item.get("fileName") == record["path"]] - if len(matches) != 1 or not _matches(matches[0], plan, record): - raise HelperFailure( - "readback-mismatch", - f"File readback failed for {record['path']}.", - blocked_at="verification", - writes=created, - resources_remaining=_remaining_files(created), - request_id=request_ids[-1] if request_ids else None, - partial=bool(created), - warnings=[*warnings, *ack_warnings, *recovery.diagnostics()], - ) - verified.append( - { - "fileId": matches[0].get("fileId"), - "fileName": record["path"], - "sha256": record["sha256"], - "size": record["size"], - } - ) - - progress.update("file-readback", files_verified=len(verified)) - return { - "status": "completed", - "outcome": str(plan.get("outcome") or "file-knowledge-source-ingestion"), - "approved_plan": {"fingerprint": fingerprint, "confirmed": True}, - "resources": { - "created": created, - "reused": reused, - "updated": [], - "skipped": [], - }, - "api_contracts": [ - {"operation": "upload-file", "version": API_VERSION, "preview": True} - ], - "data_movement": { - "boundary": {"local_root_digest": digest(str(root))}, - "result": "exact approved files uploaded directly to Search", - }, - "auth": {"mode": "entra-user", "principals": []}, - "rbac": plan.get("rbac", {"assignments": []}), - "network": plan.get("network", {"posture": "preserved", "evidence": None}), - "verification": { - "readback": verified, - "server_inventory_digest": digest(_normalize_inventory(after)), - "request_ids": request_ids, - "idempotency": "matching marker metadata is zero-write", - }, - "warnings": warnings, - "ownership": { - "run_owned": created, - "reused_not_owned": reused, - "owner": plan.get("owner"), - }, - "cleanup": { - "status": "not-requested", - "separate_confirmation_required": True, - }, - } - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--input", type=Path, required=True) - add_progress_argument(parser) - args = parser.parse_args(argv) - fingerprint: str | None = None - owner: Any = None - outcome = "file-knowledge-source-ingestion" - try: - document, plan, fingerprint = load_approved_input(args.input) - document["_computed_fingerprint"] = fingerprint - owner = plan.get("owner") - outcome = str(plan.get("outcome") or outcome) - result = execute(document, progress=Progress("file-upload", enabled=args.progress)) - except HelperFailure as failure: - result = blocked_result( - failure, - outcome=outcome, - fingerprint=fingerprint, - owner=owner, - ) - emit_result(result) - return 3 if result["status"] == "partial" else 2 - emit_result(result) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import stat +import sys +import time +from pathlib import Path, PurePosixPath +from typing import Any +from urllib.parse import parse_qs, urlencode, urlsplit + +try: + from ._progress import Progress, add_progress_argument, reporting +except ImportError: + from _progress import Progress, add_progress_argument, reporting + +try: + from ._common import ( + SEARCH_AUDIENCE, + HelperFailure, + ReadRecovery, + TokenProvider, + Transport, + azure_cli_token, + blocked_result, + digest, + emit_result, + file_digest, + http_request, + load_approved_input, + odata_name, + reject_secrets, + require_allowed_fields, + validate_search_endpoint, + ) +except ImportError: + from _common import ( # type: ignore[no-redef] + SEARCH_AUDIENCE, + HelperFailure, + ReadRecovery, + TokenProvider, + Transport, + azure_cli_token, + blocked_result, + digest, + emit_result, + file_digest, + http_request, + load_approved_input, + odata_name, + reject_secrets, + require_allowed_fields, + validate_search_endpoint, + ) + + +API_VERSION = "2026-08-01-preview" +MAX_INVENTORY_PAGES = 200 +MAX_SERVER_FILES = 200 +INVENTORY_READ_TIMEOUT_SECONDS = 60 +MAX_INVENTORY_RESPONSE_BYTES = 1024 * 1024 +MAX_FILE_BYTES = { + "free": 50 * 1024 * 1024, + "basic": 50 * 1024 * 1024, + "dedicated": 100 * 1024 * 1024, + "serverless": 100 * 1024 * 1024, +} +MEDIA_TYPE = re.compile( + r"^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]*/[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]*$" +) +FILE_MEDIA_HINTS = { + ".txt": "text/plain", ".md": "text/markdown", + ".pdf": "application/pdf", ".html": "text/html", ".htm": "text/html", + ".csv": "text/csv", ".json": "application/json", ".sh": "application/x-sh", + ".doc": "application/msword", + ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".ppt": "application/vnd.ms-powerpoint", + ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ".xls": "application/vnd.ms-excel", + ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".jpeg": "image/jpeg", ".jpg": "image/jpeg", ".png": "image/png", + ".bmp": "image/bmp", ".heif": "image/heif", ".heic": "image/heic", + ".tiff": "image/tiff", ".tif": "image/tiff", ".gif": "image/gif", + ".webp": "image/webp", ".svg": "image/svg+xml", ".avif": "image/avif", + ".ico": "image/vnd.microsoft.icon", +} + + +def validate_api_version(value: Any) -> None: + if value != API_VERSION: + raise HelperFailure( + "api-version-invalid", + f"This File helper requires {API_VERSION} for its metadata upload contract. " + "The service also supports 2026-05-01-preview minimal extraction; use a compatible client " + "or explicitly approve August, never silently change the requested API.", + blocked_at="input-resolution", + ) + + +def media_type_hint(path: str) -> str: + """Return a portable filename hint, never a claim of server content detection.""" + return FILE_MEDIA_HINTS.get(PurePosixPath(path).suffix.lower(), "application/octet-stream") + + +def _reject_credential_path(path: Path) -> None: + lowered = [part.lower() for part in path.parts] + if ( + any(part in {".ssh", ".aws", ".azure", ".git", "credentials"} or part == ".env" or part.startswith(".env.") + for part in lowered) + or lowered[-1] in {"id_rsa", "id_dsa", "id_ecdsa", "id_ed25519"} + or path.suffix.lower() in {".pem", ".key", ".pfx", ".p12", ".kdbx"} + ): + raise HelperFailure( + "credential-file-forbidden", "Credential files are outside document ingestion.", + blocked_at="input-resolution", + ) + + +def _odata_name(name: Any) -> str: + return odata_name(name) + + +def _list_url(plan: dict[str, Any]) -> str: + endpoint = validate_search_endpoint(plan.get("endpoint")) + validate_api_version(plan.get("api_version")) + name = _odata_name(plan.get("name")) + return ( + f"{endpoint}/knowledgesources('{name}')/files?" + + urlencode({"api-version": API_VERSION, "pageSize": 200}) + ) + + +def _normalize_inventory(files: list[dict[str, Any]]) -> list[dict[str, Any]]: + fields = ( + "fileId", + "fileName", + "prefix", + "metadata", + "parsingMode", + "extractionMode", + "fileSizeBytes", + "errorMessage", + ) + normalized = [ + {field: item.get(field) for field in fields if field in item} + for item in files + ] + return sorted( + normalized, + key=lambda item: (str(item.get("fileName")), str(item.get("fileId"))), + ) + + +def _list_files( + url: str, + token: str, + *, + transport: Transport, + recovery: ReadRecovery | None = None, +) -> tuple[list[dict[str, Any]], list[str]]: + try: + origin = urlsplit(url) + origin_port = origin.port + except ValueError as exc: + raise HelperFailure( + "continuation-url-invalid", "File inventory URL is malformed.", + blocked_at="verification", + ) from exc + current_url: str | None = url + seen: set[str] = set() + files: list[dict[str, Any]] = [] + request_ids: list[str] = [] + requests = 0 + deadline = time.monotonic() + INVENTORY_READ_TIMEOUT_SECONDS + if recovery is not None: + deadline = min(deadline, recovery.deadline) + + def inventory_transport(method, target, credential, **options): + nonlocal requests + if requests >= MAX_INVENTORY_PAGES: + raise HelperFailure( + "file-list-limit-exceeded", "Complete file inventory exceeds 200 pages/requests.", + blocked_at="verification", + ) + requests += 1 + return transport(method, target, credential, **options) + + while current_url: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise HelperFailure( + "file-list-timeout", "Complete file inventory read exceeded its 60-second deadline.", + blocked_at="verification", + request_id=request_ids[-1] if request_ids else None, + ) + if len(seen) >= MAX_INVENTORY_PAGES: + raise HelperFailure( + "file-list-limit-exceeded", "Complete file inventory exceeds 200 pages/requests.", + blocked_at="verification", + request_id=request_ids[-1] if request_ids else None, + ) + try: + current_url.encode("ascii") + current = urlsplit(current_url) + current_port = current.port + except (ValueError, UnicodeEncodeError) as exc: + raise HelperFailure( + "continuation-url-invalid", "Search returned a malformed continuation URL.", + blocked_at="verification", + ) from exc + query = parse_qs(current.query) + if ( + current_url in seen + or current.scheme != "https" + or current.hostname != origin.hostname + or current_port != origin_port + or current.path != origin.path + or query.get("api-version") != [API_VERSION] + or current.username + or current.password + ): + raise HelperFailure( + "continuation-url-invalid", + "Search returned a continuation URL outside the approved service.", + blocked_at="verification", + ) + seen.add(current_url) + try: + options = dict(timeout=remaining, response_deadline=deadline, + max_response_bytes=MAX_INVENTORY_RESPONSE_BYTES, follow_redirects=False) + result = (recovery.get(current_url, token, transport=inventory_transport, + max_requests=MAX_INVENTORY_PAGES - requests, **options) + if recovery is not None else inventory_transport("GET", current_url, token, **options)) + except HelperFailure as failure: + if failure.code not in {"response-deadline-exceeded", "read-recovery-budget-exhausted"}: + raise + raise HelperFailure( + "file-list-timeout", "Complete file inventory read exceeded its effective deadline (at most 60 seconds).", + blocked_at="verification", request_id=failure.request_id, + status=failure.http_status, warnings=failure.warnings, + ) from failure + if time.monotonic() >= deadline: + raise HelperFailure( + "file-list-timeout", "Complete file inventory read exceeded its 60-second deadline.", + blocked_at="verification", + request_id=result.request_id, + ) + if result.status != 200 or not isinstance(result.body, dict): + raise HelperFailure( + "file-list-invalid", + "File-list readback did not return a JSON object.", + blocked_at="reconciliation", + request_id=result.request_id, + status=result.status, + ) + values = result.body.get("value") + if isinstance(values, list) and len(files) + len(values) > MAX_SERVER_FILES: + raise HelperFailure( + "file-list-limit-exceeded", "Complete file inventory exceeds 200 records.", + blocked_at="verification", + request_id=result.request_id, + ) + if not isinstance(values, list) or not all( + isinstance(item, dict) for item in values + ): + raise HelperFailure( + "file-list-invalid", + "File-list readback did not contain an object array.", + blocked_at="reconciliation", + request_id=result.request_id, + ) + files.extend(values) + if result.request_id: + request_ids.append(result.request_id) + next_link = result.body.get("@odata.nextLink") + if next_link is not None and not isinstance(next_link, str): + raise HelperFailure( + "continuation-url-invalid", + "Search returned an invalid continuation URL.", + blocked_at="verification", + ) + current_url = next_link + return files, recovery.request_ids if recovery is not None else request_ids + + +def _validate_relative_path(value: Any) -> str: + if not isinstance(value, str) or not value or "\\" in value: + raise HelperFailure( + "inventory-path-invalid", + "Every inventory path must be a non-empty normalized POSIX path.", + blocked_at="input-resolution", + ) + path = PurePosixPath(value) + if ( + path.is_absolute() + or path.as_posix() != value + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise HelperFailure( + "inventory-path-invalid", + f"Inventory path is not safely relative: {value!r}.", + blocked_at="input-resolution", + ) + if any("\r" in part or "\n" in part or ":" in part for part in path.parts): + raise HelperFailure( + "inventory-path-invalid", + f"Inventory path contains a forbidden segment: {value!r}.", + blocked_at="input-resolution", + ) + return path.as_posix() + + +def _reject_links(path: Path) -> None: + for probe in reversed((path, *path.parents)): + attributes = getattr(probe.lstat(), "st_file_attributes", 0) + if probe.is_symlink() or attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT: + raise HelperFailure( + "inventory-path-invalid", + "Local inventory cannot traverse links or reparse points.", + blocked_at="input-resolution", + ) + + +def resolve_local_root(value: Any) -> Path: + """Resolve the explicit data boundary without following links or reparse points.""" + if not isinstance(value, str) or not os.path.isabs(value): + raise HelperFailure( + "local-root-invalid", + "local_root must be an explicit absolute path.", + blocked_at="input-resolution", + ) + try: + path = Path(value) + _reject_links(path) + root = path.resolve(strict=True) + if not root.is_dir(): + raise OSError("not a directory") + except OSError as exc: + raise HelperFailure( + "local-root-invalid", + "local_root must be an existing readable real directory.", + blocked_at="input-resolution", + ) from exc + return root + + +def _resolve_inventory_path(root: Path, value: Any) -> Path: + relative = _validate_relative_path(value) + path = root.joinpath(*PurePosixPath(relative).parts) + try: + _reject_links(path) + resolved = path.resolve(strict=True) + resolved.relative_to(root) + except (OSError, ValueError) as exc: + raise HelperFailure( + "inventory-path-invalid", + f"Inventory path escapes or is unreadable: {relative}.", + blocked_at="input-resolution", + ) from exc + if not resolved.is_file() or resolved.is_symlink(): + raise HelperFailure( + "inventory-path-invalid", + f"Inventory entry is not a regular file: {relative}.", + blocked_at="input-resolution", + ) + _reject_credential_path(resolved) + return resolved + + +def _resolve_file(root: Path, record: dict[str, Any]) -> Path: + relative = _validate_relative_path(record.get("path")) + resolved = _resolve_inventory_path(root, relative) + try: + observed = resolved.stat() + matches = ( + record.get("size") == observed.st_size + and record.get("mtime_ns") == observed.st_mtime_ns + and record.get("sha256") == file_digest(resolved) + ) + except OSError as exc: + raise HelperFailure( + "inventory-unreadable", f"Selected file cannot be read: {relative}.", + blocked_at="input-resolution", + ) from exc + if not matches: + raise HelperFailure( + "inventory-drift", + f"Size, modification time, or SHA-256 changed for {relative}.", + blocked_at="confirmation", + ) + return resolved + + +def snapshot_inventory( + root: Path, paths: list[str], *, service_tier: str +) -> list[dict[str, Any]]: + """Freeze selected files with MIME hints; Search determines actual support.""" + if not isinstance(service_tier, str) or service_tier not in MAX_FILE_BYTES: + raise HelperFailure( + "service-tier-invalid", "Select a supported service_tier.", + blocked_at="input-resolution", + ) + if not isinstance(paths, list) or not 1 <= len(paths) <= 200: + raise HelperFailure( + "inventory-invalid", "Select between 1 and 200 explicit file paths.", + blocked_at="input-resolution", + ) + relative_paths = [_validate_relative_path(path) for path in paths] + if len(set(relative_paths)) != len(relative_paths): + raise HelperFailure( + "inventory-invalid", "Selected file paths must be unique.", + blocked_at="input-resolution", + ) + records = [] + for relative in sorted(relative_paths): + path = _resolve_inventory_path(root, relative) + media_type = media_type_hint(relative) + try: + observed = path.stat() + if not 0 < observed.st_size <= MAX_FILE_BYTES[service_tier]: + raise HelperFailure( + "inventory-invalid", "File size is empty or exceeds the selected tier limit.", + blocked_at="input-resolution", + ) + record = { + "path": relative, "size": observed.st_size, + "mtime_ns": observed.st_mtime_ns, "sha256": file_digest(path), + "media_type": media_type, + } + _resolve_file(root, record) + except OSError as exc: + raise HelperFailure( + "inventory-unreadable", "Selected file cannot be read.", + blocked_at="input-resolution", + ) from exc + records.append(record) + return records + + +def read_inventory( + plan: dict[str, Any], token: str, *, transport: Transport +) -> tuple[list[dict[str, Any]], list[str]]: + """Read all pages for the exact File source, using guarded continuations.""" + return _list_files(_list_url(plan), token, transport=transport) + + +def inventory_digest(files: list[dict[str, Any]]) -> str: + return digest(_normalize_inventory(files)) + + +def reconcile_inventory( + plan: dict[str, Any], + before: list[dict[str, Any]], + *, + allow_new_uploads: bool = False, +) -> dict[str, dict[str, Any]]: + """Validate all existing markers before planning or performing any upload.""" + expected_names = {record["path"] for record in plan["files"]} + if any(item.get("fileName") not in expected_names for item in before): + raise HelperFailure( + "server-inventory-conflict", + "The server inventory contains files outside the approved inventory.", + blocked_at="reconciliation", + ) + matched = {} + for record in plan["files"]: + matches = [item for item in before if item.get("fileName") == record["path"]] + if len(matches) > 1: + raise HelperFailure( + "duplicate-file-record", f"Multiple server records exist for {record['path']}.", + blocked_at="reconciliation", + ) + if matches: + if not _matches(matches[0], plan, record): + raise HelperFailure( + "file-record-conflict", f"Server record conflicts with approved file {record['path']}.", + blocked_at="reconciliation", + ) + matched[record["path"]] = matches[0] + elif not allow_new_uploads: + raise HelperFailure( + "reused-source-upload-forbidden", + "New files cannot be uploaded into a reused source because individual-file cleanup is unsupported.", + blocked_at="reconciliation", + ) + return matched + + +def _metadata(plan: dict[str, Any], record: dict[str, Any]) -> dict[str, str]: + supplied = record.get("metadata") or {} + if not isinstance(supplied, dict) or not all( + isinstance(key, str) and isinstance(value, str) + for key, value in supplied.items() + ): + raise HelperFailure( + "file-metadata-invalid", + "File metadata must contain only string keys and values.", + blocked_at="input-resolution", + ) + metadata = dict(supplied) + metadata.update( + { + "foundryIqSha256": record["sha256"], + "foundryIqSizeBytes": str(record["size"]), + "foundryIqInventory": plan["inventory_digest"], + "foundryIqOwner": str(plan["owner"]), + "foundryIqSource": str(plan["name"]), + } + ) + return metadata + + +def _matches( + server: dict[str, Any], + plan: dict[str, Any], + record: dict[str, Any], +) -> bool: + metadata = server.get("metadata") or {} + expected = _metadata(plan, record) + return ( + server.get("fileName") == record["path"] + and server.get("fileSizeBytes") == record["size"] + and server.get("errorMessage") is None + and isinstance(metadata, dict) + and all(metadata.get(key) == value for key, value in expected.items()) + ) + + +def _multipart( + plan: dict[str, Any], + record: dict[str, Any], + content: bytes, + fingerprint: str, +) -> tuple[bytes, str]: + boundary = "foundry-iq-" + fingerprint.removeprefix("sha256:")[:24] + metadata = { + "fileName": record["path"], + "metadata": _metadata(plan, record), + } + media_type = record.get("media_type") + if not isinstance(media_type, str) or not media_type: + media_type = media_type_hint(record["path"]) + pieces = [ + f"--{boundary}\r\n".encode("ascii"), + b'Content-Disposition: form-data; name="metadata"\r\n', + b"Content-Type: application/json\r\n\r\n", + json.dumps(metadata, sort_keys=True, separators=(",", ":")).encode("utf-8"), + b"\r\n", + f"--{boundary}\r\n".encode("ascii"), + b'Content-Disposition: form-data; name="content"; filename="upload"\r\n', + f"Content-Type: {media_type}\r\n\r\n".encode("ascii"), + content, + b"\r\n", + f"--{boundary}--\r\n".encode("ascii"), + ] + return b"".join(pieces), boundary + + +def _validate_plan(plan: dict[str, Any]) -> tuple[Path, list[dict[str, Any]]]: + reject_secrets(plan) + require_allowed_fields( + plan, + { + "operation", + "outcome", + "endpoint", + "name", + "api_version", + "local_root", + "files", + "inventory_digest", + "expected_server_inventory_digest", + "service_tier", + "extraction_mode", + "rbac", + "network", + "owner", + "cleanup_approved", + }, + label="File ingestion plan", + ) + for field, allowed in ( + ("rbac", {"assignments"}), + ("network", {"posture", "evidence"}), + ): + section = plan.get(field) + if section is not None: + if not isinstance(section, dict): + raise HelperFailure( + "input-schema-invalid", + f"{field} must be an object.", + blocked_at="input-resolution", + ) + require_allowed_fields(section, allowed, label=field) + if plan.get("operation") != "ingest" or plan.get("cleanup_approved") is not False: + raise HelperFailure( + "operation-invalid", + "File ingestion requires operation ingest and cleanup_approved false.", + blocked_at="input-resolution", + ) + if not isinstance(plan.get("name"), str) or not plan["name"]: + raise HelperFailure( + "name-invalid", + "Knowledge source name is required.", + blocked_at="input-resolution", + ) + if not isinstance(plan.get("owner"), str) or not plan["owner"]: + raise HelperFailure( + "owner-invalid", + "File ingestion owner is required.", + blocked_at="input-resolution", + ) + root = resolve_local_root(plan.get("local_root")) + records = plan.get("files") + if not isinstance(records, list) or not records or len(records) > 200: + raise HelperFailure( + "inventory-invalid", + "files must contain between 1 and 200 entries.", + blocked_at="input-resolution", + ) + if not all(isinstance(record, dict) for record in records): + raise HelperFailure( + "inventory-invalid", + "Every file inventory entry must be an object.", + blocked_at="input-resolution", + ) + for record in records: + require_allowed_fields( + record, + {"path", "size", "mtime_ns", "sha256", "media_type", "metadata"}, + label="File inventory record", + ) + service_tier = plan.get("service_tier") + if service_tier not in MAX_FILE_BYTES: + raise HelperFailure( + "service-tier-invalid", + "service_tier must be free, basic, dedicated, or serverless.", + blocked_at="input-resolution", + ) + extraction_mode = plan.get("extraction_mode") + if extraction_mode not in {"minimal", "standard"}: + raise HelperFailure( + "extraction-mode-invalid", + "extraction_mode must be minimal or standard.", + blocked_at="input-resolution", + ) + paths = [_validate_relative_path(record.get("path")) for record in records] + if paths != sorted(paths) or len(paths) != len(set(paths)): + raise HelperFailure( + "inventory-invalid", + "File inventory paths must be unique and byte-sorted.", + blocked_at="input-resolution", + ) + if digest(records) != plan.get("inventory_digest"): + raise HelperFailure( + "inventory-drift", + "inventory_digest does not match the approved file records.", + blocked_at="confirmation", + ) + for record in records: + size = record.get("size") + mtime_ns = record.get("mtime_ns") + media_type = record.get("media_type") + if ( + not isinstance(size, int) + or size <= 0 + or size > MAX_FILE_BYTES[service_tier] + or not isinstance(mtime_ns, int) + or mtime_ns <= 0 + or not isinstance(media_type, str) + or MEDIA_TYPE.fullmatch(media_type) is None + ): + raise HelperFailure( + "inventory-invalid", + "Every file requires positive size/mtime, media type, and a tier-valid size.", + blocked_at="input-resolution", + ) + _resolve_file(root, record) + _metadata(plan, record) + return root, records + + +def _confirm_ambiguous_upload( + list_url: str, + token: str, + transport: Transport, + record: dict[str, Any], + plan: dict[str, Any], + request_ids: list[str], + failure: HelperFailure, + warnings: list[str], +) -> dict[str, Any] | None: + """Resolve an ambiguous upload with bounded readback, never upload replay. + + Only ambiguous outcomes (transport failure, timeout, 409/429/5xx) reach + this helper. A definitive 200/201 response never calls it. Returns the + matching server record when the readback proves the approved file + exists, otherwise ``None`` so the caller reports ``partial``. + """ + recovery = ReadRecovery() + try: + recovery.delay(failure) + observed, _ = _list_files( + list_url, token, transport=transport, recovery=recovery, + ) + except HelperFailure as read_failure: + warnings.append(f"Upload readback failed ({read_failure.code}); original upload failure retained.") + return None + finally: + request_ids.extend(recovery.request_ids) + warnings.extend(recovery.diagnostics()) + matches = [item for item in observed if item.get("fileName") == record["path"]] + if len(matches) != 1 or not _matches(matches[0], plan, record): + return None + return matches[0] + + +def _remaining_files(resources: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [ + { + "type": "knowledge-source-file", + "fileName": resource["fileName"], + "sha256": resource["sha256"], + } + for resource in resources + ] + + +@reporting("file-upload") +def execute( + document: dict[str, Any], + *, + token_provider: TokenProvider = azure_cli_token, + transport: Transport = http_request, + allow_new_uploads: bool = False, + progress: Progress | None = None, + upload_session=None, + source_check=None, + allow_upload_retry=True, +) -> dict[str, Any]: + if allow_new_uploads or upload_session is not None or len(document.get("plan", {}).get("files", [])) > 1: + try: + from .file_upload import run_batch + except ImportError: + from file_upload import run_batch + return run_batch(document, token_provider=token_provider, transport=transport, progress=progress, + allow_new_uploads=allow_new_uploads, session=upload_session, + source_check=source_check, allow_upload_retry=allow_upload_retry) + progress.update("file-inventory") + plan = document["plan"] + fingerprint = document["_computed_fingerprint"] + root, records = _validate_plan(plan) + list_url = _list_url(plan) + token = token_provider(SEARCH_AUDIENCE) + readonly = ReadRecovery() if not allow_new_uploads else None + before, request_ids = _list_files(list_url, token, transport=transport, recovery=readonly) + warnings: list[str] = list(readonly.warnings) if readonly is not None else [] + if inventory_digest(before) != plan.get("expected_server_inventory_digest"): + raise HelperFailure( + "server-inventory-drift", + "Server file inventory changed after approval.", + blocked_at="reconciliation", + warnings=readonly.diagnostics() if readonly is not None else [], + ) + matched = reconcile_inventory(plan, before, allow_new_uploads=allow_new_uploads) + + created: list[dict[str, Any]] = [] + reused: list[dict[str, Any]] = [] + acknowledged_ids: list[str] = [] + progress.update("file-upload", uploads_acknowledged=0, files_reused=0) + for record in records: + progress.update("file-upload", uploads_acknowledged=len(created), files_reused=len(reused)) + if record["path"] in matched: + reused.append( + { + "fileId": matched[record["path"]].get("fileId"), + "fileName": record["path"], + "sha256": record["sha256"], + } + ) + continue + + path = _resolve_file(root, record) + try: + content = path.read_bytes() + except OSError as exc: + raise HelperFailure( + "inventory-unreadable", + f"Approved file became unreadable: {record['path']}.", + blocked_at="execution", + writes=created, + resources_remaining=_remaining_files(created), + partial=bool(created), + ) from exc + try: + post_read_stat = path.stat() + except OSError as exc: + raise HelperFailure( + "inventory-unreadable", + f"Approved file became unreadable: {record['path']}.", + blocked_at="execution", + writes=created, + resources_remaining=_remaining_files(created), + partial=bool(created), + ) from exc + content_digest = "sha256:" + hashlib.sha256(content).hexdigest() + if ( + content_digest != record["sha256"] + or len(content) != record["size"] + or post_read_stat.st_mtime_ns != record["mtime_ns"] + ): + raise HelperFailure( + "inventory-drift", + f"Approved file changed before upload: {record['path']}.", + blocked_at="confirmation", + writes=created, + resources_remaining=_remaining_files(created), + partial=bool(created), + ) + body, boundary = _multipart(plan, record, content, fingerprint) + try: + result = transport( + "POST", + list_url, + token, + body=body, + headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, + ) + except HelperFailure as failure: + if failure.http_status is not None and 400 <= failure.http_status < 500 and failure.http_status not in {408, 409, 429}: + raise HelperFailure( + failure.code, failure.message, blocked_at=failure.blocked_at, + writes=created, resources_remaining=_remaining_files(created), + request_id=failure.request_id, status=failure.http_status, partial=bool(created), + ) from failure + # The transport outcome is ambiguous (we do not know whether the + # server received the write): attempt one readback before + # concluding partial, per the ambiguous-write contract. + confirmed = _confirm_ambiguous_upload( + list_url, token, transport, record, plan, request_ids, failure, warnings + ) + if confirmed is not None: + created.append({"fileName": record["path"], "sha256": record["sha256"]}) + if failure.request_id: + request_ids.append(failure.request_id) + continue + uncertain = { + "action": "upload-unverified", + "type": "knowledge-source-file", + "fileName": record["path"], + "sha256": record["sha256"], + } + raise HelperFailure( + failure.code, + failure.message, + blocked_at=failure.blocked_at, + writes=created + [uncertain], + resources_remaining=_remaining_files(created) + [uncertain], + request_id=failure.request_id, + status=failure.http_status, + partial=True, + warnings=warnings, + ) from failure + if result.status not in {200, 201}: + ambiguous = result.status in {408, 409, 429} or result.status >= 500 + if ambiguous: + confirmed = _confirm_ambiguous_upload( + list_url, token, transport, record, plan, request_ids, + HelperFailure("upload-failed", "Upload response is ambiguous.", + blocked_at="execution", status=result.status, + request_id=result.request_id, retry_after=result.retry_after, + recovery_deadline=result.recovery_deadline), + warnings, + ) + if confirmed is not None: + created.append( + {"fileName": record["path"], "sha256": record["sha256"]} + ) + if result.request_id: + request_ids.append(result.request_id) + continue + uncertain = { + "action": "upload-unverified", + "type": "knowledge-source-file", + "fileName": record["path"], + "sha256": record["sha256"], + } + raise HelperFailure( + "upload-failed", + f"Upload returned unexpected HTTP {result.status}.", + blocked_at="execution", + writes=created + ([uncertain] if ambiguous else []), + resources_remaining=( + _remaining_files(created) + ([uncertain] if ambiguous else []) + ), + request_id=result.request_id, + status=result.status, + partial=bool(created) or ambiguous, + warnings=warnings, + ) + # A definitive 200/201 response is not ambiguous: trust it rather than + # re-listing the whole source after every single file. The complete + # inventory is verified once, in bulk, after the loop. + created.append({"fileName": record["path"], "sha256": record["sha256"]}) + if result.request_id: + request_ids.append(result.request_id) + acknowledged_ids.append(ReadRecovery.safe_id(result.request_id)) + + progress.update("file-readback", uploads_acknowledged=len(created), files_reused=len(reused)) + ack_warnings = (["Acknowledged upload request IDs: " + ", ".join(acknowledged_ids)] + if acknowledged_ids else []) + recovery = ReadRecovery() + try: + after, after_request_ids = _list_files( + list_url, token, transport=transport, recovery=recovery, + ) + except HelperFailure as failure: + raise HelperFailure( + failure.code, + failure.message, + blocked_at=failure.blocked_at, + writes=created + failure.writes, + resources_remaining=( + _remaining_files(created) + failure.resources_remaining + ), + resources_reused=failure.resources_reused, + resources_unverified=failure.resources_unverified, + warnings=[*warnings, *ack_warnings, *failure.warnings, *recovery.diagnostics()], + request_id=failure.request_id, + status=failure.http_status, + partial=bool(created or failure.partial), + ) from failure + request_ids.extend(after_request_ids) + warnings.extend(recovery.warnings) + if len(after) != len(records): + raise HelperFailure( + "readback-mismatch", + "Final server inventory count differs from the approved inventory.", + blocked_at="verification", + writes=created, + resources_remaining=_remaining_files(created), + request_id=request_ids[-1] if request_ids else None, + partial=bool(created), + warnings=[*warnings, *ack_warnings, *recovery.diagnostics()], + ) + verified: list[dict[str, Any]] = [] + for record in records: + matches = [item for item in after if item.get("fileName") == record["path"]] + if len(matches) != 1 or not _matches(matches[0], plan, record): + raise HelperFailure( + "readback-mismatch", + f"File readback failed for {record['path']}.", + blocked_at="verification", + writes=created, + resources_remaining=_remaining_files(created), + request_id=request_ids[-1] if request_ids else None, + partial=bool(created), + warnings=[*warnings, *ack_warnings, *recovery.diagnostics()], + ) + verified.append( + { + "fileId": matches[0].get("fileId"), + "fileName": record["path"], + "sha256": record["sha256"], + "size": record["size"], + } + ) + + progress.update("file-readback", files_verified=len(verified)) + return { + "status": "completed", + "outcome": str(plan.get("outcome") or "file-knowledge-source-ingestion"), + "approved_plan": {"fingerprint": fingerprint, "confirmed": True}, + "resources": { + "created": created, + "reused": reused, + "updated": [], + "skipped": [], + }, + "api_contracts": [ + {"operation": "upload-file", "version": API_VERSION, "preview": True} + ], + "data_movement": { + "boundary": {"local_root_digest": digest(str(root))}, + "result": "exact approved files uploaded directly to Search", + }, + "auth": {"mode": "entra-user", "principals": []}, + "rbac": plan.get("rbac", {"assignments": []}), + "network": plan.get("network", {"posture": "preserved", "evidence": None}), + "verification": { + "readback": verified, + "server_inventory_digest": digest(_normalize_inventory(after)), + "request_ids": request_ids, + "idempotency": "matching marker metadata is zero-write", + }, + "warnings": warnings, + "ownership": { + "run_owned": created, + "reused_not_owned": reused, + "owner": plan.get("owner"), + }, + "cleanup": { + "status": "not-requested", + "separate_confirmation_required": True, + }, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--input", type=Path, required=True) + add_progress_argument(parser) + args = parser.parse_args(argv) + fingerprint: str | None = None + owner: Any = None + outcome = "file-knowledge-source-ingestion" + try: + document, plan, fingerprint = load_approved_input(args.input) + document["_computed_fingerprint"] = fingerprint + owner = plan.get("owner") + outcome = str(plan.get("outcome") or outcome) + result = execute(document, progress=Progress("file-upload", enabled=args.progress)) + except HelperFailure as failure: + result = blocked_result( + failure, + outcome=outcome, + fingerprint=fingerprint, + owner=owner, + ) + emit_result(result) + return 3 if result["status"] == "partial" else 2 + emit_result(result) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_source.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_source.py index 2b3f73b3..2f2e3181 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_source.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_source.py @@ -1,858 +1,858 @@ -from __future__ import annotations - -import argparse -import copy -import json -import re -import sys -from pathlib import Path -from typing import Any - -try: - from ._progress import Progress, add_progress_argument, reporting - from . import file_ingest, search_reconcile, source_vector, file_cu_mi - from . import cu_ingestion_auth as file_cu_auth - from ._common import ( - SEARCH_AUDIENCE, - MANAGEMENT_AUDIENCE, - HelperFailure, - TokenProvider, - Transport, - azure_cli_token, - blocked_result, - digest, - emit_result, - http_request, - load_approved_input, - normalize_azure_location, - reject_secrets, - require_allowed_fields, - ) -except ImportError: - from _progress import Progress, add_progress_argument, reporting - import file_ingest # type: ignore[no-redef] - import search_reconcile # type: ignore[no-redef] - import source_vector - import file_cu_mi - import cu_ingestion_auth as file_cu_auth - from _common import ( # type: ignore[no-redef] - SEARCH_AUDIENCE, - MANAGEMENT_AUDIENCE, - HelperFailure, - TokenProvider, - Transport, - azure_cli_token, - blocked_result, - digest, - emit_result, - http_request, - load_approved_input, - normalize_azure_location, - reject_secrets, - require_allowed_fields, - ) - - -def _cu_failure(code: str, message: str, *, request_id: str | None = None) -> HelperFailure: - return HelperFailure(code, message, blocked_at="cu-prerequisites", request_id=request_id) - - -def validate_content_understanding(value: Any, *, enabled: bool) -> dict[str, Any] | None: - if not enabled: - if value is not None: - raise _cu_failure("cu-choice-conflict", "Minimal extraction must omit CU choices.") - return None - if not isinstance(value, dict): - raise _cu_failure("cu-prerequisite-missing", "Standard planning requires a resolved CU account, disclosed auth channel and owner-verified prerequisites.") - value = copy.deepcopy(value) - value.setdefault("auth", "system-assigned") - reject_secrets(value) - require_allowed_fields(value, { - "endpoint", "resource_id", "auth", "api_key_environment", "prerequisites", "managed_identity", - }, label="File CU choice") - if ( - not isinstance(value.get("endpoint"), str) - or re.fullmatch(r"https://[a-z0-9][a-z0-9-]{0,62}\.services\.ai\.azure\.com/?", value["endpoint"]) is None - or not isinstance(value.get("resource_id"), str) - or re.fullmatch( - r"/subscriptions/[0-9a-fA-F-]{36}/resourceGroups/[A-Za-z0-9_.()-]{1,90}" - r"/providers/Microsoft\.CognitiveServices/accounts/[A-Za-z0-9][A-Za-z0-9_.-]{1,63}", - value["resource_id"], re.IGNORECASE, - ) is None - or value.get("auth") not in ("api-key-environment", "api-key-arm", "system-assigned") - or (value.get("auth") == "api-key-environment" and ( - not isinstance(value.get("api_key_environment"), str) - or search_reconcile.ENVIRONMENT_NAME.fullmatch(value["api_key_environment"]) is None - )) - or (value.get("auth") != "api-key-environment" and "api_key_environment" in value) - or (value.get("auth") != "system-assigned" and "managed_identity" in value) - ): - raise _cu_failure("cu-choice-invalid", "Select exact AIServices and system-assigned MI, or explicitly retain approved ARM/ENV key auth. No automatic auth fallback or setup changes.") - if value["auth"] == "system-assigned": - file_cu_mi.validate_choice(value.get("managed_identity"), value["resource_id"]) - prerequisites = value.get("prerequisites") - if not isinstance(prerequisites, dict): - raise _cu_failure("cu-prerequisite-missing", "Supply CU region/capability, selected processing/required deployments, identity/local-auth and network evidence references.") - fields = {"resource", "configuration", "identity", "network"} - require_allowed_fields(prerequisites, fields, label="File CU prerequisites") - if any(not source_vector._text(prerequisites.get(key)) for key in fields): - raise _cu_failure("cu-prerequisite-missing", "Owner-verified CU capability/region, selected processing/required deployments, auth/access and reachability evidence is required.") - source_vector._json_valid(value) - return copy.deepcopy(value) - - -def _cu_account_state(choice: dict[str, Any], account: Any) -> dict[str, Any]: - properties = account.get("properties") if isinstance(account, dict) else None - if not isinstance(properties, dict): - raise _cu_failure("cu-prerequisite-invalid", "CU account readback is incomplete.") - endpoints = properties.get("endpoints", {}) - candidates = [properties.get("endpoint")] - if isinstance(endpoints, dict): - candidates.extend(endpoints.values()) - if ( - str(account.get("id", "")).casefold() != choice["resource_id"].casefold() - or account.get("kind") != "AIServices" - or normalize_azure_location(account.get("location")) is None - or properties.get("provisioningState") != "Succeeded" - or (choice["auth"] != "system-assigned" and properties.get("disableLocalAuth") is not False) - or properties.get("publicNetworkAccess") not in ("Enabled", "Disabled") - or choice["endpoint"].rstrip("/") not in [v.rstrip("/") for v in candidates if isinstance(v, str)] - ): - raise _cu_failure("cu-prerequisite-invalid", "Readback must bind the selected ready AIServices account/endpoint/location/network. Key modes also need enabled local auth; MI does not. Any required setup change needs separate approval.") - state = { - "id": choice["resource_id"], "kind": "AIServices", "location": account["location"], - "identity": copy.deepcopy(account.get("identity")), - "properties": { - "endpoint": choice["endpoint"].rstrip("/"), "provisioningState": "Succeeded", - "disableLocalAuth": properties.get("disableLocalAuth"), "publicNetworkAccess": properties["publicNetworkAccess"], - "networkAcls": copy.deepcopy(properties.get("networkAcls")), - }, - } - if choice["auth"] == "system-assigned": - acl = properties.get("networkAcls") - if properties["publicNetworkAccess"] != "Enabled" or ( - acl is not None and (not isinstance(acl, dict) or acl.get("defaultAction") != "Allow") - ): - raise _cu_failure("cu-mi-network-unverified", "This MI path requires existing public CU reachability without default-deny ACLs. Restricted/private network compatibility needs separate verified setup and approval; no network changes or key fallback.") - reject_secrets(state) - source_vector._json_valid(state) - return state - - -def _cu_states_match(current: dict[str, Any], retained: dict[str, Any]) -> bool: - location = normalize_azure_location(retained.get("location")) - return location is not None and ( - {**current, "location": normalize_azure_location(current.get("location"))} - == {**retained, "location": location} - ) - - -def read_content_understanding( - choice: dict[str, Any], *, token_provider: TokenProvider, transport: Transport, -) -> tuple[dict[str, Any], list[str]]: - url = f"{MANAGEMENT_AUDIENCE}{choice['resource_id']}?api-version=2024-10-01" - response = transport("GET", url, token_provider(MANAGEMENT_AUDIENCE)) - try: - if response.status != 200: - raise _cu_failure("cu-prerequisite-unavailable", "Selected CU account metadata could not be read; no provisioning or auth changes are allowed.") - state = _cu_account_state(choice, response.body) - except HelperFailure as failure: - failure.request_id = response.request_id - if response.status != 200: - failure.http_status = response.status - raise - return state, [response.request_id] if response.request_id else [] - - -def verify_content_understanding_readback(choice: dict[str, Any], current: Any) -> None: - parameters = current.get("fileParameters") if isinstance(current, dict) else None - ingestion = parameters.get("ingestionParameters") if isinstance(parameters, dict) else None - ai = ingestion.get("aiServices") if isinstance(ingestion, dict) else None - if ( - not isinstance(ai, dict) or ingestion.get("contentExtractionMode") != "standard" - or not isinstance(ai.get("uri"), str) - or ai["uri"].rstrip("/") != choice["endpoint"].rstrip("/") - or ingestion.get("identity") is not None - or (choice["auth"] == "system-assigned" and ai.get("apiKey") not in file_cu_mi.REDACTED) - ): - raise _cu_failure("cu-readback-mismatch", "Observed File CU endpoint/extraction/auth conflicts with the selected configuration; credential details withheld.") - # File's approved key may be redacted in GET. It is not embedding auth, - # and source readback cannot prove its value or CU processing readiness. - - -def plan_source( - request: dict[str, Any], - *, - token_provider: TokenProvider = azure_cli_token, - transport: Transport = http_request, - context_provider=file_cu_auth.account_context, -) -> dict[str, Any]: - """Build an unapproved File workflow using only local reads and GETs.""" - if not isinstance(request, dict): - raise HelperFailure( - "input-schema-invalid", "Planning input must be an object.", - blocked_at="input-resolution", - ) - reject_secrets(request) - require_allowed_fields( - request, - {"schema_version", "api_version", "endpoint", "name", "owner", "local_root", "paths", - "service_tier", "extraction_mode", "vectorization", "embedding", "rbac", "network", - "content_understanding", "reuse_input_file", "reuse_result_file"}, - label="File planning input", - ) - if request.get("schema_version") != "1.0": - raise HelperFailure( - "input-schema-invalid", "Planning requires schema_version 1.0.", - blocked_at="input-resolution", - ) - file_ingest.validate_api_version(request.get("api_version", file_ingest.API_VERSION)) - for field in ("name", "owner"): - if not isinstance(request.get(field), str) or not request[field].strip(): - raise HelperFailure( - "input-schema-invalid", f"An explicit non-empty {field} is required.", - blocked_at="input-resolution", - ) - if request.get("extraction_mode") not in ("minimal", "standard") or request.get("vectorization") not in ("none", "azureOpenAI"): - raise HelperFailure( - "planning-processing-unsupported", - "Select minimal or standard extraction and independent vectorization none or azureOpenAI.", - blocked_at="input-resolution", - ) - embedding = source_vector.validate_choice( - request.get("embedding"), enabled=request["vectorization"] == "azureOpenAI", - api_version=file_ingest.API_VERSION, - ) - cu = validate_content_understanding( - request.get("content_understanding"), enabled=request["extraction_mode"] == "standard", - ) - if (request.get("reuse_input_file") is not None or request.get("reuse_result_file") is not None) and ( - cu is None or cu["auth"] != "system-assigned" - ): - raise _cu_failure("cu-choice-conflict", "File MI provenance inputs are only for managed-identity exact reuse.") - rbac, network = request.get("rbac"), request.get("network") - if ( - not isinstance(rbac, dict) - or not isinstance(rbac.get("assignments"), list) - or not rbac["assignments"] - or not all(isinstance(item, dict) and item for item in rbac["assignments"]) - or not isinstance(network, dict) - or not isinstance(network.get("posture"), str) - or not network["posture"].strip() - or not isinstance(network.get("evidence"), str) - or not network["evidence"].strip() - ): - raise HelperFailure( - "planning-evidence-missing", - "Supply observed RBAC assignments and network posture/evidence; the policy owner must refresh and verify them before approval.", - blocked_at="input-resolution", - ) - root = file_ingest.resolve_local_root(request.get("local_root")) - records = file_ingest.snapshot_inventory( - root, request.get("paths"), service_tier=request.get("service_tier") - ) - common = { - "endpoint": request.get("endpoint"), "name": request.get("name"), - "api_version": file_ingest.API_VERSION, "owner": request.get("owner"), - "cleanup_approved": False, "rbac": copy.deepcopy(rbac), - "network": copy.deepcopy(network), - } - source = { - **common, "operation": "reconcile", "resource_type": "knowledge-source", - "outcome": "create-file-knowledge-source", "action": "create", - "desired": { - "name": common["name"], "kind": "file", - "fileParameters": {"ingestionParameters": {"contentExtractionMode": request["extraction_mode"]}}, - }, - } - ingestion = { - **copy.deepcopy(common), "operation": "ingest", "local_root": str(root), - "files": records, "inventory_digest": digest(records), - "expected_server_inventory_digest": file_ingest.inventory_digest([]), - "service_tier": request["service_tier"], "extraction_mode": request["extraction_mode"], - } - plan = { - "operation": "reconcile-and-ingest", "outcome": "create-file-knowledge-source", - "owner": common["owner"], "cleanup_approved": False, - "source": source, "ingestion": ingestion, - } - if embedding is not None: - plan["embedding"] = embedding - source["desired"]["fileParameters"]["ingestionParameters"]["embeddingModel"] = source_vector.model_definition(embedding) - if cu is not None: - automatic = cu["auth"] == "api-key-arm" - mi = cu["auth"] == "system-assigned" - plan.update(file_cu_plan_version="1.2" if mi else "1.1" if automatic else "1.0", content_understanding=cu) - if automatic: - source["ai_services_key_acquisition"] = file_cu_auth.acquisition(cu, context_provider()) - elif mi: - source["ai_services_managed_identity"] = True - else: - source["ai_services_api_key_environment"] = cu["api_key_environment"] - source["desired"]["fileParameters"]["ingestionParameters"].update( - aiServices={"uri": cu["endpoint"].rstrip("/")}, disableImageVerbalization=True, - ) - # Validate the local inventory and all choices before any authentication. - _validate_plan(plan, require_cu_readback=False) - cu_request_ids = [] - if cu is not None: - state, cu_request_ids = read_content_understanding(cu, token_provider=token_provider, transport=transport) - plan["cu_resource_state"] = state - if cu["auth"] == "system-assigned": - plan["cu_identity_state"], ids = file_cu_mi.read_binding( - cu, common["endpoint"], token_provider=token_provider, transport=transport, - ) - cu_request_ids.extend(ids) - _validate_plan(plan) - transport = source_vector.guard_readback_transport(plan, transport) - url = search_reconcile.resource_url(source) - token = token_provider(SEARCH_AUDIENCE) - current, request_id = search_reconcile.read_resource(url, token, transport=transport) - request_ids = cu_request_ids + ([request_id] if request_id else []) - matched = {} - if current is not None: - if cu is not None and cu["auth"] == "system-assigned": - file_cu_mi.verify_reuse(request, plan, current) - if not search_reconcile.definitions_match(source["desired"], current): - raise HelperFailure( - "definition-conflict", - "The exact source has a different definition; planning never overwrites or chooses another name.", - blocked_at="reconciliation", - ) - etag = current.get("@odata.etag") - if not isinstance(etag, str) or not etag: - raise HelperFailure( - "definition-evidence-missing", "Exact reuse requires the current source ETag.", - blocked_at="reconciliation", - ) - before, ids = file_ingest.read_inventory(ingestion, token, transport=transport) - request_ids.extend(ids) - matched = file_ingest.reconcile_inventory(ingestion, before) - file_ids = [item.get("fileId") for item in matched.values()] - if ( - not all(isinstance(value, str) and value for value in file_ids) - or len(set(file_ids)) != len(file_ids) - ): - raise HelperFailure( - "file-identity-ambiguous", "Exact reuse requires unique non-empty server file IDs.", - blocked_at="reconciliation", - ) - source.update(action="reuse", expected_etag=etag) - ingestion["expected_server_inventory_digest"] = file_ingest.inventory_digest(before) - refreshed, refresh_id = search_reconcile.read_resource(url, token, transport=transport) - if refresh_id: - request_ids.append(refresh_id) - if ( - refreshed is None - or refreshed.get("@odata.etag") != etag - or not search_reconcile.definitions_match(source["desired"], refreshed) - ): - raise HelperFailure( - "definition-drift", - "Source definition or ETag changed during file inventory readback.", - blocked_at="reconciliation", - ) - # Do not return a snapshot that changed while Search discovery was running. - _validate_plan(plan) - if cu is not None: - state, ids = read_content_understanding(cu, token_provider=token_provider, transport=transport) - request_ids.extend(ids) - if not _cu_states_match(state, plan["cu_resource_state"]): - raise _cu_failure( - "cu-prerequisite-drift", "CU account access or configuration changed during planning; refresh the plan.", - request_id=ids[-1] if ids else None, - ) - if cu["auth"] == "api-key-arm": - file_cu_auth.check_context(source["ai_services_key_acquisition"]["context"], context_provider) - elif cu["auth"] == "system-assigned": - binding, ids = file_cu_mi.read_binding(cu, common["endpoint"], token_provider=token_provider, transport=transport) - request_ids.extend(ids) - if binding != plan["cu_identity_state"]: - raise _cu_failure("cu-mi-identity-drift", "Search identity, CU scoped role or network changed during planning.") - fingerprint = digest(plan) - mutation_required = source["action"] == "create" - return { - "status": "planned", "outcome": plan["outcome"], - "plan_fingerprint": fingerprint, - "execution_input": { - "schema_version": "1.0", "plan": plan, - "approval": {"confirmed": False, "fingerprint": fingerprint}, - }, - "approval_summary": { - "target": {"endpoint": common["endpoint"], "name": common["name"], - "api_version": common["api_version"], "preview": True}, - "source_action": source["action"], "owner": common["owner"], - "execution_required": mutation_required, - "mutation_approval_required": mutation_required, - "processing": ( - "standard CU extraction" + (" with source embeddings" if embedding else "; source vectors off") - if cu else "minimal extraction with embeddings" if embedding else "minimal lexical; no models" - ), - **({"content_understanding": { - "purpose": "Standard document extraction only; not source vectorization or KB answer synthesis.", - "endpoint": cu["endpoint"], - "authentication": file_cu_auth.approval_summary("file", cu["auth"], creating=mutation_required), - "auth": ( - "Search system-assigned MI for CU; no API key or credential reads, no local-auth requirement. Service implementation inspected; live compatibility unverified." - if cu["auth"] == "system-assigned" else - "Existing File CU key-auth configuration reused; no credential acquisition. Search remains keyless." - if not mutation_required else - "Approved private ARM listKeys acquisition of key1 for this source PUT only; Search remains keyless. Local authentication means key auth, not manual local setup." - if cu["auth"] == "api-key-arm" else "Explicit existing CU API-key ENV channel; Search remains keyless." - ), - **({"credential_acquisition": copy.deepcopy(source["ai_services_key_acquisition"])} if cu["auth"] == "api-key-arm" and mutation_required else {}), - **({"managed_identity": copy.deepcopy(plan["cu_identity_state"])} if cu["auth"] == "system-assigned" else {}), - "cost_and_data": "Billable CU processing, no daily free document allowance; uploaded content moves from Search to CU, possibly across regions. Search retains outputs.", - "verification": ( - "ARM verifies Search identity, exact CU-scoped role and account/network metadata, not backend MI rollout, effective access or extraction. A separately approved bounded OCR canary can validate functionality." - if cu["auth"] == "system-assigned" else - "ARM verifies account binding/local-auth/network metadata, not effective access, key validity or processing success. Verify selected processing/required deployment evidence; no blanket account-defaults confirmation." - ), - "kb_reasoning": "Unchanged; source CU does not enable KB chat or source vectors.", - }} if cu else {}), - **({"embedding": source_vector.summary(embedding)} if embedding else {}), - "data_boundary": {"paths": [r["path"] for r in records], - "file_count": len(records), "total_bytes": sum(r["size"] for r in records)}, - "uploads": 0 if matched else len(records), - "reused_files": [{"path": path, "fileId": item["fileId"]} for path, item in matched.items()], - "service_tier": request["service_tier"], - "rbac": rbac, "network": network, - "cost_and_retention": ( - "Existing Search charges remain; review File ingestion/storage charges and retention before approval." - if mutation_required else "Existing charges and retention are unchanged; no new uploads or resources." - ), - "ownership": "New source and uploaded files only; reused Search/source/files are not run-owned.", - "format_verification": "Filename/MIME hints are not detected types; Search checks actual content support during ingestion.", - "verification": ( - "Execution rechecks definition/ETag and local/server inventories, then verifies uploaded file markers." - if mutation_required else - "Fresh source definition/ETag and complete file markers/IDs match the local inventory; this does not verify ingestion readiness or retrieval." - ), - "cleanup": "Excluded; separate run-owned source cleanup plan and approval required.", - "next_step": ( - "Creation owner refreshes identity, RBAC, network, source state and cost/data consent, then obtains explicit approval of these changes before applying the unchanged execution input." - if mutation_required else - "Reuse the verified identity and file markers without mutation approval or invoking the mutation helper. Refresh discovery before later use; this observation is not future consent." - ), - }, - "read_only_evidence": {"request_ids": request_ids, "source_state": source["action"]}, - "writes_performed": [], - "warnings": [ - "Planning is not approval or completed ingestion. Uploader RBAC/network remain supplied evidence; MI identity/role metadata readbacks do not prove backend attribution or effective access." - if cu is not None and cu["auth"] == "system-assigned" else - "Planning is not approval, policy evaluation, or proof of completed ingestion. RBAC/network are caller-supplied evidence, not verified by this helper." - ], - } - - -def _validate_plan( - plan: dict[str, Any], - *, - require_cu_readback: bool = True, -) -> tuple[dict[str, Any], dict[str, Any]]: - reject_secrets(plan) - require_allowed_fields( - plan, - { - "operation", - "outcome", - "cleanup_approved", - "owner", - "source", - "ingestion", - "embedding", - "content_understanding", - "file_cu_plan_version", - "cu_resource_state", - "cu_identity_state", - }, - label="File source plan", - ) - if ( - plan.get("operation") != "reconcile-and-ingest" - or plan.get("cleanup_approved") is not False - ): - raise HelperFailure( - "operation-invalid", - "File source application requires reconcile-and-ingest with cleanup excluded.", - blocked_at="input-resolution", - ) - source = plan.get("source") - ingestion = plan.get("ingestion") - if not isinstance(source, dict) or not isinstance(ingestion, dict): - raise HelperFailure( - "input-schema-invalid", - "File source application requires source and ingestion objects.", - blocked_at="input-resolution", - ) - desired = source.get("desired") - if ( - source.get("operation") != "reconcile" - or source.get("resource_type") != "knowledge-source" - or source.get("action") not in {"create", "reuse"} - or not isinstance(desired, dict) - or desired.get("kind") != "file" - or ingestion.get("operation") != "ingest" - or plan.get("owner") != source.get("owner") - or any( - source.get(field) != ingestion.get(field) - for field in ("endpoint", "name", "api_version", "owner") - ) - ): - raise HelperFailure( - "step-contract-mismatch", - "Source reconciliation and ingestion must target the same approved File source.", - blocked_at="input-resolution", - ) - source_mode = desired.get("fileParameters", {}).get( - "ingestionParameters", {} - ).get("contentExtractionMode") - if source_mode != ingestion.get("extraction_mode"): - raise HelperFailure( - "step-contract-mismatch", - "Source and ingestion extraction modes must match exactly.", - blocked_at="input-resolution", - ) - search_reconcile._validate_plan(source) - file_ingest._validate_plan(ingestion) - source_vector.validate_plan_choice(plan) - if any(field in plan for field in ("file_cu_plan_version", "content_understanding", "cu_resource_state")): - if plan.get("file_cu_plan_version") not in ("1.0", "1.1", "1.2") or source_mode != "standard": - raise _cu_failure("cu-plan-mismatch", "New File CU plans require their supported CU-specific version and standard extraction.") - cu = validate_content_understanding(plan.get("content_understanding"), enabled=True) - automatic = cu["auth"] == "api-key-arm" - mi = cu["auth"] == "system-assigned" - acquisition = source.get("ai_services_key_acquisition") - if automatic: - file_cu_auth.validate_acquisition(acquisition, cu["endpoint"]) - if ( - plan["file_cu_plan_version"] != ("1.2" if mi else "1.1" if automatic else "1.0") - or (automatic and acquisition["resource_id"] != cu["resource_id"]) - or (not automatic and acquisition is not None) - or source.get("ai_services_managed_identity") is not (True if mi else None) - or (not mi and "cu_identity_state" in plan) - ): - raise _cu_failure("cu-plan-mismatch", "CU auth mode, version and exact acquisition scope must match the approved plan.") - settings = desired["fileParameters"]["ingestionParameters"] - if ( - settings.get("aiServices") != {"uri": cu["endpoint"].rstrip("/")} - or settings.get("identity") is not None - or settings.get("disableImageVerbalization") is not True - or settings.get("chatCompletionModel") is not None - or source.get("ai_services_api_key_environment") != cu.get("api_key_environment") - or ("embedding" in plan) != (settings.get("embeddingModel") is not None) - ): - raise _cu_failure("cu-plan-mismatch", "CU/embedding choices, credential channel and source processing must match the approved definition.") - if require_cu_readback: - state = plan.get("cu_resource_state") - if not isinstance(state, dict) or state != _cu_account_state(cu, state): - raise _cu_failure("cu-prerequisite-missing", "Retain the planner's selected CU account readback.") - if mi: - file_cu_mi.validate_state(plan.get("cu_identity_state"), cu, source["endpoint"]) - elif source.get("ai_services_key_acquisition") is not None or source.get("ai_services_managed_identity") is not None or "cu_identity_state" in plan: - raise _cu_failure("cu-plan-mismatch", "Automatic acquisition requires the complete versioned File CU workflow.") - return source, ingestion - - -def _writes(result: dict[str, Any]) -> list[dict[str, Any]]: - writes: list[dict[str, Any]] = [] - for action in ("created", "updated"): - for resource in result["resources"].get(action, []): - writes.append( - { - "action": action, - "type": resource["type"], - "name": resource["name"], - } - ) - return writes - - -@reporting("file-source") -def execute( - document: dict[str, Any], - *, - token_provider: TokenProvider = azure_cli_token, - transport: Transport = http_request, - progress: Progress | None = None, - context_provider=file_cu_auth.account_context, - mi_on_created=None, - cleanup_capture=None, - upload_receipt_dir=None, - allow_upload_retry=True, -) -> dict[str, Any]: - progress.update("validation") - plan = document["plan"] - fingerprint = document["_computed_fingerprint"] - source, ingestion = _validate_plan(plan) - upload_session = None - if upload_receipt_dir is not None: - try: - from .file_upload import Session - except ImportError: - from file_upload import Session - upload_session = Session(upload_receipt_dir, document, context_provider=context_provider) - private_key = None - acquisition = source.get("ai_services_key_acquisition") - mi = source.get("ai_services_managed_identity") is True - mi_callback = None - if mi_on_created is not None and (not mi or not callable(mi_on_created)): - raise _cu_failure("creation-callback-unsupported", "Private MI checkpoints cannot receive File key-auth wire.") - if acquisition is not None or mi: - approval = document.get("approval") - if ( - not isinstance(approval, dict) or approval.get("confirmed") is not True - or approval.get("fingerprint") != digest(plan) or fingerprint != digest(plan) - ): - raise _cu_failure("approval-missing", "Private credential acquisition requires the unchanged fingerprinted source approval.") - if acquisition is not None: - file_cu_auth.check_context(acquisition["context"], context_provider) - child = {"_computed_fingerprint": fingerprint} - cu_ids = [] - if "content_understanding" in plan: - state, cu_ids = read_content_understanding( - plan["content_understanding"], token_provider=token_provider, transport=transport, - ) - if not _cu_states_match(state, plan["cu_resource_state"]): - raise _cu_failure( - "cu-prerequisite-drift", "CU account access or configuration changed since approval; refresh the plan.", - request_id=cu_ids[-1] if cu_ids else None, - ) - if mi: - binding, ids = file_cu_mi.read_binding( - plan["content_understanding"], source["endpoint"], token_provider=token_provider, transport=transport, - ) - cu_ids.extend(ids) - if binding != plan["cu_identity_state"]: - raise _cu_failure("cu-mi-identity-drift", "Search identity, CU role assignment or network changed since approval; refresh the concrete plan.") - raw_transport = transport - - def recheck_mi(): - state, _ = read_content_understanding( - plan["content_understanding"], token_provider=token_provider, transport=raw_transport, - ) - binding, _ = file_cu_mi.read_binding( - plan["content_understanding"], source["endpoint"], token_provider=token_provider, transport=raw_transport, - ) - if not _cu_states_match(state, plan["cu_resource_state"]) or binding != plan["cu_identity_state"]: - raise _cu_failure("cu-mi-identity-drift", "CU account or Search identity/role/network changed immediately before source PUT.") - - transport, mi_callback = file_cu_mi.guard_create(plan, transport, recheck_mi, mi_on_created) - if acquisition is not None: - def recheck(): - current, _ = read_content_understanding( - plan["content_understanding"], token_provider=token_provider, transport=transport, - ) - if not _cu_states_match(current, plan["cu_resource_state"]): - raise _cu_failure("cu-prerequisite-drift", "CU account changed before credential acquisition; refresh the plan and approval.") - - private_key = file_cu_auth.PrivateKey( - acquisition, token_provider=token_provider, transport=transport, - context_provider=context_provider, recheck=recheck, - ) - transport = private_key.transport - - if upload_session is not None: - transport = upload_session.transport(transport) - progress.update("source-reconciliation") - source_result = search_reconcile.execute( - {**child, "plan": source}, - token_provider=token_provider, - transport=source_vector.guard_readback_transport(plan, transport), - credential_provider=private_key.acquire if private_key else None, - **({"managed_identity_verified": True, "on_created": mi_callback} if mi else {}), - **({"cleanup_capture": cleanup_capture} if cleanup_capture is not None else {}), - **({"on_file_acknowledged": upload_session.acknowledge} if upload_session is not None else {}), - ) - source_writes = _writes(source_result) - - def check_retry_source(recovery, token): - _validate_plan(plan) - current, _ = search_reconcile._get( - search_reconcile.resource_url(source), token, - transport=source_vector.guard_readback_transport(plan, transport), recovery=recovery, - ) - etag = source_result["verification"]["readback"]["etag"] - if (not etag or current is None or current.get("@odata.etag") != etag - or not search_reconcile.definitions_match(source["desired"], current)): - raise HelperFailure("file-upload-source-drift", "Retry requires the unchanged acknowledged source version/definition.", - blocked_at="verification") - - try: - if upload_session is not None: - etag = upload_session.require_ack() - if source_result["verification"]["readback"]["etag"] != etag: - raise HelperFailure("file-upload-source-drift", "Source readback differs from the retained creation ACK; retain the source.", - blocked_at="verification") - ingestion_result = file_ingest.execute( - {**child, "plan": ingestion}, - token_provider=token_provider, - transport=transport, - allow_new_uploads=bool(source_result["resources"]["created"]), - progress=progress, - source_check=check_retry_source, - allow_upload_retry=allow_upload_retry, - **({"upload_session": upload_session} if upload_session is not None else {}), - ) - except HelperFailure as failure: - combined = HelperFailure( - failure.code, - failure.message, - blocked_at=failure.blocked_at, - writes=source_writes + failure.writes, - resources_remaining=( - [ - { - "type": str(write["type"]), - "name": str(write["name"]), - } - for write in source_writes - ] - + failure.resources_remaining - ), - resources_reused=failure.resources_reused, - resources_unverified=failure.resources_unverified, - warnings=failure.warnings, - request_id=failure.request_id, - status=failure.http_status, - partial=bool(source_writes or failure.writes or failure.partial), - ) - combined.file_batch = failure.file_batch - raise combined from failure - - resources = {"created": [], "reused": [], "updated": [], "skipped": []} - for action in resources: - resources[action].extend(source_result["resources"].get(action, [])) - resources[action].extend(ingestion_result["resources"].get(action, [])) - return { - "status": "completed", - "outcome": str(plan.get("outcome") or "create-file-knowledge-source"), - "approved_plan": {"fingerprint": fingerprint, "confirmed": True}, - "resources": resources, - "api_contracts": source_result["api_contracts"] - + ingestion_result["api_contracts"], - "data_movement": ingestion_result["data_movement"], - "auth": ingestion_result["auth"], - "rbac": ingestion_result["rbac"], - "network": ingestion_result["network"], - "verification": { - **({"cu_managed_identity": { - "mode": "system-assigned", "binding_digest": digest(plan["cu_identity_state"]), - "processing": "unverified until indexed OCR marker validation", - "principal_attribution": "unverified; no backend identity telemetry collected", - }} if mi else {}), - "readback": { - "source": source_result["verification"]["readback"], - "files": ingestion_result["verification"]["readback"], - }, - "request_ids": cu_ids + source_result["verification"]["request_ids"] - + ingestion_result["verification"]["request_ids"], - "idempotency": ( - "exact source and file marker readback is zero-write" - ), - }, - "warnings": source_result["warnings"] + ingestion_result["warnings"], - **({"file_batch": ingestion_result["file_batch"]} if "file_batch" in ingestion_result else {}), - "ownership": { - "run_owned": source_result["ownership"]["run_owned"] - + ingestion_result["ownership"]["run_owned"], - "reused_not_owned": source_result["ownership"]["reused_not_owned"] - + ingestion_result["ownership"]["reused_not_owned"], - "owner": plan.get("owner"), - }, - "cleanup": { - "status": "not-requested", - "separate_confirmation_required": True, - }, - } - - -def main(argv: list[str] | None = None) -> int: - try: - from . import _cleanup_receipts as cleanup_receipts - from .private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode - except ImportError: - import _cleanup_receipts as cleanup_receipts - from private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode - parser = argparse.ArgumentParser() - modes = parser.add_mutually_exclusive_group(required=True) - modes.add_argument("--input", type=Path) - modes.add_argument("--plan", type=Path) - add_execution_output_argument(parser) - add_progress_argument(parser) - cleanup_receipts.add_argument(parser) - parser.add_argument("--upload-receipt-dir", type=Path, - help="Required for creation: existing empty private directory for original File ACK and pre-upload attempts.") - args = parser.parse_args(argv) - fingerprint: str | None = None - owner: Any = None - outcome = "create-file-knowledge-source" - capture = None - try: - validate_execution_output_mode(args) - if args.plan and (args.cleanup_receipt_dir or args.upload_receipt_dir): - raise HelperFailure("input-schema-invalid", "Creation capture requires approved --input.", blocked_at="confirmation") - if args.plan: - try: - request = json.loads(args.plan.read_text(encoding="utf-8")) - except (OSError, UnicodeError, json.JSONDecodeError) as exc: - raise HelperFailure( - "input-unreadable", "Planning input must be readable UTF-8 JSON.", - blocked_at="input-resolution", - ) from exc - result = plan_source(request) - emit_plan_result(result, args.execution_output) - return 0 - document, plan, fingerprint = load_approved_input(args.input) - document["_computed_fingerprint"] = fingerprint - owner = plan.get("owner") - outcome = str(plan.get("outcome") or outcome) - source, _ = _validate_plan(plan) - if source["action"] == "create" and args.cleanup_receipt_dir is None: - raise HelperFailure( - "file-creation-receipt-required", - "File creation requires explicit --cleanup-receipt-dir pointing to an existing protected private directory; no default is inferred.", - blocked_at="confirmation", - ) - if args.cleanup_receipt_dir is not None: - directory = cleanup_receipts.private_io.validate_private_artifact_directory(str(args.cleanup_receipt_dir)) - if args.upload_receipt_dir is not None: - upload_directory = cleanup_receipts.private_io.validate_private_artifact_directory(str(args.upload_receipt_dir)) - if directory == upload_directory: - raise HelperFailure("file-receipt-directory-conflict", - "Use separate private cleanup and upload journal directories.", - blocked_at="confirmation") - elif source["action"] == "create": - raise HelperFailure( - "file-upload-receipt-required", - "File creation requires explicit --upload-receipt-dir for durable original ACK and pre-upload attempt records; use a separate existing empty private directory.", - blocked_at="confirmation", - ) - capture = cleanup_receipts.Capture(directory, document) - result = execute(document, progress=Progress("file-source", enabled=args.progress), - **({"cleanup_capture": capture} if capture else {}), - **({"upload_receipt_dir": args.upload_receipt_dir} if args.upload_receipt_dir else {})) - except HelperFailure as failure: - result = blocked_result( - failure, - outcome=outcome, - fingerprint=fingerprint, - owner=owner, - ) - if capture is not None: - result["cleanup_receipts"] = capture.summaries - result["safe_next_decision"] = ( - "Retain any acknowledged source and confirmed uploads. Do not rerun the creation envelope, " - "replay uncertain uploads, reset or delete resources. Use file_upload.py --plan with the " - "original upload journal for newly approved never-attempted files; missing evidence blocks continuation, not retention." - ) - emit_result(result) - return 3 if result["status"] == "partial" else 2 - if capture is not None: - result["cleanup_receipts"] = capture.summaries - emit_result(result) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) +from __future__ import annotations + +import argparse +import copy +import json +import re +import sys +from pathlib import Path +from typing import Any + +try: + from ._progress import Progress, add_progress_argument, reporting + from . import file_ingest, search_reconcile, source_vector, file_cu_mi + from . import cu_ingestion_auth as file_cu_auth + from ._common import ( + SEARCH_AUDIENCE, + MANAGEMENT_AUDIENCE, + HelperFailure, + TokenProvider, + Transport, + azure_cli_token, + blocked_result, + digest, + emit_result, + http_request, + load_approved_input, + normalize_azure_location, + reject_secrets, + require_allowed_fields, + ) +except ImportError: + from _progress import Progress, add_progress_argument, reporting + import file_ingest # type: ignore[no-redef] + import search_reconcile # type: ignore[no-redef] + import source_vector + import file_cu_mi + import cu_ingestion_auth as file_cu_auth + from _common import ( # type: ignore[no-redef] + SEARCH_AUDIENCE, + MANAGEMENT_AUDIENCE, + HelperFailure, + TokenProvider, + Transport, + azure_cli_token, + blocked_result, + digest, + emit_result, + http_request, + load_approved_input, + normalize_azure_location, + reject_secrets, + require_allowed_fields, + ) + + +def _cu_failure(code: str, message: str, *, request_id: str | None = None) -> HelperFailure: + return HelperFailure(code, message, blocked_at="cu-prerequisites", request_id=request_id) + + +def validate_content_understanding(value: Any, *, enabled: bool) -> dict[str, Any] | None: + if not enabled: + if value is not None: + raise _cu_failure("cu-choice-conflict", "Minimal extraction must omit CU choices.") + return None + if not isinstance(value, dict): + raise _cu_failure("cu-prerequisite-missing", "Standard planning requires a resolved CU account, disclosed auth channel and owner-verified prerequisites.") + value = copy.deepcopy(value) + value.setdefault("auth", "system-assigned") + reject_secrets(value) + require_allowed_fields(value, { + "endpoint", "resource_id", "auth", "api_key_environment", "prerequisites", "managed_identity", + }, label="File CU choice") + if ( + not isinstance(value.get("endpoint"), str) + or re.fullmatch(r"https://[a-z0-9][a-z0-9-]{0,62}\.services\.ai\.azure\.com/?", value["endpoint"]) is None + or not isinstance(value.get("resource_id"), str) + or re.fullmatch( + r"/subscriptions/[0-9a-fA-F-]{36}/resourceGroups/[A-Za-z0-9_.()-]{1,90}" + r"/providers/Microsoft\.CognitiveServices/accounts/[A-Za-z0-9][A-Za-z0-9_.-]{1,63}", + value["resource_id"], re.IGNORECASE, + ) is None + or value.get("auth") not in ("api-key-environment", "api-key-arm", "system-assigned") + or (value.get("auth") == "api-key-environment" and ( + not isinstance(value.get("api_key_environment"), str) + or search_reconcile.ENVIRONMENT_NAME.fullmatch(value["api_key_environment"]) is None + )) + or (value.get("auth") != "api-key-environment" and "api_key_environment" in value) + or (value.get("auth") != "system-assigned" and "managed_identity" in value) + ): + raise _cu_failure("cu-choice-invalid", "Select exact AIServices and system-assigned MI, or explicitly retain approved ARM/ENV key auth. No automatic auth fallback or setup changes.") + if value["auth"] == "system-assigned": + file_cu_mi.validate_choice(value.get("managed_identity"), value["resource_id"]) + prerequisites = value.get("prerequisites") + if not isinstance(prerequisites, dict): + raise _cu_failure("cu-prerequisite-missing", "Supply CU region/capability, selected processing/required deployments, identity/local-auth and network evidence references.") + fields = {"resource", "configuration", "identity", "network"} + require_allowed_fields(prerequisites, fields, label="File CU prerequisites") + if any(not source_vector._text(prerequisites.get(key)) for key in fields): + raise _cu_failure("cu-prerequisite-missing", "Owner-verified CU capability/region, selected processing/required deployments, auth/access and reachability evidence is required.") + source_vector._json_valid(value) + return copy.deepcopy(value) + + +def _cu_account_state(choice: dict[str, Any], account: Any) -> dict[str, Any]: + properties = account.get("properties") if isinstance(account, dict) else None + if not isinstance(properties, dict): + raise _cu_failure("cu-prerequisite-invalid", "CU account readback is incomplete.") + endpoints = properties.get("endpoints", {}) + candidates = [properties.get("endpoint")] + if isinstance(endpoints, dict): + candidates.extend(endpoints.values()) + if ( + str(account.get("id", "")).casefold() != choice["resource_id"].casefold() + or account.get("kind") != "AIServices" + or normalize_azure_location(account.get("location")) is None + or properties.get("provisioningState") != "Succeeded" + or (choice["auth"] != "system-assigned" and properties.get("disableLocalAuth") is not False) + or properties.get("publicNetworkAccess") not in ("Enabled", "Disabled") + or choice["endpoint"].rstrip("/") not in [v.rstrip("/") for v in candidates if isinstance(v, str)] + ): + raise _cu_failure("cu-prerequisite-invalid", "Readback must bind the selected ready AIServices account/endpoint/location/network. Key modes also need enabled local auth; MI does not. Any required setup change needs separate approval.") + state = { + "id": choice["resource_id"], "kind": "AIServices", "location": account["location"], + "identity": copy.deepcopy(account.get("identity")), + "properties": { + "endpoint": choice["endpoint"].rstrip("/"), "provisioningState": "Succeeded", + "disableLocalAuth": properties.get("disableLocalAuth"), "publicNetworkAccess": properties["publicNetworkAccess"], + "networkAcls": copy.deepcopy(properties.get("networkAcls")), + }, + } + if choice["auth"] == "system-assigned": + acl = properties.get("networkAcls") + if properties["publicNetworkAccess"] != "Enabled" or ( + acl is not None and (not isinstance(acl, dict) or acl.get("defaultAction") != "Allow") + ): + raise _cu_failure("cu-mi-network-unverified", "This MI path requires existing public CU reachability without default-deny ACLs. Restricted/private network compatibility needs separate verified setup and approval; no network changes or key fallback.") + reject_secrets(state) + source_vector._json_valid(state) + return state + + +def _cu_states_match(current: dict[str, Any], retained: dict[str, Any]) -> bool: + location = normalize_azure_location(retained.get("location")) + return location is not None and ( + {**current, "location": normalize_azure_location(current.get("location"))} + == {**retained, "location": location} + ) + + +def read_content_understanding( + choice: dict[str, Any], *, token_provider: TokenProvider, transport: Transport, +) -> tuple[dict[str, Any], list[str]]: + url = f"{MANAGEMENT_AUDIENCE}{choice['resource_id']}?api-version=2024-10-01" + response = transport("GET", url, token_provider(MANAGEMENT_AUDIENCE)) + try: + if response.status != 200: + raise _cu_failure("cu-prerequisite-unavailable", "Selected CU account metadata could not be read; no provisioning or auth changes are allowed.") + state = _cu_account_state(choice, response.body) + except HelperFailure as failure: + failure.request_id = response.request_id + if response.status != 200: + failure.http_status = response.status + raise + return state, [response.request_id] if response.request_id else [] + + +def verify_content_understanding_readback(choice: dict[str, Any], current: Any) -> None: + parameters = current.get("fileParameters") if isinstance(current, dict) else None + ingestion = parameters.get("ingestionParameters") if isinstance(parameters, dict) else None + ai = ingestion.get("aiServices") if isinstance(ingestion, dict) else None + if ( + not isinstance(ai, dict) or ingestion.get("contentExtractionMode") != "standard" + or not isinstance(ai.get("uri"), str) + or ai["uri"].rstrip("/") != choice["endpoint"].rstrip("/") + or ingestion.get("identity") is not None + or (choice["auth"] == "system-assigned" and ai.get("apiKey") not in file_cu_mi.REDACTED) + ): + raise _cu_failure("cu-readback-mismatch", "Observed File CU endpoint/extraction/auth conflicts with the selected configuration; credential details withheld.") + # File's approved key may be redacted in GET. It is not embedding auth, + # and source readback cannot prove its value or CU processing readiness. + + +def plan_source( + request: dict[str, Any], + *, + token_provider: TokenProvider = azure_cli_token, + transport: Transport = http_request, + context_provider=file_cu_auth.account_context, +) -> dict[str, Any]: + """Build an unapproved File workflow using only local reads and GETs.""" + if not isinstance(request, dict): + raise HelperFailure( + "input-schema-invalid", "Planning input must be an object.", + blocked_at="input-resolution", + ) + reject_secrets(request) + require_allowed_fields( + request, + {"schema_version", "api_version", "endpoint", "name", "owner", "local_root", "paths", + "service_tier", "extraction_mode", "vectorization", "embedding", "rbac", "network", + "content_understanding", "reuse_input_file", "reuse_result_file"}, + label="File planning input", + ) + if request.get("schema_version") != "1.0": + raise HelperFailure( + "input-schema-invalid", "Planning requires schema_version 1.0.", + blocked_at="input-resolution", + ) + file_ingest.validate_api_version(request.get("api_version", file_ingest.API_VERSION)) + for field in ("name", "owner"): + if not isinstance(request.get(field), str) or not request[field].strip(): + raise HelperFailure( + "input-schema-invalid", f"An explicit non-empty {field} is required.", + blocked_at="input-resolution", + ) + if request.get("extraction_mode") not in ("minimal", "standard") or request.get("vectorization") not in ("none", "azureOpenAI"): + raise HelperFailure( + "planning-processing-unsupported", + "Select minimal or standard extraction and independent vectorization none or azureOpenAI.", + blocked_at="input-resolution", + ) + embedding = source_vector.validate_choice( + request.get("embedding"), enabled=request["vectorization"] == "azureOpenAI", + api_version=file_ingest.API_VERSION, + ) + cu = validate_content_understanding( + request.get("content_understanding"), enabled=request["extraction_mode"] == "standard", + ) + if (request.get("reuse_input_file") is not None or request.get("reuse_result_file") is not None) and ( + cu is None or cu["auth"] != "system-assigned" + ): + raise _cu_failure("cu-choice-conflict", "File MI provenance inputs are only for managed-identity exact reuse.") + rbac, network = request.get("rbac"), request.get("network") + if ( + not isinstance(rbac, dict) + or not isinstance(rbac.get("assignments"), list) + or not rbac["assignments"] + or not all(isinstance(item, dict) and item for item in rbac["assignments"]) + or not isinstance(network, dict) + or not isinstance(network.get("posture"), str) + or not network["posture"].strip() + or not isinstance(network.get("evidence"), str) + or not network["evidence"].strip() + ): + raise HelperFailure( + "planning-evidence-missing", + "Supply observed RBAC assignments and network posture/evidence; the policy owner must refresh and verify them before approval.", + blocked_at="input-resolution", + ) + root = file_ingest.resolve_local_root(request.get("local_root")) + records = file_ingest.snapshot_inventory( + root, request.get("paths"), service_tier=request.get("service_tier") + ) + common = { + "endpoint": request.get("endpoint"), "name": request.get("name"), + "api_version": file_ingest.API_VERSION, "owner": request.get("owner"), + "cleanup_approved": False, "rbac": copy.deepcopy(rbac), + "network": copy.deepcopy(network), + } + source = { + **common, "operation": "reconcile", "resource_type": "knowledge-source", + "outcome": "create-file-knowledge-source", "action": "create", + "desired": { + "name": common["name"], "kind": "file", + "fileParameters": {"ingestionParameters": {"contentExtractionMode": request["extraction_mode"]}}, + }, + } + ingestion = { + **copy.deepcopy(common), "operation": "ingest", "local_root": str(root), + "files": records, "inventory_digest": digest(records), + "expected_server_inventory_digest": file_ingest.inventory_digest([]), + "service_tier": request["service_tier"], "extraction_mode": request["extraction_mode"], + } + plan = { + "operation": "reconcile-and-ingest", "outcome": "create-file-knowledge-source", + "owner": common["owner"], "cleanup_approved": False, + "source": source, "ingestion": ingestion, + } + if embedding is not None: + plan["embedding"] = embedding + source["desired"]["fileParameters"]["ingestionParameters"]["embeddingModel"] = source_vector.model_definition(embedding) + if cu is not None: + automatic = cu["auth"] == "api-key-arm" + mi = cu["auth"] == "system-assigned" + plan.update(file_cu_plan_version="1.2" if mi else "1.1" if automatic else "1.0", content_understanding=cu) + if automatic: + source["ai_services_key_acquisition"] = file_cu_auth.acquisition(cu, context_provider()) + elif mi: + source["ai_services_managed_identity"] = True + else: + source["ai_services_api_key_environment"] = cu["api_key_environment"] + source["desired"]["fileParameters"]["ingestionParameters"].update( + aiServices={"uri": cu["endpoint"].rstrip("/")}, disableImageVerbalization=True, + ) + # Validate the local inventory and all choices before any authentication. + _validate_plan(plan, require_cu_readback=False) + cu_request_ids = [] + if cu is not None: + state, cu_request_ids = read_content_understanding(cu, token_provider=token_provider, transport=transport) + plan["cu_resource_state"] = state + if cu["auth"] == "system-assigned": + plan["cu_identity_state"], ids = file_cu_mi.read_binding( + cu, common["endpoint"], token_provider=token_provider, transport=transport, + ) + cu_request_ids.extend(ids) + _validate_plan(plan) + transport = source_vector.guard_readback_transport(plan, transport) + url = search_reconcile.resource_url(source) + token = token_provider(SEARCH_AUDIENCE) + current, request_id = search_reconcile.read_resource(url, token, transport=transport) + request_ids = cu_request_ids + ([request_id] if request_id else []) + matched = {} + if current is not None: + if cu is not None and cu["auth"] == "system-assigned": + file_cu_mi.verify_reuse(request, plan, current) + if not search_reconcile.definitions_match(source["desired"], current): + raise HelperFailure( + "definition-conflict", + "The exact source has a different definition; planning never overwrites or chooses another name.", + blocked_at="reconciliation", + ) + etag = current.get("@odata.etag") + if not isinstance(etag, str) or not etag: + raise HelperFailure( + "definition-evidence-missing", "Exact reuse requires the current source ETag.", + blocked_at="reconciliation", + ) + before, ids = file_ingest.read_inventory(ingestion, token, transport=transport) + request_ids.extend(ids) + matched = file_ingest.reconcile_inventory(ingestion, before) + file_ids = [item.get("fileId") for item in matched.values()] + if ( + not all(isinstance(value, str) and value for value in file_ids) + or len(set(file_ids)) != len(file_ids) + ): + raise HelperFailure( + "file-identity-ambiguous", "Exact reuse requires unique non-empty server file IDs.", + blocked_at="reconciliation", + ) + source.update(action="reuse", expected_etag=etag) + ingestion["expected_server_inventory_digest"] = file_ingest.inventory_digest(before) + refreshed, refresh_id = search_reconcile.read_resource(url, token, transport=transport) + if refresh_id: + request_ids.append(refresh_id) + if ( + refreshed is None + or refreshed.get("@odata.etag") != etag + or not search_reconcile.definitions_match(source["desired"], refreshed) + ): + raise HelperFailure( + "definition-drift", + "Source definition or ETag changed during file inventory readback.", + blocked_at="reconciliation", + ) + # Do not return a snapshot that changed while Search discovery was running. + _validate_plan(plan) + if cu is not None: + state, ids = read_content_understanding(cu, token_provider=token_provider, transport=transport) + request_ids.extend(ids) + if not _cu_states_match(state, plan["cu_resource_state"]): + raise _cu_failure( + "cu-prerequisite-drift", "CU account access or configuration changed during planning; refresh the plan.", + request_id=ids[-1] if ids else None, + ) + if cu["auth"] == "api-key-arm": + file_cu_auth.check_context(source["ai_services_key_acquisition"]["context"], context_provider) + elif cu["auth"] == "system-assigned": + binding, ids = file_cu_mi.read_binding(cu, common["endpoint"], token_provider=token_provider, transport=transport) + request_ids.extend(ids) + if binding != plan["cu_identity_state"]: + raise _cu_failure("cu-mi-identity-drift", "Search identity, CU scoped role or network changed during planning.") + fingerprint = digest(plan) + mutation_required = source["action"] == "create" + return { + "status": "planned", "outcome": plan["outcome"], + "plan_fingerprint": fingerprint, + "execution_input": { + "schema_version": "1.0", "plan": plan, + "approval": {"confirmed": False, "fingerprint": fingerprint}, + }, + "approval_summary": { + "target": {"endpoint": common["endpoint"], "name": common["name"], + "api_version": common["api_version"], "preview": True}, + "source_action": source["action"], "owner": common["owner"], + "execution_required": mutation_required, + "mutation_approval_required": mutation_required, + "processing": ( + "standard CU extraction" + (" with source embeddings" if embedding else "; source vectors off") + if cu else "minimal extraction with embeddings" if embedding else "minimal lexical; no models" + ), + **({"content_understanding": { + "purpose": "Standard document extraction only; not source vectorization or KB answer synthesis.", + "endpoint": cu["endpoint"], + "authentication": file_cu_auth.approval_summary("file", cu["auth"], creating=mutation_required), + "auth": ( + "Search system-assigned MI for CU; no API key or credential reads, no local-auth requirement. Service implementation inspected; live compatibility unverified." + if cu["auth"] == "system-assigned" else + "Existing File CU key-auth configuration reused; no credential acquisition. Search remains keyless." + if not mutation_required else + "Approved private ARM listKeys acquisition of key1 for this source PUT only; Search remains keyless. Local authentication means key auth, not manual local setup." + if cu["auth"] == "api-key-arm" else "Explicit existing CU API-key ENV channel; Search remains keyless." + ), + **({"credential_acquisition": copy.deepcopy(source["ai_services_key_acquisition"])} if cu["auth"] == "api-key-arm" and mutation_required else {}), + **({"managed_identity": copy.deepcopy(plan["cu_identity_state"])} if cu["auth"] == "system-assigned" else {}), + "cost_and_data": "Billable CU processing, no daily free document allowance; uploaded content moves from Search to CU, possibly across regions. Search retains outputs.", + "verification": ( + "ARM verifies Search identity, exact CU-scoped role and account/network metadata, not backend MI rollout, effective access or extraction. A separately approved bounded OCR canary can validate functionality." + if cu["auth"] == "system-assigned" else + "ARM verifies account binding/local-auth/network metadata, not effective access, key validity or processing success. Verify selected processing/required deployment evidence; no blanket account-defaults confirmation." + ), + "kb_reasoning": "Unchanged; source CU does not enable KB chat or source vectors.", + }} if cu else {}), + **({"embedding": source_vector.summary(embedding)} if embedding else {}), + "data_boundary": {"paths": [r["path"] for r in records], + "file_count": len(records), "total_bytes": sum(r["size"] for r in records)}, + "uploads": 0 if matched else len(records), + "reused_files": [{"path": path, "fileId": item["fileId"]} for path, item in matched.items()], + "service_tier": request["service_tier"], + "rbac": rbac, "network": network, + "cost_and_retention": ( + "Existing Search charges remain; review File ingestion/storage charges and retention before approval." + if mutation_required else "Existing charges and retention are unchanged; no new uploads or resources." + ), + "ownership": "New source and uploaded files only; reused Search/source/files are not run-owned.", + "format_verification": "Filename/MIME hints are not detected types; Search checks actual content support during ingestion.", + "verification": ( + "Execution rechecks definition/ETag and local/server inventories, then verifies uploaded file markers." + if mutation_required else + "Fresh source definition/ETag and complete file markers/IDs match the local inventory; this does not verify ingestion readiness or retrieval." + ), + "cleanup": "Excluded; separate run-owned source cleanup plan and approval required.", + "next_step": ( + "Creation owner refreshes identity, RBAC, network, source state and cost/data consent, then obtains explicit approval of these changes before applying the unchanged execution input." + if mutation_required else + "Reuse the verified identity and file markers without mutation approval or invoking the mutation helper. Refresh discovery before later use; this observation is not future consent." + ), + }, + "read_only_evidence": {"request_ids": request_ids, "source_state": source["action"]}, + "writes_performed": [], + "warnings": [ + "Planning is not approval or completed ingestion. Uploader RBAC/network remain supplied evidence; MI identity/role metadata readbacks do not prove backend attribution or effective access." + if cu is not None and cu["auth"] == "system-assigned" else + "Planning is not approval, policy evaluation, or proof of completed ingestion. RBAC/network are caller-supplied evidence, not verified by this helper." + ], + } + + +def _validate_plan( + plan: dict[str, Any], + *, + require_cu_readback: bool = True, +) -> tuple[dict[str, Any], dict[str, Any]]: + reject_secrets(plan) + require_allowed_fields( + plan, + { + "operation", + "outcome", + "cleanup_approved", + "owner", + "source", + "ingestion", + "embedding", + "content_understanding", + "file_cu_plan_version", + "cu_resource_state", + "cu_identity_state", + }, + label="File source plan", + ) + if ( + plan.get("operation") != "reconcile-and-ingest" + or plan.get("cleanup_approved") is not False + ): + raise HelperFailure( + "operation-invalid", + "File source application requires reconcile-and-ingest with cleanup excluded.", + blocked_at="input-resolution", + ) + source = plan.get("source") + ingestion = plan.get("ingestion") + if not isinstance(source, dict) or not isinstance(ingestion, dict): + raise HelperFailure( + "input-schema-invalid", + "File source application requires source and ingestion objects.", + blocked_at="input-resolution", + ) + desired = source.get("desired") + if ( + source.get("operation") != "reconcile" + or source.get("resource_type") != "knowledge-source" + or source.get("action") not in {"create", "reuse"} + or not isinstance(desired, dict) + or desired.get("kind") != "file" + or ingestion.get("operation") != "ingest" + or plan.get("owner") != source.get("owner") + or any( + source.get(field) != ingestion.get(field) + for field in ("endpoint", "name", "api_version", "owner") + ) + ): + raise HelperFailure( + "step-contract-mismatch", + "Source reconciliation and ingestion must target the same approved File source.", + blocked_at="input-resolution", + ) + source_mode = desired.get("fileParameters", {}).get( + "ingestionParameters", {} + ).get("contentExtractionMode") + if source_mode != ingestion.get("extraction_mode"): + raise HelperFailure( + "step-contract-mismatch", + "Source and ingestion extraction modes must match exactly.", + blocked_at="input-resolution", + ) + search_reconcile._validate_plan(source) + file_ingest._validate_plan(ingestion) + source_vector.validate_plan_choice(plan) + if any(field in plan for field in ("file_cu_plan_version", "content_understanding", "cu_resource_state")): + if plan.get("file_cu_plan_version") not in ("1.0", "1.1", "1.2") or source_mode != "standard": + raise _cu_failure("cu-plan-mismatch", "New File CU plans require their supported CU-specific version and standard extraction.") + cu = validate_content_understanding(plan.get("content_understanding"), enabled=True) + automatic = cu["auth"] == "api-key-arm" + mi = cu["auth"] == "system-assigned" + acquisition = source.get("ai_services_key_acquisition") + if automatic: + file_cu_auth.validate_acquisition(acquisition, cu["endpoint"]) + if ( + plan["file_cu_plan_version"] != ("1.2" if mi else "1.1" if automatic else "1.0") + or (automatic and acquisition["resource_id"] != cu["resource_id"]) + or (not automatic and acquisition is not None) + or source.get("ai_services_managed_identity") is not (True if mi else None) + or (not mi and "cu_identity_state" in plan) + ): + raise _cu_failure("cu-plan-mismatch", "CU auth mode, version and exact acquisition scope must match the approved plan.") + settings = desired["fileParameters"]["ingestionParameters"] + if ( + settings.get("aiServices") != {"uri": cu["endpoint"].rstrip("/")} + or settings.get("identity") is not None + or settings.get("disableImageVerbalization") is not True + or settings.get("chatCompletionModel") is not None + or source.get("ai_services_api_key_environment") != cu.get("api_key_environment") + or ("embedding" in plan) != (settings.get("embeddingModel") is not None) + ): + raise _cu_failure("cu-plan-mismatch", "CU/embedding choices, credential channel and source processing must match the approved definition.") + if require_cu_readback: + state = plan.get("cu_resource_state") + if not isinstance(state, dict) or state != _cu_account_state(cu, state): + raise _cu_failure("cu-prerequisite-missing", "Retain the planner's selected CU account readback.") + if mi: + file_cu_mi.validate_state(plan.get("cu_identity_state"), cu, source["endpoint"]) + elif source.get("ai_services_key_acquisition") is not None or source.get("ai_services_managed_identity") is not None or "cu_identity_state" in plan: + raise _cu_failure("cu-plan-mismatch", "Automatic acquisition requires the complete versioned File CU workflow.") + return source, ingestion + + +def _writes(result: dict[str, Any]) -> list[dict[str, Any]]: + writes: list[dict[str, Any]] = [] + for action in ("created", "updated"): + for resource in result["resources"].get(action, []): + writes.append( + { + "action": action, + "type": resource["type"], + "name": resource["name"], + } + ) + return writes + + +@reporting("file-source") +def execute( + document: dict[str, Any], + *, + token_provider: TokenProvider = azure_cli_token, + transport: Transport = http_request, + progress: Progress | None = None, + context_provider=file_cu_auth.account_context, + mi_on_created=None, + cleanup_capture=None, + upload_receipt_dir=None, + allow_upload_retry=True, +) -> dict[str, Any]: + progress.update("validation") + plan = document["plan"] + fingerprint = document["_computed_fingerprint"] + source, ingestion = _validate_plan(plan) + upload_session = None + if upload_receipt_dir is not None: + try: + from .file_upload import Session + except ImportError: + from file_upload import Session + upload_session = Session(upload_receipt_dir, document, context_provider=context_provider) + private_key = None + acquisition = source.get("ai_services_key_acquisition") + mi = source.get("ai_services_managed_identity") is True + mi_callback = None + if mi_on_created is not None and (not mi or not callable(mi_on_created)): + raise _cu_failure("creation-callback-unsupported", "Private MI checkpoints cannot receive File key-auth wire.") + if acquisition is not None or mi: + approval = document.get("approval") + if ( + not isinstance(approval, dict) or approval.get("confirmed") is not True + or approval.get("fingerprint") != digest(plan) or fingerprint != digest(plan) + ): + raise _cu_failure("approval-missing", "Private credential acquisition requires the unchanged fingerprinted source approval.") + if acquisition is not None: + file_cu_auth.check_context(acquisition["context"], context_provider) + child = {"_computed_fingerprint": fingerprint} + cu_ids = [] + if "content_understanding" in plan: + state, cu_ids = read_content_understanding( + plan["content_understanding"], token_provider=token_provider, transport=transport, + ) + if not _cu_states_match(state, plan["cu_resource_state"]): + raise _cu_failure( + "cu-prerequisite-drift", "CU account access or configuration changed since approval; refresh the plan.", + request_id=cu_ids[-1] if cu_ids else None, + ) + if mi: + binding, ids = file_cu_mi.read_binding( + plan["content_understanding"], source["endpoint"], token_provider=token_provider, transport=transport, + ) + cu_ids.extend(ids) + if binding != plan["cu_identity_state"]: + raise _cu_failure("cu-mi-identity-drift", "Search identity, CU role assignment or network changed since approval; refresh the concrete plan.") + raw_transport = transport + + def recheck_mi(): + state, _ = read_content_understanding( + plan["content_understanding"], token_provider=token_provider, transport=raw_transport, + ) + binding, _ = file_cu_mi.read_binding( + plan["content_understanding"], source["endpoint"], token_provider=token_provider, transport=raw_transport, + ) + if not _cu_states_match(state, plan["cu_resource_state"]) or binding != plan["cu_identity_state"]: + raise _cu_failure("cu-mi-identity-drift", "CU account or Search identity/role/network changed immediately before source PUT.") + + transport, mi_callback = file_cu_mi.guard_create(plan, transport, recheck_mi, mi_on_created) + if acquisition is not None: + def recheck(): + current, _ = read_content_understanding( + plan["content_understanding"], token_provider=token_provider, transport=transport, + ) + if not _cu_states_match(current, plan["cu_resource_state"]): + raise _cu_failure("cu-prerequisite-drift", "CU account changed before credential acquisition; refresh the plan and approval.") + + private_key = file_cu_auth.PrivateKey( + acquisition, token_provider=token_provider, transport=transport, + context_provider=context_provider, recheck=recheck, + ) + transport = private_key.transport + + if upload_session is not None: + transport = upload_session.transport(transport) + progress.update("source-reconciliation") + source_result = search_reconcile.execute( + {**child, "plan": source}, + token_provider=token_provider, + transport=source_vector.guard_readback_transport(plan, transport), + credential_provider=private_key.acquire if private_key else None, + **({"managed_identity_verified": True, "on_created": mi_callback} if mi else {}), + **({"cleanup_capture": cleanup_capture} if cleanup_capture is not None else {}), + **({"on_file_acknowledged": upload_session.acknowledge} if upload_session is not None else {}), + ) + source_writes = _writes(source_result) + + def check_retry_source(recovery, token): + _validate_plan(plan) + current, _ = search_reconcile._get( + search_reconcile.resource_url(source), token, + transport=source_vector.guard_readback_transport(plan, transport), recovery=recovery, + ) + etag = source_result["verification"]["readback"]["etag"] + if (not etag or current is None or current.get("@odata.etag") != etag + or not search_reconcile.definitions_match(source["desired"], current)): + raise HelperFailure("file-upload-source-drift", "Retry requires the unchanged acknowledged source version/definition.", + blocked_at="verification") + + try: + if upload_session is not None: + etag = upload_session.require_ack() + if source_result["verification"]["readback"]["etag"] != etag: + raise HelperFailure("file-upload-source-drift", "Source readback differs from the retained creation ACK; retain the source.", + blocked_at="verification") + ingestion_result = file_ingest.execute( + {**child, "plan": ingestion}, + token_provider=token_provider, + transport=transport, + allow_new_uploads=bool(source_result["resources"]["created"]), + progress=progress, + source_check=check_retry_source, + allow_upload_retry=allow_upload_retry, + **({"upload_session": upload_session} if upload_session is not None else {}), + ) + except HelperFailure as failure: + combined = HelperFailure( + failure.code, + failure.message, + blocked_at=failure.blocked_at, + writes=source_writes + failure.writes, + resources_remaining=( + [ + { + "type": str(write["type"]), + "name": str(write["name"]), + } + for write in source_writes + ] + + failure.resources_remaining + ), + resources_reused=failure.resources_reused, + resources_unverified=failure.resources_unverified, + warnings=failure.warnings, + request_id=failure.request_id, + status=failure.http_status, + partial=bool(source_writes or failure.writes or failure.partial), + ) + combined.file_batch = failure.file_batch + raise combined from failure + + resources = {"created": [], "reused": [], "updated": [], "skipped": []} + for action in resources: + resources[action].extend(source_result["resources"].get(action, [])) + resources[action].extend(ingestion_result["resources"].get(action, [])) + return { + "status": "completed", + "outcome": str(plan.get("outcome") or "create-file-knowledge-source"), + "approved_plan": {"fingerprint": fingerprint, "confirmed": True}, + "resources": resources, + "api_contracts": source_result["api_contracts"] + + ingestion_result["api_contracts"], + "data_movement": ingestion_result["data_movement"], + "auth": ingestion_result["auth"], + "rbac": ingestion_result["rbac"], + "network": ingestion_result["network"], + "verification": { + **({"cu_managed_identity": { + "mode": "system-assigned", "binding_digest": digest(plan["cu_identity_state"]), + "processing": "unverified until indexed OCR marker validation", + "principal_attribution": "unverified; no backend identity telemetry collected", + }} if mi else {}), + "readback": { + "source": source_result["verification"]["readback"], + "files": ingestion_result["verification"]["readback"], + }, + "request_ids": cu_ids + source_result["verification"]["request_ids"] + + ingestion_result["verification"]["request_ids"], + "idempotency": ( + "exact source and file marker readback is zero-write" + ), + }, + "warnings": source_result["warnings"] + ingestion_result["warnings"], + **({"file_batch": ingestion_result["file_batch"]} if "file_batch" in ingestion_result else {}), + "ownership": { + "run_owned": source_result["ownership"]["run_owned"] + + ingestion_result["ownership"]["run_owned"], + "reused_not_owned": source_result["ownership"]["reused_not_owned"] + + ingestion_result["ownership"]["reused_not_owned"], + "owner": plan.get("owner"), + }, + "cleanup": { + "status": "not-requested", + "separate_confirmation_required": True, + }, + } + + +def main(argv: list[str] | None = None) -> int: + try: + from . import _cleanup_receipts as cleanup_receipts + from .private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode + except ImportError: + import _cleanup_receipts as cleanup_receipts + from private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode + parser = argparse.ArgumentParser() + modes = parser.add_mutually_exclusive_group(required=True) + modes.add_argument("--input", type=Path) + modes.add_argument("--plan", type=Path) + add_execution_output_argument(parser) + add_progress_argument(parser) + cleanup_receipts.add_argument(parser) + parser.add_argument("--upload-receipt-dir", type=Path, + help="Required for creation: existing empty private directory for original File ACK and pre-upload attempts.") + args = parser.parse_args(argv) + fingerprint: str | None = None + owner: Any = None + outcome = "create-file-knowledge-source" + capture = None + try: + validate_execution_output_mode(args) + if args.plan and (args.cleanup_receipt_dir or args.upload_receipt_dir): + raise HelperFailure("input-schema-invalid", "Creation capture requires approved --input.", blocked_at="confirmation") + if args.plan: + try: + request = json.loads(args.plan.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise HelperFailure( + "input-unreadable", "Planning input must be readable UTF-8 JSON.", + blocked_at="input-resolution", + ) from exc + result = plan_source(request) + emit_plan_result(result, args.execution_output) + return 0 + document, plan, fingerprint = load_approved_input(args.input) + document["_computed_fingerprint"] = fingerprint + owner = plan.get("owner") + outcome = str(plan.get("outcome") or outcome) + source, _ = _validate_plan(plan) + if source["action"] == "create" and args.cleanup_receipt_dir is None: + raise HelperFailure( + "file-creation-receipt-required", + "File creation requires explicit --cleanup-receipt-dir pointing to an existing protected private directory; no default is inferred.", + blocked_at="confirmation", + ) + if args.cleanup_receipt_dir is not None: + directory = cleanup_receipts.private_io.validate_private_artifact_directory(str(args.cleanup_receipt_dir)) + if args.upload_receipt_dir is not None: + upload_directory = cleanup_receipts.private_io.validate_private_artifact_directory(str(args.upload_receipt_dir)) + if directory == upload_directory: + raise HelperFailure("file-receipt-directory-conflict", + "Use separate private cleanup and upload journal directories.", + blocked_at="confirmation") + elif source["action"] == "create": + raise HelperFailure( + "file-upload-receipt-required", + "File creation requires explicit --upload-receipt-dir for durable original ACK and pre-upload attempt records; use a separate existing empty private directory.", + blocked_at="confirmation", + ) + capture = cleanup_receipts.Capture(directory, document) + result = execute(document, progress=Progress("file-source", enabled=args.progress), + **({"cleanup_capture": capture} if capture else {}), + **({"upload_receipt_dir": args.upload_receipt_dir} if args.upload_receipt_dir else {})) + except HelperFailure as failure: + result = blocked_result( + failure, + outcome=outcome, + fingerprint=fingerprint, + owner=owner, + ) + if capture is not None: + result["cleanup_receipts"] = capture.summaries + result["safe_next_decision"] = ( + "Retain any acknowledged source and confirmed uploads. Do not rerun the creation envelope, " + "replay uncertain uploads, reset or delete resources. Use file_upload.py --plan with the " + "original upload journal for newly approved never-attempted files; missing evidence blocks continuation, not retention." + ) + emit_result(result) + return 3 if result["status"] == "partial" else 2 + if capture is not None: + result["cleanup_receipts"] = capture.summaries + emit_result(result) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_upload.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_upload.py index 9a95a62f..a027eea3 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_upload.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_upload.py @@ -1,853 +1,853 @@ -"""Synchronous File batches; one in-operation queue-429 retry, never unknown-outcome replay.""" -from __future__ import annotations - -import argparse -import copy -import hashlib -import re -import sys -import time -import uuid -from dataclasses import replace -from pathlib import Path - -try: - from . import _bootstrap_io as private_io, cu_ingestion_auth, file_ingest, search_reconcile - from ._common import (HelperFailure, ReadRecovery, RetryAfter, RetryAfterTiming, SEARCH_AUDIENCE, - retry_after_timing, retry_after_not_before, valid_utc_timestamp, azure_cli_token, blocked_result, - digest, emit_result, http_request, load_approved_input, reject_secrets, require_allowed_fields) - from ._progress import Progress, add_progress_argument, reporting -except ImportError: - import _bootstrap_io as private_io, cu_ingestion_auth, file_ingest, search_reconcile - from _common import (HelperFailure, ReadRecovery, RetryAfter, RetryAfterTiming, SEARCH_AUDIENCE, - retry_after_timing, retry_after_not_before, valid_utc_timestamp, azure_cli_token, blocked_result, - digest, emit_result, http_request, load_approved_input, reject_secrets, require_allowed_fields) - from _progress import Progress, add_progress_argument, reporting - - -RETAIN = " Retain the source and evidence; no creation replay, reset, deletion or new roles." -UPLOAD_TIMEOUT = 180 -MAX_BACKOFF_RECORDS = 200 - - -def _rejected(status): - return type(status) is int and 400 <= status < 500 and status not in (408, 409) - - -def fail(code, message): - return HelperFailure(code, message + RETAIN, blocked_at="file-upload-resume") - - -class Session: - def __init__(self, directory, document=None, *, context_provider=cu_ingestion_auth.account_context): - self.directory = private_io.validate_private_artifact_directory(str(directory)) - self.context_provider = context_provider - self.records = {} - self.network_failure = None - self.active_attempt = None - if document is not None: - self._validate_document(document) - if any(self.directory.iterdir()): - raise fail("file-upload-journal-exists", "Use a new empty private receipt directory for original creation.") - context = context_provider() - cu_ingestion_auth.validate_context(context) - self.seed = {"document": {key: copy.deepcopy(document[key]) for key in ("schema_version", "plan", "approval")}, - "context": context, "backoff_version": "1.0"} - self._write("run.json", self.seed) - else: - self._load() - self.seed = self.records["run.json"] - self._validate_document(self.seed["document"]) - cu_ingestion_auth.validate_context(self.seed["context"]) - self.document = self.seed["document"] - self.plan = self.document["plan"] - self.ingestion = self.plan["ingestion"] - - @staticmethod - def _validate_document(document): - try: - from . import file_source - except ImportError: - import file_source - if (not isinstance(document, dict) or document.get("schema_version") != "1.0" - or not isinstance(document.get("plan"), dict) - or "_computed_fingerprint" in document and document["_computed_fingerprint"] != digest(document.get("plan")) - or not isinstance(document.get("approval"), dict) or document["approval"].get("confirmed") is not True - or document.get("approval") != {"confirmed": True, "fingerprint": digest(document.get("plan"))}): - raise fail("file-upload-approval-missing", "Original unchanged File creation approval is required.") - require_allowed_fields(document, {"schema_version", "plan", "approval", "_computed_fingerprint"}, label="original File envelope") - file_source._validate_plan(document["plan"]) - if document["plan"]["source"]["action"] != "create": - raise fail("file-upload-provenance-missing", "This continuation is only for original acknowledged creation, not generic reuse.") - - def _write(self, name, payload): - value = {"schema_version": "1.0", "kind": "file-upload-journal", "payload": payload, "integrity": digest(payload)} - reject_secrets(value) - try: - private_io.atomic_private_file(self.directory, name, value, max_bytes=private_io.MAX_BYTES) - except HelperFailure as error: - raise HelperFailure("file-upload-receipt-failed", "Private upload evidence persistence failed." + RETAIN, - blocked_at="local-persistence", warnings=error.warnings) from error - self.records[name] = copy.deepcopy(payload) - - def _load(self): - # Reuse the existing bounded, private, no-link evidence reader. - try: - from .blob_recheck import read_private - except ImportError: - from blob_recheck import read_private - try: - paths = list(self.directory.iterdir()) - except OSError as error: - raise fail("file-upload-evidence-unreadable", "Original upload evidence is inaccessible.") from error - if len(paths) > 803 + MAX_BACKOFF_RECORDS or not {"run.json", "source-ack.json"}.issubset({p.name for p in paths}): - raise fail("file-upload-provenance-missing", "Necessary original ACK and journal were not retained.") - for path in paths: - if (path.name not in {"run.json", "source-ack.json", "pending-request.json"} - and not re.fullmatch(r"(?:[0-9]{4}-(retry-)?(attempt|result)|backoff-[0-9]{4})\.json", path.name)): - raise fail("file-upload-evidence-invalid", "Unexpected or incomplete journal entry; do not discard it to resume.") - value = read_private(path) - if (set(value) != {"schema_version", "kind", "payload", "integrity"} - or value["schema_version"] != "1.0" or value["kind"] != "file-upload-journal" - or value["integrity"] != digest(value["payload"])): - raise fail("file-upload-evidence-invalid", "Original private upload evidence changed.") - self.records[path.name] = value["payload"] - seed = self.records["run.json"] - if (not isinstance(seed, dict) or set(seed) not in ({"document", "context"}, {"document", "context", "backoff_version"}) - or "backoff_version" in seed and seed["backoff_version"] != "1.0"): - raise fail("file-upload-evidence-invalid", "Original approval/context evidence is incomplete.") - - def acknowledge(self, metadata): - self._write("source-ack.json", {"original_plan_digest": digest(self.plan), - "source_url": search_reconcile.resource_url(self.plan["source"]), - "acknowledgement": metadata}) - - def require_ack(self): - value = self.records.get("source-ack.json") - if (not isinstance(value, dict) or set(value) != {"original_plan_digest", "source_url", "acknowledgement"} - or value["original_plan_digest"] != digest(self.plan) - or value["source_url"] != search_reconcile.resource_url(self.plan["source"])): - raise fail("file-upload-provenance-missing", "An original acknowledged conditional source creation must be retained.") - ack = value["acknowledgement"] - if (not isinstance(ack, dict) or set(ack) != {"status", "request_id", "etag_evidence"} - or ack["status"] not in (200, 201) or not ack["request_id"] - or not isinstance(ack["request_id"], str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}", ack["request_id"]) - or not isinstance(ack["etag_evidence"], dict) or set(ack["etag_evidence"]) != {"body", "headers"} - or not isinstance(ack["etag_evidence"]["headers"], list)): - raise fail("file-upload-provenance-missing", "Original successful creation ACK/request ID is required, not GET ownership.") - etag = search_reconcile.resolve_etag(ack["etag_evidence"], ack["request_id"]) - if not etag: - raise fail("file-upload-provenance-missing", "The original creation ACK lacks version proof; GET cannot supply it.") - return etag - - def states(self): - states = [] - events = self.backoff_events() - allowed = {"run.json", "source-ack.json", "pending-request.json"} | set(events) - pending = self.records.get("pending-request.json") - if "pending-request.json" in self.records and ( - not isinstance(pending, dict) or set(pending) != {"version", "plan_digest", "nonce", "method", "url_digest"} - or pending["version"] != "1.0" or pending["plan_digest"] != digest(self.plan) - or not isinstance(pending["nonce"], str) or not re.fullmatch(r"[0-9a-f]{32}", pending["nonce"]) - or pending["method"] not in ("GET", "POST", "PUT", "HEAD") - or not isinstance(pending["url_digest"], str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", pending["url_digest"]) - ): - raise fail("file-upload-evidence-invalid", "Pending request evidence is invalid.") - for index, record in enumerate(self.ingestion["files"]): - state = {"upload": "not_attempted", "request_ids": [], "ingested": False} - for retry in (False, True): - prefix = f"{index:04}-" + ("retry-" if retry else "") - attempt_name, result_name = prefix + "attempt.json", prefix + "result.json" - allowed.update((attempt_name, result_name)) - attempt, result = self.records.get(attempt_name), self.records.get(result_name) - expected = self._attempt_record(index, retry=retry) if attempt is not None else None - if attempt is not None and attempt != expected or result is not None and attempt is None: - raise fail("file-upload-evidence-invalid", "Upload attempt evidence does not bind the original source/corpus.") - if attempt is not None: - state.update(upload="unverified", ingested=False) - state.pop("file_proof", None) - state.pop("ack_file_id", None) - if result is not None: - if (not isinstance(result, dict) or not {"upload", "status", "request_id"} <= set(result) - or set(result) - {"upload", "status", "request_id", "file_proof", "file_id", "throttle_event"} - or not isinstance(result["upload"], str) - or result["upload"] not in {"accepted", "rejected", "unverified"} - or result["upload"] == "accepted" and result["status"] not in (200, 201) - or result["upload"] == "rejected" and not _rejected(result["status"]) - or result["status"] is not None and (type(result["status"]) is not int or not 100 <= result["status"] <= 599) - or result["request_id"] is not None and ReadRecovery.safe_id(result["request_id"]) != result["request_id"]): - raise fail("file-upload-evidence-invalid", "Upload result is not an original supported ACK or failure.") - linked = [event for event in events.values() if event["attempt"] == attempt_name] - if "throttle_event" in result: - if (result["status"] != 429 or len(linked) != 1 - or result["throttle_event"] != digest(linked[0]) - or result["request_id"] != linked[0]["request_id"]): - raise fail("file-upload-evidence-invalid", "Upload 429 timing does not bind its original attempt/result.") - elif result["status"] == 429 and self.seed.get("backoff_version") is not None: - raise fail("file-upload-evidence-invalid", "Required original upload 429 timing evidence is missing.") - if "file_id" in result: - if result["upload"] != "accepted" or not _file_id(result["file_id"]): - raise fail("file-upload-evidence-invalid", "Original upload ACK file identity is invalid.") - state["ack_file_id"] = result["file_id"] - proof = result.get("file_proof") - if proof is not None: - if (result["upload"] != "accepted" or not isinstance(proof, dict) - or set(proof) != {"file_id", "record_digest"} or not _file_id(proof["file_id"]) - or proof["record_digest"] != digest(record) - or result.get("file_id", proof["file_id"]) != proof["file_id"]): - raise fail("file-upload-evidence-invalid", "Persisted File completion proof does not match the approved record.") - state.update(file_proof=proof, ingested=True) - state.update(upload=result["upload"], status=result["status"]) - if result["request_id"]: - state["request_ids"].append(result["request_id"]) - states.append(state) - if set(self.records) - allowed: - raise fail("file-upload-evidence-invalid", "Upload journal contains records outside the approved corpus.") - return states - - def _attempt_record(self, index, *, retry=False): - record = { - "original_plan_digest": digest(self.plan), "record_digest": digest(self.ingestion["files"][index]), - "creation_ack_digest": digest(self.records["source-ack.json"]), - } - if retry: - rejection = self.records.get(f"{index:04}-result.json") - if not isinstance(rejection, dict) or rejection.get("status") != 429 or rejection.get("upload") != "rejected": - raise fail("file-upload-replay-forbidden", "A retry attempt requires the same operation's retained File upload 429.") - record["rejected_result_digest"] = digest(rejection) - return record - - def attempt(self, index, *, retry=False): - prefix = f"{index:04}-" + ("retry-" if retry else "") - self._write(prefix + "attempt.json", self._attempt_record(index, retry=retry)) - self.active_attempt = prefix + "attempt.json" - - def result(self, index, upload, status, request_id, *, retry=False, proof=None, file_id=None): - prefix = f"{index:04}-" + ("retry-" if retry else "") - event = {} - if status == 429: - linked = [value for value in self.backoff_events().values() if value["attempt"] == prefix + "attempt.json"] - if len(linked) != 1: - raise fail("file-upload-evidence-invalid", "Original upload 429 timing was not retained.") - event["throttle_event"] = digest(linked[0]) - self._write(prefix + "result.json", {"upload": upload, "status": status, - "request_id": ReadRecovery.safe_id(request_id) if request_id else None, - **({"file_id": file_id} if file_id is not None else {}), - **({"file_proof": proof} if proof is not None else {}), **event}) - self.active_attempt = None - - def backoff_events(self): - events = {key: self.records[key] for key in sorted(self.records) if key.startswith("backoff-")} - if len(events) > MAX_BACKOFF_RECORDS: - raise fail("file-upload-evidence-invalid", "Too many retained backoff records.") - plan_digest = digest(self.plan) - previous = None - for index, (name, event) in enumerate(events.items()): - if (name != f"backoff-{index:04}.json" or not isinstance(event, dict) - or set(event) != {"version", "plan_digest", "previous", "method", "url_digest", - "attempt", "attempt_digest", "request_id", "metadata", "timing", "origin"} - or event["version"] != "1.0" or event["plan_digest"] != plan_digest - or event["previous"] != previous or event["method"] not in ("GET", "POST", "PUT", "HEAD") - or not isinstance(event["url_digest"], str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", event["url_digest"]) - or event["request_id"] is not None and ReadRecovery.safe_id(event["request_id"]) != event["request_id"] - or event["origin"] not in ("response-headers", "native-http-error", "typed-metadata", "unavailable")): - raise fail("file-upload-evidence-invalid", "Original backoff provenance is invalid or incomplete.") - if event["attempt"] is not None: - attempt = self.records.get(event["attempt"]) if isinstance(event["attempt"], str) else None - if (attempt is None or not re.fullmatch(r"[0-9]{4}-(retry-)?attempt\.json", event["attempt"]) - or event["attempt_digest"] != digest(attempt) or event["method"] != "POST" - or event["url_digest"] != digest(file_ingest._list_url(self.ingestion))): - raise fail("file-upload-evidence-invalid", "Backoff does not bind the original upload attempt.") - elif event["attempt_digest"] is not None: - raise fail("file-upload-evidence-invalid", "Backoff attempt provenance is inconsistent.") - metadata, timing = event["metadata"], event["timing"] - if (not isinstance(metadata, dict) or set(metadata) != {"kind", "value"} - or metadata["kind"] not in ("seconds", "date", "date-rfc850", "missing", "invalid", "overlong") - or not valid_utc_timestamp(metadata["value"]) - or metadata["kind"] == "seconds" and (type(metadata["value"]) is not int or not 0 <= metadata["value"] <= 30) - or metadata["kind"] in ("missing", "invalid", "overlong") and metadata["value"] != 0 - or not isinstance(timing, dict) or set(timing) != {"received_at_utc", "not_before_utc", "server_delay_seconds"}): - raise fail("file-upload-evidence-invalid", "Retained Retry-After metadata is invalid.") - received, deadline = timing["received_at_utc"], timing["not_before_utc"] - seconds = timing["server_delay_seconds"] - if (received is not None and not valid_utc_timestamp(received) - or deadline is not None and (received is None or not valid_utc_timestamp(deadline)) - or seconds is not None and (type(seconds) is not int or not 0 <= seconds < 315537897600 - or metadata["kind"] not in ("seconds", "overlong")) - or metadata["kind"] == "seconds" and seconds not in (None, metadata["value"])): - raise fail("file-upload-evidence-invalid", "Retained backoff UTC timing is invalid.") - if deadline is not None: - if (event["origin"] == "unavailable" - or event["origin"] == "typed-metadata" and metadata["kind"] not in ("seconds", "date", "date-rfc850") - or metadata["kind"] == "overlong" and (seconds is None or seconds <= 30 or deadline != received + seconds) - or metadata["kind"] != "overlong" - and deadline != retry_after_not_before(RetryAfter(**metadata), received)): - raise fail("file-upload-evidence-invalid", "Retained backoff deadline contradicts original metadata.") - previous = digest(event) - return events - - def backoff_status(self): - events = self.backoff_events() - attempts = {name: value for name, value in self.records.items() - if re.fullmatch(r"[0-9]{4}-(retry-)?(attempt|result)\.json", name)} - if any(not isinstance(value, dict) for value in attempts.values()): - raise fail("file-upload-evidence-invalid", "Attempt/result evidence is malformed.") - # Legacy 429 or an interrupted receipt cannot prove what restriction was received. - unknown = "pending-request.json" in self.records or any( - name.endswith("result.json") and value.get("status") == 429 and "throttle_event" not in value - or name.endswith("attempt.json") and name.replace("attempt.json", "result.json") not in self.records - and name != self.active_attempt - for name, value in attempts.items() - ) - now = time.time() - if unknown or any(event["timing"]["not_before_utc"] is None for event in events.values()): - return {"status": "unresolved", "reason": "Original response timing is unavailable; no deadline is inferred."} - if not events: - return {"status": "clear"} - if (not valid_utc_timestamp(now) - or any(now < event["timing"]["received_at_utc"] for event in events.values())): - return {"status": "unresolved", "reason": "UTC clock is invalid or precedes retained response receipt."} - deadline = max(event["timing"]["not_before_utc"] for event in events.values()) - return {"status": "waiting" if now < deadline else "elapsed", "not_before_utc": deadline, - "seconds_remaining": max(0, deadline - now)} - - def require_backoff(self): - status = self.backoff_status() - if status["status"] in ("waiting", "unresolved"): - error = fail("file-upload-backoff-" + status["status"], - "Server backoff still applies; fresh approval does not waive it. " - + ("Wait until the retained UTC not-before time." if status["status"] == "waiting" - else "Original timing evidence or scoped operator recovery is required.")) - error.partial = "source-ack.json" in self.records - error.file_batch = _summary(self.ingestion, self.states(), self) - raise error - - def transport(self, raw): - if getattr(raw, "_file_journal_owner", None) is self: - return raw - - def invoke(method, url, token, **kwargs): - if self.network_failure is not None: - raise self.network_failure - self.require_backoff() - if len(self.backoff_events()) >= MAX_BACKOFF_RECORDS: - raise fail("file-upload-backoff-limit", "Backoff journal limit reached; no more requests.") - pending = {"version": "1.0", "plan_digest": digest(self.plan), "nonce": uuid.uuid4().hex, - "method": method, "url_digest": digest(url)} - self._write("pending-request.json", pending) - try: - response = raw(method, url, token, **kwargs) - except HelperFailure as error: - if error.http_status == 429: - self.record_backoff(method, url, error, None) - self.finish_request(error) - raise - if response.status == 429: - error = HelperFailure("azure-http-error", "Azure request failed with HTTP 429.", - blocked_at="execution", status=429, request_id=response.request_id, - retry_after=response.retry_after, recovery_deadline=response.recovery_deadline) - self.record_backoff(method, url, error, response.headers) - try: - self.finish_request() - except HelperFailure as persistence: - if method in ("PUT", "POST") and response.status in (200, 201): - return replace(response, ack_failure=persistence) - raise - return response - - invoke._file_journal_owner = self - return invoke - - def finish_request(self, original=None): - try: - path = self.directory / "pending-request.json" - expected = self.records["pending-request.json"] - retained = private_io.read_json(path) - if not isinstance(retained, dict) or retained.get("payload") != expected or retained.get("integrity") != digest(expected): - raise fail("file-upload-evidence-invalid", "Pending request receipt changed.") - path.unlink() - self.records.pop("pending-request.json") - except (OSError, HelperFailure) as cleanup: - error = original or HelperFailure("file-upload-receipt-failed", "Pending request evidence could not be finalized.", - blocked_at="local-persistence") - error.blocked_at = "local-persistence" - error.warnings.append("Pending request evidence remains unresolved; no further requests.") - self.network_failure = error - raise error from cleanup - - def record_backoff(self, method, url, error, headers): - metadata = error.retry_after - origin = "response-headers" if headers is not None else "native-http-error" - timing = retry_after_timing(headers) if headers is not None else error.retry_after_timing - if timing is None: - received = time.time() - received = received if valid_utc_timestamp(received) else None - known = isinstance(metadata, RetryAfter) and metadata.kind in ("seconds", "date", "date-rfc850") - timing = RetryAfterTiming(received, retry_after_not_before(metadata, received) if known else None, - metadata.value if known and metadata.kind == "seconds" else None) - origin = "typed-metadata" if known else "unavailable" - if not isinstance(metadata, RetryAfter): - metadata, origin = RetryAfter("missing"), "unavailable" - timing = RetryAfterTiming(timing.received_at_utc, None) - events = self.backoff_events() - attempt = self.active_attempt if method == "POST" and url == file_ingest._list_url(self.ingestion) else None - event = {"version": "1.0", "plan_digest": digest(self.plan), - "previous": digest(next(reversed(events.values()))) if events else None, - "method": method, "url_digest": digest(url), "attempt": attempt, - "attempt_digest": digest(self.records[attempt]) if attempt else None, - "request_id": ReadRecovery.safe_id(error.request_id) if error.request_id else None, - "metadata": metadata._asdict(), "timing": timing._asdict(), "origin": origin} - try: - self._write(f"backoff-{len(events):04}.json", event) - except HelperFailure as persistence: - error.blocked_at = "local-persistence" - error.warnings.extend(persistence.warnings) - error.warnings.append("Original HTTP 429 retained; backoff receipt persistence failed. No further requests.") - self.network_failure = error - raise error from persistence - - -def _file_id(value): - return isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}", value) is not None - - -def _proof(plan, record, item): - if isinstance(item, dict) and _file_id(item.get("fileId")) and file_ingest._matches(item, plan, record): - return {"file_id": item["fileId"], "record_digest": digest(record)} - return None - - -def _observe(plan, states, files): - names = {record["path"] for record in plan["files"]} - if any(item.get("fileName") not in names for item in files): - raise fail("server-inventory-conflict", "File inventory contains an unapproved identity.") - observed = {} - for index, record in enumerate(plan["files"]): - matches = [item for item in files if item.get("fileName") == record["path"]] - if len(matches) > 1 or matches and not file_ingest._matches({**matches[0], "errorMessage": None}, plan, record): - raise fail("file-record-conflict", "File inventory has duplicate or conflicting approved markers.") - state = states[index] - state["verification"] = {"accepted": "pending", "rejected": "failed"}.get(state["upload"], state["upload"]) - if matches: - item = matches[0] - proof = _proof(plan, record, item) - if (proof and state.get("file_proof") and proof != state["file_proof"] - or state.get("ack_file_id") and item.get("fileId") is not None - and item["fileId"] != state["ack_file_id"]): - raise fail("file-record-conflict", "Current File identity differs from the persisted upload ACK.") - state["verification"] = ("failed" if state["upload"] == "rejected" and state.get("status") != 429 - or item.get("errorMessage") is not None else - "confirmed" if proof else "unverified") - state["ingested"] = state["verification"] == "confirmed" - observed[index] = item - return observed - - -def _summary(plan, states, session): - counts = {name: 0 for name in ("accepted", "confirmed", "failed", "pending", "unverified", "not_attempted")} - files = [] - for record, state in zip(plan["files"], states): - counts["accepted"] += state["upload"] == "accepted" - verification = state.get("verification", {"accepted": "pending", "rejected": "failed"}.get(state["upload"], state["upload"])) - counts[verification] += 1 - files.append({"fileName": record["path"], "sha256": record["sha256"], "upload": state["upload"], - "verification": verification, "http_status": state.get("status"), - "request_ids": state["request_ids"][:3]}) - ingested = sum(bool(state.get("ingested")) for state in states) - return {"counts": counts, "files": files, "ingested": ingested, - "readiness": "ingested" if ingested == len(states) and counts["confirmed"] == len(states) else "partial", - "retrieval": "unverified", - **({"backoff": session.backoff_status()} if session else {}), - **({"upload_retry": { - "status": "blocked", "code": "file-upload-retry-safety-unproven", - "reason": "Unknown upload outcomes cannot be replayed; only a received File upload 429 permits one bounded in-operation retry.", - }} if counts["unverified"] else {}), - "resume": ("Plan newly approved upload-only continuation for never-attempted files; uncertain attempts are never replayed." - if session else "Original durable ACK/attempt evidence was not retained; uploads cannot safely resume from GET alone."), - "subset_handoff": "Report the proved ingested subset and remaining outcomes; subset isolation and KB retrieval are separate. KB changes need separate approval."} - - -def _progress(progress, plan, states): - summary = _summary(plan, states, None) - counts = summary["counts"] - progress.update("file-upload", uploads_acknowledged=counts["accepted"], files_verified=counts["confirmed"], - files_ingested=summary["ingested"], - files_reused=sum(state["upload"] == "not_attempted" and state.get("verification") == "confirmed" for state in states), - files_failed=counts["failed"], files_unverified=counts["unverified"], - files_not_attempted=counts["not_attempted"], files_pending=counts["pending"]) - - -def run_batch(document, *, token_provider, transport, progress, allow_new_uploads=False, session=None, - eligible=None, source_check=None, allow_upload_retry=True): - plan, fingerprint = document["plan"], document["_computed_fingerprint"] - root, records = file_ingest._validate_plan(plan) - states = session.states() if session else [{"upload": "not_attempted", "request_ids": []} for _ in records] - if session: - session.require_backoff() - transport = session.transport(transport) - created, reused, request_ids, warnings = [], [], [], [] - primary = None - stopped = False - retry_used = False - readback_failed = False - token = token_provider(SEARCH_AUDIENCE) - url = file_ingest._list_url(plan) - recovery = ReadRecovery(on_wait=progress.waiting) - progress.update("file-inventory") - try: - before, ids = file_ingest._list_files(url, token, transport=transport, recovery=recovery) - request_ids.extend(ids) - if session is None and file_ingest.inventory_digest(before) != plan["expected_server_inventory_digest"]: - raise fail("server-inventory-drift", "Approved initial file inventory changed.") - observed = _observe(plan, states, before) - if not allow_new_uploads and len(observed) != len(records): - raise fail("reused-source-upload-forbidden", "Generic reuse cannot authorize new uploads.") - except HelperFailure as failure: - failure.file_batch = _summary(plan, states, session) - failure.file_batch["request_ids"] = recovery.request_ids - raise - warnings.extend(recovery.warnings) - eligible = set(range(len(records))) if eligible is None else set(eligible) - after = None - for index, record in enumerate(records): - _progress(progress, plan, states) - state = states[index] - if index in observed: - reused.append({"fileId": observed[index].get("fileId"), "fileName": record["path"], "sha256": record["sha256"]}) - continue - if index not in eligible or state["upload"] != "not_attempted": - continue - try: - path = file_ingest._resolve_file(root, record) - content = path.read_bytes() - if (len(content) != record["size"] or "sha256:" + hashlib.sha256(content).hexdigest() != record["sha256"] - or path.stat().st_mtime_ns != record["mtime_ns"]): - raise fail("inventory-drift", "Approved corpus changed before upload.") - except OSError: - primary = primary or fail("inventory-unreadable", "Approved corpus became inaccessible.") - warnings.append("Batch stopped: approved corpus became inaccessible.") - stopped = True - break - except HelperFailure as failure: - primary = primary or failure - warnings.append(f"Batch stopped ({failure.code}); no further uploads were attempted.") - stopped = True - break - body, boundary = file_ingest._multipart(plan, record, content, fingerprint) - for attempt in range(2): - try: - if attempt: - file_ingest._resolve_file(root, record) - if session: - session.attempt(index, retry=bool(attempt)) - except HelperFailure as failure: - primary = primary or failure - warnings.append(f"Upload attempt evidence/drift check failed ({failure.code}); no further requests.") - stopped = True - break - state.update(upload="unverified", verification="unverified", status=None) - _progress(progress, plan, states) - response = None - transport_failed = False - try: - with progress.processing_file(index + 1, len(records), attempt + 1): - response = transport("POST", url, token, body=body, - headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, - follow_redirects=False, timeout=UPLOAD_TIMEOUT, - response_deadline=time.monotonic() + UPLOAD_TIMEOUT, - max_response_bytes=1024 * 1024) - except HelperFailure as caught: - transport_failed = True - failure = caught - failure_status, failure_id = failure.http_status, failure.request_id - else: - failure_status, failure_id = response.status, response.request_id - failure = HelperFailure("upload-failed", f"Upload returned HTTP {response.status}.", - blocked_at="execution", status=response.status, request_id=response.request_id, - retry_after=response.retry_after, recovery_deadline=response.recovery_deadline) - if failure_id: - safe_id = ReadRecovery.safe_id(failure_id) - failure.request_id = safe_id - state["request_ids"].append(safe_id) - request_ids.append(safe_id) - state["status"] = failure_status - proof = None - if failure_status in (200, 201): - state.update(upload="accepted", verification="pending") - created.append({"fileName": record["path"], "sha256": record["sha256"]}) - if response is not None: - proof = _proof(plan, record, response.body) - if isinstance(response.body, dict) and _file_id(response.body.get("fileId")): - state["ack_file_id"] = response.body["fileId"] - elif _rejected(failure_status): - state.update(upload="rejected", verification="failed") - try: - if response is not None and response.ack_failure is not None: - raise response.ack_failure - if session: - session.result(index, state["upload"], failure_status, failure_id, retry=bool(attempt), - proof=proof, file_id=state.get("ack_file_id")) - except HelperFailure as persistence: - primary = primary or (failure if failure_status not in (200, 201) else persistence) - primary.warnings.extend(persistence.warnings) - warnings.append("Upload journal persistence failed; no further requests were issued.") - stopped = True - break - if failure_status in (200, 201) and not transport_failed: - if proof: - state.update(file_proof=proof, ingested=True, verification="confirmed") - elif isinstance(response.body, dict) and ( - response.body.get("errorMessage") is not None - or "fileName" in response.body and not file_ingest._matches(response.body, plan, record) - ): - primary = primary or fail("upload-metadata-unverified", "Upload ACK metadata conflicts with the approved file.") - stopped = True - _progress(progress, plan, states) - break - primary = primary or failure - if failure_status == 415: - break - _progress(progress, plan, states) - if failure_status == 429 and not retry_used and allow_upload_retry and failure.blocked_at != "local-persistence": - retry_used = True - recovery = ReadRecovery(on_wait=progress.waiting) - try: - recovery.delay(failure) - if source_check is not None: - source_check(recovery, token) - inventory, _ = file_ingest._list_files(url, token, transport=transport, recovery=recovery) - seen = _observe(plan, states, inventory) - if index in seen: - if state["verification"] != "confirmed": - raise fail("file-upload-retry-conflict", "Existing file does not prove the approved completed upload.") - created.append({"fileName": record["path"], "sha256": record["sha256"]}) - break - file_ingest._resolve_file(root, record) - if time.monotonic() >= recovery.deadline: - raise fail("read-recovery-budget-exhausted", "Retry preflight exceeded its complete read budget.") - warnings.append("File upload HTTP 429: one same-operation retry after bounded backoff; no source recreation.") - except HelperFailure as read_failure: - warnings.append(f"File retry stopped ({read_failure.code}; HTTP {read_failure.http_status}).") - warnings.extend(read_failure.warnings) - stopped = True - finally: - request_ids.extend(recovery.request_ids) - warnings.extend(recovery.diagnostics()) - if not stopped: - continue - else: - stopped = True - warnings.append(f"Batch stopped ({failure.code}; HTTP {failure_status}); no further uploads.") - if failure.blocked_at != "local-persistence" and ( - failure_status in (408, 409) or failure_status is None or failure_status >= 500 - ): - recovery = ReadRecovery(on_wait=progress.waiting) - try: - after, _ = file_ingest._list_files(url, token, transport=transport, recovery=recovery) - _observe(plan, states, after) - if state["verification"] == "confirmed": - created.append({"fileName": record["path"], "sha256": record["sha256"]}) - except HelperFailure as read_failure: - warnings.append(f"File readback stopped ({read_failure.code}; HTTP {read_failure.http_status}).") - warnings.extend(read_failure.warnings) - request_ids.extend(recovery.request_ids) - warnings.extend(recovery.diagnostics()) - break - if stopped: - break - if session: - session.active_attempt = None - _progress(progress, plan, states) - progress.update("file-readback") - if all(state["upload"] == "rejected" and state["verification"] != "confirmed" for state in states): - stopped = True - if after is None and not stopped: - recovery = ReadRecovery(on_wait=progress.waiting) - try: - after, ids = file_ingest._list_files(url, token, transport=transport, recovery=recovery) - _observe(plan, states, after) - except HelperFailure as failure: - primary = primary or failure - readback_failed = True - warnings.append(f"File readback stopped ({failure.code}; HTTP {failure.http_status}).") - request_ids.extend(recovery.request_ids) - warnings.extend(recovery.diagnostics()) - batch = _summary(plan, states, session) - batch["request_ids"] = request_ids[:804] - batch["upload_retry_used"] = retry_used - progress.update("file-readback", files_verified=batch["counts"]["confirmed"], files_failed=batch["counts"]["failed"], - files_pending=batch["counts"]["pending"], files_unverified=batch["counts"]["unverified"], - files_not_attempted=batch["counts"]["not_attempted"], files_ingested=batch["ingested"]) - if batch["counts"]["confirmed"] != len(records) or readback_failed: - failure = primary or fail("readback-mismatch", "Some approved files remain pending, failed or unverified.") - uncertain = [{"action": "upload-unverified", "type": "knowledge-source-file", - "fileName": record["path"], "sha256": record["sha256"]} - for record, state in zip(records, states) - if state["upload"] == "unverified" and state["verification"] != "confirmed"] - failure.writes = created + uncertain + failure.writes - failure.resources_remaining = file_ingest._remaining_files(created) + uncertain + failure.resources_remaining - failure.partial = bool(created) or any(s["upload"] == "unverified" for s in states) or ( - failure.partial and not _rejected(failure.http_status)) - failure.warnings.extend(warnings) - failure.file_batch = batch - raise failure - verified = [{"fileId": item.get("fileId"), "fileName": record["path"], "sha256": record["sha256"], "size": record["size"]} - for record in records for item in after or before if item.get("fileName") == record["path"]] - return {"status": "completed", "outcome": plan.get("outcome", "file-knowledge-source-ingestion"), - "approved_plan": {"fingerprint": fingerprint, "confirmed": True}, - "resources": {"created": created, "reused": reused, "updated": [], "skipped": []}, - "api_contracts": [{"operation": "upload-file", "version": file_ingest.API_VERSION, "preview": True}], - "data_movement": {"boundary": {"local_root_digest": digest(str(root))}, "result": "Approved direct File uploads"}, - "auth": {"mode": "entra-user", "principals": []}, "rbac": plan.get("rbac", {"assignments": []}), - "network": plan.get("network", {"posture": "preserved", "evidence": None}), - "verification": {"readback": verified, "request_ids": request_ids, "server_inventory_digest": file_ingest.inventory_digest(after or before), - "idempotency": "Only a received File upload 429 permits one bounded same-operation retry; unknown outcomes are never replayed."}, - "warnings": warnings, "file_batch": batch, - "ownership": {"run_owned": created, "reused_not_owned": reused, "owner": plan["owner"]}, - "cleanup": {"status": "not-requested", "separate_confirmation_required": True}} - - -def _source_check(session, token, transport, recovery): - try: - from . import source_vector - except ImportError: - import source_vector - plan = session.plan - file_ingest._validate_plan(session.ingestion) - etag = session.require_ack() - current, _ = search_reconcile._get(search_reconcile.resource_url(plan["source"]), token, - transport=source_vector.guard_readback_transport(plan, transport), recovery=recovery) - if (current is None or current.get("@odata.etag") != etag - or not search_reconcile.definitions_match(plan["source"]["desired"], current)): - raise fail("file-upload-source-drift", "Current source does not match original acknowledged identity/version/definition.") - return current - - -def _verify(session, token_provider, transport, context_provider): - try: - from . import file_source, file_cu_mi - except ImportError: - import file_source, file_cu_mi - session.require_ack() - session.states() - session.require_backoff() - transport = session.transport(transport) - context = context_provider() - cu_ingestion_auth.validate_context(context) - if context != session.seed["context"]: - raise fail("file-upload-context-drift", "Original tenant/subscription/principal changed.") - plan = session.plan - if "content_understanding" in plan: - state, _ = file_source.read_content_understanding(plan["content_understanding"], token_provider=token_provider, transport=transport) - if not file_source._cu_states_match(state, plan["cu_resource_state"]): - raise fail("file-upload-auth-drift", "Original CU account/auth/network state changed.") - if plan["source"].get("ai_services_managed_identity"): - binding, _ = file_cu_mi.read_binding(plan["content_understanding"], plan["source"]["endpoint"], - token_provider=token_provider, transport=transport) - if binding != plan["cu_identity_state"]: - raise fail("file-upload-auth-drift", "Original Search MI/CU role/network binding changed.") - return _source_check(session, token_provider(SEARCH_AUDIENCE), transport, ReadRecovery()) - - -def plan_resume(request, *, token_provider=azure_cli_token, transport=http_request, - context_provider=cu_ingestion_auth.account_context): - if not isinstance(request, dict) or not isinstance(request.get("receipt_directory"), str): - raise fail("file-upload-input-invalid", "Use an object with the original private receipt directory.") - require_allowed_fields(request, {"schema_version", "receipt_directory"}, label="upload resume request") - if request.get("schema_version") != "1.0": - raise fail("file-upload-input-invalid", "Use the supported resume request schema.") - session = Session(request.get("receipt_directory")) - transport = session.transport(transport) - _verify(session, token_provider, transport, context_provider) - states = session.states() - inventory, _ = file_ingest._list_files(file_ingest._list_url(session.ingestion), token_provider(SEARCH_AUDIENCE), - transport=transport, recovery=ReadRecovery()) - observed = _observe(session.ingestion, states, inventory) - eligible = [i for i, state in enumerate(states) if state["upload"] == "not_attempted" and i not in observed] - plan = {"operation": "resume-file-uploads", "version": "1.0", "receipt_directory": str(session.directory), - "original_plan_digest": digest(session.plan), "creation_ack_digest": digest(session.records["source-ack.json"]), - "journal_digest": digest(session.records), "eligible": eligible, "owner": session.plan["owner"], "cleanup_approved": False} - return {"status": "planned", "execution_input": {"schema_version": "1.0", "plan": plan, - "approval": {"confirmed": False, "fingerprint": digest(plan)}}, - "approval_summary": {"execution_required": bool(eligible), "mutation_approval_required": bool(eligible), "uploads": len(eligible)}, - "file_batch": _summary(session.ingestion, states, session), "writes_performed": [], - "next_step": "Approve only never-attempted uploads. No eligible files: retain this observation; do not replay uncertain files."} - - -@reporting("file-source") -def execute(document, *, token_provider=azure_cli_token, transport=http_request, - context_provider=cu_ingestion_auth.account_context, progress=None): - if not isinstance(document, dict) or not isinstance(document.get("plan"), dict): - raise fail("file-upload-input-invalid", "Use the newly approved upload-only envelope.") - require_allowed_fields(document, {"schema_version", "plan", "approval", "_computed_fingerprint"}, label="upload resume envelope") - plan = document.get("plan") - reject_secrets(document) - require_allowed_fields(plan, {"operation", "version", "receipt_directory", "original_plan_digest", "creation_ack_digest", - "journal_digest", "eligible", "owner", "cleanup_approved"}, label="upload resume plan") - if (document.get("schema_version") != "1.0" or plan.get("operation") != "resume-file-uploads" or plan.get("version") != "1.0" - or set(plan) != {"operation", "version", "receipt_directory", "original_plan_digest", "creation_ack_digest", - "journal_digest", "eligible", "owner", "cleanup_approved"} - or not isinstance(document.get("approval"), dict) or document["approval"].get("confirmed") is not True - or plan.get("cleanup_approved") is not False or document.get("approval") != {"confirmed": True, "fingerprint": digest(plan)} - or document.get("_computed_fingerprint") != digest(plan)): - raise fail("file-upload-approval-missing", "A newly approved unchanged upload-only plan is required.") - session = Session(plan.get("receipt_directory")) - transport = session.transport(transport) - if (plan["original_plan_digest"] != digest(session.plan) or plan["journal_digest"] != digest(session.records) - or plan["creation_ack_digest"] != digest(session.records["source-ack.json"]) or plan["owner"] != session.plan["owner"] - or not isinstance(plan["eligible"], list) or any(type(i) is not int or not 0 <= i < len(session.ingestion["files"]) for i in plan["eligible"]) - or len(set(plan["eligible"])) != len(plan["eligible"])): - raise fail("file-upload-plan-drift", "Original approval/ACK/journal changed; refresh the upload-only plan.") - states = session.states() - if any(states[i]["upload"] != "not_attempted" for i in plan["eligible"]): - raise fail("file-upload-replay-forbidden", "Attempted files cannot be submitted again, even after an empty inventory.") - progress.update("source-reconciliation") - try: - _verify(session, token_provider, transport, context_provider) - result = run_batch({"plan": session.ingestion, "_computed_fingerprint": digest(session.plan)}, token_provider=token_provider, - transport=transport, progress=progress, allow_new_uploads=True, session=session, eligible=plan["eligible"], - source_check=lambda recovery, token: _source_check(session, token, transport, recovery)) - except HelperFailure as failure: - if failure.file_batch is None: - failure.file_batch = _summary(session.ingestion, states, session) - failure.partial = True - failure.resources_remaining.insert(0, {"type": "knowledge-source", "name": session.plan["source"]["name"]}) - raise - result["approved_plan"] = document["approval"] - result["original_run"] = {"plan_digest": digest(session.plan), "creation_ack_digest": plan["creation_ack_digest"], - "source_retained": session.plan["source"]["name"]} - return result - - -def main(argv=None): - try: - from .private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode - except ImportError: - from private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode - parser = argparse.ArgumentParser() - modes = parser.add_mutually_exclusive_group(required=True) - modes.add_argument("--plan", type=Path) - modes.add_argument("--input", type=Path) - add_execution_output_argument(parser) - add_progress_argument(parser) - args = parser.parse_args(argv) - fingerprint = None - try: - validate_execution_output_mode(args) - if args.plan: - emit_plan_result(plan_resume(private_io.read_json(args.plan)), args.execution_output) - return 0 - document, _, fingerprint = load_approved_input(args.input) - document["_computed_fingerprint"] = fingerprint - result = execute(document, progress=Progress("file-source", enabled=args.progress)) - except HelperFailure as failure: - result = blocked_result(failure, outcome="resume-file-uploads", fingerprint=fingerprint) - result["safe_next_decision"] = "Retain the source. Missing/changed original evidence blocks upload continuation, not retention; no creation replay or cleanup workaround." - emit_result(result) - return 3 if result["status"] == "partial" else 2 - emit_result(result) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) +"""Synchronous File batches; one in-operation queue-429 retry, never unknown-outcome replay.""" +from __future__ import annotations + +import argparse +import copy +import hashlib +import re +import sys +import time +import uuid +from dataclasses import replace +from pathlib import Path + +try: + from . import _bootstrap_io as private_io, cu_ingestion_auth, file_ingest, search_reconcile + from ._common import (HelperFailure, ReadRecovery, RetryAfter, RetryAfterTiming, SEARCH_AUDIENCE, + retry_after_timing, retry_after_not_before, valid_utc_timestamp, azure_cli_token, blocked_result, + digest, emit_result, http_request, load_approved_input, reject_secrets, require_allowed_fields) + from ._progress import Progress, add_progress_argument, reporting +except ImportError: + import _bootstrap_io as private_io, cu_ingestion_auth, file_ingest, search_reconcile + from _common import (HelperFailure, ReadRecovery, RetryAfter, RetryAfterTiming, SEARCH_AUDIENCE, + retry_after_timing, retry_after_not_before, valid_utc_timestamp, azure_cli_token, blocked_result, + digest, emit_result, http_request, load_approved_input, reject_secrets, require_allowed_fields) + from _progress import Progress, add_progress_argument, reporting + + +RETAIN = " Retain the source and evidence; no creation replay, reset, deletion or new roles." +UPLOAD_TIMEOUT = 180 +MAX_BACKOFF_RECORDS = 200 + + +def _rejected(status): + return type(status) is int and 400 <= status < 500 and status not in (408, 409) + + +def fail(code, message): + return HelperFailure(code, message + RETAIN, blocked_at="file-upload-resume") + + +class Session: + def __init__(self, directory, document=None, *, context_provider=cu_ingestion_auth.account_context): + self.directory = private_io.validate_private_artifact_directory(str(directory)) + self.context_provider = context_provider + self.records = {} + self.network_failure = None + self.active_attempt = None + if document is not None: + self._validate_document(document) + if any(self.directory.iterdir()): + raise fail("file-upload-journal-exists", "Use a new empty private receipt directory for original creation.") + context = context_provider() + cu_ingestion_auth.validate_context(context) + self.seed = {"document": {key: copy.deepcopy(document[key]) for key in ("schema_version", "plan", "approval")}, + "context": context, "backoff_version": "1.0"} + self._write("run.json", self.seed) + else: + self._load() + self.seed = self.records["run.json"] + self._validate_document(self.seed["document"]) + cu_ingestion_auth.validate_context(self.seed["context"]) + self.document = self.seed["document"] + self.plan = self.document["plan"] + self.ingestion = self.plan["ingestion"] + + @staticmethod + def _validate_document(document): + try: + from . import file_source + except ImportError: + import file_source + if (not isinstance(document, dict) or document.get("schema_version") != "1.0" + or not isinstance(document.get("plan"), dict) + or "_computed_fingerprint" in document and document["_computed_fingerprint"] != digest(document.get("plan")) + or not isinstance(document.get("approval"), dict) or document["approval"].get("confirmed") is not True + or document.get("approval") != {"confirmed": True, "fingerprint": digest(document.get("plan"))}): + raise fail("file-upload-approval-missing", "Original unchanged File creation approval is required.") + require_allowed_fields(document, {"schema_version", "plan", "approval", "_computed_fingerprint"}, label="original File envelope") + file_source._validate_plan(document["plan"]) + if document["plan"]["source"]["action"] != "create": + raise fail("file-upload-provenance-missing", "This continuation is only for original acknowledged creation, not generic reuse.") + + def _write(self, name, payload): + value = {"schema_version": "1.0", "kind": "file-upload-journal", "payload": payload, "integrity": digest(payload)} + reject_secrets(value) + try: + private_io.atomic_private_file(self.directory, name, value, max_bytes=private_io.MAX_BYTES) + except HelperFailure as error: + raise HelperFailure("file-upload-receipt-failed", "Private upload evidence persistence failed." + RETAIN, + blocked_at="local-persistence", warnings=error.warnings) from error + self.records[name] = copy.deepcopy(payload) + + def _load(self): + # Reuse the existing bounded, private, no-link evidence reader. + try: + from .blob_recheck import read_private + except ImportError: + from blob_recheck import read_private + try: + paths = list(self.directory.iterdir()) + except OSError as error: + raise fail("file-upload-evidence-unreadable", "Original upload evidence is inaccessible.") from error + if len(paths) > 803 + MAX_BACKOFF_RECORDS or not {"run.json", "source-ack.json"}.issubset({p.name for p in paths}): + raise fail("file-upload-provenance-missing", "Necessary original ACK and journal were not retained.") + for path in paths: + if (path.name not in {"run.json", "source-ack.json", "pending-request.json"} + and not re.fullmatch(r"(?:[0-9]{4}-(retry-)?(attempt|result)|backoff-[0-9]{4})\.json", path.name)): + raise fail("file-upload-evidence-invalid", "Unexpected or incomplete journal entry; do not discard it to resume.") + value = read_private(path) + if (set(value) != {"schema_version", "kind", "payload", "integrity"} + or value["schema_version"] != "1.0" or value["kind"] != "file-upload-journal" + or value["integrity"] != digest(value["payload"])): + raise fail("file-upload-evidence-invalid", "Original private upload evidence changed.") + self.records[path.name] = value["payload"] + seed = self.records["run.json"] + if (not isinstance(seed, dict) or set(seed) not in ({"document", "context"}, {"document", "context", "backoff_version"}) + or "backoff_version" in seed and seed["backoff_version"] != "1.0"): + raise fail("file-upload-evidence-invalid", "Original approval/context evidence is incomplete.") + + def acknowledge(self, metadata): + self._write("source-ack.json", {"original_plan_digest": digest(self.plan), + "source_url": search_reconcile.resource_url(self.plan["source"]), + "acknowledgement": metadata}) + + def require_ack(self): + value = self.records.get("source-ack.json") + if (not isinstance(value, dict) or set(value) != {"original_plan_digest", "source_url", "acknowledgement"} + or value["original_plan_digest"] != digest(self.plan) + or value["source_url"] != search_reconcile.resource_url(self.plan["source"])): + raise fail("file-upload-provenance-missing", "An original acknowledged conditional source creation must be retained.") + ack = value["acknowledgement"] + if (not isinstance(ack, dict) or set(ack) != {"status", "request_id", "etag_evidence"} + or ack["status"] not in (200, 201) or not ack["request_id"] + or not isinstance(ack["request_id"], str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}", ack["request_id"]) + or not isinstance(ack["etag_evidence"], dict) or set(ack["etag_evidence"]) != {"body", "headers"} + or not isinstance(ack["etag_evidence"]["headers"], list)): + raise fail("file-upload-provenance-missing", "Original successful creation ACK/request ID is required, not GET ownership.") + etag = search_reconcile.resolve_etag(ack["etag_evidence"], ack["request_id"]) + if not etag: + raise fail("file-upload-provenance-missing", "The original creation ACK lacks version proof; GET cannot supply it.") + return etag + + def states(self): + states = [] + events = self.backoff_events() + allowed = {"run.json", "source-ack.json", "pending-request.json"} | set(events) + pending = self.records.get("pending-request.json") + if "pending-request.json" in self.records and ( + not isinstance(pending, dict) or set(pending) != {"version", "plan_digest", "nonce", "method", "url_digest"} + or pending["version"] != "1.0" or pending["plan_digest"] != digest(self.plan) + or not isinstance(pending["nonce"], str) or not re.fullmatch(r"[0-9a-f]{32}", pending["nonce"]) + or pending["method"] not in ("GET", "POST", "PUT", "HEAD") + or not isinstance(pending["url_digest"], str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", pending["url_digest"]) + ): + raise fail("file-upload-evidence-invalid", "Pending request evidence is invalid.") + for index, record in enumerate(self.ingestion["files"]): + state = {"upload": "not_attempted", "request_ids": [], "ingested": False} + for retry in (False, True): + prefix = f"{index:04}-" + ("retry-" if retry else "") + attempt_name, result_name = prefix + "attempt.json", prefix + "result.json" + allowed.update((attempt_name, result_name)) + attempt, result = self.records.get(attempt_name), self.records.get(result_name) + expected = self._attempt_record(index, retry=retry) if attempt is not None else None + if attempt is not None and attempt != expected or result is not None and attempt is None: + raise fail("file-upload-evidence-invalid", "Upload attempt evidence does not bind the original source/corpus.") + if attempt is not None: + state.update(upload="unverified", ingested=False) + state.pop("file_proof", None) + state.pop("ack_file_id", None) + if result is not None: + if (not isinstance(result, dict) or not {"upload", "status", "request_id"} <= set(result) + or set(result) - {"upload", "status", "request_id", "file_proof", "file_id", "throttle_event"} + or not isinstance(result["upload"], str) + or result["upload"] not in {"accepted", "rejected", "unverified"} + or result["upload"] == "accepted" and result["status"] not in (200, 201) + or result["upload"] == "rejected" and not _rejected(result["status"]) + or result["status"] is not None and (type(result["status"]) is not int or not 100 <= result["status"] <= 599) + or result["request_id"] is not None and ReadRecovery.safe_id(result["request_id"]) != result["request_id"]): + raise fail("file-upload-evidence-invalid", "Upload result is not an original supported ACK or failure.") + linked = [event for event in events.values() if event["attempt"] == attempt_name] + if "throttle_event" in result: + if (result["status"] != 429 or len(linked) != 1 + or result["throttle_event"] != digest(linked[0]) + or result["request_id"] != linked[0]["request_id"]): + raise fail("file-upload-evidence-invalid", "Upload 429 timing does not bind its original attempt/result.") + elif result["status"] == 429 and self.seed.get("backoff_version") is not None: + raise fail("file-upload-evidence-invalid", "Required original upload 429 timing evidence is missing.") + if "file_id" in result: + if result["upload"] != "accepted" or not _file_id(result["file_id"]): + raise fail("file-upload-evidence-invalid", "Original upload ACK file identity is invalid.") + state["ack_file_id"] = result["file_id"] + proof = result.get("file_proof") + if proof is not None: + if (result["upload"] != "accepted" or not isinstance(proof, dict) + or set(proof) != {"file_id", "record_digest"} or not _file_id(proof["file_id"]) + or proof["record_digest"] != digest(record) + or result.get("file_id", proof["file_id"]) != proof["file_id"]): + raise fail("file-upload-evidence-invalid", "Persisted File completion proof does not match the approved record.") + state.update(file_proof=proof, ingested=True) + state.update(upload=result["upload"], status=result["status"]) + if result["request_id"]: + state["request_ids"].append(result["request_id"]) + states.append(state) + if set(self.records) - allowed: + raise fail("file-upload-evidence-invalid", "Upload journal contains records outside the approved corpus.") + return states + + def _attempt_record(self, index, *, retry=False): + record = { + "original_plan_digest": digest(self.plan), "record_digest": digest(self.ingestion["files"][index]), + "creation_ack_digest": digest(self.records["source-ack.json"]), + } + if retry: + rejection = self.records.get(f"{index:04}-result.json") + if not isinstance(rejection, dict) or rejection.get("status") != 429 or rejection.get("upload") != "rejected": + raise fail("file-upload-replay-forbidden", "A retry attempt requires the same operation's retained File upload 429.") + record["rejected_result_digest"] = digest(rejection) + return record + + def attempt(self, index, *, retry=False): + prefix = f"{index:04}-" + ("retry-" if retry else "") + self._write(prefix + "attempt.json", self._attempt_record(index, retry=retry)) + self.active_attempt = prefix + "attempt.json" + + def result(self, index, upload, status, request_id, *, retry=False, proof=None, file_id=None): + prefix = f"{index:04}-" + ("retry-" if retry else "") + event = {} + if status == 429: + linked = [value for value in self.backoff_events().values() if value["attempt"] == prefix + "attempt.json"] + if len(linked) != 1: + raise fail("file-upload-evidence-invalid", "Original upload 429 timing was not retained.") + event["throttle_event"] = digest(linked[0]) + self._write(prefix + "result.json", {"upload": upload, "status": status, + "request_id": ReadRecovery.safe_id(request_id) if request_id else None, + **({"file_id": file_id} if file_id is not None else {}), + **({"file_proof": proof} if proof is not None else {}), **event}) + self.active_attempt = None + + def backoff_events(self): + events = {key: self.records[key] for key in sorted(self.records) if key.startswith("backoff-")} + if len(events) > MAX_BACKOFF_RECORDS: + raise fail("file-upload-evidence-invalid", "Too many retained backoff records.") + plan_digest = digest(self.plan) + previous = None + for index, (name, event) in enumerate(events.items()): + if (name != f"backoff-{index:04}.json" or not isinstance(event, dict) + or set(event) != {"version", "plan_digest", "previous", "method", "url_digest", + "attempt", "attempt_digest", "request_id", "metadata", "timing", "origin"} + or event["version"] != "1.0" or event["plan_digest"] != plan_digest + or event["previous"] != previous or event["method"] not in ("GET", "POST", "PUT", "HEAD") + or not isinstance(event["url_digest"], str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", event["url_digest"]) + or event["request_id"] is not None and ReadRecovery.safe_id(event["request_id"]) != event["request_id"] + or event["origin"] not in ("response-headers", "native-http-error", "typed-metadata", "unavailable")): + raise fail("file-upload-evidence-invalid", "Original backoff provenance is invalid or incomplete.") + if event["attempt"] is not None: + attempt = self.records.get(event["attempt"]) if isinstance(event["attempt"], str) else None + if (attempt is None or not re.fullmatch(r"[0-9]{4}-(retry-)?attempt\.json", event["attempt"]) + or event["attempt_digest"] != digest(attempt) or event["method"] != "POST" + or event["url_digest"] != digest(file_ingest._list_url(self.ingestion))): + raise fail("file-upload-evidence-invalid", "Backoff does not bind the original upload attempt.") + elif event["attempt_digest"] is not None: + raise fail("file-upload-evidence-invalid", "Backoff attempt provenance is inconsistent.") + metadata, timing = event["metadata"], event["timing"] + if (not isinstance(metadata, dict) or set(metadata) != {"kind", "value"} + or metadata["kind"] not in ("seconds", "date", "date-rfc850", "missing", "invalid", "overlong") + or not valid_utc_timestamp(metadata["value"]) + or metadata["kind"] == "seconds" and (type(metadata["value"]) is not int or not 0 <= metadata["value"] <= 30) + or metadata["kind"] in ("missing", "invalid", "overlong") and metadata["value"] != 0 + or not isinstance(timing, dict) or set(timing) != {"received_at_utc", "not_before_utc", "server_delay_seconds"}): + raise fail("file-upload-evidence-invalid", "Retained Retry-After metadata is invalid.") + received, deadline = timing["received_at_utc"], timing["not_before_utc"] + seconds = timing["server_delay_seconds"] + if (received is not None and not valid_utc_timestamp(received) + or deadline is not None and (received is None or not valid_utc_timestamp(deadline)) + or seconds is not None and (type(seconds) is not int or not 0 <= seconds < 315537897600 + or metadata["kind"] not in ("seconds", "overlong")) + or metadata["kind"] == "seconds" and seconds not in (None, metadata["value"])): + raise fail("file-upload-evidence-invalid", "Retained backoff UTC timing is invalid.") + if deadline is not None: + if (event["origin"] == "unavailable" + or event["origin"] == "typed-metadata" and metadata["kind"] not in ("seconds", "date", "date-rfc850") + or metadata["kind"] == "overlong" and (seconds is None or seconds <= 30 or deadline != received + seconds) + or metadata["kind"] != "overlong" + and deadline != retry_after_not_before(RetryAfter(**metadata), received)): + raise fail("file-upload-evidence-invalid", "Retained backoff deadline contradicts original metadata.") + previous = digest(event) + return events + + def backoff_status(self): + events = self.backoff_events() + attempts = {name: value for name, value in self.records.items() + if re.fullmatch(r"[0-9]{4}-(retry-)?(attempt|result)\.json", name)} + if any(not isinstance(value, dict) for value in attempts.values()): + raise fail("file-upload-evidence-invalid", "Attempt/result evidence is malformed.") + # Legacy 429 or an interrupted receipt cannot prove what restriction was received. + unknown = "pending-request.json" in self.records or any( + name.endswith("result.json") and value.get("status") == 429 and "throttle_event" not in value + or name.endswith("attempt.json") and name.replace("attempt.json", "result.json") not in self.records + and name != self.active_attempt + for name, value in attempts.items() + ) + now = time.time() + if unknown or any(event["timing"]["not_before_utc"] is None for event in events.values()): + return {"status": "unresolved", "reason": "Original response timing is unavailable; no deadline is inferred."} + if not events: + return {"status": "clear"} + if (not valid_utc_timestamp(now) + or any(now < event["timing"]["received_at_utc"] for event in events.values())): + return {"status": "unresolved", "reason": "UTC clock is invalid or precedes retained response receipt."} + deadline = max(event["timing"]["not_before_utc"] for event in events.values()) + return {"status": "waiting" if now < deadline else "elapsed", "not_before_utc": deadline, + "seconds_remaining": max(0, deadline - now)} + + def require_backoff(self): + status = self.backoff_status() + if status["status"] in ("waiting", "unresolved"): + error = fail("file-upload-backoff-" + status["status"], + "Server backoff still applies; fresh approval does not waive it. " + + ("Wait until the retained UTC not-before time." if status["status"] == "waiting" + else "Original timing evidence or scoped operator recovery is required.")) + error.partial = "source-ack.json" in self.records + error.file_batch = _summary(self.ingestion, self.states(), self) + raise error + + def transport(self, raw): + if getattr(raw, "_file_journal_owner", None) is self: + return raw + + def invoke(method, url, token, **kwargs): + if self.network_failure is not None: + raise self.network_failure + self.require_backoff() + if len(self.backoff_events()) >= MAX_BACKOFF_RECORDS: + raise fail("file-upload-backoff-limit", "Backoff journal limit reached; no more requests.") + pending = {"version": "1.0", "plan_digest": digest(self.plan), "nonce": uuid.uuid4().hex, + "method": method, "url_digest": digest(url)} + self._write("pending-request.json", pending) + try: + response = raw(method, url, token, **kwargs) + except HelperFailure as error: + if error.http_status == 429: + self.record_backoff(method, url, error, None) + self.finish_request(error) + raise + if response.status == 429: + error = HelperFailure("azure-http-error", "Azure request failed with HTTP 429.", + blocked_at="execution", status=429, request_id=response.request_id, + retry_after=response.retry_after, recovery_deadline=response.recovery_deadline) + self.record_backoff(method, url, error, response.headers) + try: + self.finish_request() + except HelperFailure as persistence: + if method in ("PUT", "POST") and response.status in (200, 201): + return replace(response, ack_failure=persistence) + raise + return response + + invoke._file_journal_owner = self + return invoke + + def finish_request(self, original=None): + try: + path = self.directory / "pending-request.json" + expected = self.records["pending-request.json"] + retained = private_io.read_json(path) + if not isinstance(retained, dict) or retained.get("payload") != expected or retained.get("integrity") != digest(expected): + raise fail("file-upload-evidence-invalid", "Pending request receipt changed.") + path.unlink() + self.records.pop("pending-request.json") + except (OSError, HelperFailure) as cleanup: + error = original or HelperFailure("file-upload-receipt-failed", "Pending request evidence could not be finalized.", + blocked_at="local-persistence") + error.blocked_at = "local-persistence" + error.warnings.append("Pending request evidence remains unresolved; no further requests.") + self.network_failure = error + raise error from cleanup + + def record_backoff(self, method, url, error, headers): + metadata = error.retry_after + origin = "response-headers" if headers is not None else "native-http-error" + timing = retry_after_timing(headers) if headers is not None else error.retry_after_timing + if timing is None: + received = time.time() + received = received if valid_utc_timestamp(received) else None + known = isinstance(metadata, RetryAfter) and metadata.kind in ("seconds", "date", "date-rfc850") + timing = RetryAfterTiming(received, retry_after_not_before(metadata, received) if known else None, + metadata.value if known and metadata.kind == "seconds" else None) + origin = "typed-metadata" if known else "unavailable" + if not isinstance(metadata, RetryAfter): + metadata, origin = RetryAfter("missing"), "unavailable" + timing = RetryAfterTiming(timing.received_at_utc, None) + events = self.backoff_events() + attempt = self.active_attempt if method == "POST" and url == file_ingest._list_url(self.ingestion) else None + event = {"version": "1.0", "plan_digest": digest(self.plan), + "previous": digest(next(reversed(events.values()))) if events else None, + "method": method, "url_digest": digest(url), "attempt": attempt, + "attempt_digest": digest(self.records[attempt]) if attempt else None, + "request_id": ReadRecovery.safe_id(error.request_id) if error.request_id else None, + "metadata": metadata._asdict(), "timing": timing._asdict(), "origin": origin} + try: + self._write(f"backoff-{len(events):04}.json", event) + except HelperFailure as persistence: + error.blocked_at = "local-persistence" + error.warnings.extend(persistence.warnings) + error.warnings.append("Original HTTP 429 retained; backoff receipt persistence failed. No further requests.") + self.network_failure = error + raise error from persistence + + +def _file_id(value): + return isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}", value) is not None + + +def _proof(plan, record, item): + if isinstance(item, dict) and _file_id(item.get("fileId")) and file_ingest._matches(item, plan, record): + return {"file_id": item["fileId"], "record_digest": digest(record)} + return None + + +def _observe(plan, states, files): + names = {record["path"] for record in plan["files"]} + if any(item.get("fileName") not in names for item in files): + raise fail("server-inventory-conflict", "File inventory contains an unapproved identity.") + observed = {} + for index, record in enumerate(plan["files"]): + matches = [item for item in files if item.get("fileName") == record["path"]] + if len(matches) > 1 or matches and not file_ingest._matches({**matches[0], "errorMessage": None}, plan, record): + raise fail("file-record-conflict", "File inventory has duplicate or conflicting approved markers.") + state = states[index] + state["verification"] = {"accepted": "pending", "rejected": "failed"}.get(state["upload"], state["upload"]) + if matches: + item = matches[0] + proof = _proof(plan, record, item) + if (proof and state.get("file_proof") and proof != state["file_proof"] + or state.get("ack_file_id") and item.get("fileId") is not None + and item["fileId"] != state["ack_file_id"]): + raise fail("file-record-conflict", "Current File identity differs from the persisted upload ACK.") + state["verification"] = ("failed" if state["upload"] == "rejected" and state.get("status") != 429 + or item.get("errorMessage") is not None else + "confirmed" if proof else "unverified") + state["ingested"] = state["verification"] == "confirmed" + observed[index] = item + return observed + + +def _summary(plan, states, session): + counts = {name: 0 for name in ("accepted", "confirmed", "failed", "pending", "unverified", "not_attempted")} + files = [] + for record, state in zip(plan["files"], states): + counts["accepted"] += state["upload"] == "accepted" + verification = state.get("verification", {"accepted": "pending", "rejected": "failed"}.get(state["upload"], state["upload"])) + counts[verification] += 1 + files.append({"fileName": record["path"], "sha256": record["sha256"], "upload": state["upload"], + "verification": verification, "http_status": state.get("status"), + "request_ids": state["request_ids"][:3]}) + ingested = sum(bool(state.get("ingested")) for state in states) + return {"counts": counts, "files": files, "ingested": ingested, + "readiness": "ingested" if ingested == len(states) and counts["confirmed"] == len(states) else "partial", + "retrieval": "unverified", + **({"backoff": session.backoff_status()} if session else {}), + **({"upload_retry": { + "status": "blocked", "code": "file-upload-retry-safety-unproven", + "reason": "Unknown upload outcomes cannot be replayed; only a received File upload 429 permits one bounded in-operation retry.", + }} if counts["unverified"] else {}), + "resume": ("Plan newly approved upload-only continuation for never-attempted files; uncertain attempts are never replayed." + if session else "Original durable ACK/attempt evidence was not retained; uploads cannot safely resume from GET alone."), + "subset_handoff": "Report the proved ingested subset and remaining outcomes; subset isolation and KB retrieval are separate. KB changes need separate approval."} + + +def _progress(progress, plan, states): + summary = _summary(plan, states, None) + counts = summary["counts"] + progress.update("file-upload", uploads_acknowledged=counts["accepted"], files_verified=counts["confirmed"], + files_ingested=summary["ingested"], + files_reused=sum(state["upload"] == "not_attempted" and state.get("verification") == "confirmed" for state in states), + files_failed=counts["failed"], files_unverified=counts["unverified"], + files_not_attempted=counts["not_attempted"], files_pending=counts["pending"]) + + +def run_batch(document, *, token_provider, transport, progress, allow_new_uploads=False, session=None, + eligible=None, source_check=None, allow_upload_retry=True): + plan, fingerprint = document["plan"], document["_computed_fingerprint"] + root, records = file_ingest._validate_plan(plan) + states = session.states() if session else [{"upload": "not_attempted", "request_ids": []} for _ in records] + if session: + session.require_backoff() + transport = session.transport(transport) + created, reused, request_ids, warnings = [], [], [], [] + primary = None + stopped = False + retry_used = False + readback_failed = False + token = token_provider(SEARCH_AUDIENCE) + url = file_ingest._list_url(plan) + recovery = ReadRecovery(on_wait=progress.waiting) + progress.update("file-inventory") + try: + before, ids = file_ingest._list_files(url, token, transport=transport, recovery=recovery) + request_ids.extend(ids) + if session is None and file_ingest.inventory_digest(before) != plan["expected_server_inventory_digest"]: + raise fail("server-inventory-drift", "Approved initial file inventory changed.") + observed = _observe(plan, states, before) + if not allow_new_uploads and len(observed) != len(records): + raise fail("reused-source-upload-forbidden", "Generic reuse cannot authorize new uploads.") + except HelperFailure as failure: + failure.file_batch = _summary(plan, states, session) + failure.file_batch["request_ids"] = recovery.request_ids + raise + warnings.extend(recovery.warnings) + eligible = set(range(len(records))) if eligible is None else set(eligible) + after = None + for index, record in enumerate(records): + _progress(progress, plan, states) + state = states[index] + if index in observed: + reused.append({"fileId": observed[index].get("fileId"), "fileName": record["path"], "sha256": record["sha256"]}) + continue + if index not in eligible or state["upload"] != "not_attempted": + continue + try: + path = file_ingest._resolve_file(root, record) + content = path.read_bytes() + if (len(content) != record["size"] or "sha256:" + hashlib.sha256(content).hexdigest() != record["sha256"] + or path.stat().st_mtime_ns != record["mtime_ns"]): + raise fail("inventory-drift", "Approved corpus changed before upload.") + except OSError: + primary = primary or fail("inventory-unreadable", "Approved corpus became inaccessible.") + warnings.append("Batch stopped: approved corpus became inaccessible.") + stopped = True + break + except HelperFailure as failure: + primary = primary or failure + warnings.append(f"Batch stopped ({failure.code}); no further uploads were attempted.") + stopped = True + break + body, boundary = file_ingest._multipart(plan, record, content, fingerprint) + for attempt in range(2): + try: + if attempt: + file_ingest._resolve_file(root, record) + if session: + session.attempt(index, retry=bool(attempt)) + except HelperFailure as failure: + primary = primary or failure + warnings.append(f"Upload attempt evidence/drift check failed ({failure.code}); no further requests.") + stopped = True + break + state.update(upload="unverified", verification="unverified", status=None) + _progress(progress, plan, states) + response = None + transport_failed = False + try: + with progress.processing_file(index + 1, len(records), attempt + 1): + response = transport("POST", url, token, body=body, + headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, + follow_redirects=False, timeout=UPLOAD_TIMEOUT, + response_deadline=time.monotonic() + UPLOAD_TIMEOUT, + max_response_bytes=1024 * 1024) + except HelperFailure as caught: + transport_failed = True + failure = caught + failure_status, failure_id = failure.http_status, failure.request_id + else: + failure_status, failure_id = response.status, response.request_id + failure = HelperFailure("upload-failed", f"Upload returned HTTP {response.status}.", + blocked_at="execution", status=response.status, request_id=response.request_id, + retry_after=response.retry_after, recovery_deadline=response.recovery_deadline) + if failure_id: + safe_id = ReadRecovery.safe_id(failure_id) + failure.request_id = safe_id + state["request_ids"].append(safe_id) + request_ids.append(safe_id) + state["status"] = failure_status + proof = None + if failure_status in (200, 201): + state.update(upload="accepted", verification="pending") + created.append({"fileName": record["path"], "sha256": record["sha256"]}) + if response is not None: + proof = _proof(plan, record, response.body) + if isinstance(response.body, dict) and _file_id(response.body.get("fileId")): + state["ack_file_id"] = response.body["fileId"] + elif _rejected(failure_status): + state.update(upload="rejected", verification="failed") + try: + if response is not None and response.ack_failure is not None: + raise response.ack_failure + if session: + session.result(index, state["upload"], failure_status, failure_id, retry=bool(attempt), + proof=proof, file_id=state.get("ack_file_id")) + except HelperFailure as persistence: + primary = primary or (failure if failure_status not in (200, 201) else persistence) + primary.warnings.extend(persistence.warnings) + warnings.append("Upload journal persistence failed; no further requests were issued.") + stopped = True + break + if failure_status in (200, 201) and not transport_failed: + if proof: + state.update(file_proof=proof, ingested=True, verification="confirmed") + elif isinstance(response.body, dict) and ( + response.body.get("errorMessage") is not None + or "fileName" in response.body and not file_ingest._matches(response.body, plan, record) + ): + primary = primary or fail("upload-metadata-unverified", "Upload ACK metadata conflicts with the approved file.") + stopped = True + _progress(progress, plan, states) + break + primary = primary or failure + if failure_status == 415: + break + _progress(progress, plan, states) + if failure_status == 429 and not retry_used and allow_upload_retry and failure.blocked_at != "local-persistence": + retry_used = True + recovery = ReadRecovery(on_wait=progress.waiting) + try: + recovery.delay(failure) + if source_check is not None: + source_check(recovery, token) + inventory, _ = file_ingest._list_files(url, token, transport=transport, recovery=recovery) + seen = _observe(plan, states, inventory) + if index in seen: + if state["verification"] != "confirmed": + raise fail("file-upload-retry-conflict", "Existing file does not prove the approved completed upload.") + created.append({"fileName": record["path"], "sha256": record["sha256"]}) + break + file_ingest._resolve_file(root, record) + if time.monotonic() >= recovery.deadline: + raise fail("read-recovery-budget-exhausted", "Retry preflight exceeded its complete read budget.") + warnings.append("File upload HTTP 429: one same-operation retry after bounded backoff; no source recreation.") + except HelperFailure as read_failure: + warnings.append(f"File retry stopped ({read_failure.code}; HTTP {read_failure.http_status}).") + warnings.extend(read_failure.warnings) + stopped = True + finally: + request_ids.extend(recovery.request_ids) + warnings.extend(recovery.diagnostics()) + if not stopped: + continue + else: + stopped = True + warnings.append(f"Batch stopped ({failure.code}; HTTP {failure_status}); no further uploads.") + if failure.blocked_at != "local-persistence" and ( + failure_status in (408, 409) or failure_status is None or failure_status >= 500 + ): + recovery = ReadRecovery(on_wait=progress.waiting) + try: + after, _ = file_ingest._list_files(url, token, transport=transport, recovery=recovery) + _observe(plan, states, after) + if state["verification"] == "confirmed": + created.append({"fileName": record["path"], "sha256": record["sha256"]}) + except HelperFailure as read_failure: + warnings.append(f"File readback stopped ({read_failure.code}; HTTP {read_failure.http_status}).") + warnings.extend(read_failure.warnings) + request_ids.extend(recovery.request_ids) + warnings.extend(recovery.diagnostics()) + break + if stopped: + break + if session: + session.active_attempt = None + _progress(progress, plan, states) + progress.update("file-readback") + if all(state["upload"] == "rejected" and state["verification"] != "confirmed" for state in states): + stopped = True + if after is None and not stopped: + recovery = ReadRecovery(on_wait=progress.waiting) + try: + after, ids = file_ingest._list_files(url, token, transport=transport, recovery=recovery) + _observe(plan, states, after) + except HelperFailure as failure: + primary = primary or failure + readback_failed = True + warnings.append(f"File readback stopped ({failure.code}; HTTP {failure.http_status}).") + request_ids.extend(recovery.request_ids) + warnings.extend(recovery.diagnostics()) + batch = _summary(plan, states, session) + batch["request_ids"] = request_ids[:804] + batch["upload_retry_used"] = retry_used + progress.update("file-readback", files_verified=batch["counts"]["confirmed"], files_failed=batch["counts"]["failed"], + files_pending=batch["counts"]["pending"], files_unverified=batch["counts"]["unverified"], + files_not_attempted=batch["counts"]["not_attempted"], files_ingested=batch["ingested"]) + if batch["counts"]["confirmed"] != len(records) or readback_failed: + failure = primary or fail("readback-mismatch", "Some approved files remain pending, failed or unverified.") + uncertain = [{"action": "upload-unverified", "type": "knowledge-source-file", + "fileName": record["path"], "sha256": record["sha256"]} + for record, state in zip(records, states) + if state["upload"] == "unverified" and state["verification"] != "confirmed"] + failure.writes = created + uncertain + failure.writes + failure.resources_remaining = file_ingest._remaining_files(created) + uncertain + failure.resources_remaining + failure.partial = bool(created) or any(s["upload"] == "unverified" for s in states) or ( + failure.partial and not _rejected(failure.http_status)) + failure.warnings.extend(warnings) + failure.file_batch = batch + raise failure + verified = [{"fileId": item.get("fileId"), "fileName": record["path"], "sha256": record["sha256"], "size": record["size"]} + for record in records for item in after or before if item.get("fileName") == record["path"]] + return {"status": "completed", "outcome": plan.get("outcome", "file-knowledge-source-ingestion"), + "approved_plan": {"fingerprint": fingerprint, "confirmed": True}, + "resources": {"created": created, "reused": reused, "updated": [], "skipped": []}, + "api_contracts": [{"operation": "upload-file", "version": file_ingest.API_VERSION, "preview": True}], + "data_movement": {"boundary": {"local_root_digest": digest(str(root))}, "result": "Approved direct File uploads"}, + "auth": {"mode": "entra-user", "principals": []}, "rbac": plan.get("rbac", {"assignments": []}), + "network": plan.get("network", {"posture": "preserved", "evidence": None}), + "verification": {"readback": verified, "request_ids": request_ids, "server_inventory_digest": file_ingest.inventory_digest(after or before), + "idempotency": "Only a received File upload 429 permits one bounded same-operation retry; unknown outcomes are never replayed."}, + "warnings": warnings, "file_batch": batch, + "ownership": {"run_owned": created, "reused_not_owned": reused, "owner": plan["owner"]}, + "cleanup": {"status": "not-requested", "separate_confirmation_required": True}} + + +def _source_check(session, token, transport, recovery): + try: + from . import source_vector + except ImportError: + import source_vector + plan = session.plan + file_ingest._validate_plan(session.ingestion) + etag = session.require_ack() + current, _ = search_reconcile._get(search_reconcile.resource_url(plan["source"]), token, + transport=source_vector.guard_readback_transport(plan, transport), recovery=recovery) + if (current is None or current.get("@odata.etag") != etag + or not search_reconcile.definitions_match(plan["source"]["desired"], current)): + raise fail("file-upload-source-drift", "Current source does not match original acknowledged identity/version/definition.") + return current + + +def _verify(session, token_provider, transport, context_provider): + try: + from . import file_source, file_cu_mi + except ImportError: + import file_source, file_cu_mi + session.require_ack() + session.states() + session.require_backoff() + transport = session.transport(transport) + context = context_provider() + cu_ingestion_auth.validate_context(context) + if context != session.seed["context"]: + raise fail("file-upload-context-drift", "Original tenant/subscription/principal changed.") + plan = session.plan + if "content_understanding" in plan: + state, _ = file_source.read_content_understanding(plan["content_understanding"], token_provider=token_provider, transport=transport) + if not file_source._cu_states_match(state, plan["cu_resource_state"]): + raise fail("file-upload-auth-drift", "Original CU account/auth/network state changed.") + if plan["source"].get("ai_services_managed_identity"): + binding, _ = file_cu_mi.read_binding(plan["content_understanding"], plan["source"]["endpoint"], + token_provider=token_provider, transport=transport) + if binding != plan["cu_identity_state"]: + raise fail("file-upload-auth-drift", "Original Search MI/CU role/network binding changed.") + return _source_check(session, token_provider(SEARCH_AUDIENCE), transport, ReadRecovery()) + + +def plan_resume(request, *, token_provider=azure_cli_token, transport=http_request, + context_provider=cu_ingestion_auth.account_context): + if not isinstance(request, dict) or not isinstance(request.get("receipt_directory"), str): + raise fail("file-upload-input-invalid", "Use an object with the original private receipt directory.") + require_allowed_fields(request, {"schema_version", "receipt_directory"}, label="upload resume request") + if request.get("schema_version") != "1.0": + raise fail("file-upload-input-invalid", "Use the supported resume request schema.") + session = Session(request.get("receipt_directory")) + transport = session.transport(transport) + _verify(session, token_provider, transport, context_provider) + states = session.states() + inventory, _ = file_ingest._list_files(file_ingest._list_url(session.ingestion), token_provider(SEARCH_AUDIENCE), + transport=transport, recovery=ReadRecovery()) + observed = _observe(session.ingestion, states, inventory) + eligible = [i for i, state in enumerate(states) if state["upload"] == "not_attempted" and i not in observed] + plan = {"operation": "resume-file-uploads", "version": "1.0", "receipt_directory": str(session.directory), + "original_plan_digest": digest(session.plan), "creation_ack_digest": digest(session.records["source-ack.json"]), + "journal_digest": digest(session.records), "eligible": eligible, "owner": session.plan["owner"], "cleanup_approved": False} + return {"status": "planned", "execution_input": {"schema_version": "1.0", "plan": plan, + "approval": {"confirmed": False, "fingerprint": digest(plan)}}, + "approval_summary": {"execution_required": bool(eligible), "mutation_approval_required": bool(eligible), "uploads": len(eligible)}, + "file_batch": _summary(session.ingestion, states, session), "writes_performed": [], + "next_step": "Approve only never-attempted uploads. No eligible files: retain this observation; do not replay uncertain files."} + + +@reporting("file-source") +def execute(document, *, token_provider=azure_cli_token, transport=http_request, + context_provider=cu_ingestion_auth.account_context, progress=None): + if not isinstance(document, dict) or not isinstance(document.get("plan"), dict): + raise fail("file-upload-input-invalid", "Use the newly approved upload-only envelope.") + require_allowed_fields(document, {"schema_version", "plan", "approval", "_computed_fingerprint"}, label="upload resume envelope") + plan = document.get("plan") + reject_secrets(document) + require_allowed_fields(plan, {"operation", "version", "receipt_directory", "original_plan_digest", "creation_ack_digest", + "journal_digest", "eligible", "owner", "cleanup_approved"}, label="upload resume plan") + if (document.get("schema_version") != "1.0" or plan.get("operation") != "resume-file-uploads" or plan.get("version") != "1.0" + or set(plan) != {"operation", "version", "receipt_directory", "original_plan_digest", "creation_ack_digest", + "journal_digest", "eligible", "owner", "cleanup_approved"} + or not isinstance(document.get("approval"), dict) or document["approval"].get("confirmed") is not True + or plan.get("cleanup_approved") is not False or document.get("approval") != {"confirmed": True, "fingerprint": digest(plan)} + or document.get("_computed_fingerprint") != digest(plan)): + raise fail("file-upload-approval-missing", "A newly approved unchanged upload-only plan is required.") + session = Session(plan.get("receipt_directory")) + transport = session.transport(transport) + if (plan["original_plan_digest"] != digest(session.plan) or plan["journal_digest"] != digest(session.records) + or plan["creation_ack_digest"] != digest(session.records["source-ack.json"]) or plan["owner"] != session.plan["owner"] + or not isinstance(plan["eligible"], list) or any(type(i) is not int or not 0 <= i < len(session.ingestion["files"]) for i in plan["eligible"]) + or len(set(plan["eligible"])) != len(plan["eligible"])): + raise fail("file-upload-plan-drift", "Original approval/ACK/journal changed; refresh the upload-only plan.") + states = session.states() + if any(states[i]["upload"] != "not_attempted" for i in plan["eligible"]): + raise fail("file-upload-replay-forbidden", "Attempted files cannot be submitted again, even after an empty inventory.") + progress.update("source-reconciliation") + try: + _verify(session, token_provider, transport, context_provider) + result = run_batch({"plan": session.ingestion, "_computed_fingerprint": digest(session.plan)}, token_provider=token_provider, + transport=transport, progress=progress, allow_new_uploads=True, session=session, eligible=plan["eligible"], + source_check=lambda recovery, token: _source_check(session, token, transport, recovery)) + except HelperFailure as failure: + if failure.file_batch is None: + failure.file_batch = _summary(session.ingestion, states, session) + failure.partial = True + failure.resources_remaining.insert(0, {"type": "knowledge-source", "name": session.plan["source"]["name"]}) + raise + result["approved_plan"] = document["approval"] + result["original_run"] = {"plan_digest": digest(session.plan), "creation_ack_digest": plan["creation_ack_digest"], + "source_retained": session.plan["source"]["name"]} + return result + + +def main(argv=None): + try: + from .private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode + except ImportError: + from private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode + parser = argparse.ArgumentParser() + modes = parser.add_mutually_exclusive_group(required=True) + modes.add_argument("--plan", type=Path) + modes.add_argument("--input", type=Path) + add_execution_output_argument(parser) + add_progress_argument(parser) + args = parser.parse_args(argv) + fingerprint = None + try: + validate_execution_output_mode(args) + if args.plan: + emit_plan_result(plan_resume(private_io.read_json(args.plan)), args.execution_output) + return 0 + document, _, fingerprint = load_approved_input(args.input) + document["_computed_fingerprint"] = fingerprint + result = execute(document, progress=Progress("file-source", enabled=args.progress)) + except HelperFailure as failure: + result = blocked_result(failure, outcome="resume-file-uploads", fingerprint=fingerprint) + result["safe_next_decision"] = "Retain the source. Missing/changed original evidence blocks upload continuation, not retention; no creation replay or cleanup workaround." + emit_result(result) + return 3 if result["status"] == "partial" else 2 + emit_result(result) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/hosted_connect.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/hosted_connect.py index 524e3306..9d5be754 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/hosted_connect.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/hosted_connect.py @@ -1,488 +1,488 @@ -"""GET-only assessment of an existing Hosted agent's toolbox connection. - -Promotion is deliberately unavailable until a conditional shared-default update -contract is verified. No intermediate resources are created while that gate blocks. -""" -from __future__ import annotations - -import argparse -import re -import sys -from pathlib import Path -from urllib.parse import parse_qs, quote, unquote, urlsplit - -try: - from . import _prompt_read as read - from ._bootstrap_io import read_json, run_cli - from ._common import ( - MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, HelperFailure, azure_cli_token, - blocked_result, digest, emit_result, http_request, reject_secrets, - ) - from .prompt_connect import _project_identity, _project_endpoint, PROJECT_ID -except ImportError: - import _prompt_read as read - from _bootstrap_io import read_json, run_cli - from _common import ( - MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, HelperFailure, azure_cli_token, - blocked_result, digest, emit_result, http_request, reject_secrets, - ) - from prompt_connect import _project_identity, _project_endpoint, PROJECT_ID - -AI_AUDIENCE = "https://ai.azure.com" -FOUNDRY_USER = "53ca6127-db72-4b80-b1b0-d745d6d5456d" -OUTCOME = "assess-existing-hosted-toolbox" -FIELDS = { - "schema_version", "scope", "project", "search_service", "knowledge_base", - "agent_name", "agent_version", "toolbox_name", "tool_label", "connection_name", - "reader_assignment_id", "project_assignment_id", "retention_owner", -} -OPTIONAL = {"known_agents", "supported_question", "unrelated_question"} - - -class ProjectSelectionFailure(HelperFailure): - def __init__(self, candidates): - super().__init__( - "project-ambiguous" if candidates else "project-absent", - "Scoped project name did not resolve uniquely; select an exact account/project without widening scope.", - blocked_at="input-resolution", - ) - self.candidates = candidates - - -def fail(code, message, **metadata): - return read.fail(code, message, **metadata) - - -def name(value): - return isinstance(value, str) and read.NAME.fullmatch(value) is not None - - -def url(value): - try: - parsed = urlsplit(value) - if (parsed.scheme != "https" or not parsed.hostname or parsed.username - or parsed.password or parsed.port not in (None, 443) or parsed.fragment): - raise ValueError() - return parsed - except ValueError as error: - raise fail("selection-invalid", "Use an exact public-cloud HTTPS endpoint without credentials or fragments.") from error - - -def validate(request): - if not isinstance(request, dict): - raise fail("input-schema-invalid", "Hosted assessment requires a resolved intent object.") - reject_secrets(request) - if set(request) - FIELDS - OPTIONAL or not FIELDS <= set(request) or request["schema_version"] != "1.0": - raise fail("input-schema-invalid", "Use only the documented Hosted assessment decisions.") - scope = request["scope"] - if (not isinstance(scope, dict) or not {"subscription_id", "resource_group"} <= set(scope) - or set(scope) - {"subscription_id", "resource_group", "account_name"} - or any(not name(value) for value in scope.values())): - raise fail("scope-invalid", "Select one subscription/resource group and optionally one account; no automatic widening.") - for key in ("agent_name", "agent_version", "toolbox_name", "tool_label", "connection_name"): - if not name(request[key]): - raise fail("input-schema-invalid", "Select bounded exact agent/version, toolbox and connection names.") - if request["agent_version"].casefold() in {"latest", "default"}: - raise fail("agent-version-unresolved", "Pin the observed agent version, never latest.") - for key in ("project", "search_service", "knowledge_base"): - value = request[key] - if not isinstance(value, str) or not 1 <= len(value) <= 2048: - raise fail("selection-invalid", "Supply a name, exact resource ID or documented endpoint.") - if value.startswith("https:"): - parsed = url(value) - if key == "project": - _project_endpoint(value) - if (not name(parsed.hostname.removesuffix(".services.ai.azure.com")) - or not name(unquote(parsed.path.rstrip("/").rsplit("/", 1)[-1]))): - raise fail("selection-invalid", "The project endpoint must identify a valid account/project name.") - elif key == "search_service": - if (not parsed.hostname.endswith(".search.windows.net") or parsed.path not in ("", "/") - or parsed.query): - raise fail("selection-invalid", "Select one Search service endpoint, not an index.") - elif (not parsed.hostname.endswith(".search.windows.net") - or re.fullmatch(r"/knowledgebases/[A-Za-z0-9_.-]+/mcp", parsed.path) is None - or parse_qs(parsed.query) != {"api-version": ["2026-08-01-preview"]}): - raise fail("selection-invalid", "Select an exact preview KB MCP endpoint.") - elif value.startswith("/"): - pattern = PROJECT_ID if key == "project" else read.SEARCH_ID if key == "search_service" else None - if pattern is None or pattern.fullmatch(value) is None: - raise fail("selection-invalid", "The supplied resource ID does not identify the selected resource kind.") - elif not name(value): - raise fail("selection-invalid", "The selected name is malformed.") - if key == "search_service" and not value.startswith("/"): - service = url(value).hostname.removesuffix(".search.windows.net") if value.startswith("https:") else value - resource_id = (f"/subscriptions/{scope['subscription_id']}/resourceGroups/{scope['resource_group']}" - "/providers/Microsoft.Search/searchServices/" + service) - if read.SEARCH_ID.fullmatch(resource_id) is None: - raise fail("selection-invalid", "Use a valid Search service name, not a display label or index.") - for key in ("reader_assignment_id", "project_assignment_id"): - value = request[key] - if (not isinstance(value, str) or len(value) > 2048 - or re.fullmatch(r"/subscriptions/[^/?#\s]+/resourceGroups/[^?#\s]+/providers/" - r"Microsoft.Authorization/roleAssignments/[0-9a-fA-F-]+", value, re.I) is None - or not read.GUID.fullmatch(value.rsplit("/", 1)[-1])): - raise fail("role-selection-invalid", "Select exact resource-scoped role-assignment IDs, not principals to invent.") - if (not isinstance(request["retention_owner"], str) or not request["retention_owner"].strip() - or len(request["retention_owner"]) > 256): - raise fail("owner-unresolved", "Name the retained connection/toolbox-version owner; cleanup is unsupported.") - agents = request.get("known_agents", []) - if (not isinstance(agents, list) or len(agents) > 20 - or any(not isinstance(item, dict) or set(item) != {"name", "version"} - or not name(item["name"]) or not name(item["version"]) - or item["version"].casefold() in {"latest", "default"} for item in agents)): - raise fail("known-bindings-invalid", "Supply at most 20 known exact agent/version bindings; no exclusive-consumer inference.") - for key in ("supported_question", "unrelated_question"): - if key in request and (not isinstance(request[key], str) or not request[key].strip() or len(request[key]) > 4096): - raise fail("acceptance-invalid", "Questions are optional resolved candidates, never implicit invocation approval.") - - -class Reads: - def __init__(self, token_provider, transport): - self.token_provider, self.transport = token_provider, transport - self.tokens, self.request_ids = {}, [] - self.last_request_id = None - - def get(self, endpoint, audience, *, absent=False, label="resource"): - if audience not in self.tokens: - self.tokens[audience] = self.token_provider(audience) - value, ids = read.get_object(endpoint, self.tokens[audience], self.transport, absent=absent, label=label) - self.request_ids.extend(ids) - self.last_request_id = ids[-1] if ids else None - return value - - def collection(self, endpoint): - initial = urlsplit(endpoint) - result, seen = [], set() - while endpoint: - current = url(endpoint) - if (endpoint in seen or len(seen) >= 20 or current.netloc != initial.netloc - or current.path != initial.path - or parse_qs(current.query).get("api-version") != parse_qs(initial.query).get("api-version")): - raise fail("inventory-unverified", "Scoped inventory continuation is unsafe or exceeds 20 pages.") - seen.add(endpoint) - page = self.get(endpoint, MANAGEMENT_AUDIENCE, label="scoped-inventory") - items = page.get("value") - if not isinstance(items, list) or any(not isinstance(item, dict) for item in items): - raise fail("inventory-unverified", "Scoped inventory is malformed; no inferred absence.") - result.extend(items) - if len(result) > 100: - raise fail("inventory-limit", "Scoped inventory exceeds 100 resources; select an exact parent instead.") - endpoint = page.get("nextLink") - if endpoint is not None and not isinstance(endpoint, str): - raise fail("inventory-unverified", "Scoped continuation must be a URL.") - return result - - -def resolve(request, reads): - scope = request["scope"] - prefix = f"/subscriptions/{scope['subscription_id']}/resourceGroups/{scope['resource_group']}" - account_prefix = prefix + "/providers/Microsoft.CognitiveServices/accounts/" - selection = request["project"] - if selection.startswith("/"): - project_id = selection - elif selection.startswith("https:"): - parsed = url(selection) - project_id = account_prefix + parsed.hostname.removesuffix(".services.ai.azure.com") + "/projects/" + unquote(parsed.path.rstrip("/").rsplit("/", 1)[-1]) - elif scope.get("account_name"): - project_id = account_prefix + scope["account_name"] + "/projects/" + selection - else: - accounts = reads.collection(MANAGEMENT_AUDIENCE + account_prefix.rstrip("/") + "?api-version=2025-06-01") - if len(accounts) > 20: - raise fail("account-selection-required", "Select one account; do not enumerate more than 20 accounts for a project name.") - matches, seen = [], set() - for account in accounts: - account_id = account.get("id", "") - if (not isinstance(account_id, str) or not account_id.casefold().startswith(account_prefix.casefold()) - or not name(account_id[len(account_prefix):]) or account_id.casefold() in seen): - raise fail("inventory-unverified", "Account inventory contains a duplicate or out-of-scope resource.") - seen.add(account_id.casefold()) - projects_seen = set() - for project in reads.collection(MANAGEMENT_AUDIENCE + account_id + "/projects?api-version=" + read.PROJECT_API): - candidate = project.get("id", "") - if (not isinstance(candidate, str) or PROJECT_ID.fullmatch(candidate) is None - or not candidate.casefold().startswith((account_id + "/projects/").casefold()) - or candidate.casefold() in projects_seen): - raise fail("inventory-unverified", "Project inventory contains an unverified identity.") - projects_seen.add(candidate.casefold()) - if candidate.rsplit("/", 1)[-1].casefold() == selection.casefold(): - matches.append(candidate) - if len(matches) != 1: - raise ProjectSelectionFailure(matches) - project_id = matches[0] - match = PROJECT_ID.fullmatch(project_id) - if match is None: - raise fail("selection-invalid", "The resolved project resource identity is malformed.") - endpoint = f"https://{match['account']}.services.ai.azure.com/api/projects/{quote(match['project'], safe='')}" - _project_identity({"project_resource_id": project_id, "project_endpoint": endpoint}) - selection = request["search_service"] - if selection.startswith("/"): - search_id = selection - else: - service = url(selection).hostname.removesuffix(".search.windows.net") if selection.startswith("https:") else selection - search_id = prefix + "/providers/Microsoft.Search/searchServices/" + service - match = read.SEARCH_ID.fullmatch(search_id) - if match is None: - raise fail("selection-invalid", "The selected Search service identity is invalid.") - search_endpoint = "https://" + match["name"].lower() + ".search.windows.net" - selection = request["knowledge_base"] - if selection.startswith("https:"): - parsed = url(selection) - if parsed.hostname != urlsplit(search_endpoint).hostname: - raise fail("kb-binding-invalid", "The KB endpoint belongs to another selected Search service.") - kb_name = parsed.path.split("/")[2] - else: - kb_name = selection - return project_id, endpoint, search_id, search_endpoint, kb_name - - -def role(reads, assignment_id, scope, principal, role_id): - prefix = scope + "/providers/Microsoft.Authorization/roleAssignments/" - if (not assignment_id.casefold().startswith(prefix.casefold()) - or not read.GUID.fullmatch(assignment_id[len(prefix):])): - raise fail("role-scope-invalid", "Role assignment must use the exact selected resource scope.") - value = reads.get(MANAGEMENT_AUDIENCE + assignment_id + "?api-version=" + read.ROLE_API, - MANAGEMENT_AUDIENCE, label="role-assignment") - properties = value.get("properties") - definitions = { - "/providers/microsoft.authorization/roledefinitions/" + role_id, - "/subscriptions/" + scope.split("/")[2].lower() + "/providers/microsoft.authorization/roledefinitions/" + role_id, - } - if (str(value.get("id", "")).casefold() != assignment_id.casefold() or not isinstance(properties, dict) - or str(properties.get("principalId", "")).lower() != principal - or str(properties.get("scope", "")).casefold() != scope.casefold() - or str(properties.get("roleDefinitionId", "")).casefold() not in definitions - or properties.get("principalType", "ServicePrincipal") != "ServicePrincipal" - or properties.get("condition") not in (None, "")): - raise fail("hosted-runtime-role-unverified", "Require exact-scope grants to the published Hosted principal, not the project or blueprint.") - return value - - -def agent(reads, endpoint, agent_name, version): - value = reads.get(endpoint + f"/agents/{quote(agent_name, safe='')}/versions/{quote(version, safe='')}?api-version=v1", - AI_AUDIENCE, label="hosted-agent") - if value.get("name") != agent_name or str(value.get("version")) != version or not isinstance(value.get("definition"), dict): - raise fail("agent-identity-unverified", "Read the exact observed agent/version; never select latest.") - return value - - -def toolbox_binding(environment, endpoint, toolbox_name): - """Match the first-party FoundryToolbox environment resolver, not arbitrary code.""" - if not isinstance(environment, dict): - return None - consumer = endpoint + "/toolboxes/" + toolbox_name + "/mcp?api-version=v1" - if "TOOLBOX_ENDPOINT" in environment: - return "endpoint" if environment["TOOLBOX_ENDPOINT"] == consumer else None - project = environment.get("FOUNDRY_PROJECT_ENDPOINT") - if (isinstance(project, str) and project.rstrip("/") == endpoint - and environment.get("TOOLBOX_NAME") == toolbox_name): - return "name" - return None - - -def assess(request, *, token_provider=azure_cli_token, transport=http_request, cli=run_cli): - validate(request) - scope = request["scope"] - selected_project = request["project"] - context_id = selected_project if selected_project.startswith("/") else "/subscriptions/" + scope["subscription_id"] - context = read.cli_context(context_id, cli=cli) - reads = Reads(token_provider, transport) - project_id, endpoint, search_id, search_endpoint, kb_name = resolve(request, reads) - project = reads.get(MANAGEMENT_AUDIENCE + project_id + "?api-version=" + read.PROJECT_API, MANAGEMENT_AUDIENCE, label="project") - properties = project.get("properties", {}) - endpoints = properties.get("endpoints") if isinstance(properties, dict) else None - if (str(project.get("id", "")).casefold() != project_id.casefold() or not isinstance(properties, dict) - or str(properties.get("provisioningState", "")).lower() != "succeeded" - or not isinstance(endpoints, dict) or endpoint not in endpoints.values()): - raise fail("project-unverified", "The exact ready project endpoint must be confirmed remotely.") - selected = agent(reads, endpoint, request["agent_name"], request["agent_version"]) - definition, identity = selected["definition"], selected.get("instance_identity") - other_principals = [ - value.get("principalId") or value.get("principal_id") - for value in (project.get("identity"), selected.get("blueprint")) - if isinstance(value, dict) - ] - if (definition.get("kind") != "hosted" or selected.get("status") != "active" - or not isinstance(identity, dict) or identity.get("status") != "active" - or not isinstance(identity.get("principal_id"), str) or not read.GUID.fullmatch(identity["principal_id"]) - or not isinstance(identity.get("client_id"), str) or not read.GUID.fullmatch(identity["client_id"]) - or identity["principal_id"].casefold() in { - value.casefold() for value in other_principals if isinstance(value, str) - }): - raise fail("hosted-principal-unverified", "Require an active published Hosted agent and observed instance principal.") - principal = identity["principal_id"].lower() - consumer = endpoint + "/toolboxes/" + request["toolbox_name"] + "/mcp?api-version=v1" - environment = definition.get("environment_variables") - binding_mode = toolbox_binding(environment, endpoint, request["toolbox_name"]) - if binding_mode is None: - raise fail("hosted-runtime-change-required", "Observed settings do not establish this unversioned FoundryToolbox binding. The runtime owner must verify custom/overridden behavior or the actual configuration change before requesting source; a version-pinned developer endpoint is not equivalent.") - search = reads.get(MANAGEMENT_AUDIENCE + search_id + "?api-version=" + read.SEARCH_API, MANAGEMENT_AUDIENCE, label="search") - properties = search.get("properties") - if (str(search.get("id", "")).casefold() != search_id.casefold() or not isinstance(properties, dict) - or str(properties.get("status", "")).lower() not in {"running", "provisioning", "degraded"} - or str(properties.get("provisioningState", "")).lower() not in {"succeeded", "provisioning"}): - raise fail("search-operation-blocked", "Search failed/deleting/disabled/unresolved state blocks this assessment.") - kb = reads.get(search_endpoint + "/knowledgebases('" + quote(kb_name, safe="") + "')?api-version=2026-08-01-preview", - SEARCH_AUDIENCE, absent=True, label="knowledge-base") - if kb is None: - raise fail("knowledge-base-absent", "The exact KB is absent; Search indexes are not KB evidence.", - status=404, request_id=reads.last_request_id) - _, profile = read.kb_state(kb, kb_name) - reader = role(reads, request["reader_assignment_id"], search_id, principal, read.READER_ROLE) - project_role = role(reads, request["project_assignment_id"], project_id, principal, FOUNDRY_USER) - toolbox_url = endpoint + "/toolboxes/" + request["toolbox_name"] - toolbox = reads.get(toolbox_url + "?api-version=v1", AI_AUDIENCE, label="toolbox") - default = toolbox.get("default_version") - if (toolbox.get("name") != request["toolbox_name"] or not name(default) - or default.casefold() in {"latest", "default"}): - raise fail("toolbox-default-unverified", "Read the exact toolbox and its current default version.") - version = reads.get(toolbox_url + "/versions/" + default + "?api-version=v1", AI_AUDIENCE, label="toolbox-version") - tools = version.get("tools") - if (version.get("name") != request["toolbox_name"] or str(version.get("version")) != default - or not isinstance(tools, list) or len(tools) > 200 or any(not isinstance(tool, dict) for tool in tools)): - raise fail("toolbox-version-unverified", "The default immutable version and its complete tool list must be verified.") - matching = [tool for tool in tools if tool.get("server_label") == request["tool_label"]] - if len(matching) > 1: - raise fail("toolbox-binding-ambiguous", "Multiple selected KB tool labels require explicit reconciliation.") - if matching: - tool = matching[0] - allowed = tool.get("allowed_tools") or [] - allowed = allowed.get("tool_names", []) if isinstance(allowed, dict) else allowed - if (tool.get("type") != "mcp" or not isinstance(allowed, list) - or any(not isinstance(item, str) for item in allowed) - or not isinstance(tool.get("headers") or {}, dict) - or not isinstance(tool.get("server_url"), str) or len(tool["server_url"]) > 2048 - or not isinstance(tool.get("project_connection_id"), str)): - raise fail("toolbox-binding-unverified", "The selected MCP tool definition is malformed or has another type.") - if tool.get("authorization") is not None or tool.get("connector_id") is not None or tool.get("headers"): - raise fail("toolbox-auth-unverified", "Inline authorization, connectors or custom headers are outside this agentic-identity recipe; preserve them rather than silently replacing their auth.") - if "allowed_tools" in tool and tool["allowed_tools"] is not None and "knowledge_base_retrieve" not in allowed: - raise fail("toolbox-policy-unverified", "The explicit tool filter does not establish access to knowledge_base_retrieve; review the policy delta separately, never silently widen it.") - old_endpoint = url(tool["server_url"]) - api = parse_qs(old_endpoint.query) - reference = tool["project_connection_id"] - prefix = project_id + "/connections/" - if (not old_endpoint.hostname.endswith(".search.windows.net") - or re.fullmatch(r"/knowledgebases/[A-Za-z0-9_.-]+/mcp", old_endpoint.path) is None - or set(api) != {"api-version"} or len(api["api-version"]) != 1 - or re.fullmatch(r"\d{4}-\d{2}-\d{2}(?:-preview)?", api["api-version"][0]) is None - or not (name(reference) or (reference.casefold().startswith(prefix.casefold()) - and name(reference[len(prefix):])))): - raise fail("toolbox-binding-unverified", "The selected label is not a verified Search KB MCP binding.") - target = search_endpoint + "/knowledgebases/" + kb_name + "/mcp?api-version=2026-08-01-preview" - connection_url = MANAGEMENT_AUDIENCE + project_id + "/connections/" + request["connection_name"] + "?api-version=" + read.PROJECT_API - connection = reads.get(connection_url, MANAGEMENT_AUDIENCE, absent=True, label="connection") - if connection is not None: - props = connection.get("properties") - expected_id = project_id + "/connections/" + request["connection_name"] - if (connection.get("name") != request["connection_name"] or not isinstance(props, dict) - or str(connection.get("id", expected_id)).casefold() != expected_id.casefold() - or any(props.get(key) != value for key, value in { - "category": "RemoteTool", "authType": "AgenticIdentityToken", - "target": target, "audience": "https://search.azure.com/", - }.items())): - raise fail("connection-conflict", "This exact connection NAME has incompatible Hosted recipe auth/binding. Preserve it; explicitly choose a new name, even for the same KB endpoint.") - old_connection, old_connection_url = None, None - if matching: - old_name = matching[0]["project_connection_id"].rsplit("/", 1)[-1] - old_connection_url = MANAGEMENT_AUDIENCE + project_id + "/connections/" + old_name + "?api-version=" + read.PROJECT_API - old_connection = connection if old_name == request["connection_name"] else reads.get( - old_connection_url, MANAGEMENT_AUDIENCE, label="previous-connection", - ) - old_id = project_id + "/connections/" + old_name - if (old_connection is None or old_connection.get("name") != old_name - or str(old_connection.get("id", old_id)).casefold() != old_id.casefold() - or not isinstance(old_connection.get("properties"), dict) - or old_connection["properties"].get("target") != matching[0]["server_url"]): - raise fail("toolbox-existing-binding-unverified", "The previous connection and selected toolbox tool do not establish one consistent KB binding.") - exact = bool(connection and matching - and matching[0].get("project_connection_id") in { - request["connection_name"], project_id + "/connections/" + request["connection_name"], - } - and matching[0]["server_url"] == target) - known = [{"name": request["agent_name"], "version": request["agent_version"]}] - known_states = [] - for item in request.get("known_agents", []): - observed = agent(reads, endpoint, item["name"], item["version"]) - known_states.append(observed) - env = observed["definition"].get("environment_variables", {}) - configured_tools = observed["definition"].get("tools") or [] - tool_binding = isinstance(configured_tools, list) and any( - isinstance(tool, dict) and tool.get("type") == "mcp" and tool.get("server_url") == consumer - for tool in configured_tools - ) - if (toolbox_binding(env, endpoint, request["toolbox_name"]) or tool_binding) and item not in known: - known.append(item) - refreshed = reads.get(toolbox_url + "?api-version=v1", AI_AUDIENCE, label="toolbox") - refreshed_agent = agent(reads, endpoint, request["agent_name"], request["agent_version"]) - refreshed_connection = reads.get(connection_url, MANAGEMENT_AUDIENCE, absent=True, label="connection") - refreshed_old = old_connection - if old_connection_url and old_connection_url != connection_url: - refreshed_old = reads.get(old_connection_url, MANAGEMENT_AUDIENCE, label="previous-connection") - refreshed_version = reads.get(toolbox_url + "/versions/" + default + "?api-version=v1", AI_AUDIENCE, label="toolbox-version") - if (refreshed != toolbox or refreshed_agent != selected or refreshed_connection != connection - or refreshed_old != old_connection or refreshed_version != version - or read.cli_context(project_id, cli=cli) != context): - raise fail("hosted-protected-state-drift", "Agent, toolbox default/metadata or CLI context changed during assessment.") - summary = { - "branch": "toolbox-only", "agent": request["agent_name"], "agent_version": request["agent_version"], - "project_resource_id": project_id, "project_endpoint": endpoint, "search_resource_id": search_id, - "toolbox_name": request["toolbox_name"], "consumer_endpoint": consumer, - "runtime_binding": {"resolver": "FoundryToolbox environment", "mode": binding_mode, - "runtime_usage_verified": False}, - "agent_change": "none", "runtime_principal_id": principal, "kb_profile": profile, - "before": {"kb_endpoint": matching[0].get("server_url") if matching else None, "default_version": default, - "connection": matching[0].get("project_connection_id") if matching else None}, - "after": {"kb_endpoint": target, "default_version": default if exact else "new immutable version (not created)", - "connection": request["connection_name"]}, - "connection_action": "reuse" if connection else "create (blocked)", - "known_consumers": known, "unknown_consumers": True, - "shared_default_approval_required": not exact, "mutation_approval_required": not exact, - "retention_owner": request["retention_owner"], "cleanup": "unsupported; retain previous versions and legacy connections", - "rollback": "separate explicit plan; never automatic", - "tool_policy": "Preserve existing tool approval/filter/configuration; runtime enforcement is not proven by this assessment.", - "acceptance": { - "supported_candidate_needed": "supported_question" not in request, - "unrelated_question_needed": "unrelated_question" not in request, - "invocation_approval": "not granted by assessment", "agent_tool_retrieval": "not-run", - }, - } - result = { - "status": "planned" if exact else "blocked", "outcome": OUTCOME, "approval_summary": summary, - "writes_performed": [], "execution_input": None, "execution_available": False, - "private_evidence": {"fingerprint": digest({ - "project": project, "agent": selected, "search": search, "kb": kb, - "reader": reader, "project_role": project_role, "toolbox": toolbox, - "version": version, "connection": connection, "previous_connection": old_connection, - "known_agents": known_states, "cli": context, - })}, - "request_ids": reads.request_ids, - "warnings": ["Configured runtime binding is not actual agent tool-use proof; acceptance invocations remain separate.", - "Known bindings are not a complete consumer inventory; external consumers may follow this default."], - } - if (str(search["properties"].get("status")).lower() != "running" - or str(search["properties"].get("provisioningState")).lower() != "succeeded"): - result["warnings"].append("Search is provisioning/degraded; healthy KB GET is configuration evidence, not retrieval readiness.") - if not exact: - result["first_blocker"] = { - "code": "toolbox-promotion-concurrency-unverified", - "message": "SDK/REST version creation and default promotion exist, but their conditional ETag/default update contract is unverified. Obtain service-owner confirmation before any connection/version creation; arbitrary If-Match headers are not proof.", - } - return result - - -def main(argv=None): - parser = argparse.ArgumentParser() - parser.add_argument("--plan", type=Path, required=True) - args = parser.parse_args(argv) - try: - result = assess(read_json(args.plan)) - except HelperFailure as error: - result = blocked_result(error, outcome=OUTCOME, fingerprint=None, owner=None) - if isinstance(error, ProjectSelectionFailure): - result["selection_candidates"] = error.candidates - emit_result(result) - return 3 if result["status"] == "partial" else 2 if result["status"] == "blocked" else 0 - - -if __name__ == "__main__": - sys.exit(main()) +"""GET-only assessment of an existing Hosted agent's toolbox connection. + +Promotion is deliberately unavailable until a conditional shared-default update +contract is verified. No intermediate resources are created while that gate blocks. +""" +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path +from urllib.parse import parse_qs, quote, unquote, urlsplit + +try: + from . import _prompt_read as read + from ._bootstrap_io import read_json, run_cli + from ._common import ( + MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, HelperFailure, azure_cli_token, + blocked_result, digest, emit_result, http_request, reject_secrets, + ) + from .prompt_connect import _project_identity, _project_endpoint, PROJECT_ID +except ImportError: + import _prompt_read as read + from _bootstrap_io import read_json, run_cli + from _common import ( + MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, HelperFailure, azure_cli_token, + blocked_result, digest, emit_result, http_request, reject_secrets, + ) + from prompt_connect import _project_identity, _project_endpoint, PROJECT_ID + +AI_AUDIENCE = "https://ai.azure.com" +FOUNDRY_USER = "53ca6127-db72-4b80-b1b0-d745d6d5456d" +OUTCOME = "assess-existing-hosted-toolbox" +FIELDS = { + "schema_version", "scope", "project", "search_service", "knowledge_base", + "agent_name", "agent_version", "toolbox_name", "tool_label", "connection_name", + "reader_assignment_id", "project_assignment_id", "retention_owner", +} +OPTIONAL = {"known_agents", "supported_question", "unrelated_question"} + + +class ProjectSelectionFailure(HelperFailure): + def __init__(self, candidates): + super().__init__( + "project-ambiguous" if candidates else "project-absent", + "Scoped project name did not resolve uniquely; select an exact account/project without widening scope.", + blocked_at="input-resolution", + ) + self.candidates = candidates + + +def fail(code, message, **metadata): + return read.fail(code, message, **metadata) + + +def name(value): + return isinstance(value, str) and read.NAME.fullmatch(value) is not None + + +def url(value): + try: + parsed = urlsplit(value) + if (parsed.scheme != "https" or not parsed.hostname or parsed.username + or parsed.password or parsed.port not in (None, 443) or parsed.fragment): + raise ValueError() + return parsed + except ValueError as error: + raise fail("selection-invalid", "Use an exact public-cloud HTTPS endpoint without credentials or fragments.") from error + + +def validate(request): + if not isinstance(request, dict): + raise fail("input-schema-invalid", "Hosted assessment requires a resolved intent object.") + reject_secrets(request) + if set(request) - FIELDS - OPTIONAL or not FIELDS <= set(request) or request["schema_version"] != "1.0": + raise fail("input-schema-invalid", "Use only the documented Hosted assessment decisions.") + scope = request["scope"] + if (not isinstance(scope, dict) or not {"subscription_id", "resource_group"} <= set(scope) + or set(scope) - {"subscription_id", "resource_group", "account_name"} + or any(not name(value) for value in scope.values())): + raise fail("scope-invalid", "Select one subscription/resource group and optionally one account; no automatic widening.") + for key in ("agent_name", "agent_version", "toolbox_name", "tool_label", "connection_name"): + if not name(request[key]): + raise fail("input-schema-invalid", "Select bounded exact agent/version, toolbox and connection names.") + if request["agent_version"].casefold() in {"latest", "default"}: + raise fail("agent-version-unresolved", "Pin the observed agent version, never latest.") + for key in ("project", "search_service", "knowledge_base"): + value = request[key] + if not isinstance(value, str) or not 1 <= len(value) <= 2048: + raise fail("selection-invalid", "Supply a name, exact resource ID or documented endpoint.") + if value.startswith("https:"): + parsed = url(value) + if key == "project": + _project_endpoint(value) + if (not name(parsed.hostname.removesuffix(".services.ai.azure.com")) + or not name(unquote(parsed.path.rstrip("/").rsplit("/", 1)[-1]))): + raise fail("selection-invalid", "The project endpoint must identify a valid account/project name.") + elif key == "search_service": + if (not parsed.hostname.endswith(".search.windows.net") or parsed.path not in ("", "/") + or parsed.query): + raise fail("selection-invalid", "Select one Search service endpoint, not an index.") + elif (not parsed.hostname.endswith(".search.windows.net") + or re.fullmatch(r"/knowledgebases/[A-Za-z0-9_.-]+/mcp", parsed.path) is None + or parse_qs(parsed.query) != {"api-version": ["2026-08-01-preview"]}): + raise fail("selection-invalid", "Select an exact preview KB MCP endpoint.") + elif value.startswith("/"): + pattern = PROJECT_ID if key == "project" else read.SEARCH_ID if key == "search_service" else None + if pattern is None or pattern.fullmatch(value) is None: + raise fail("selection-invalid", "The supplied resource ID does not identify the selected resource kind.") + elif not name(value): + raise fail("selection-invalid", "The selected name is malformed.") + if key == "search_service" and not value.startswith("/"): + service = url(value).hostname.removesuffix(".search.windows.net") if value.startswith("https:") else value + resource_id = (f"/subscriptions/{scope['subscription_id']}/resourceGroups/{scope['resource_group']}" + "/providers/Microsoft.Search/searchServices/" + service) + if read.SEARCH_ID.fullmatch(resource_id) is None: + raise fail("selection-invalid", "Use a valid Search service name, not a display label or index.") + for key in ("reader_assignment_id", "project_assignment_id"): + value = request[key] + if (not isinstance(value, str) or len(value) > 2048 + or re.fullmatch(r"/subscriptions/[^/?#\s]+/resourceGroups/[^?#\s]+/providers/" + r"Microsoft.Authorization/roleAssignments/[0-9a-fA-F-]+", value, re.I) is None + or not read.GUID.fullmatch(value.rsplit("/", 1)[-1])): + raise fail("role-selection-invalid", "Select exact resource-scoped role-assignment IDs, not principals to invent.") + if (not isinstance(request["retention_owner"], str) or not request["retention_owner"].strip() + or len(request["retention_owner"]) > 256): + raise fail("owner-unresolved", "Name the retained connection/toolbox-version owner; cleanup is unsupported.") + agents = request.get("known_agents", []) + if (not isinstance(agents, list) or len(agents) > 20 + or any(not isinstance(item, dict) or set(item) != {"name", "version"} + or not name(item["name"]) or not name(item["version"]) + or item["version"].casefold() in {"latest", "default"} for item in agents)): + raise fail("known-bindings-invalid", "Supply at most 20 known exact agent/version bindings; no exclusive-consumer inference.") + for key in ("supported_question", "unrelated_question"): + if key in request and (not isinstance(request[key], str) or not request[key].strip() or len(request[key]) > 4096): + raise fail("acceptance-invalid", "Questions are optional resolved candidates, never implicit invocation approval.") + + +class Reads: + def __init__(self, token_provider, transport): + self.token_provider, self.transport = token_provider, transport + self.tokens, self.request_ids = {}, [] + self.last_request_id = None + + def get(self, endpoint, audience, *, absent=False, label="resource"): + if audience not in self.tokens: + self.tokens[audience] = self.token_provider(audience) + value, ids = read.get_object(endpoint, self.tokens[audience], self.transport, absent=absent, label=label) + self.request_ids.extend(ids) + self.last_request_id = ids[-1] if ids else None + return value + + def collection(self, endpoint): + initial = urlsplit(endpoint) + result, seen = [], set() + while endpoint: + current = url(endpoint) + if (endpoint in seen or len(seen) >= 20 or current.netloc != initial.netloc + or current.path != initial.path + or parse_qs(current.query).get("api-version") != parse_qs(initial.query).get("api-version")): + raise fail("inventory-unverified", "Scoped inventory continuation is unsafe or exceeds 20 pages.") + seen.add(endpoint) + page = self.get(endpoint, MANAGEMENT_AUDIENCE, label="scoped-inventory") + items = page.get("value") + if not isinstance(items, list) or any(not isinstance(item, dict) for item in items): + raise fail("inventory-unverified", "Scoped inventory is malformed; no inferred absence.") + result.extend(items) + if len(result) > 100: + raise fail("inventory-limit", "Scoped inventory exceeds 100 resources; select an exact parent instead.") + endpoint = page.get("nextLink") + if endpoint is not None and not isinstance(endpoint, str): + raise fail("inventory-unverified", "Scoped continuation must be a URL.") + return result + + +def resolve(request, reads): + scope = request["scope"] + prefix = f"/subscriptions/{scope['subscription_id']}/resourceGroups/{scope['resource_group']}" + account_prefix = prefix + "/providers/Microsoft.CognitiveServices/accounts/" + selection = request["project"] + if selection.startswith("/"): + project_id = selection + elif selection.startswith("https:"): + parsed = url(selection) + project_id = account_prefix + parsed.hostname.removesuffix(".services.ai.azure.com") + "/projects/" + unquote(parsed.path.rstrip("/").rsplit("/", 1)[-1]) + elif scope.get("account_name"): + project_id = account_prefix + scope["account_name"] + "/projects/" + selection + else: + accounts = reads.collection(MANAGEMENT_AUDIENCE + account_prefix.rstrip("/") + "?api-version=2025-06-01") + if len(accounts) > 20: + raise fail("account-selection-required", "Select one account; do not enumerate more than 20 accounts for a project name.") + matches, seen = [], set() + for account in accounts: + account_id = account.get("id", "") + if (not isinstance(account_id, str) or not account_id.casefold().startswith(account_prefix.casefold()) + or not name(account_id[len(account_prefix):]) or account_id.casefold() in seen): + raise fail("inventory-unverified", "Account inventory contains a duplicate or out-of-scope resource.") + seen.add(account_id.casefold()) + projects_seen = set() + for project in reads.collection(MANAGEMENT_AUDIENCE + account_id + "/projects?api-version=" + read.PROJECT_API): + candidate = project.get("id", "") + if (not isinstance(candidate, str) or PROJECT_ID.fullmatch(candidate) is None + or not candidate.casefold().startswith((account_id + "/projects/").casefold()) + or candidate.casefold() in projects_seen): + raise fail("inventory-unverified", "Project inventory contains an unverified identity.") + projects_seen.add(candidate.casefold()) + if candidate.rsplit("/", 1)[-1].casefold() == selection.casefold(): + matches.append(candidate) + if len(matches) != 1: + raise ProjectSelectionFailure(matches) + project_id = matches[0] + match = PROJECT_ID.fullmatch(project_id) + if match is None: + raise fail("selection-invalid", "The resolved project resource identity is malformed.") + endpoint = f"https://{match['account']}.services.ai.azure.com/api/projects/{quote(match['project'], safe='')}" + _project_identity({"project_resource_id": project_id, "project_endpoint": endpoint}) + selection = request["search_service"] + if selection.startswith("/"): + search_id = selection + else: + service = url(selection).hostname.removesuffix(".search.windows.net") if selection.startswith("https:") else selection + search_id = prefix + "/providers/Microsoft.Search/searchServices/" + service + match = read.SEARCH_ID.fullmatch(search_id) + if match is None: + raise fail("selection-invalid", "The selected Search service identity is invalid.") + search_endpoint = "https://" + match["name"].lower() + ".search.windows.net" + selection = request["knowledge_base"] + if selection.startswith("https:"): + parsed = url(selection) + if parsed.hostname != urlsplit(search_endpoint).hostname: + raise fail("kb-binding-invalid", "The KB endpoint belongs to another selected Search service.") + kb_name = parsed.path.split("/")[2] + else: + kb_name = selection + return project_id, endpoint, search_id, search_endpoint, kb_name + + +def role(reads, assignment_id, scope, principal, role_id): + prefix = scope + "/providers/Microsoft.Authorization/roleAssignments/" + if (not assignment_id.casefold().startswith(prefix.casefold()) + or not read.GUID.fullmatch(assignment_id[len(prefix):])): + raise fail("role-scope-invalid", "Role assignment must use the exact selected resource scope.") + value = reads.get(MANAGEMENT_AUDIENCE + assignment_id + "?api-version=" + read.ROLE_API, + MANAGEMENT_AUDIENCE, label="role-assignment") + properties = value.get("properties") + definitions = { + "/providers/microsoft.authorization/roledefinitions/" + role_id, + "/subscriptions/" + scope.split("/")[2].lower() + "/providers/microsoft.authorization/roledefinitions/" + role_id, + } + if (str(value.get("id", "")).casefold() != assignment_id.casefold() or not isinstance(properties, dict) + or str(properties.get("principalId", "")).lower() != principal + or str(properties.get("scope", "")).casefold() != scope.casefold() + or str(properties.get("roleDefinitionId", "")).casefold() not in definitions + or properties.get("principalType", "ServicePrincipal") != "ServicePrincipal" + or properties.get("condition") not in (None, "")): + raise fail("hosted-runtime-role-unverified", "Require exact-scope grants to the published Hosted principal, not the project or blueprint.") + return value + + +def agent(reads, endpoint, agent_name, version): + value = reads.get(endpoint + f"/agents/{quote(agent_name, safe='')}/versions/{quote(version, safe='')}?api-version=v1", + AI_AUDIENCE, label="hosted-agent") + if value.get("name") != agent_name or str(value.get("version")) != version or not isinstance(value.get("definition"), dict): + raise fail("agent-identity-unverified", "Read the exact observed agent/version; never select latest.") + return value + + +def toolbox_binding(environment, endpoint, toolbox_name): + """Match the first-party FoundryToolbox environment resolver, not arbitrary code.""" + if not isinstance(environment, dict): + return None + consumer = endpoint + "/toolboxes/" + toolbox_name + "/mcp?api-version=v1" + if "TOOLBOX_ENDPOINT" in environment: + return "endpoint" if environment["TOOLBOX_ENDPOINT"] == consumer else None + project = environment.get("FOUNDRY_PROJECT_ENDPOINT") + if (isinstance(project, str) and project.rstrip("/") == endpoint + and environment.get("TOOLBOX_NAME") == toolbox_name): + return "name" + return None + + +def assess(request, *, token_provider=azure_cli_token, transport=http_request, cli=run_cli): + validate(request) + scope = request["scope"] + selected_project = request["project"] + context_id = selected_project if selected_project.startswith("/") else "/subscriptions/" + scope["subscription_id"] + context = read.cli_context(context_id, cli=cli) + reads = Reads(token_provider, transport) + project_id, endpoint, search_id, search_endpoint, kb_name = resolve(request, reads) + project = reads.get(MANAGEMENT_AUDIENCE + project_id + "?api-version=" + read.PROJECT_API, MANAGEMENT_AUDIENCE, label="project") + properties = project.get("properties", {}) + endpoints = properties.get("endpoints") if isinstance(properties, dict) else None + if (str(project.get("id", "")).casefold() != project_id.casefold() or not isinstance(properties, dict) + or str(properties.get("provisioningState", "")).lower() != "succeeded" + or not isinstance(endpoints, dict) or endpoint not in endpoints.values()): + raise fail("project-unverified", "The exact ready project endpoint must be confirmed remotely.") + selected = agent(reads, endpoint, request["agent_name"], request["agent_version"]) + definition, identity = selected["definition"], selected.get("instance_identity") + other_principals = [ + value.get("principalId") or value.get("principal_id") + for value in (project.get("identity"), selected.get("blueprint")) + if isinstance(value, dict) + ] + if (definition.get("kind") != "hosted" or selected.get("status") != "active" + or not isinstance(identity, dict) or identity.get("status") != "active" + or not isinstance(identity.get("principal_id"), str) or not read.GUID.fullmatch(identity["principal_id"]) + or not isinstance(identity.get("client_id"), str) or not read.GUID.fullmatch(identity["client_id"]) + or identity["principal_id"].casefold() in { + value.casefold() for value in other_principals if isinstance(value, str) + }): + raise fail("hosted-principal-unverified", "Require an active published Hosted agent and observed instance principal.") + principal = identity["principal_id"].lower() + consumer = endpoint + "/toolboxes/" + request["toolbox_name"] + "/mcp?api-version=v1" + environment = definition.get("environment_variables") + binding_mode = toolbox_binding(environment, endpoint, request["toolbox_name"]) + if binding_mode is None: + raise fail("hosted-runtime-change-required", "Observed settings do not establish this unversioned FoundryToolbox binding. The runtime owner must verify custom/overridden behavior or the actual configuration change before requesting source; a version-pinned developer endpoint is not equivalent.") + search = reads.get(MANAGEMENT_AUDIENCE + search_id + "?api-version=" + read.SEARCH_API, MANAGEMENT_AUDIENCE, label="search") + properties = search.get("properties") + if (str(search.get("id", "")).casefold() != search_id.casefold() or not isinstance(properties, dict) + or str(properties.get("status", "")).lower() not in {"running", "provisioning", "degraded"} + or str(properties.get("provisioningState", "")).lower() not in {"succeeded", "provisioning"}): + raise fail("search-operation-blocked", "Search failed/deleting/disabled/unresolved state blocks this assessment.") + kb = reads.get(search_endpoint + "/knowledgebases('" + quote(kb_name, safe="") + "')?api-version=2026-08-01-preview", + SEARCH_AUDIENCE, absent=True, label="knowledge-base") + if kb is None: + raise fail("knowledge-base-absent", "The exact KB is absent; Search indexes are not KB evidence.", + status=404, request_id=reads.last_request_id) + _, profile = read.kb_state(kb, kb_name) + reader = role(reads, request["reader_assignment_id"], search_id, principal, read.READER_ROLE) + project_role = role(reads, request["project_assignment_id"], project_id, principal, FOUNDRY_USER) + toolbox_url = endpoint + "/toolboxes/" + request["toolbox_name"] + toolbox = reads.get(toolbox_url + "?api-version=v1", AI_AUDIENCE, label="toolbox") + default = toolbox.get("default_version") + if (toolbox.get("name") != request["toolbox_name"] or not name(default) + or default.casefold() in {"latest", "default"}): + raise fail("toolbox-default-unverified", "Read the exact toolbox and its current default version.") + version = reads.get(toolbox_url + "/versions/" + default + "?api-version=v1", AI_AUDIENCE, label="toolbox-version") + tools = version.get("tools") + if (version.get("name") != request["toolbox_name"] or str(version.get("version")) != default + or not isinstance(tools, list) or len(tools) > 200 or any(not isinstance(tool, dict) for tool in tools)): + raise fail("toolbox-version-unverified", "The default immutable version and its complete tool list must be verified.") + matching = [tool for tool in tools if tool.get("server_label") == request["tool_label"]] + if len(matching) > 1: + raise fail("toolbox-binding-ambiguous", "Multiple selected KB tool labels require explicit reconciliation.") + if matching: + tool = matching[0] + allowed = tool.get("allowed_tools") or [] + allowed = allowed.get("tool_names", []) if isinstance(allowed, dict) else allowed + if (tool.get("type") != "mcp" or not isinstance(allowed, list) + or any(not isinstance(item, str) for item in allowed) + or not isinstance(tool.get("headers") or {}, dict) + or not isinstance(tool.get("server_url"), str) or len(tool["server_url"]) > 2048 + or not isinstance(tool.get("project_connection_id"), str)): + raise fail("toolbox-binding-unverified", "The selected MCP tool definition is malformed or has another type.") + if tool.get("authorization") is not None or tool.get("connector_id") is not None or tool.get("headers"): + raise fail("toolbox-auth-unverified", "Inline authorization, connectors or custom headers are outside this agentic-identity recipe; preserve them rather than silently replacing their auth.") + if "allowed_tools" in tool and tool["allowed_tools"] is not None and "knowledge_base_retrieve" not in allowed: + raise fail("toolbox-policy-unverified", "The explicit tool filter does not establish access to knowledge_base_retrieve; review the policy delta separately, never silently widen it.") + old_endpoint = url(tool["server_url"]) + api = parse_qs(old_endpoint.query) + reference = tool["project_connection_id"] + prefix = project_id + "/connections/" + if (not old_endpoint.hostname.endswith(".search.windows.net") + or re.fullmatch(r"/knowledgebases/[A-Za-z0-9_.-]+/mcp", old_endpoint.path) is None + or set(api) != {"api-version"} or len(api["api-version"]) != 1 + or re.fullmatch(r"\d{4}-\d{2}-\d{2}(?:-preview)?", api["api-version"][0]) is None + or not (name(reference) or (reference.casefold().startswith(prefix.casefold()) + and name(reference[len(prefix):])))): + raise fail("toolbox-binding-unverified", "The selected label is not a verified Search KB MCP binding.") + target = search_endpoint + "/knowledgebases/" + kb_name + "/mcp?api-version=2026-08-01-preview" + connection_url = MANAGEMENT_AUDIENCE + project_id + "/connections/" + request["connection_name"] + "?api-version=" + read.PROJECT_API + connection = reads.get(connection_url, MANAGEMENT_AUDIENCE, absent=True, label="connection") + if connection is not None: + props = connection.get("properties") + expected_id = project_id + "/connections/" + request["connection_name"] + if (connection.get("name") != request["connection_name"] or not isinstance(props, dict) + or str(connection.get("id", expected_id)).casefold() != expected_id.casefold() + or any(props.get(key) != value for key, value in { + "category": "RemoteTool", "authType": "AgenticIdentityToken", + "target": target, "audience": "https://search.azure.com/", + }.items())): + raise fail("connection-conflict", "This exact connection NAME has incompatible Hosted recipe auth/binding. Preserve it; explicitly choose a new name, even for the same KB endpoint.") + old_connection, old_connection_url = None, None + if matching: + old_name = matching[0]["project_connection_id"].rsplit("/", 1)[-1] + old_connection_url = MANAGEMENT_AUDIENCE + project_id + "/connections/" + old_name + "?api-version=" + read.PROJECT_API + old_connection = connection if old_name == request["connection_name"] else reads.get( + old_connection_url, MANAGEMENT_AUDIENCE, label="previous-connection", + ) + old_id = project_id + "/connections/" + old_name + if (old_connection is None or old_connection.get("name") != old_name + or str(old_connection.get("id", old_id)).casefold() != old_id.casefold() + or not isinstance(old_connection.get("properties"), dict) + or old_connection["properties"].get("target") != matching[0]["server_url"]): + raise fail("toolbox-existing-binding-unverified", "The previous connection and selected toolbox tool do not establish one consistent KB binding.") + exact = bool(connection and matching + and matching[0].get("project_connection_id") in { + request["connection_name"], project_id + "/connections/" + request["connection_name"], + } + and matching[0]["server_url"] == target) + known = [{"name": request["agent_name"], "version": request["agent_version"]}] + known_states = [] + for item in request.get("known_agents", []): + observed = agent(reads, endpoint, item["name"], item["version"]) + known_states.append(observed) + env = observed["definition"].get("environment_variables", {}) + configured_tools = observed["definition"].get("tools") or [] + tool_binding = isinstance(configured_tools, list) and any( + isinstance(tool, dict) and tool.get("type") == "mcp" and tool.get("server_url") == consumer + for tool in configured_tools + ) + if (toolbox_binding(env, endpoint, request["toolbox_name"]) or tool_binding) and item not in known: + known.append(item) + refreshed = reads.get(toolbox_url + "?api-version=v1", AI_AUDIENCE, label="toolbox") + refreshed_agent = agent(reads, endpoint, request["agent_name"], request["agent_version"]) + refreshed_connection = reads.get(connection_url, MANAGEMENT_AUDIENCE, absent=True, label="connection") + refreshed_old = old_connection + if old_connection_url and old_connection_url != connection_url: + refreshed_old = reads.get(old_connection_url, MANAGEMENT_AUDIENCE, label="previous-connection") + refreshed_version = reads.get(toolbox_url + "/versions/" + default + "?api-version=v1", AI_AUDIENCE, label="toolbox-version") + if (refreshed != toolbox or refreshed_agent != selected or refreshed_connection != connection + or refreshed_old != old_connection or refreshed_version != version + or read.cli_context(project_id, cli=cli) != context): + raise fail("hosted-protected-state-drift", "Agent, toolbox default/metadata or CLI context changed during assessment.") + summary = { + "branch": "toolbox-only", "agent": request["agent_name"], "agent_version": request["agent_version"], + "project_resource_id": project_id, "project_endpoint": endpoint, "search_resource_id": search_id, + "toolbox_name": request["toolbox_name"], "consumer_endpoint": consumer, + "runtime_binding": {"resolver": "FoundryToolbox environment", "mode": binding_mode, + "runtime_usage_verified": False}, + "agent_change": "none", "runtime_principal_id": principal, "kb_profile": profile, + "before": {"kb_endpoint": matching[0].get("server_url") if matching else None, "default_version": default, + "connection": matching[0].get("project_connection_id") if matching else None}, + "after": {"kb_endpoint": target, "default_version": default if exact else "new immutable version (not created)", + "connection": request["connection_name"]}, + "connection_action": "reuse" if connection else "create (blocked)", + "known_consumers": known, "unknown_consumers": True, + "shared_default_approval_required": not exact, "mutation_approval_required": not exact, + "retention_owner": request["retention_owner"], "cleanup": "unsupported; retain previous versions and legacy connections", + "rollback": "separate explicit plan; never automatic", + "tool_policy": "Preserve existing tool approval/filter/configuration; runtime enforcement is not proven by this assessment.", + "acceptance": { + "supported_candidate_needed": "supported_question" not in request, + "unrelated_question_needed": "unrelated_question" not in request, + "invocation_approval": "not granted by assessment", "agent_tool_retrieval": "not-run", + }, + } + result = { + "status": "planned" if exact else "blocked", "outcome": OUTCOME, "approval_summary": summary, + "writes_performed": [], "execution_input": None, "execution_available": False, + "private_evidence": {"fingerprint": digest({ + "project": project, "agent": selected, "search": search, "kb": kb, + "reader": reader, "project_role": project_role, "toolbox": toolbox, + "version": version, "connection": connection, "previous_connection": old_connection, + "known_agents": known_states, "cli": context, + })}, + "request_ids": reads.request_ids, + "warnings": ["Configured runtime binding is not actual agent tool-use proof; acceptance invocations remain separate.", + "Known bindings are not a complete consumer inventory; external consumers may follow this default."], + } + if (str(search["properties"].get("status")).lower() != "running" + or str(search["properties"].get("provisioningState")).lower() != "succeeded"): + result["warnings"].append("Search is provisioning/degraded; healthy KB GET is configuration evidence, not retrieval readiness.") + if not exact: + result["first_blocker"] = { + "code": "toolbox-promotion-concurrency-unverified", + "message": "SDK/REST version creation and default promotion exist, but their conditional ETag/default update contract is unverified. Obtain service-owner confirmation before any connection/version creation; arbitrary If-Match headers are not proof.", + } + return result + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument("--plan", type=Path, required=True) + args = parser.parse_args(argv) + try: + result = assess(read_json(args.plan)) + except HelperFailure as error: + result = blocked_result(error, outcome=OUTCOME, fingerprint=None, owner=None) + if isinstance(error, ProjectSelectionFailure): + result["selection_candidates"] = error.candidates + emit_result(result) + return 3 if result["status"] == "partial" else 2 if result["status"] == "blocked" else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/kb-contracts.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/kb-contracts.md index eaf6fb0c..f0f9d494 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/kb-contracts.md +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/kb-contracts.md @@ -1,132 +1,132 @@ -# Knowledge-base planning contract - -Use for KB creation/reuse or the explicit model-free agent transition. -The [KB owner](../knowledge-bases/create.md) resolves goals, source readiness, -access and costs; [common contracts](contracts.md) govern approval and execution. -No helper source inspection, hand-built `desired`/models or hash scripts on the -normal path. Reuse supplied choices instead of asking another questionnaire. - -## Read-only planning - -```text -python helpers/search_reconcile.py --plan -``` - -Closed UTF-8 input, at most one MiB: - -```json -{"schema_version":"1.0","endpoint":"https://svc.search.windows.net","name":"docs-kb","owner":"operator@example.com","source_name":"docs-file","api_version":"2026-08-01-preview","reasoning_effort":"minimal","output_mode":"extractiveData"} -``` - -All shown fields are required. `name`/`source_name` are exact names, not patterns; -owner is workflow metadata, not wire data. Names are URL-encoded, never code. -`action` defaults to `create-or-reuse`. Renaming after collision needs approval. - -| Choice | Supported shape | -|---|---| -| API | `2026-04-01` for Blob minimal/extractive; `2026-08-01-preview` for File/ADLS, reasoning/synthesis or agents | -| Reasoning/output | Preview `minimal`/`low`/`medium`: `extractiveData` or `answerSynthesis`; GA minimal/extractive only | -| Model | Omit/null for minimal/extractive; one explicit `model` below for low/medium or synthesis | -| Optional text | `description`, `retrieval_instructions`, `answer_instructions`: text/null, each at most 4096 characters; instructions preview-only | -| Optional controls | Existing closed `data_movement`, `rbac`, `network` objects from common contracts | - -No implicit API/mode/model choice, escalation or CU/vector inference. Source -processing and embeddings remain unchanged. Unknown fields and invalid choices -block before authentication. No keys, secret environments, model provisioning, -network/RBAC changes or model calls. - -### Reasoning or synthesis with a chat model - -Use resolved deployment/model choices, not guesses or a hardcoded default: - -```json -{"schema_version":"1.0","endpoint":"https://svc.search.windows.net","name":"docs-kb-synthesis","owner":"operator@example.com","source_name":"docs-file","api_version":"2026-08-01-preview","reasoning_effort":"low","output_mode":"answerSynthesis","model":{"endpoint":"https://models.services.ai.azure.com","deployment":"chat-deployment","model":"gpt-4.1-mini","auth":"system-assigned","prerequisites":{"deployment":"","identity":"","network":""}}} -``` - -`model` accepts only those five fields. Prerequisites are nonempty text, at most -4096 characters each: owner attestations, not helper proof of capacity/readiness/access. -Use a supported Azure OpenAI chat model, exact public-cloud account endpoint and deployment. -The Search system identity needs model access. CU/embeddings are not KB chat. -For medium or minimal synthesis, change `reasoning_effort` accordingly; retain -the model. Minimal synthesis adds model cost without query planning; use `intents`, -not `messages`, for retrieval. Never auto-escalate or change extractive defaults. - -The helper generates the complete wire definition, including: - -```json -{"name":"docs-kb-synthesis","knowledgeSources":[{"name":"docs-file"}],"retrievalReasoningEffort":{"kind":"low"},"outputMode":"answerSynthesis","models":[{"kind":"azureOpenAI","azureOpenAIParameters":{"resourceUri":"https://models.services.ai.azure.com","deploymentId":"chat-deployment","modelName":"gpt-4.1-mini","authIdentity":null}}]} -``` - -No `apiKey`. Only minimal/extractive omits models. GA omits preview fields/models. -New artifacts bind -`kb_plan_version: "1.0"` and `kb_model` (null or the selected choice) to that wire -definition. Existing approved artifacts retain their fingerprints; preview -minimal/synthesis is now admitted with one model. -Masked/nonempty keys or a different model identity in KB readback block; they -cannot be hidden by generic definition normalization. - -## Observation, result and approval - -Only selected Search `GET knowledgesources('{source}')` and -`GET knowledgebases('{name}')`, using the requested API and signed-in CLI token. -Source GETs bracket KB observation; existing KBs are reread for ETag/definition -stability. Three GETs for an absent KB, four for existing; no resource enumeration, -status polling, uploads, retrieval, PUT/PATCH/DELETE or writes to local artifacts. -These observations are not locks or ingestion/retrieval proof. - -The source must exist and return the exact selected File/Blob/ADLS definition. -The helper derives `verified_source` name/normalized definition digest from -those reads; never supply or fabricate it in the intent. Finish source-owner -readiness and prerequisite verification before mutation approval. - -Exit `0`: `status: planned`, `execution_input`, `plan_fingerprint`, -`approval_summary`, read request IDs and `writes_performed: []`. -Save only `execution_input` in a private UTF-8 JSON file. It already contains -the complete plan and computed fingerprint, with `approval.confirmed: false`. -Do not wrap it again, edit its generated body or calculate a fingerprint. -Summaries omit prerequisite references; retain private artifact data privately. - -An exact existing KB is observed reuse, not a prediction: stable name/definition/ -ETag, `action: reuse`, `execution_required: false`, -`mutation_approval_required: false`. Do not execute an unapproved envelope just -to repeat this read-only verification. Continue authorized retrieval; ingestion -and retrieval remain explicitly unverified by planning. - -Otherwise present concrete actions/source/mode/model, access, cost/data movement, -acceptance and cleanup. An already-confirmed concrete plan is not a reason to -repeat choice questions; confirm any material difference before proceeding. -Only actual approval of the unchanged generated plan permits setting its existing -`approval.confirmed` to true; preserve the supplied fingerprint. Tool permission, -bootstrap consent or a proposed future source does not establish that approval. -The planner never infers consent from the input. - -```text -python helpers/search_reconcile.py --input -``` - -Apply freshly verifies the source binding, conditional create or current ETag -update, exact readback and keyless model auth. Source/KB drift blocks; regenerate -the proposal and review changes, never hand-repair nested JSON/hashes. -Exit `2` is structured blocked/no-write; post-write failures are `3`/partial -with retained ownership. No automatic retries of writes or cleanup. -An updated KB remains pre-existing/reused, including failed or ambiguous -post-write verification; an update never establishes creation ownership. - -## Explicit agent-minimal transition - -This is optional KB-side model-free normalization, not an MCP prerequisite. -Never downgrade a valid reasoning/synthesis KB to connect an agent. -For the owner's existing model-free GA/extractive KB transition, set -`action: agent-minimal-transition`, preview API, minimal effort and extractive -output. Omit model and optional text changes. The source/KB must exist and match. -Existing absent/minimal/low effort is allowed only without models and synthesis; -the summary discloses prior mode fields, including an observed default low. -Only the two preview mode fields change, guarded by the current ETag; unrelated -fields/models remain. The update needs separate concrete approval. -An equivalent KB is reuse. Verify actual agent MCP invocation separately from GA REST retrieval. - -## Authorities - -Authorities: failure/conflict/uncertainty only. -[preview KB wire/model fields](https://learn.microsoft.com/rest/api/searchservice/knowledge-bases/create-or-update?view=rest-searchservice-2026-08-01-preview), -[KB creation and model access](https://learn.microsoft.com/azure/search/agentic-retrieval-how-to-create-knowledge-base). +# Knowledge-base planning contract + +Use for KB creation/reuse or the explicit model-free agent transition. +The [KB owner](../knowledge-bases/create.md) resolves goals, source readiness, +access and costs; [common contracts](contracts.md) govern approval and execution. +No helper source inspection, hand-built `desired`/models or hash scripts on the +normal path. Reuse supplied choices instead of asking another questionnaire. + +## Read-only planning + +```text +python helpers/search_reconcile.py --plan +``` + +Closed UTF-8 input, at most one MiB: + +```json +{"schema_version":"1.0","endpoint":"https://svc.search.windows.net","name":"docs-kb","owner":"operator@example.com","source_name":"docs-file","api_version":"2026-08-01-preview","reasoning_effort":"minimal","output_mode":"extractiveData"} +``` + +All shown fields are required. `name`/`source_name` are exact names, not patterns; +owner is workflow metadata, not wire data. Names are URL-encoded, never code. +`action` defaults to `create-or-reuse`. Renaming after collision needs approval. + +| Choice | Supported shape | +|---|---| +| API | `2026-04-01` for Blob minimal/extractive; `2026-08-01-preview` for File/ADLS, reasoning/synthesis or agents | +| Reasoning/output | Preview `minimal`/`low`/`medium`: `extractiveData` or `answerSynthesis`; GA minimal/extractive only | +| Model | Omit/null for minimal/extractive; one explicit `model` below for low/medium or synthesis | +| Optional text | `description`, `retrieval_instructions`, `answer_instructions`: text/null, each at most 4096 characters; instructions preview-only | +| Optional controls | Existing closed `data_movement`, `rbac`, `network` objects from common contracts | + +No implicit API/mode/model choice, escalation or CU/vector inference. Source +processing and embeddings remain unchanged. Unknown fields and invalid choices +block before authentication. No keys, secret environments, model provisioning, +network/RBAC changes or model calls. + +### Reasoning or synthesis with a chat model + +Use resolved deployment/model choices, not guesses or a hardcoded default: + +```json +{"schema_version":"1.0","endpoint":"https://svc.search.windows.net","name":"docs-kb-synthesis","owner":"operator@example.com","source_name":"docs-file","api_version":"2026-08-01-preview","reasoning_effort":"low","output_mode":"answerSynthesis","model":{"endpoint":"https://models.services.ai.azure.com","deployment":"chat-deployment","model":"gpt-4.1-mini","auth":"system-assigned","prerequisites":{"deployment":"","identity":"","network":""}}} +``` + +`model` accepts only those five fields. Prerequisites are nonempty text, at most +4096 characters each: owner attestations, not helper proof of capacity/readiness/access. +Use a supported Azure OpenAI chat model, exact public-cloud account endpoint and deployment. +The Search system identity needs model access. CU/embeddings are not KB chat. +For medium or minimal synthesis, change `reasoning_effort` accordingly; retain +the model. Minimal synthesis adds model cost without query planning; use `intents`, +not `messages`, for retrieval. Never auto-escalate or change extractive defaults. + +The helper generates the complete wire definition, including: + +```json +{"name":"docs-kb-synthesis","knowledgeSources":[{"name":"docs-file"}],"retrievalReasoningEffort":{"kind":"low"},"outputMode":"answerSynthesis","models":[{"kind":"azureOpenAI","azureOpenAIParameters":{"resourceUri":"https://models.services.ai.azure.com","deploymentId":"chat-deployment","modelName":"gpt-4.1-mini","authIdentity":null}}]} +``` + +No `apiKey`. Only minimal/extractive omits models. GA omits preview fields/models. +New artifacts bind +`kb_plan_version: "1.0"` and `kb_model` (null or the selected choice) to that wire +definition. Existing approved artifacts retain their fingerprints; preview +minimal/synthesis is now admitted with one model. +Masked/nonempty keys or a different model identity in KB readback block; they +cannot be hidden by generic definition normalization. + +## Observation, result and approval + +Only selected Search `GET knowledgesources('{source}')` and +`GET knowledgebases('{name}')`, using the requested API and signed-in CLI token. +Source GETs bracket KB observation; existing KBs are reread for ETag/definition +stability. Three GETs for an absent KB, four for existing; no resource enumeration, +status polling, uploads, retrieval, PUT/PATCH/DELETE or writes to local artifacts. +These observations are not locks or ingestion/retrieval proof. + +The source must exist and return the exact selected File/Blob/ADLS definition. +The helper derives `verified_source` name/normalized definition digest from +those reads; never supply or fabricate it in the intent. Finish source-owner +readiness and prerequisite verification before mutation approval. + +Exit `0`: `status: planned`, `execution_input`, `plan_fingerprint`, +`approval_summary`, read request IDs and `writes_performed: []`. +Save only `execution_input` in a private UTF-8 JSON file. It already contains +the complete plan and computed fingerprint, with `approval.confirmed: false`. +Do not wrap it again, edit its generated body or calculate a fingerprint. +Summaries omit prerequisite references; retain private artifact data privately. + +An exact existing KB is observed reuse, not a prediction: stable name/definition/ +ETag, `action: reuse`, `execution_required: false`, +`mutation_approval_required: false`. Do not execute an unapproved envelope just +to repeat this read-only verification. Continue authorized retrieval; ingestion +and retrieval remain explicitly unverified by planning. + +Otherwise present concrete actions/source/mode/model, access, cost/data movement, +acceptance and cleanup. An already-confirmed concrete plan is not a reason to +repeat choice questions; confirm any material difference before proceeding. +Only actual approval of the unchanged generated plan permits setting its existing +`approval.confirmed` to true; preserve the supplied fingerprint. Tool permission, +bootstrap consent or a proposed future source does not establish that approval. +The planner never infers consent from the input. + +```text +python helpers/search_reconcile.py --input +``` + +Apply freshly verifies the source binding, conditional create or current ETag +update, exact readback and keyless model auth. Source/KB drift blocks; regenerate +the proposal and review changes, never hand-repair nested JSON/hashes. +Exit `2` is structured blocked/no-write; post-write failures are `3`/partial +with retained ownership. No automatic retries of writes or cleanup. +An updated KB remains pre-existing/reused, including failed or ambiguous +post-write verification; an update never establishes creation ownership. + +## Explicit agent-minimal transition + +This is optional KB-side model-free normalization, not an MCP prerequisite. +Never downgrade a valid reasoning/synthesis KB to connect an agent. +For the owner's existing model-free GA/extractive KB transition, set +`action: agent-minimal-transition`, preview API, minimal effort and extractive +output. Omit model and optional text changes. The source/KB must exist and match. +Existing absent/minimal/low effort is allowed only without models and synthesis; +the summary discloses prior mode fields, including an observed default low. +Only the two preview mode fields change, guarded by the current ETag; unrelated +fields/models remain. The update needs separate concrete approval. +An equivalent KB is reuse. Verify actual agent MCP invocation separately from GA REST retrieval. + +## Authorities + +Authorities: failure/conflict/uncertainty only. +[preview KB wire/model fields](https://learn.microsoft.com/rest/api/searchservice/knowledge-bases/create-or-update?view=rest-searchservice-2026-08-01-preview), +[KB creation and model access](https://learn.microsoft.com/azure/search/agentic-retrieval-how-to-create-knowledge-base). diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/knowledge_base_retrieve.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/knowledge_base_retrieve.py index 080db5aa..f7f92291 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/knowledge_base_retrieve.py +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/knowledge_base_retrieve.py @@ -1,187 +1,187 @@ -from __future__ import annotations - -import argparse -import json -import shutil -import subprocess -import sys -from pathlib import Path -from typing import Any -from urllib.parse import quote - -try: - from ._common import ( - SEARCH_AUDIENCE, HelperFailure, HttpResult, TokenProvider, Transport, - azure_cli_token, blocked_result, emit_result, http_request, - reject_secrets, require_allowed_fields, validate_search_endpoint, - ) -except ImportError: - from _common import ( # type: ignore[no-redef] - SEARCH_AUDIENCE, HelperFailure, HttpResult, TokenProvider, Transport, - azure_cli_token, blocked_result, emit_result, http_request, - reject_secrets, require_allowed_fields, validate_search_endpoint, - ) - - -API_VERSIONS = {"2026-04-01", "2026-08-01-preview"} - - -def _invalid(message: str) -> HelperFailure: - return HelperFailure("input-schema-invalid", message, blocked_at="input-resolution") - - -def _target(endpoint: Any, name: Any, api_version: Any) -> str: - if not isinstance(api_version, str) or api_version not in API_VERSIONS: - raise _invalid("Unsupported API version.") - if not isinstance(name, str) or not name.strip() or any(ord(c) < 32 for c in name): - raise _invalid("An exact nonempty resource name is required.") - try: - return validate_search_endpoint(endpoint) - except ValueError as exc: - raise _invalid("Invalid Search endpoint.") from exc - - -def retrieval_request( - endpoint: str, name: str, api_version: str, effort: str, query: str, -) -> tuple[str, str, dict[str, Any]]: - base = _target(endpoint, name, api_version) + "/knowledgebases" - if not isinstance(effort, str) or effort not in {"minimal", "low", "medium"}: - raise _invalid("Unsupported reasoning effort.") - if api_version == "2026-04-01" and effort != "minimal": - raise _invalid("GA requires minimal retrieval.") - if not isinstance(query, str) or not query.strip(): - raise _invalid("A nonempty query is required.") - definition_url = base + "('" + quote(name.replace("'", "''"), safe="") + "')" - retrieve_url = base + "/" + quote(name, safe="") + "/retrieve" - suffix = "?api-version=" + api_version - body = ( - {"intents": [{"type": "semantic", "search": query}]} - if effort == "minimal" else - {"messages": [{"role": "user", "content": [{"type": "text", "text": query}]}]} - ) - return definition_url + suffix, retrieve_url + suffix, body - - -def read_json( - method: str, url: str, body: dict[str, Any] | None = None, - forward_permissions: bool = False, *, - token_provider: TokenProvider = azure_cli_token, - transport: Transport = http_request, -) -> HttpResult: - token = token_provider(SEARCH_AUDIENCE) - headers = {"Content-Type": "application/json"} - if forward_permissions: - headers["x-ms-query-source-authorization"] = token - result = transport( - method, url, token, - body=json.dumps(body).encode("utf-8") if body is not None else None, - headers=headers, follow_redirects=False, max_response_bytes=5 * 1024 * 1024, - ) - if result.status != 200 or not isinstance(result.body, dict): - raise HelperFailure( - "retrieval-incomplete", "Expected complete JSON response.", - blocked_at="verification", status=result.status, request_id=result.request_id, - ) - return result - - -def _verify_signed_in_user() -> None: - executable = shutil.which("az") - if executable is None: - raise HelperFailure( - "azure-cli-unavailable", "Azure CLI is required for keyless authentication.", - blocked_at="execution", - ) - try: - result = subprocess.run( - [executable, "account", "show", "--query", "user.type", "--output", "tsv"], - check=True, capture_output=True, text=True, timeout=60, - ) - except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc: - raise HelperFailure( - "permission-forwarding-unavailable", "Could not verify the signed-in user.", - blocked_at="input-resolution", - ) from exc - if result.stdout.strip().casefold() != "user": - raise HelperFailure( - "permission-forwarding-unavailable", "Permission forwarding requires a signed-in user.", - blocked_at="input-resolution", - ) - - -def execute( - document: Any, *, token_provider: TokenProvider = azure_cli_token, - transport: Transport = http_request, -) -> dict[str, Any]: - if not isinstance(document, dict): - raise _invalid("Input must be a JSON object.") - reject_secrets(document) - operation = document.get("operation") - if not isinstance(operation, str) or operation not in { - "get-knowledge-base", "get-knowledge-source", "retrieve", - }: - raise _invalid("Choose get-knowledge-base, get-knowledge-source, or retrieve.") - fields = {"operation", "endpoint", "name", "api_version"} - if operation == "retrieve": - fields |= {"effort", "query", "forward_permissions"} - require_allowed_fields(document, fields, label="read-only request") - endpoint = _target(document.get("endpoint"), document.get("name"), document.get("api_version")) - name, api_version = document["name"], document["api_version"] - body = None - forwarding = False - if operation == "retrieve": - forwarding = document.get("forward_permissions") - if type(forwarding) is not bool: - raise _invalid("Explicit forward_permissions true or false is required.") - _, url, body = retrieval_request( - endpoint, name, api_version, document.get("effort"), document.get("query"), - ) - if forwarding: - _verify_signed_in_user() - method = "POST" - else: - collection = "knowledgebases" if operation == "get-knowledge-base" else "knowledgesources" - url = endpoint + "/" + collection + "('" + quote(name.replace("'", "''"), safe="") + "')" - url += "?api-version=" + api_version - method = "GET" - result = read_json( - method, url, body, forwarding, token_provider=token_provider, transport=transport, - ) - return { - "status": "response-received", "operation": operation, "mutation": "none", - "writes_performed": [], "http_status": result.status, "request_id": result.request_id, - "response_body": result.body, "verification": "not-performed", "cleanup": "not-applicable", - } - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description="Read Search KB/source definitions or retrieve from one KB.") - parser.add_argument("--input", required=True, type=Path) - args = parser.parse_args(argv) - try: - try: - document = json.loads(args.input.read_text(encoding="utf-8")) - except OSError as exc: - raise HelperFailure("input-unreadable", "Input file cannot be read.", blocked_at="input-resolution") from exc - except (UnicodeError, json.JSONDecodeError) as exc: - raise HelperFailure("input-invalid-json", "Input must be UTF-8 JSON.", blocked_at="input-resolution") from exc - result = execute(document) - except HelperFailure as failure: - # POST retrieve is read-only even when the generic transport marks it ambiguous. - readonly_failure = HelperFailure( - failure.code, failure.message, blocked_at=failure.blocked_at, - status=failure.http_status, request_id=failure.request_id, - ) - result = blocked_result(readonly_failure, outcome="knowledge-base-retrieval", fingerprint=None) - result.update( - mutation="none", cleanup="not-applicable", - safe_next_decision="Resolve the first blocker before repeating this read-only request.", - ) - emit_result(result) - return 2 - emit_result(result) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any +from urllib.parse import quote + +try: + from ._common import ( + SEARCH_AUDIENCE, HelperFailure, HttpResult, TokenProvider, Transport, + azure_cli_token, blocked_result, emit_result, http_request, + reject_secrets, require_allowed_fields, validate_search_endpoint, + ) +except ImportError: + from _common import ( # type: ignore[no-redef] + SEARCH_AUDIENCE, HelperFailure, HttpResult, TokenProvider, Transport, + azure_cli_token, blocked_result, emit_result, http_request, + reject_secrets, require_allowed_fields, validate_search_endpoint, + ) + + +API_VERSIONS = {"2026-04-01", "2026-08-01-preview"} + + +def _invalid(message: str) -> HelperFailure: + return HelperFailure("input-schema-invalid", message, blocked_at="input-resolution") + + +def _target(endpoint: Any, name: Any, api_version: Any) -> str: + if not isinstance(api_version, str) or api_version not in API_VERSIONS: + raise _invalid("Unsupported API version.") + if not isinstance(name, str) or not name.strip() or any(ord(c) < 32 for c in name): + raise _invalid("An exact nonempty resource name is required.") + try: + return validate_search_endpoint(endpoint) + except ValueError as exc: + raise _invalid("Invalid Search endpoint.") from exc + + +def retrieval_request( + endpoint: str, name: str, api_version: str, effort: str, query: str, +) -> tuple[str, str, dict[str, Any]]: + base = _target(endpoint, name, api_version) + "/knowledgebases" + if not isinstance(effort, str) or effort not in {"minimal", "low", "medium"}: + raise _invalid("Unsupported reasoning effort.") + if api_version == "2026-04-01" and effort != "minimal": + raise _invalid("GA requires minimal retrieval.") + if not isinstance(query, str) or not query.strip(): + raise _invalid("A nonempty query is required.") + definition_url = base + "('" + quote(name.replace("'", "''"), safe="") + "')" + retrieve_url = base + "/" + quote(name, safe="") + "/retrieve" + suffix = "?api-version=" + api_version + body = ( + {"intents": [{"type": "semantic", "search": query}]} + if effort == "minimal" else + {"messages": [{"role": "user", "content": [{"type": "text", "text": query}]}]} + ) + return definition_url + suffix, retrieve_url + suffix, body + + +def read_json( + method: str, url: str, body: dict[str, Any] | None = None, + forward_permissions: bool = False, *, + token_provider: TokenProvider = azure_cli_token, + transport: Transport = http_request, +) -> HttpResult: + token = token_provider(SEARCH_AUDIENCE) + headers = {"Content-Type": "application/json"} + if forward_permissions: + headers["x-ms-query-source-authorization"] = token + result = transport( + method, url, token, + body=json.dumps(body).encode("utf-8") if body is not None else None, + headers=headers, follow_redirects=False, max_response_bytes=5 * 1024 * 1024, + ) + if result.status != 200 or not isinstance(result.body, dict): + raise HelperFailure( + "retrieval-incomplete", "Expected complete JSON response.", + blocked_at="verification", status=result.status, request_id=result.request_id, + ) + return result + + +def _verify_signed_in_user() -> None: + executable = shutil.which("az") + if executable is None: + raise HelperFailure( + "azure-cli-unavailable", "Azure CLI is required for keyless authentication.", + blocked_at="execution", + ) + try: + result = subprocess.run( + [executable, "account", "show", "--query", "user.type", "--output", "tsv"], + check=True, capture_output=True, text=True, timeout=60, + ) + except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc: + raise HelperFailure( + "permission-forwarding-unavailable", "Could not verify the signed-in user.", + blocked_at="input-resolution", + ) from exc + if result.stdout.strip().casefold() != "user": + raise HelperFailure( + "permission-forwarding-unavailable", "Permission forwarding requires a signed-in user.", + blocked_at="input-resolution", + ) + + +def execute( + document: Any, *, token_provider: TokenProvider = azure_cli_token, + transport: Transport = http_request, +) -> dict[str, Any]: + if not isinstance(document, dict): + raise _invalid("Input must be a JSON object.") + reject_secrets(document) + operation = document.get("operation") + if not isinstance(operation, str) or operation not in { + "get-knowledge-base", "get-knowledge-source", "retrieve", + }: + raise _invalid("Choose get-knowledge-base, get-knowledge-source, or retrieve.") + fields = {"operation", "endpoint", "name", "api_version"} + if operation == "retrieve": + fields |= {"effort", "query", "forward_permissions"} + require_allowed_fields(document, fields, label="read-only request") + endpoint = _target(document.get("endpoint"), document.get("name"), document.get("api_version")) + name, api_version = document["name"], document["api_version"] + body = None + forwarding = False + if operation == "retrieve": + forwarding = document.get("forward_permissions") + if type(forwarding) is not bool: + raise _invalid("Explicit forward_permissions true or false is required.") + _, url, body = retrieval_request( + endpoint, name, api_version, document.get("effort"), document.get("query"), + ) + if forwarding: + _verify_signed_in_user() + method = "POST" + else: + collection = "knowledgebases" if operation == "get-knowledge-base" else "knowledgesources" + url = endpoint + "/" + collection + "('" + quote(name.replace("'", "''"), safe="") + "')" + url += "?api-version=" + api_version + method = "GET" + result = read_json( + method, url, body, forwarding, token_provider=token_provider, transport=transport, + ) + return { + "status": "response-received", "operation": operation, "mutation": "none", + "writes_performed": [], "http_status": result.status, "request_id": result.request_id, + "response_body": result.body, "verification": "not-performed", "cleanup": "not-applicable", + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Read Search KB/source definitions or retrieve from one KB.") + parser.add_argument("--input", required=True, type=Path) + args = parser.parse_args(argv) + try: + try: + document = json.loads(args.input.read_text(encoding="utf-8")) + except OSError as exc: + raise HelperFailure("input-unreadable", "Input file cannot be read.", blocked_at="input-resolution") from exc + except (UnicodeError, json.JSONDecodeError) as exc: + raise HelperFailure("input-invalid-json", "Input must be UTF-8 JSON.", blocked_at="input-resolution") from exc + result = execute(document) + except HelperFailure as failure: + # POST retrieve is read-only even when the generic transport marks it ambiguous. + readonly_failure = HelperFailure( + failure.code, failure.message, blocked_at=failure.blocked_at, + status=failure.http_status, request_id=failure.request_id, + ) + result = blocked_result(readonly_failure, outcome="knowledge-base-retrieval", fingerprint=None) + result.update( + mutation="none", cleanup="not-applicable", + safe_next_decision="Resolve the first blocker before repeating this read-only request.", + ) + emit_result(result) + return 2 + emit_result(result) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/model-discovery-contracts.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/model-discovery-contracts.md index 7f0f8759..0e7e98ba 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/model-discovery-contracts.md +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/model-discovery-contracts.md @@ -1,150 +1,150 @@ -# Read-only model and CU selection - -`purpose: none` means no dependencies, -not minimal retrieval effort. Minimal+extractive can be model-free; -minimal+answerSynthesis needs `purpose: chat`. -Only FIND inventories; supplied IDs exact-read. KB/retrieve verifies API; -no readiness/access/approval proof. - -## Invocation and closed input - -From skill directory: - -```text -python helpers/model_discovery.py --input choices.json -``` - -Required JSON (UTF-8): - -```json -{"schema_version":"1.0","purpose":"embedding","subscription_id":null,"resource_group":null,"account_id":null,"account_name":null,"deployment":null} -``` - -`purpose`: `none`, `embedding`, `chat`, `cu`. Null is unresolved. -GUID subscription; ARM names. -RGs: 1-90 Unicode letters/decimal digits or `_-.()`; no final period. -`account_id`: exact Cognitive Services account ARM ID. `deployment`: name or -deployment ARM ID. IDs supply parent/scope; conflicting fields block before CLI. -CU/none reject deployment selectors. -Optional `endpoint`: observed HTTPS origin, only with an exact account selector. -Null/omitted requires a unique origin; explicit selection must match ARM. -Reuse IDs/choices; no setup attestations during discovery. - -Three dependency types: CU service capability (Content Understanding, Standard extraction); -embedding deployment (vectorization); chat deployment (enabled image verbalization/KB synthesis). -No project/agent gate. -Keep separate CU, embedding and chat selections. CU never selects models. -Unresolved models keep scope; set -`account_id`, `account_name`, `deployment` and `endpoint` to null; never repurpose -CU's `selection_input`. Explicit model selectors win. -Prefer one compatible existing Foundry/AIServices account: -CU capability + two model deployments. -Separate embedding/KB-chat: per consumer contract. CU image-description models -must be in the Foundry resource attached to the skillset. -Never provision/migrate/redeploy to consolidate. -CU proves no model deployments. Empty model lists are not scope-wide absence: -inspect another selected candidate before proposing creation. -A 403/timeout is unresolved, not absence; no denied-scope broadening. - -- Known account ID or account name plus group: exact account GET. -- Name without group: selected-subscription typed listing; unique match exact GET, - otherwise choices. -- Unknown account: group-scoped, otherwise subscription-scoped Cognitive Services - accounts: AIServices/OpenAI for models; AIServices for CU. - No deployment listing until an account is selected, even if only one exists. -- Selected model account: exact deployment GET when named, else its deployment list. - Validate each role's deployment/capabilities/auth; never infer support - from names. Missing capabilities need owner validation. -- CU: exact account metadata only; no deployment reads. - -Bootstrap's shell-free runner disables installs/auto-upgrade/telemetry and raw logs. -No login/context changes, secrets, writes, role/Policy/catalog/quota scans, -content reads/uploads or model calls. - -## Exact CLI surfaces - -Unresolved subscription only: `az account show --query "{id:id}" --output json --only-show-errors`. -Otherwise skip context lookup; explicit scope wins. -ARM calls: - -```text -az rest --method get --url --query --subscription --output json --only-show-errors -``` - -`URL`: `https://management.azure.com?api-version=2024-10-01`. -Encode paths, not IDs. `A` is the exact account path: -`/subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts/`. - -| Read | Path | -| --- | --- | -| Accounts in subscription | `/subscriptions//providers/Microsoft.CognitiveServices/accounts` | -| Accounts in group | `/subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts` | -| Exact account | `A` | -| Selected account deployments | `A/deployments` | -| Exact deployment | `A/deployments/` | - -Complete pagination: 10 pages/200 raw rows aggregate; 20 seconds/command, -120 seconds total. Partial results block; narrow `resource_group` or use an exact ID. -At most ten supported endpoint origins per account. -Continuation retains ARM host/collection/API; only skip-token parameters. -Native CLI JMESPath projects before capture: inventory IDs/names/kinds/locations, -exact account endpoints/state, deployment identity/model/state/public capabilities. -No ARM `$select`; all rows and `nextLink` survive. One MiB per stream, aggregate -captured bytes and final JSON; post-capture checks are not memory bounds. -No retries, fanout, denied-scope broadening or absence inference. - -## Closed output - -Fields: `schema_version`, `status`, `purpose`, `scope`, -`accounts`, `deployments`, `selected`, `limits`, `warnings`, `first_failure`, -`writes_performed` (always `[]`). Exit 0 returns `skipped`, `no-candidates`, -`account-choice-required`, `deployment-choice-required` or `selected`; -exit 2: `blocked`. No-candidates covers only the completed query. -Scope: null or `{subscription_id, resource_group}`; `limits`: bounds above. -`first_failure` is null or `{code, status, message, request_id, message_digest}`. -Original service code/status/request ID and message digest survive CLI failure; -No raw text/invented IDs; unreadable is not absent. - -Account rows: `{account_id, name, resource_group, location, kind, -provisioning_state, endpoint, endpoint_sources, endpoint_state, endpoint_candidates, selection_input}`. -Deployment rows: `{deployment_id, name, model_name, model_version, model_format, -capabilities, provisioning_state, selection_input}`. Metadata fields may be null; -capabilities: safe string map, not support/capacity proof: -`embeddings`, `chatCompletion` and validated numeric `maxContextToken`, `maxOutputToken` -only. Missing/invalid values remain unknown; internal/unknown fields never emit. -`selected` is null or `{account, deployment}` (deployment null for CU). -`endpoint_candidates` is the sorted observed-origin list. Endpoint resolution -failure retains the exact account row, never deployments or a selected result. -Other failures discard choices. -Inventory `endpoint_state` is `not-assessed`; endpoints/state are null/empty until GET. - -Present observed compatible deployments together in existing flow. -Keep complete scoped rows internally for ambiguity checks; never paste full -inventories/JSON to the customer. Show at most five concise account/deployment -choices, total count and a more/other choice from retained rows, not new discovery. -Show account/group/region and deployment/model names distinctly. Known Storage -location or selected region can rank suggestions, not prove CU/model availability. -Explicit locations/reuse win; no hidden regional filter or absence claim. -For choices, save the chosen row's **`selection_input`** unchanged. -For ambiguous origins, copy it and set `endpoint` to a listed origin; rerun. -No caller filtering, ID stitching or CLI glue. -Exact readback revalidates choices. Selection is metadata only: owner verifies -model suitability, version/capacity/cost/residency, -selected CU dependencies, actual access and network before concrete approval. - -## Endpoint metadata, not a CU readiness assertion - -ARM `2024-10-01` does **not** document `endpoints["ContentUnderstanding"]` as a -stable CU discriminator. Never infer hosts from map keys/customSubDomainName. - -Inspect the exact account's primary endpoint and every map **value** locally. -CU: `services.ai.azure.com`; embedding/chat also accept `openai.azure.com` and -`cognitiveservices.azure.com`. HTTPS origins only; strip only a trailing slash; -without a selector require exactly one matching origin. -Missing/multiple matching origins block; no rewriting, arbitrary fetch, -paths/query/user-info/ports or readiness inference. For diagnostics: -[scope/endpoint rules](model-discovery-scopes.md). - -File CU MI/explicit key and Blob Search identity stay distinct; neither is accessed. -Existing source planners own exact-name collision checks; no extra preflight, -suffixes, writes or auth changes. +# Read-only model and CU selection + +`purpose: none` means no dependencies, +not minimal retrieval effort. Minimal+extractive can be model-free; +minimal+answerSynthesis needs `purpose: chat`. +Only FIND inventories; supplied IDs exact-read. KB/retrieve verifies API; +no readiness/access/approval proof. + +## Invocation and closed input + +From skill directory: + +```text +python helpers/model_discovery.py --input choices.json +``` + +Required JSON (UTF-8): + +```json +{"schema_version":"1.0","purpose":"embedding","subscription_id":null,"resource_group":null,"account_id":null,"account_name":null,"deployment":null} +``` + +`purpose`: `none`, `embedding`, `chat`, `cu`. Null is unresolved. +GUID subscription; ARM names. +RGs: 1-90 Unicode letters/decimal digits or `_-.()`; no final period. +`account_id`: exact Cognitive Services account ARM ID. `deployment`: name or +deployment ARM ID. IDs supply parent/scope; conflicting fields block before CLI. +CU/none reject deployment selectors. +Optional `endpoint`: observed HTTPS origin, only with an exact account selector. +Null/omitted requires a unique origin; explicit selection must match ARM. +Reuse IDs/choices; no setup attestations during discovery. + +Three dependency types: CU service capability (Content Understanding, Standard extraction); +embedding deployment (vectorization); chat deployment (enabled image verbalization/KB synthesis). +No project/agent gate. +Keep separate CU, embedding and chat selections. CU never selects models. +Unresolved models keep scope; set +`account_id`, `account_name`, `deployment` and `endpoint` to null; never repurpose +CU's `selection_input`. Explicit model selectors win. +Prefer one compatible existing Foundry/AIServices account: +CU capability + two model deployments. +Separate embedding/KB-chat: per consumer contract. CU image-description models +must be in the Foundry resource attached to the skillset. +Never provision/migrate/redeploy to consolidate. +CU proves no model deployments. Empty model lists are not scope-wide absence: +inspect another selected candidate before proposing creation. +A 403/timeout is unresolved, not absence; no denied-scope broadening. + +- Known account ID or account name plus group: exact account GET. +- Name without group: selected-subscription typed listing; unique match exact GET, + otherwise choices. +- Unknown account: group-scoped, otherwise subscription-scoped Cognitive Services + accounts: AIServices/OpenAI for models; AIServices for CU. + No deployment listing until an account is selected, even if only one exists. +- Selected model account: exact deployment GET when named, else its deployment list. + Validate each role's deployment/capabilities/auth; never infer support + from names. Missing capabilities need owner validation. +- CU: exact account metadata only; no deployment reads. + +Bootstrap's shell-free runner disables installs/auto-upgrade/telemetry and raw logs. +No login/context changes, secrets, writes, role/Policy/catalog/quota scans, +content reads/uploads or model calls. + +## Exact CLI surfaces + +Unresolved subscription only: `az account show --query "{id:id}" --output json --only-show-errors`. +Otherwise skip context lookup; explicit scope wins. +ARM calls: + +```text +az rest --method get --url --query --subscription --output json --only-show-errors +``` + +`URL`: `https://management.azure.com?api-version=2024-10-01`. +Encode paths, not IDs. `A` is the exact account path: +`/subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts/`. + +| Read | Path | +| --- | --- | +| Accounts in subscription | `/subscriptions//providers/Microsoft.CognitiveServices/accounts` | +| Accounts in group | `/subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts` | +| Exact account | `A` | +| Selected account deployments | `A/deployments` | +| Exact deployment | `A/deployments/` | + +Complete pagination: 10 pages/200 raw rows aggregate; 20 seconds/command, +120 seconds total. Partial results block; narrow `resource_group` or use an exact ID. +At most ten supported endpoint origins per account. +Continuation retains ARM host/collection/API; only skip-token parameters. +Native CLI JMESPath projects before capture: inventory IDs/names/kinds/locations, +exact account endpoints/state, deployment identity/model/state/public capabilities. +No ARM `$select`; all rows and `nextLink` survive. One MiB per stream, aggregate +captured bytes and final JSON; post-capture checks are not memory bounds. +No retries, fanout, denied-scope broadening or absence inference. + +## Closed output + +Fields: `schema_version`, `status`, `purpose`, `scope`, +`accounts`, `deployments`, `selected`, `limits`, `warnings`, `first_failure`, +`writes_performed` (always `[]`). Exit 0 returns `skipped`, `no-candidates`, +`account-choice-required`, `deployment-choice-required` or `selected`; +exit 2: `blocked`. No-candidates covers only the completed query. +Scope: null or `{subscription_id, resource_group}`; `limits`: bounds above. +`first_failure` is null or `{code, status, message, request_id, message_digest}`. +Original service code/status/request ID and message digest survive CLI failure; +No raw text/invented IDs; unreadable is not absent. + +Account rows: `{account_id, name, resource_group, location, kind, +provisioning_state, endpoint, endpoint_sources, endpoint_state, endpoint_candidates, selection_input}`. +Deployment rows: `{deployment_id, name, model_name, model_version, model_format, +capabilities, provisioning_state, selection_input}`. Metadata fields may be null; +capabilities: safe string map, not support/capacity proof: +`embeddings`, `chatCompletion` and validated numeric `maxContextToken`, `maxOutputToken` +only. Missing/invalid values remain unknown; internal/unknown fields never emit. +`selected` is null or `{account, deployment}` (deployment null for CU). +`endpoint_candidates` is the sorted observed-origin list. Endpoint resolution +failure retains the exact account row, never deployments or a selected result. +Other failures discard choices. +Inventory `endpoint_state` is `not-assessed`; endpoints/state are null/empty until GET. + +Present observed compatible deployments together in existing flow. +Keep complete scoped rows internally for ambiguity checks; never paste full +inventories/JSON to the customer. Show at most five concise account/deployment +choices, total count and a more/other choice from retained rows, not new discovery. +Show account/group/region and deployment/model names distinctly. Known Storage +location or selected region can rank suggestions, not prove CU/model availability. +Explicit locations/reuse win; no hidden regional filter or absence claim. +For choices, save the chosen row's **`selection_input`** unchanged. +For ambiguous origins, copy it and set `endpoint` to a listed origin; rerun. +No caller filtering, ID stitching or CLI glue. +Exact readback revalidates choices. Selection is metadata only: owner verifies +model suitability, version/capacity/cost/residency, +selected CU dependencies, actual access and network before concrete approval. + +## Endpoint metadata, not a CU readiness assertion + +ARM `2024-10-01` does **not** document `endpoints["ContentUnderstanding"]` as a +stable CU discriminator. Never infer hosts from map keys/customSubDomainName. + +Inspect the exact account's primary endpoint and every map **value** locally. +CU: `services.ai.azure.com`; embedding/chat also accept `openai.azure.com` and +`cognitiveservices.azure.com`. HTTPS origins only; strip only a trailing slash; +without a selector require exactly one matching origin. +Missing/multiple matching origins block; no rewriting, arbitrary fetch, +paths/query/user-info/ports or readiness inference. For diagnostics: +[scope/endpoint rules](model-discovery-scopes.md). + +File CU MI/explicit key and Blob Search identity stay distinct; neither is accessed. +Existing source planners own exact-name collision checks; no extra preflight, +suffixes, writes or auth changes. diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/model-discovery-scopes.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/model-discovery-scopes.md index 7598e428..28e8b223 100644 --- a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/model-discovery-scopes.md +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/model-discovery-scopes.md @@ -1,131 +1,131 @@ -# Discovery scope and endpoint diagnostics - -Read for ambiguous/missing endpoints, CU/model handoff confusion, unusual RG names -or selector/readback errors. -Metadata only; no setup or billable probes. - -## Separate dependency selections - -CU extraction uses `content_understanding.endpoint` / `aiServices.uri`; source -vectors use `embedding.endpoint` / `embeddingModel.azureOpenAIParameters.resourceUri`. -Retain each role's account ID, observed origin and selected deployment separately. -The CU key/identity is not evidence of embedding access. - -For example, CU account A can have no embedding deployment while model account B -already hosts a suitable one. Leave A selected for CU; start unresolved -`purpose: embedding` discovery in the agreed scope without A's account/name/endpoint. -The returned model candidates include `OpenAI` and `AIServices` accounts. A Foundry -resource is an `AIServices` model-hosting account, not another mandatory resource. -Choose a model account before listing its deployments. An account-list result is -not a deployment search; one empty deployment list does not exhaust the scope. -Keep the shortlist and choose another candidate, not repeated reads of A. - -An explicit embedding deployment fixes its own parent account; never overwrite -that binding with CU's. Same-account reuse remains valid when both roles are -independently verified. Carry the selected embedding origin/deployment unchanged -into the source plan; never rewrite hosts or substitute the CU origin. - -Prefer compatible existing deployments. If the completed agreed search finds -none suitable, propose a deployment in a compatible existing account before a -new account, with separate approval and model/SKU/quota/access/cost checks. -An explicitly selected target failure, denial, timeout or incomplete inventory -remains unresolved: no automatic scope expansion, creation or silent fallback. - -## Resource groups are not account names - -`Microsoft.Resources/resourcegroups` permits 1-90 characters: underscores, -hyphens, periods, parentheses, and Unicode categories Lu/Ll/Lt/Lm/Lo/Nd -(letters or decimal digits). No final period. No alphanumeric-first requirement: -`_shared-ai`, `.shared` and `(ops)` are valid. Combining marks, nondecimal numeric -characters, whitespace, slashes, percent escapes and controls are not allowed. -Use literal names in selectors/ARM IDs, not URL-encoded strings. - -The helper uses one RG predicate for supplied groups, account/deployment IDs and -observed IDs, before filtering account kinds. Thus an unrelated Speech account -in a valid RG does not poison a model listing; malformed/foreign scope still -blocks the entire result, never silently skips a row. Account/deployment name -rules remain separate. Unicode case handling must not merge distinct groups -through multi-character folds. - -Only HTTP paths are UTF-8 percent-encoded. Returned IDs and selection inputs stay -literal. Continuations must decode to the same collection and retain the exact -ARM host/API and allowed skip-token parameters; they are sent on the validated -encoded path. No context change, broader listing or Unicode transliteration. - -## Purpose-specific observed origins - -| Purpose | Accepted suffixes | -|---|---| -| `embedding` | `openai.azure.com`, `services.ai.azure.com`, `cognitiveservices.azure.com` | -| `chat` | Same model origins, independently checked against the KB consumer | -| `cu` | Only `services.ai.azure.com`; no model-host substitution | - -Require `https://