From 8cb1011d54bf3e6e78ab510817bcf3685db28187 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 17 Sep 2026 11:05:24 +0000 Subject: [PATCH] Sync plugin files from GitHub-Copilot-for-Azure --- .claude-plugin/marketplace.json | 11 + .cursor-plugin/marketplace.json | 11 + .../hooks/scripts/pluginPathAllowPattern.ps1 | 47 + .../hooks/scripts/pluginPathAllowPattern.sh | 27 + .../hooks/scripts/track-telemetry.ps1 | 76 +- .../hooks/scripts/track-telemetry.sh | 23 +- .../hooks/scripts/pluginPathAllowPattern.ps1 | 47 + .../hooks/scripts/pluginPathAllowPattern.sh | 27 + .../hooks/scripts/track-telemetry.ps1 | 76 +- .../hooks/scripts/track-telemetry.sh | 23 +- .../hooks/scripts/pluginPathAllowPattern.ps1 | 47 + .../hooks/scripts/pluginPathAllowPattern.sh | 27 + .../hooks/scripts/track-telemetry.ps1 | 76 +- .../hooks/scripts/track-telemetry.sh | 23 +- .../.claude-plugin/plugin.json | 30 + .../.cursor-plugin/plugin.json | 30 + .github/plugins/foundry-iq-skills/.mcp.json | 3 + .../foundry-iq-skills/.plugin/plugin.json | 30 + .../plugins/foundry-iq-skills/CHANGELOG.md | 5 + .github/plugins/foundry-iq-skills/LICENSE | 21 + .github/plugins/foundry-iq-skills/README.md | 82 + .../foundry-iq-skills/hooks/claude-hooks.json | 14 + .../hooks/copilot-hooks.json | 16 + .../foundry-iq-skills/hooks/cursor-hooks.json | 17 + .../hooks/scripts/pluginPathAllowPattern.ps1 | 47 + .../hooks/scripts/pluginPathAllowPattern.sh | 27 + .../hooks/scripts/track-telemetry.js | 42 + .../hooks/scripts/track-telemetry.ps1 | 527 ++++++ .../hooks/scripts/track-telemetry.sh | 480 +++++ .../skills/foundry-iq/SKILL.md | 59 + .../agents/connect-hosted-toolbox.md | 140 ++ .../foundry-iq/agents/connect-hosted.md | 168 ++ .../agents/connect-prompt-sdk-fallback.md | 139 ++ .../skills/foundry-iq/agents/connect.md | 146 ++ .../foundry-iq/agents/create-missing-agent.md | 163 ++ .../foundry-iq/helpers/_blob_observation.py | 83 + .../foundry-iq/helpers/_bootstrap_io.py | 589 +++++++ .../helpers/_cleanup_dependencies.py | 435 +++++ .../foundry-iq/helpers/_cleanup_receipts.py | 239 +++ .../skills/foundry-iq/helpers/_common.py | 949 ++++++++++ .../foundry-iq/helpers/_document_adapters.py | 336 ++++ .../foundry-iq/helpers/_document_limits.py | 95 + .../helpers/_indexer_observation.py | 135 ++ .../foundry-iq/helpers/_initial_prompt.py | 181 ++ .../skills/foundry-iq/helpers/_progress.py | 259 +++ .../skills/foundry-iq/helpers/_prompt_read.py | 230 +++ .../foundry-iq/helpers/blob-contracts.md | 142 ++ .../foundry-iq/helpers/blob-cu-contracts.md | 132 ++ .../foundry-iq/helpers/blob_inventory.py | 330 ++++ .../skills/foundry-iq/helpers/blob_recheck.py | 1131 ++++++++++++ .../skills/foundry-iq/helpers/blob_source.py | 1346 ++++++++++++++ .../foundry-iq/helpers/bootstrap-contracts.md | 150 ++ .../foundry-iq/helpers/bootstrap_azure.py | 716 ++++++++ .../skills/foundry-iq/helpers/cleanup_plan.py | 735 ++++++++ .../skills/foundry-iq/helpers/contracts.md | 154 ++ .../foundry-iq/helpers/cu_ingestion_auth.py | 245 +++ .../foundry-iq/helpers/document-assessment.md | 124 ++ .../foundry-iq/helpers/document_assess.py | 353 ++++ .../helpers/file-upload-recovery.md | 135 ++ .../foundry-iq/helpers/file_cu_canary.py | 540 ++++++ .../skills/foundry-iq/helpers/file_cu_mi.py | 206 +++ .../skills/foundry-iq/helpers/file_ingest.py | 1040 +++++++++++ .../skills/foundry-iq/helpers/file_source.py | 858 +++++++++ .../skills/foundry-iq/helpers/file_upload.py | 853 +++++++++ .../foundry-iq/helpers/hosted_connect.py | 488 ++++++ .../skills/foundry-iq/helpers/kb-contracts.md | 132 ++ .../helpers/knowledge_base_retrieve.py | 187 ++ .../helpers/model-discovery-contracts.md | 150 ++ .../helpers/model-discovery-scopes.md | 131 ++ .../foundry-iq/helpers/model_discovery.py | 451 +++++ .../foundry-iq/helpers/private-artifacts.md | 115 ++ .../foundry-iq/helpers/private_artifacts.py | 103 ++ .../foundry-iq/helpers/prompt_cleanup.py | 621 +++++++ .../foundry-iq/helpers/prompt_connect.py | 1541 +++++++++++++++++ .../helpers/requirements-assessment.txt | 3 + .../retrieval-verification-contracts.md | 141 ++ .../foundry-iq/helpers/retrieval_verify.py | 418 +++++ .../helpers/search-intake-contracts.md | 134 ++ .../foundry-iq/helpers/search_intake.py | 303 ++++ .../foundry-iq/helpers/search_reconcile.py | 1465 ++++++++++++++++ .../foundry-iq/helpers/source_vector.py | 706 ++++++++ .../foundry-iq/helpers/throttle-recovery.md | 80 + .../foundry-iq/helpers/vector-contracts.md | 143 ++ .../foundry-iq/knowledge-bases/create.md | 160 ++ .../foundry-iq/knowledge-bases/retrieve.md | 167 ++ .../knowledge-sources/create-azure-blob.md | 170 ++ .../knowledge-sources/create-file.md | 171 ++ .../skills/foundry-iq/lifecycle/cleanup.md | 163 ++ .../foundry-iq/references/abstention.md | 18 + .../references/blob-binding-evidence.md | 139 ++ .../references/blob-indexer-observation.md | 135 ++ .../references/blob-ingestion-progress.md | 114 ++ .../references/blob-readiness-recheck.md | 138 ++ .../references/blob-vector-readiness.md | 42 + .../foundry-iq/references/bootstrap-azure.md | 138 ++ .../foundry-iq/references/content-fit.md | 126 ++ .../foundry-iq/references/cu-ingestion.md | 124 ++ .../foundry-iq/references/file-cu-canary.md | 163 ++ .../foundry-iq/references/intent-routing.md | 111 ++ .../skills/foundry-iq/references/owner.md | 36 + .../references/platform-contracts.md | 155 ++ .../references/platform-interfaces.md | 119 ++ .../foundry-iq/references/resource-intake.md | 130 ++ .../foundry-iq/references/search-substrate.md | 124 ++ .../foundry-iq/references/standard-cu.md | 129 ++ .../foundry-iq/search-services/create.md | 163 ++ .../foundry-iq/troubleshooting/diagnose.md | 142 ++ hooks/scripts/pluginPathAllowPattern.ps1 | 47 + hooks/scripts/pluginPathAllowPattern.sh | 27 + hooks/scripts/track-telemetry.ps1 | 76 +- hooks/scripts/track-telemetry.sh | 23 +- 111 files changed, 25786 insertions(+), 224 deletions(-) create mode 100644 .github/plugins/azure-kusto-graph-skills/hooks/scripts/pluginPathAllowPattern.ps1 create mode 100644 .github/plugins/azure-kusto-graph-skills/hooks/scripts/pluginPathAllowPattern.sh create mode 100644 .github/plugins/azure-local-skills/hooks/scripts/pluginPathAllowPattern.ps1 create mode 100644 .github/plugins/azure-local-skills/hooks/scripts/pluginPathAllowPattern.sh create mode 100644 .github/plugins/azure-skills/hooks/scripts/pluginPathAllowPattern.ps1 create mode 100644 .github/plugins/azure-skills/hooks/scripts/pluginPathAllowPattern.sh create mode 100644 .github/plugins/foundry-iq-skills/.claude-plugin/plugin.json create mode 100644 .github/plugins/foundry-iq-skills/.cursor-plugin/plugin.json create mode 100644 .github/plugins/foundry-iq-skills/.mcp.json create mode 100644 .github/plugins/foundry-iq-skills/.plugin/plugin.json create mode 100644 .github/plugins/foundry-iq-skills/CHANGELOG.md create mode 100644 .github/plugins/foundry-iq-skills/LICENSE create mode 100644 .github/plugins/foundry-iq-skills/README.md create mode 100644 .github/plugins/foundry-iq-skills/hooks/claude-hooks.json create mode 100644 .github/plugins/foundry-iq-skills/hooks/copilot-hooks.json create mode 100644 .github/plugins/foundry-iq-skills/hooks/cursor-hooks.json create mode 100644 .github/plugins/foundry-iq-skills/hooks/scripts/pluginPathAllowPattern.ps1 create mode 100644 .github/plugins/foundry-iq-skills/hooks/scripts/pluginPathAllowPattern.sh create mode 100644 .github/plugins/foundry-iq-skills/hooks/scripts/track-telemetry.js create mode 100644 .github/plugins/foundry-iq-skills/hooks/scripts/track-telemetry.ps1 create mode 100755 .github/plugins/foundry-iq-skills/hooks/scripts/track-telemetry.sh create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/SKILL.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-hosted-toolbox.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-hosted.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-prompt-sdk-fallback.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/agents/create-missing-agent.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_blob_observation.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_bootstrap_io.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_cleanup_dependencies.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_cleanup_receipts.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_common.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_document_adapters.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_document_limits.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_indexer_observation.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_initial_prompt.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_progress.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_prompt_read.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob-contracts.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob-cu-contracts.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_inventory.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_recheck.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_source.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/bootstrap-contracts.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/bootstrap_azure.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/cleanup_plan.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/contracts.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/cu_ingestion_auth.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/document-assessment.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/document_assess.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file-upload-recovery.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_cu_canary.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_cu_mi.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_ingest.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_source.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_upload.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/hosted_connect.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/kb-contracts.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/knowledge_base_retrieve.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/model-discovery-contracts.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/model-discovery-scopes.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/model_discovery.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/private-artifacts.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/private_artifacts.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/prompt_cleanup.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/prompt_connect.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/requirements-assessment.txt create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/retrieval-verification-contracts.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/retrieval_verify.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/search-intake-contracts.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/search_intake.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/search_reconcile.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/source_vector.py create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/throttle-recovery.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/vector-contracts.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/knowledge-bases/create.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/knowledge-bases/retrieve.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/knowledge-sources/create-azure-blob.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/knowledge-sources/create-file.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/lifecycle/cleanup.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/references/abstention.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/references/blob-binding-evidence.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/references/blob-indexer-observation.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/references/blob-ingestion-progress.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/references/blob-readiness-recheck.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/references/blob-vector-readiness.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/references/bootstrap-azure.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/references/content-fit.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/references/cu-ingestion.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/references/file-cu-canary.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/references/intent-routing.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/references/owner.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/references/platform-contracts.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/references/platform-interfaces.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/references/resource-intake.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/references/search-substrate.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/references/standard-cu.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/search-services/create.md create mode 100644 .github/plugins/foundry-iq-skills/skills/foundry-iq/troubleshooting/diagnose.md create mode 100644 hooks/scripts/pluginPathAllowPattern.ps1 create mode 100644 hooks/scripts/pluginPathAllowPattern.sh diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 144651b5..128dbf64 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -37,6 +37,17 @@ }, "homepage": "https://github.com/microsoft/azure-skills", "repository": "https://github.com/microsoft/github-copilot-for-azure" + }, + { + "name": "foundry-iq-skills", + "source": "./.github/plugins/foundry-iq-skills", + "description": "Create and retrieve from a Foundry IQ knowledge base using File, Azure Blob, or ADLS Gen2, then connect an existing Prompt or Hosted Agent.", + "author": { + "name": "Microsoft", + "url": "https://www.microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-skills", + "repository": "https://github.com/microsoft/github-copilot-for-azure" } ] } diff --git a/.cursor-plugin/marketplace.json b/.cursor-plugin/marketplace.json index f0fc903f..13b282c3 100644 --- a/.cursor-plugin/marketplace.json +++ b/.cursor-plugin/marketplace.json @@ -39,6 +39,17 @@ }, "homepage": "https://github.com/microsoft/azure-skills", "repository": "https://github.com/microsoft/github-copilot-for-azure" + }, + { + "name": "foundry-iq-skills", + "source": "./.github/plugins/foundry-iq-skills", + "description": "Create and retrieve from a Foundry IQ knowledge base using File, Azure Blob, or ADLS Gen2, then connect an existing Prompt or Hosted Agent.", + "author": { + "name": "Microsoft", + "url": "https://www.microsoft.com" + }, + "homepage": "https://github.com/microsoft/azure-skills", + "repository": "https://github.com/microsoft/github-copilot-for-azure" } ] } diff --git a/.github/plugins/azure-kusto-graph-skills/hooks/scripts/pluginPathAllowPattern.ps1 b/.github/plugins/azure-kusto-graph-skills/hooks/scripts/pluginPathAllowPattern.ps1 new file mode 100644 index 00000000..2031bce2 --- /dev/null +++ b/.github/plugins/azure-kusto-graph-skills/hooks/scripts/pluginPathAllowPattern.ps1 @@ -0,0 +1,47 @@ +$pluginPathPatterns = @() + +# --- azure-skills plugin --- +# The Copilot CLI pattern wildcards the catalog/marketplace folder name +# (e.g. "awesome-copilot") since it does not necessarily match the plugin's +# own name ("azure"). +$pathPatternCopilot = '\.copilot/installed-plugins/[^/]+/azure/skills/' +$pathPatternClaude = '\.claude/plugins/cache/(azure-skills|claude-plugins-official)/azure/[0-9.]+/skills/' +$pathPatternCursor = '\.cursor/plugins/cache/[^/]+/azure/[^/]+/skills/' +$pathPatternVscodeAgentPlugins = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-skills/skills/' + +# --- azure-kusto-graph-skills plugin --- +$pathPatternCopilotKustoGraph = '\.copilot/installed-plugins/[^/]+/azure-kusto-graph-skills/skills/' +$pathPatternClaudeKustoGraph = '\.claude/plugins/cache/azure-skills/azure-kusto-graph-skills/[0-9.]+/skills/' +$pathPatternCursorKustoGraph = '\.cursor/plugins/cache/[^/]+/azure-kusto-graph-skills/[^/]+/skills/' +$pathPatternVscodeAgentPluginsKustoGraph = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-kusto-graph-skills/skills/' + +# --- azure-local-skills plugin --- +$pathPatternCopilotAzureLocal = '\.copilot/installed-plugins/[^/]+/azure-local-skills/skills/' +$pathPatternClaudeAzureLocal = '\.claude/plugins/cache/azure-skills/azure-local-skills/[0-9.]+/skills/' +$pathPatternCursorAzureLocal = '\.cursor/plugins/cache/[^/]+/azure-local-skills/[^/]+/skills/' +$pathPatternVscodeAgentPluginsAzureLocal = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-local-skills/skills/' + +# --- foundry-iq-skills plugin --- +$pathPatternCopilotFoundryIq = '\.copilot/installed-plugins/[^/]+/foundry-iq-skills/skills/' +$pathPatternClaudeFoundryIq = '\.claude/plugins/cache/azure-skills/foundry-iq-skills/[0-9.]+/skills/' +$pathPatternCursorFoundryIq = '\.cursor/plugins/cache/[^/]+/foundry-iq-skills/[^/]+/skills/' +$pathPatternVscodeAgentPluginsFoundryIq = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/foundry-iq-skills/skills/' + +$pluginPathPatterns += @( + $pathPatternCopilot, + $pathPatternClaude, + $pathPatternCursor, + $pathPatternVscodeAgentPlugins, + $pathPatternCopilotKustoGraph, + $pathPatternClaudeKustoGraph, + $pathPatternCursorKustoGraph, + $pathPatternVscodeAgentPluginsKustoGraph, + $pathPatternCopilotAzureLocal, + $pathPatternClaudeAzureLocal, + $pathPatternCursorAzureLocal, + $pathPatternVscodeAgentPluginsAzureLocal, + $pathPatternCopilotFoundryIq, + $pathPatternClaudeFoundryIq, + $pathPatternCursorFoundryIq, + $pathPatternVscodeAgentPluginsFoundryIq +) diff --git a/.github/plugins/azure-kusto-graph-skills/hooks/scripts/pluginPathAllowPattern.sh b/.github/plugins/azure-kusto-graph-skills/hooks/scripts/pluginPathAllowPattern.sh new file mode 100644 index 00000000..1acfcc1f --- /dev/null +++ b/.github/plugins/azure-kusto-graph-skills/hooks/scripts/pluginPathAllowPattern.sh @@ -0,0 +1,27 @@ +# --- azure-skills plugin --- +# The Copilot CLI pattern wildcards the catalog/marketplace folder name +# (e.g. "awesome-copilot") since it does not necessarily match the +# plugin's own name ("azure"). +[[ "$p" == *".copilot/installed-plugins/"*"/azure/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/azure-skills/azure/"*"/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/claude-plugins-official/azure/"*"/skills/"* ]] && return 0 +[[ "$p" == *".cursor/plugins/cache/"*"/azure/"*"/skills/"* ]] && return 0 +[[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-skills/skills/"* ]] && return 0 + +# --- azure-kusto-graph-skills plugin --- +[[ "$p" == *".copilot/installed-plugins/"*"/azure-kusto-graph-skills/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/azure-skills/azure-kusto-graph-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *".cursor/plugins/cache/"*"/azure-kusto-graph-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-kusto-graph-skills/skills/"* ]] && return 0 + +# --- azure-local-skills plugin --- +[[ "$p" == *".copilot/installed-plugins/"*"/azure-local-skills/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/azure-skills/azure-local-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *".cursor/plugins/cache/"*"/azure-local-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-local-skills/skills/"* ]] && return 0 + +# --- foundry-iq-skills plugin --- +[[ "$p" == *".copilot/installed-plugins/"*"/foundry-iq-skills/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/azure-skills/foundry-iq-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *".cursor/plugins/cache/"*"/foundry-iq-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/foundry-iq-skills/skills/"* ]] && return 0 diff --git a/.github/plugins/azure-kusto-graph-skills/hooks/scripts/track-telemetry.ps1 b/.github/plugins/azure-kusto-graph-skills/hooks/scripts/track-telemetry.ps1 index 2f584ac3..30484e84 100644 --- a/.github/plugins/azure-kusto-graph-skills/hooks/scripts/track-telemetry.ps1 +++ b/.github/plugins/azure-kusto-graph-skills/hooks/scripts/track-telemetry.ps1 @@ -168,12 +168,40 @@ function Write-Success { exit 0 } +# Removes UTF-8 BOM markers and the common Windows mojibake forms that can +# precede Cursor hook JSON after stdin passes through Windows PowerShell. +function Remove-LeadingUtf8BomArtifacts { + param([AllowEmptyString()][string]$Value) + + if ($null -eq $Value) { return $Value } + + $bomArtifacts = @( + [string][char]0xFEFF, + (-join ([char[]]@(0x00EF, 0x00BB, 0x00BF))), + (-join ([char[]]@(0x2229, 0x2557, 0x2510))) + ) + + do { + $removedArtifact = $false + foreach ($artifact in $bomArtifacts) { + if ($Value.StartsWith($artifact, [System.StringComparison]::Ordinal)) { + $Value = $Value.Substring($artifact.Length) + $removedArtifact = $true + break + } + } + } while ($removedArtifact) + + return $Value +} + # Resolve this script's directory so we can locate bundled skills. In the # installed plugin, hooks/ and skills/ are siblings under the plugin root, so # /skills//SKILL.md is the skill definition. $scriptDir = $PSScriptRoot if (-not $scriptDir) { $scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path } $skillsDir = Join-Path (Split-Path -Parent (Split-Path -Parent $scriptDir)) 'skills' +$pluginPathAllowPattern = Join-Path $scriptDir 'pluginPathAllowPattern.ps1' # Return true only when a target belongs to this hook's plugin. Since this hook # is copied into every plugin, comparing through the skills directory prevents @@ -226,21 +254,23 @@ function Get-PluginVersion { # === Main Processing === -# Read entire stdin at once - hooks send one complete JSON per invocation +# Read stdin as bytes and decode it as UTF-8. Reading through Console.In and +# re-encoding with Console.InputEncoding can introduce code-page mojibake. try { - $stdinEncoding = [Console]::InputEncoding - $rawInput = [Console]::In.ReadToEnd() + $stdinStream = [Console]::OpenStandardInput() + $inputBuffer = New-Object System.IO.MemoryStream + $stdinStream.CopyTo($inputBuffer) $utf8WithoutBom = New-Object System.Text.UTF8Encoding($false) - # Recover the original UTF-8 bytes when Windows PowerShell decoded stdin with its OEM code page. - $rawInput = $utf8WithoutBom.GetString($stdinEncoding.GetBytes($rawInput)) + $rawInput = $utf8WithoutBom.GetString($inputBuffer.ToArray()) } catch { Write-Success +} finally { + if ($inputBuffer) { $inputBuffer.Dispose() } } -# Some clients prefix the JSON stream with a UTF-8 BOM; remove that marker before parsing. -if ($rawInput.Length -gt 0 -and [int]$rawInput[0] -eq 0xFEFF) { - $rawInput = $rawInput.Substring(1) -} +# Some clients prefix the JSON stream with a UTF-8 BOM. Cursor on Windows can +# surface an additional mojibake copy of that marker after stdin decoding. +$rawInput = Remove-LeadingUtf8BomArtifacts -Value $rawInput # Return success and exit if no input if ([string]::IsNullOrWhiteSpace($rawInput)) { @@ -339,37 +369,13 @@ function Get-ToolInputPath { # swapping both the catalog/plugin segments (e.g. "azure" and "azure-skills") # for the new plugin's name. -# --- azure-skills plugin --- -# The Copilot CLI pattern wildcards the catalog/marketplace folder name -# (e.g. "awesome-copilot") since it does not necessarily match the plugin's -# own name ("azure"). -$pathPatternCopilot = '\.copilot/installed-plugins/[^/]+/azure/skills/' -$pathPatternClaude = '\.claude/plugins/cache/(azure-skills|claude-plugins-official)/azure/[0-9.]+/skills/' -$pathPatternCursor = '\.cursor/plugins/cache/[^/]+/azure/[^/]+/skills/' -$pathPatternVscodeAgentPlugins = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-skills/skills/' - -# --- azure-kusto-graph-skills plugin --- -$pathPatternCopilotKustoGraph = '\.copilot/installed-plugins/[^/]+/azure-kusto-graph-skills/skills/' -$pathPatternClaudeKustoGraph = '\.claude/plugins/cache/azure-skills/azure-kusto-graph-skills/[0-9.]+/skills/' -$pathPatternCursorKustoGraph = '\.cursor/plugins/cache/[^/]+/azure-kusto-graph-skills/[^/]+/skills/' -$pathPatternVscodeAgentPluginsKustoGraph = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-kusto-graph-skills/skills/' - -# --- azure-local-skills plugin --- -$pathPatternCopilotAzureLocal = '\.copilot/installed-plugins/[^/]+/azure-local-skills/skills/' -$pathPatternClaudeAzureLocal = '\.claude/plugins/cache/azure-skills/azure-local-skills/[0-9.]+/skills/' -$pathPatternCursorAzureLocal = '\.cursor/plugins/cache/[^/]+/azure-local-skills/[^/]+/skills/' -$pathPatternVscodeAgentPluginsAzureLocal = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-local-skills/skills/' +. $pluginPathAllowPattern # --- shared across all plugins --- $pathPatternAgentsSkills = '\.agents/skills/' # Put the path patterns into an array for easier iteration -$pathPatterns = @( - $pathPatternCopilot, $pathPatternClaude, $pathPatternCursor, $pathPatternVscodeAgentPlugins, - $pathPatternCopilotKustoGraph, $pathPatternClaudeKustoGraph, $pathPatternCursorKustoGraph, $pathPatternVscodeAgentPluginsKustoGraph, - $pathPatternCopilotAzureLocal, $pathPatternClaudeAzureLocal, $pathPatternCursorAzureLocal, $pathPatternVscodeAgentPluginsAzureLocal, - $pathPatternAgentsSkills -) +$pathPatterns = @($pluginPathPatterns) + @($pathPatternAgentsSkills) # If $env:AZURE_SKILLS_PLUGIN_ROOT is set, add it to the path patterns for local skill development if ($env:AZURE_SKILLS_PLUGIN_ROOT) { diff --git a/.github/plugins/azure-kusto-graph-skills/hooks/scripts/track-telemetry.sh b/.github/plugins/azure-kusto-graph-skills/hooks/scripts/track-telemetry.sh index b7038f3c..e7327301 100755 --- a/.github/plugins/azure-kusto-graph-skills/hooks/scripts/track-telemetry.sh +++ b/.github/plugins/azure-kusto-graph-skills/hooks/scripts/track-telemetry.sh @@ -162,6 +162,7 @@ write_telemetry_debug_log() { # /../../skills//SKILL.md is the skill definition. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" 2>/dev/null && pwd)" SKILLS_DIR="$(cd "$SCRIPT_DIR/../.." 2>/dev/null && pwd)/skills" +PLUGIN_PATH_ALLOW_PATTERN="$SCRIPT_DIR/pluginPathAllowPattern.sh" # Return true only when a target belongs to this hook's plugin. Since this hook # is copied into every plugin, comparing through the skills directory prevents @@ -339,27 +340,7 @@ fi is_azure_skills_path() { local p="$1" - # --- azure-skills plugin --- - # The Copilot CLI pattern wildcards the catalog/marketplace folder name - # (e.g. "awesome-copilot") since it does not necessarily match the - # plugin's own name ("azure"). - [[ "$p" == *".copilot/installed-plugins/"*"/azure/skills/"* ]] && return 0 - [[ "$p" == *".claude/plugins/cache/azure-skills/azure/"*"/skills/"* ]] && return 0 - [[ "$p" == *".claude/plugins/cache/claude-plugins-official/azure/"*"/skills/"* ]] && return 0 - [[ "$p" == *".cursor/plugins/cache/"*"/azure/"*"/skills/"* ]] && return 0 - [[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-skills/skills/"* ]] && return 0 - - # --- azure-kusto-graph-skills plugin --- - [[ "$p" == *".copilot/installed-plugins/"*"/azure-kusto-graph-skills/skills/"* ]] && return 0 - [[ "$p" == *".claude/plugins/cache/azure-skills/azure-kusto-graph-skills/"*"/skills/"* ]] && return 0 - [[ "$p" == *".cursor/plugins/cache/"*"/azure-kusto-graph-skills/"*"/skills/"* ]] && return 0 - [[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-kusto-graph-skills/skills/"* ]] && return 0 - - # --- azure-local-skills plugin --- - [[ "$p" == *".copilot/installed-plugins/"*"/azure-local-skills/skills/"* ]] && return 0 - [[ "$p" == *".claude/plugins/cache/azure-skills/azure-local-skills/"*"/skills/"* ]] && return 0 - [[ "$p" == *".cursor/plugins/cache/"*"/azure-local-skills/"*"/skills/"* ]] && return 0 - [[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-local-skills/skills/"* ]] && return 0 + if . "$PLUGIN_PATH_ALLOW_PATTERN"; then return 0; fi # --- shared across all plugins --- [[ "$p" == *".agents/skills/"* ]] && return 0 diff --git a/.github/plugins/azure-local-skills/hooks/scripts/pluginPathAllowPattern.ps1 b/.github/plugins/azure-local-skills/hooks/scripts/pluginPathAllowPattern.ps1 new file mode 100644 index 00000000..2031bce2 --- /dev/null +++ b/.github/plugins/azure-local-skills/hooks/scripts/pluginPathAllowPattern.ps1 @@ -0,0 +1,47 @@ +$pluginPathPatterns = @() + +# --- azure-skills plugin --- +# The Copilot CLI pattern wildcards the catalog/marketplace folder name +# (e.g. "awesome-copilot") since it does not necessarily match the plugin's +# own name ("azure"). +$pathPatternCopilot = '\.copilot/installed-plugins/[^/]+/azure/skills/' +$pathPatternClaude = '\.claude/plugins/cache/(azure-skills|claude-plugins-official)/azure/[0-9.]+/skills/' +$pathPatternCursor = '\.cursor/plugins/cache/[^/]+/azure/[^/]+/skills/' +$pathPatternVscodeAgentPlugins = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-skills/skills/' + +# --- azure-kusto-graph-skills plugin --- +$pathPatternCopilotKustoGraph = '\.copilot/installed-plugins/[^/]+/azure-kusto-graph-skills/skills/' +$pathPatternClaudeKustoGraph = '\.claude/plugins/cache/azure-skills/azure-kusto-graph-skills/[0-9.]+/skills/' +$pathPatternCursorKustoGraph = '\.cursor/plugins/cache/[^/]+/azure-kusto-graph-skills/[^/]+/skills/' +$pathPatternVscodeAgentPluginsKustoGraph = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-kusto-graph-skills/skills/' + +# --- azure-local-skills plugin --- +$pathPatternCopilotAzureLocal = '\.copilot/installed-plugins/[^/]+/azure-local-skills/skills/' +$pathPatternClaudeAzureLocal = '\.claude/plugins/cache/azure-skills/azure-local-skills/[0-9.]+/skills/' +$pathPatternCursorAzureLocal = '\.cursor/plugins/cache/[^/]+/azure-local-skills/[^/]+/skills/' +$pathPatternVscodeAgentPluginsAzureLocal = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-local-skills/skills/' + +# --- foundry-iq-skills plugin --- +$pathPatternCopilotFoundryIq = '\.copilot/installed-plugins/[^/]+/foundry-iq-skills/skills/' +$pathPatternClaudeFoundryIq = '\.claude/plugins/cache/azure-skills/foundry-iq-skills/[0-9.]+/skills/' +$pathPatternCursorFoundryIq = '\.cursor/plugins/cache/[^/]+/foundry-iq-skills/[^/]+/skills/' +$pathPatternVscodeAgentPluginsFoundryIq = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/foundry-iq-skills/skills/' + +$pluginPathPatterns += @( + $pathPatternCopilot, + $pathPatternClaude, + $pathPatternCursor, + $pathPatternVscodeAgentPlugins, + $pathPatternCopilotKustoGraph, + $pathPatternClaudeKustoGraph, + $pathPatternCursorKustoGraph, + $pathPatternVscodeAgentPluginsKustoGraph, + $pathPatternCopilotAzureLocal, + $pathPatternClaudeAzureLocal, + $pathPatternCursorAzureLocal, + $pathPatternVscodeAgentPluginsAzureLocal, + $pathPatternCopilotFoundryIq, + $pathPatternClaudeFoundryIq, + $pathPatternCursorFoundryIq, + $pathPatternVscodeAgentPluginsFoundryIq +) diff --git a/.github/plugins/azure-local-skills/hooks/scripts/pluginPathAllowPattern.sh b/.github/plugins/azure-local-skills/hooks/scripts/pluginPathAllowPattern.sh new file mode 100644 index 00000000..1acfcc1f --- /dev/null +++ b/.github/plugins/azure-local-skills/hooks/scripts/pluginPathAllowPattern.sh @@ -0,0 +1,27 @@ +# --- azure-skills plugin --- +# The Copilot CLI pattern wildcards the catalog/marketplace folder name +# (e.g. "awesome-copilot") since it does not necessarily match the +# plugin's own name ("azure"). +[[ "$p" == *".copilot/installed-plugins/"*"/azure/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/azure-skills/azure/"*"/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/claude-plugins-official/azure/"*"/skills/"* ]] && return 0 +[[ "$p" == *".cursor/plugins/cache/"*"/azure/"*"/skills/"* ]] && return 0 +[[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-skills/skills/"* ]] && return 0 + +# --- azure-kusto-graph-skills plugin --- +[[ "$p" == *".copilot/installed-plugins/"*"/azure-kusto-graph-skills/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/azure-skills/azure-kusto-graph-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *".cursor/plugins/cache/"*"/azure-kusto-graph-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-kusto-graph-skills/skills/"* ]] && return 0 + +# --- azure-local-skills plugin --- +[[ "$p" == *".copilot/installed-plugins/"*"/azure-local-skills/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/azure-skills/azure-local-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *".cursor/plugins/cache/"*"/azure-local-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-local-skills/skills/"* ]] && return 0 + +# --- foundry-iq-skills plugin --- +[[ "$p" == *".copilot/installed-plugins/"*"/foundry-iq-skills/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/azure-skills/foundry-iq-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *".cursor/plugins/cache/"*"/foundry-iq-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/foundry-iq-skills/skills/"* ]] && return 0 diff --git a/.github/plugins/azure-local-skills/hooks/scripts/track-telemetry.ps1 b/.github/plugins/azure-local-skills/hooks/scripts/track-telemetry.ps1 index 2f584ac3..30484e84 100644 --- a/.github/plugins/azure-local-skills/hooks/scripts/track-telemetry.ps1 +++ b/.github/plugins/azure-local-skills/hooks/scripts/track-telemetry.ps1 @@ -168,12 +168,40 @@ function Write-Success { exit 0 } +# Removes UTF-8 BOM markers and the common Windows mojibake forms that can +# precede Cursor hook JSON after stdin passes through Windows PowerShell. +function Remove-LeadingUtf8BomArtifacts { + param([AllowEmptyString()][string]$Value) + + if ($null -eq $Value) { return $Value } + + $bomArtifacts = @( + [string][char]0xFEFF, + (-join ([char[]]@(0x00EF, 0x00BB, 0x00BF))), + (-join ([char[]]@(0x2229, 0x2557, 0x2510))) + ) + + do { + $removedArtifact = $false + foreach ($artifact in $bomArtifacts) { + if ($Value.StartsWith($artifact, [System.StringComparison]::Ordinal)) { + $Value = $Value.Substring($artifact.Length) + $removedArtifact = $true + break + } + } + } while ($removedArtifact) + + return $Value +} + # Resolve this script's directory so we can locate bundled skills. In the # installed plugin, hooks/ and skills/ are siblings under the plugin root, so # /skills//SKILL.md is the skill definition. $scriptDir = $PSScriptRoot if (-not $scriptDir) { $scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path } $skillsDir = Join-Path (Split-Path -Parent (Split-Path -Parent $scriptDir)) 'skills' +$pluginPathAllowPattern = Join-Path $scriptDir 'pluginPathAllowPattern.ps1' # Return true only when a target belongs to this hook's plugin. Since this hook # is copied into every plugin, comparing through the skills directory prevents @@ -226,21 +254,23 @@ function Get-PluginVersion { # === Main Processing === -# Read entire stdin at once - hooks send one complete JSON per invocation +# Read stdin as bytes and decode it as UTF-8. Reading through Console.In and +# re-encoding with Console.InputEncoding can introduce code-page mojibake. try { - $stdinEncoding = [Console]::InputEncoding - $rawInput = [Console]::In.ReadToEnd() + $stdinStream = [Console]::OpenStandardInput() + $inputBuffer = New-Object System.IO.MemoryStream + $stdinStream.CopyTo($inputBuffer) $utf8WithoutBom = New-Object System.Text.UTF8Encoding($false) - # Recover the original UTF-8 bytes when Windows PowerShell decoded stdin with its OEM code page. - $rawInput = $utf8WithoutBom.GetString($stdinEncoding.GetBytes($rawInput)) + $rawInput = $utf8WithoutBom.GetString($inputBuffer.ToArray()) } catch { Write-Success +} finally { + if ($inputBuffer) { $inputBuffer.Dispose() } } -# Some clients prefix the JSON stream with a UTF-8 BOM; remove that marker before parsing. -if ($rawInput.Length -gt 0 -and [int]$rawInput[0] -eq 0xFEFF) { - $rawInput = $rawInput.Substring(1) -} +# Some clients prefix the JSON stream with a UTF-8 BOM. Cursor on Windows can +# surface an additional mojibake copy of that marker after stdin decoding. +$rawInput = Remove-LeadingUtf8BomArtifacts -Value $rawInput # Return success and exit if no input if ([string]::IsNullOrWhiteSpace($rawInput)) { @@ -339,37 +369,13 @@ function Get-ToolInputPath { # swapping both the catalog/plugin segments (e.g. "azure" and "azure-skills") # for the new plugin's name. -# --- azure-skills plugin --- -# The Copilot CLI pattern wildcards the catalog/marketplace folder name -# (e.g. "awesome-copilot") since it does not necessarily match the plugin's -# own name ("azure"). -$pathPatternCopilot = '\.copilot/installed-plugins/[^/]+/azure/skills/' -$pathPatternClaude = '\.claude/plugins/cache/(azure-skills|claude-plugins-official)/azure/[0-9.]+/skills/' -$pathPatternCursor = '\.cursor/plugins/cache/[^/]+/azure/[^/]+/skills/' -$pathPatternVscodeAgentPlugins = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-skills/skills/' - -# --- azure-kusto-graph-skills plugin --- -$pathPatternCopilotKustoGraph = '\.copilot/installed-plugins/[^/]+/azure-kusto-graph-skills/skills/' -$pathPatternClaudeKustoGraph = '\.claude/plugins/cache/azure-skills/azure-kusto-graph-skills/[0-9.]+/skills/' -$pathPatternCursorKustoGraph = '\.cursor/plugins/cache/[^/]+/azure-kusto-graph-skills/[^/]+/skills/' -$pathPatternVscodeAgentPluginsKustoGraph = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-kusto-graph-skills/skills/' - -# --- azure-local-skills plugin --- -$pathPatternCopilotAzureLocal = '\.copilot/installed-plugins/[^/]+/azure-local-skills/skills/' -$pathPatternClaudeAzureLocal = '\.claude/plugins/cache/azure-skills/azure-local-skills/[0-9.]+/skills/' -$pathPatternCursorAzureLocal = '\.cursor/plugins/cache/[^/]+/azure-local-skills/[^/]+/skills/' -$pathPatternVscodeAgentPluginsAzureLocal = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-local-skills/skills/' +. $pluginPathAllowPattern # --- shared across all plugins --- $pathPatternAgentsSkills = '\.agents/skills/' # Put the path patterns into an array for easier iteration -$pathPatterns = @( - $pathPatternCopilot, $pathPatternClaude, $pathPatternCursor, $pathPatternVscodeAgentPlugins, - $pathPatternCopilotKustoGraph, $pathPatternClaudeKustoGraph, $pathPatternCursorKustoGraph, $pathPatternVscodeAgentPluginsKustoGraph, - $pathPatternCopilotAzureLocal, $pathPatternClaudeAzureLocal, $pathPatternCursorAzureLocal, $pathPatternVscodeAgentPluginsAzureLocal, - $pathPatternAgentsSkills -) +$pathPatterns = @($pluginPathPatterns) + @($pathPatternAgentsSkills) # If $env:AZURE_SKILLS_PLUGIN_ROOT is set, add it to the path patterns for local skill development if ($env:AZURE_SKILLS_PLUGIN_ROOT) { diff --git a/.github/plugins/azure-local-skills/hooks/scripts/track-telemetry.sh b/.github/plugins/azure-local-skills/hooks/scripts/track-telemetry.sh index b7038f3c..e7327301 100755 --- a/.github/plugins/azure-local-skills/hooks/scripts/track-telemetry.sh +++ b/.github/plugins/azure-local-skills/hooks/scripts/track-telemetry.sh @@ -162,6 +162,7 @@ write_telemetry_debug_log() { # /../../skills//SKILL.md is the skill definition. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" 2>/dev/null && pwd)" SKILLS_DIR="$(cd "$SCRIPT_DIR/../.." 2>/dev/null && pwd)/skills" +PLUGIN_PATH_ALLOW_PATTERN="$SCRIPT_DIR/pluginPathAllowPattern.sh" # Return true only when a target belongs to this hook's plugin. Since this hook # is copied into every plugin, comparing through the skills directory prevents @@ -339,27 +340,7 @@ fi is_azure_skills_path() { local p="$1" - # --- azure-skills plugin --- - # The Copilot CLI pattern wildcards the catalog/marketplace folder name - # (e.g. "awesome-copilot") since it does not necessarily match the - # plugin's own name ("azure"). - [[ "$p" == *".copilot/installed-plugins/"*"/azure/skills/"* ]] && return 0 - [[ "$p" == *".claude/plugins/cache/azure-skills/azure/"*"/skills/"* ]] && return 0 - [[ "$p" == *".claude/plugins/cache/claude-plugins-official/azure/"*"/skills/"* ]] && return 0 - [[ "$p" == *".cursor/plugins/cache/"*"/azure/"*"/skills/"* ]] && return 0 - [[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-skills/skills/"* ]] && return 0 - - # --- azure-kusto-graph-skills plugin --- - [[ "$p" == *".copilot/installed-plugins/"*"/azure-kusto-graph-skills/skills/"* ]] && return 0 - [[ "$p" == *".claude/plugins/cache/azure-skills/azure-kusto-graph-skills/"*"/skills/"* ]] && return 0 - [[ "$p" == *".cursor/plugins/cache/"*"/azure-kusto-graph-skills/"*"/skills/"* ]] && return 0 - [[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-kusto-graph-skills/skills/"* ]] && return 0 - - # --- azure-local-skills plugin --- - [[ "$p" == *".copilot/installed-plugins/"*"/azure-local-skills/skills/"* ]] && return 0 - [[ "$p" == *".claude/plugins/cache/azure-skills/azure-local-skills/"*"/skills/"* ]] && return 0 - [[ "$p" == *".cursor/plugins/cache/"*"/azure-local-skills/"*"/skills/"* ]] && return 0 - [[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-local-skills/skills/"* ]] && return 0 + if . "$PLUGIN_PATH_ALLOW_PATTERN"; then return 0; fi # --- shared across all plugins --- [[ "$p" == *".agents/skills/"* ]] && return 0 diff --git a/.github/plugins/azure-skills/hooks/scripts/pluginPathAllowPattern.ps1 b/.github/plugins/azure-skills/hooks/scripts/pluginPathAllowPattern.ps1 new file mode 100644 index 00000000..2031bce2 --- /dev/null +++ b/.github/plugins/azure-skills/hooks/scripts/pluginPathAllowPattern.ps1 @@ -0,0 +1,47 @@ +$pluginPathPatterns = @() + +# --- azure-skills plugin --- +# The Copilot CLI pattern wildcards the catalog/marketplace folder name +# (e.g. "awesome-copilot") since it does not necessarily match the plugin's +# own name ("azure"). +$pathPatternCopilot = '\.copilot/installed-plugins/[^/]+/azure/skills/' +$pathPatternClaude = '\.claude/plugins/cache/(azure-skills|claude-plugins-official)/azure/[0-9.]+/skills/' +$pathPatternCursor = '\.cursor/plugins/cache/[^/]+/azure/[^/]+/skills/' +$pathPatternVscodeAgentPlugins = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-skills/skills/' + +# --- azure-kusto-graph-skills plugin --- +$pathPatternCopilotKustoGraph = '\.copilot/installed-plugins/[^/]+/azure-kusto-graph-skills/skills/' +$pathPatternClaudeKustoGraph = '\.claude/plugins/cache/azure-skills/azure-kusto-graph-skills/[0-9.]+/skills/' +$pathPatternCursorKustoGraph = '\.cursor/plugins/cache/[^/]+/azure-kusto-graph-skills/[^/]+/skills/' +$pathPatternVscodeAgentPluginsKustoGraph = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-kusto-graph-skills/skills/' + +# --- azure-local-skills plugin --- +$pathPatternCopilotAzureLocal = '\.copilot/installed-plugins/[^/]+/azure-local-skills/skills/' +$pathPatternClaudeAzureLocal = '\.claude/plugins/cache/azure-skills/azure-local-skills/[0-9.]+/skills/' +$pathPatternCursorAzureLocal = '\.cursor/plugins/cache/[^/]+/azure-local-skills/[^/]+/skills/' +$pathPatternVscodeAgentPluginsAzureLocal = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-local-skills/skills/' + +# --- foundry-iq-skills plugin --- +$pathPatternCopilotFoundryIq = '\.copilot/installed-plugins/[^/]+/foundry-iq-skills/skills/' +$pathPatternClaudeFoundryIq = '\.claude/plugins/cache/azure-skills/foundry-iq-skills/[0-9.]+/skills/' +$pathPatternCursorFoundryIq = '\.cursor/plugins/cache/[^/]+/foundry-iq-skills/[^/]+/skills/' +$pathPatternVscodeAgentPluginsFoundryIq = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/foundry-iq-skills/skills/' + +$pluginPathPatterns += @( + $pathPatternCopilot, + $pathPatternClaude, + $pathPatternCursor, + $pathPatternVscodeAgentPlugins, + $pathPatternCopilotKustoGraph, + $pathPatternClaudeKustoGraph, + $pathPatternCursorKustoGraph, + $pathPatternVscodeAgentPluginsKustoGraph, + $pathPatternCopilotAzureLocal, + $pathPatternClaudeAzureLocal, + $pathPatternCursorAzureLocal, + $pathPatternVscodeAgentPluginsAzureLocal, + $pathPatternCopilotFoundryIq, + $pathPatternClaudeFoundryIq, + $pathPatternCursorFoundryIq, + $pathPatternVscodeAgentPluginsFoundryIq +) diff --git a/.github/plugins/azure-skills/hooks/scripts/pluginPathAllowPattern.sh b/.github/plugins/azure-skills/hooks/scripts/pluginPathAllowPattern.sh new file mode 100644 index 00000000..1acfcc1f --- /dev/null +++ b/.github/plugins/azure-skills/hooks/scripts/pluginPathAllowPattern.sh @@ -0,0 +1,27 @@ +# --- azure-skills plugin --- +# The Copilot CLI pattern wildcards the catalog/marketplace folder name +# (e.g. "awesome-copilot") since it does not necessarily match the +# plugin's own name ("azure"). +[[ "$p" == *".copilot/installed-plugins/"*"/azure/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/azure-skills/azure/"*"/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/claude-plugins-official/azure/"*"/skills/"* ]] && return 0 +[[ "$p" == *".cursor/plugins/cache/"*"/azure/"*"/skills/"* ]] && return 0 +[[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-skills/skills/"* ]] && return 0 + +# --- azure-kusto-graph-skills plugin --- +[[ "$p" == *".copilot/installed-plugins/"*"/azure-kusto-graph-skills/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/azure-skills/azure-kusto-graph-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *".cursor/plugins/cache/"*"/azure-kusto-graph-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-kusto-graph-skills/skills/"* ]] && return 0 + +# --- azure-local-skills plugin --- +[[ "$p" == *".copilot/installed-plugins/"*"/azure-local-skills/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/azure-skills/azure-local-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *".cursor/plugins/cache/"*"/azure-local-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-local-skills/skills/"* ]] && return 0 + +# --- foundry-iq-skills plugin --- +[[ "$p" == *".copilot/installed-plugins/"*"/foundry-iq-skills/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/azure-skills/foundry-iq-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *".cursor/plugins/cache/"*"/foundry-iq-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/foundry-iq-skills/skills/"* ]] && return 0 diff --git a/.github/plugins/azure-skills/hooks/scripts/track-telemetry.ps1 b/.github/plugins/azure-skills/hooks/scripts/track-telemetry.ps1 index 2f584ac3..30484e84 100644 --- a/.github/plugins/azure-skills/hooks/scripts/track-telemetry.ps1 +++ b/.github/plugins/azure-skills/hooks/scripts/track-telemetry.ps1 @@ -168,12 +168,40 @@ function Write-Success { exit 0 } +# Removes UTF-8 BOM markers and the common Windows mojibake forms that can +# precede Cursor hook JSON after stdin passes through Windows PowerShell. +function Remove-LeadingUtf8BomArtifacts { + param([AllowEmptyString()][string]$Value) + + if ($null -eq $Value) { return $Value } + + $bomArtifacts = @( + [string][char]0xFEFF, + (-join ([char[]]@(0x00EF, 0x00BB, 0x00BF))), + (-join ([char[]]@(0x2229, 0x2557, 0x2510))) + ) + + do { + $removedArtifact = $false + foreach ($artifact in $bomArtifacts) { + if ($Value.StartsWith($artifact, [System.StringComparison]::Ordinal)) { + $Value = $Value.Substring($artifact.Length) + $removedArtifact = $true + break + } + } + } while ($removedArtifact) + + return $Value +} + # Resolve this script's directory so we can locate bundled skills. In the # installed plugin, hooks/ and skills/ are siblings under the plugin root, so # /skills//SKILL.md is the skill definition. $scriptDir = $PSScriptRoot if (-not $scriptDir) { $scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path } $skillsDir = Join-Path (Split-Path -Parent (Split-Path -Parent $scriptDir)) 'skills' +$pluginPathAllowPattern = Join-Path $scriptDir 'pluginPathAllowPattern.ps1' # Return true only when a target belongs to this hook's plugin. Since this hook # is copied into every plugin, comparing through the skills directory prevents @@ -226,21 +254,23 @@ function Get-PluginVersion { # === Main Processing === -# Read entire stdin at once - hooks send one complete JSON per invocation +# Read stdin as bytes and decode it as UTF-8. Reading through Console.In and +# re-encoding with Console.InputEncoding can introduce code-page mojibake. try { - $stdinEncoding = [Console]::InputEncoding - $rawInput = [Console]::In.ReadToEnd() + $stdinStream = [Console]::OpenStandardInput() + $inputBuffer = New-Object System.IO.MemoryStream + $stdinStream.CopyTo($inputBuffer) $utf8WithoutBom = New-Object System.Text.UTF8Encoding($false) - # Recover the original UTF-8 bytes when Windows PowerShell decoded stdin with its OEM code page. - $rawInput = $utf8WithoutBom.GetString($stdinEncoding.GetBytes($rawInput)) + $rawInput = $utf8WithoutBom.GetString($inputBuffer.ToArray()) } catch { Write-Success +} finally { + if ($inputBuffer) { $inputBuffer.Dispose() } } -# Some clients prefix the JSON stream with a UTF-8 BOM; remove that marker before parsing. -if ($rawInput.Length -gt 0 -and [int]$rawInput[0] -eq 0xFEFF) { - $rawInput = $rawInput.Substring(1) -} +# Some clients prefix the JSON stream with a UTF-8 BOM. Cursor on Windows can +# surface an additional mojibake copy of that marker after stdin decoding. +$rawInput = Remove-LeadingUtf8BomArtifacts -Value $rawInput # Return success and exit if no input if ([string]::IsNullOrWhiteSpace($rawInput)) { @@ -339,37 +369,13 @@ function Get-ToolInputPath { # swapping both the catalog/plugin segments (e.g. "azure" and "azure-skills") # for the new plugin's name. -# --- azure-skills plugin --- -# The Copilot CLI pattern wildcards the catalog/marketplace folder name -# (e.g. "awesome-copilot") since it does not necessarily match the plugin's -# own name ("azure"). -$pathPatternCopilot = '\.copilot/installed-plugins/[^/]+/azure/skills/' -$pathPatternClaude = '\.claude/plugins/cache/(azure-skills|claude-plugins-official)/azure/[0-9.]+/skills/' -$pathPatternCursor = '\.cursor/plugins/cache/[^/]+/azure/[^/]+/skills/' -$pathPatternVscodeAgentPlugins = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-skills/skills/' - -# --- azure-kusto-graph-skills plugin --- -$pathPatternCopilotKustoGraph = '\.copilot/installed-plugins/[^/]+/azure-kusto-graph-skills/skills/' -$pathPatternClaudeKustoGraph = '\.claude/plugins/cache/azure-skills/azure-kusto-graph-skills/[0-9.]+/skills/' -$pathPatternCursorKustoGraph = '\.cursor/plugins/cache/[^/]+/azure-kusto-graph-skills/[^/]+/skills/' -$pathPatternVscodeAgentPluginsKustoGraph = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-kusto-graph-skills/skills/' - -# --- azure-local-skills plugin --- -$pathPatternCopilotAzureLocal = '\.copilot/installed-plugins/[^/]+/azure-local-skills/skills/' -$pathPatternClaudeAzureLocal = '\.claude/plugins/cache/azure-skills/azure-local-skills/[0-9.]+/skills/' -$pathPatternCursorAzureLocal = '\.cursor/plugins/cache/[^/]+/azure-local-skills/[^/]+/skills/' -$pathPatternVscodeAgentPluginsAzureLocal = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-local-skills/skills/' +. $pluginPathAllowPattern # --- shared across all plugins --- $pathPatternAgentsSkills = '\.agents/skills/' # Put the path patterns into an array for easier iteration -$pathPatterns = @( - $pathPatternCopilot, $pathPatternClaude, $pathPatternCursor, $pathPatternVscodeAgentPlugins, - $pathPatternCopilotKustoGraph, $pathPatternClaudeKustoGraph, $pathPatternCursorKustoGraph, $pathPatternVscodeAgentPluginsKustoGraph, - $pathPatternCopilotAzureLocal, $pathPatternClaudeAzureLocal, $pathPatternCursorAzureLocal, $pathPatternVscodeAgentPluginsAzureLocal, - $pathPatternAgentsSkills -) +$pathPatterns = @($pluginPathPatterns) + @($pathPatternAgentsSkills) # If $env:AZURE_SKILLS_PLUGIN_ROOT is set, add it to the path patterns for local skill development if ($env:AZURE_SKILLS_PLUGIN_ROOT) { diff --git a/.github/plugins/azure-skills/hooks/scripts/track-telemetry.sh b/.github/plugins/azure-skills/hooks/scripts/track-telemetry.sh index b7038f3c..e7327301 100755 --- a/.github/plugins/azure-skills/hooks/scripts/track-telemetry.sh +++ b/.github/plugins/azure-skills/hooks/scripts/track-telemetry.sh @@ -162,6 +162,7 @@ write_telemetry_debug_log() { # /../../skills//SKILL.md is the skill definition. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" 2>/dev/null && pwd)" SKILLS_DIR="$(cd "$SCRIPT_DIR/../.." 2>/dev/null && pwd)/skills" +PLUGIN_PATH_ALLOW_PATTERN="$SCRIPT_DIR/pluginPathAllowPattern.sh" # Return true only when a target belongs to this hook's plugin. Since this hook # is copied into every plugin, comparing through the skills directory prevents @@ -339,27 +340,7 @@ fi is_azure_skills_path() { local p="$1" - # --- azure-skills plugin --- - # The Copilot CLI pattern wildcards the catalog/marketplace folder name - # (e.g. "awesome-copilot") since it does not necessarily match the - # plugin's own name ("azure"). - [[ "$p" == *".copilot/installed-plugins/"*"/azure/skills/"* ]] && return 0 - [[ "$p" == *".claude/plugins/cache/azure-skills/azure/"*"/skills/"* ]] && return 0 - [[ "$p" == *".claude/plugins/cache/claude-plugins-official/azure/"*"/skills/"* ]] && return 0 - [[ "$p" == *".cursor/plugins/cache/"*"/azure/"*"/skills/"* ]] && return 0 - [[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-skills/skills/"* ]] && return 0 - - # --- azure-kusto-graph-skills plugin --- - [[ "$p" == *".copilot/installed-plugins/"*"/azure-kusto-graph-skills/skills/"* ]] && return 0 - [[ "$p" == *".claude/plugins/cache/azure-skills/azure-kusto-graph-skills/"*"/skills/"* ]] && return 0 - [[ "$p" == *".cursor/plugins/cache/"*"/azure-kusto-graph-skills/"*"/skills/"* ]] && return 0 - [[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-kusto-graph-skills/skills/"* ]] && return 0 - - # --- azure-local-skills plugin --- - [[ "$p" == *".copilot/installed-plugins/"*"/azure-local-skills/skills/"* ]] && return 0 - [[ "$p" == *".claude/plugins/cache/azure-skills/azure-local-skills/"*"/skills/"* ]] && return 0 - [[ "$p" == *".cursor/plugins/cache/"*"/azure-local-skills/"*"/skills/"* ]] && return 0 - [[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-local-skills/skills/"* ]] && return 0 + if . "$PLUGIN_PATH_ALLOW_PATTERN"; then return 0; fi # --- shared across all plugins --- [[ "$p" == *".agents/skills/"* ]] && return 0 diff --git a/.github/plugins/foundry-iq-skills/.claude-plugin/plugin.json b/.github/plugins/foundry-iq-skills/.claude-plugin/plugin.json new file mode 100644 index 00000000..59780579 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/.claude-plugin/plugin.json @@ -0,0 +1,30 @@ +{ + "name": "foundry-iq-skills", + "description": "Create and retrieve from a Foundry IQ knowledge base using File, Azure Blob, or ADLS Gen2, then connect an existing Prompt or Hosted Agent.", + "version": "0.1.0", + "author": { + "name": "Microsoft", + "url": "https://www.microsoft.com" + }, + "homepage": "https://github.com/microsoft/github-copilot-for-azure", + "repository": "https://github.com/microsoft/GitHub-Copilot-for-Azure", + "license": "MIT", + "keywords": [ + "azure", + "azure-ai-search", + "rag", + "grounding", + "foundry-iq", + "knowledge-base", + "file-knowledge-source", + "azure-blob", + "adls-gen2", + "retrieval", + "citations", + "prompt-agent", + "hosted-agent" + ], + "skills": "./skills/", + "mcpServers": "./.mcp.json", + "hooks": "./hooks/claude-hooks.json" +} \ No newline at end of file diff --git a/.github/plugins/foundry-iq-skills/.cursor-plugin/plugin.json b/.github/plugins/foundry-iq-skills/.cursor-plugin/plugin.json new file mode 100644 index 00000000..a1e13280 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/.cursor-plugin/plugin.json @@ -0,0 +1,30 @@ +{ + "name": "foundry-iq-skills", + "description": "Create and retrieve from a Foundry IQ knowledge base using File, Azure Blob, or ADLS Gen2, then connect an existing Prompt or Hosted Agent.", + "version": "0.1.0", + "author": { + "name": "Microsoft", + "url": "https://www.microsoft.com" + }, + "homepage": "https://github.com/microsoft/github-copilot-for-azure", + "repository": "https://github.com/microsoft/GitHub-Copilot-for-Azure", + "license": "MIT", + "keywords": [ + "azure", + "azure-ai-search", + "rag", + "grounding", + "foundry-iq", + "knowledge-base", + "file-knowledge-source", + "azure-blob", + "adls-gen2", + "retrieval", + "citations", + "prompt-agent", + "hosted-agent" + ], + "skills": "./skills/", + "mcpServers": "./.mcp.json", + "hooks": "./hooks/cursor-hooks.json" +} \ No newline at end of file diff --git a/.github/plugins/foundry-iq-skills/.mcp.json b/.github/plugins/foundry-iq-skills/.mcp.json new file mode 100644 index 00000000..70011302 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/.mcp.json @@ -0,0 +1,3 @@ +{ + "mcpServers": {} +} \ No newline at end of file diff --git a/.github/plugins/foundry-iq-skills/.plugin/plugin.json b/.github/plugins/foundry-iq-skills/.plugin/plugin.json new file mode 100644 index 00000000..c8693517 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/.plugin/plugin.json @@ -0,0 +1,30 @@ +{ + "name": "foundry-iq-skills", + "description": "Create and retrieve from a Foundry IQ knowledge base using File, Azure Blob, or ADLS Gen2, then connect an existing Prompt or Hosted Agent.", + "version": "0.1.0", + "author": { + "name": "Microsoft", + "url": "https://www.microsoft.com" + }, + "homepage": "https://github.com/microsoft/github-copilot-for-azure", + "repository": "https://github.com/microsoft/GitHub-Copilot-for-Azure", + "license": "MIT", + "keywords": [ + "azure", + "azure-ai-search", + "rag", + "grounding", + "foundry-iq", + "knowledge-base", + "file-knowledge-source", + "azure-blob", + "adls-gen2", + "retrieval", + "citations", + "prompt-agent", + "hosted-agent" + ], + "skills": "./skills/", + "mcpServers": "./.mcp.json", + "hooks": "./hooks/copilot-hooks.json" +} \ No newline at end of file diff --git a/.github/plugins/foundry-iq-skills/CHANGELOG.md b/.github/plugins/foundry-iq-skills/CHANGELOG.md new file mode 100644 index 00000000..5e53757a --- /dev/null +++ b/.github/plugins/foundry-iq-skills/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +## 0.1.0 + +- feat: add Foundry IQ skills plugin ([#3209](https://github.com/microsoft/GitHub-Copilot-for-Azure/pull/3209)) diff --git a/.github/plugins/foundry-iq-skills/LICENSE b/.github/plugins/foundry-iq-skills/LICENSE new file mode 100644 index 00000000..0cd75c0a --- /dev/null +++ b/.github/plugins/foundry-iq-skills/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright 2025 (c) Microsoft Corporation. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE \ No newline at end of file diff --git a/.github/plugins/foundry-iq-skills/README.md b/.github/plugins/foundry-iq-skills/README.md new file mode 100644 index 00000000..b189d75b --- /dev/null +++ b/.github/plugins/foundry-iq-skills/README.md @@ -0,0 +1,82 @@ +# Foundry IQ Skills + +Build grounded knowledge experiences with [Azure AI Search (Foundry IQ)](https://learn.microsoft.com/azure/search/agentic-retrieval-overview). + +This plugin helps GitHub Copilot CLI create and use Foundry IQ knowledge bases, using your existing Azure resources when possible. + +## Skill + +- **foundry-iq**: Build, connect, query, and troubleshoot Foundry IQ knowledge experiences using supported Azure resources. + +## What it helps with + +- Create or reuse an Azure AI Search (Foundry IQ) service +- Create a File knowledge source from local files +- Create a knowledge source from Azure Blob Storage or Azure Data Lake Storage +- Create a knowledge base and validate that its content is searchable +- Query an existing knowledge base and return citations +- Connect an existing knowledge base to an agent +- Diagnose knowledge base failures and unsupported requests +- Prepare a cleanup plan for resources created by the workflow + +The skill is intended for Foundry IQ knowledge-base workflows. It does not support classic Azure AI Search (Foundry IQ) application, index, or query development, generic agent creation, or repository-file search. + +## Prerequisites + +- [Git](https://git-scm.com/downloads), required to add the plugin marketplace +- [GitHub Copilot CLI](https://github.com/github/copilot-cli) +- [Python 3.12 or later](https://www.python.org/downloads/), required to run the plugin's helper scripts +- Access to the Azure subscriptions and resources involved in your request +- An authenticated Azure identity with the permissions required for the requested read or change + +If you use Azure CLI for authentication, install the [Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli) and sign in: + +```bash +az login +``` + +The plugin supports Copilot CLI on Windows and Linux. Depending on the task, it uses supported Azure SDK, REST, infrastructure-as-code, Azure MCP Server, or native knowledge-base MCP interfaces. + +## Installation + +Run these commands in Copilot CLI: + +```text +/plugin marketplace add microsoft/azure-skills +/plugin install foundry-iq-skills@azure-skills +``` + +To update the plugin: + +```text +/plugin update foundry-iq-skills@azure-skills +``` + +## Example prompts + +- "Create a knowledge base from `./docs`." +- "Create a knowledge base from this Blob container." +- "Query this knowledge base with citations." +- "Connect this knowledge base to my existing agent." +- "Why is retrieval from this knowledge base failing?" + +## Before changes are made + +The skill starts with read-only checks and can reuse Azure resources you identify. It asks whether to use an existing service, find compatible services, or create a new one before performing broader discovery. + +It asks for your approval before it: + +- Creates an Azure AI Search (Foundry IQ) service, knowledge source, or knowledge base +- Connects a knowledge base to an agent +- Makes another planned change to Azure resources + +Approval applies to the plan shown to you. If the plan changes, the skill asks again. Cleanup is handled as a separate planning workflow. The skill can also perform supported deletion of workflow-owned resources after separate approval. + +## Learn more + +- [Azure AI Search (Foundry IQ) overview](https://learn.microsoft.com/azure/search/search-what-is-azure-search) +- [Foundry IQ and agentic retrieval](https://learn.microsoft.com/azure/search/agentic-retrieval-overview) +- [Azure AI Search (Foundry IQ) API and SDK versions](https://learn.microsoft.com/azure/search/search-api-versions) +- [Azure AI Search (Foundry IQ) REST API](https://learn.microsoft.com/en-us/rest/api/searchservice/?source=recommendations) +- [Azure MCP Server tools for Azure AI Search (Foundry IQ)](https://learn.microsoft.com/azure/developer/azure-mcp-server/tools/azure-ai-search) +- [Foundry IQ skill details](skills/foundry-iq/SKILL.md) diff --git a/.github/plugins/foundry-iq-skills/hooks/claude-hooks.json b/.github/plugins/foundry-iq-skills/hooks/claude-hooks.json new file mode 100644 index 00000000..284be250 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/hooks/claude-hooks.json @@ -0,0 +1,14 @@ +{ + "hooks": { + "PostToolUse": [ + { + "hooks": [ + { + "type": "command", + "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/scripts/track-telemetry.sh\"" + } + ] + } + ] + } +} diff --git a/.github/plugins/foundry-iq-skills/hooks/copilot-hooks.json b/.github/plugins/foundry-iq-skills/hooks/copilot-hooks.json new file mode 100644 index 00000000..39587eb9 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/hooks/copilot-hooks.json @@ -0,0 +1,16 @@ +{ + "hooks": { + "PostToolUse": [ + { + "type": "command", + + "windows": "${PLUGIN_ROOT}/hooks/scripts/track-telemetry.ps1", + "osx": "${PLUGIN_ROOT}/hooks/scripts/track-telemetry.sh", + "linux": "${PLUGIN_ROOT}/hooks/scripts/track-telemetry.sh", + + "bash": "${PLUGIN_ROOT}/hooks/scripts/track-telemetry.sh", + "powershell": "${PLUGIN_ROOT}/hooks/scripts/track-telemetry.ps1" + } + ] + } +} diff --git a/.github/plugins/foundry-iq-skills/hooks/cursor-hooks.json b/.github/plugins/foundry-iq-skills/hooks/cursor-hooks.json new file mode 100644 index 00000000..b20f8b4e --- /dev/null +++ b/.github/plugins/foundry-iq-skills/hooks/cursor-hooks.json @@ -0,0 +1,17 @@ +{ + "version": 1, + "hooks": { + "postToolUse": [ + { + "type": "command", + "command": "node \"${CURSOR_PLUGIN_ROOT}/hooks/scripts/track-telemetry.js\"" + } + ], + "afterMCPExecution": [ + { + "type": "command", + "command": "node \"${CURSOR_PLUGIN_ROOT}/hooks/scripts/track-telemetry.js\"" + } + ] + } +} diff --git a/.github/plugins/foundry-iq-skills/hooks/scripts/pluginPathAllowPattern.ps1 b/.github/plugins/foundry-iq-skills/hooks/scripts/pluginPathAllowPattern.ps1 new file mode 100644 index 00000000..2031bce2 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/hooks/scripts/pluginPathAllowPattern.ps1 @@ -0,0 +1,47 @@ +$pluginPathPatterns = @() + +# --- azure-skills plugin --- +# The Copilot CLI pattern wildcards the catalog/marketplace folder name +# (e.g. "awesome-copilot") since it does not necessarily match the plugin's +# own name ("azure"). +$pathPatternCopilot = '\.copilot/installed-plugins/[^/]+/azure/skills/' +$pathPatternClaude = '\.claude/plugins/cache/(azure-skills|claude-plugins-official)/azure/[0-9.]+/skills/' +$pathPatternCursor = '\.cursor/plugins/cache/[^/]+/azure/[^/]+/skills/' +$pathPatternVscodeAgentPlugins = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-skills/skills/' + +# --- azure-kusto-graph-skills plugin --- +$pathPatternCopilotKustoGraph = '\.copilot/installed-plugins/[^/]+/azure-kusto-graph-skills/skills/' +$pathPatternClaudeKustoGraph = '\.claude/plugins/cache/azure-skills/azure-kusto-graph-skills/[0-9.]+/skills/' +$pathPatternCursorKustoGraph = '\.cursor/plugins/cache/[^/]+/azure-kusto-graph-skills/[^/]+/skills/' +$pathPatternVscodeAgentPluginsKustoGraph = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-kusto-graph-skills/skills/' + +# --- azure-local-skills plugin --- +$pathPatternCopilotAzureLocal = '\.copilot/installed-plugins/[^/]+/azure-local-skills/skills/' +$pathPatternClaudeAzureLocal = '\.claude/plugins/cache/azure-skills/azure-local-skills/[0-9.]+/skills/' +$pathPatternCursorAzureLocal = '\.cursor/plugins/cache/[^/]+/azure-local-skills/[^/]+/skills/' +$pathPatternVscodeAgentPluginsAzureLocal = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-local-skills/skills/' + +# --- foundry-iq-skills plugin --- +$pathPatternCopilotFoundryIq = '\.copilot/installed-plugins/[^/]+/foundry-iq-skills/skills/' +$pathPatternClaudeFoundryIq = '\.claude/plugins/cache/azure-skills/foundry-iq-skills/[0-9.]+/skills/' +$pathPatternCursorFoundryIq = '\.cursor/plugins/cache/[^/]+/foundry-iq-skills/[^/]+/skills/' +$pathPatternVscodeAgentPluginsFoundryIq = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/foundry-iq-skills/skills/' + +$pluginPathPatterns += @( + $pathPatternCopilot, + $pathPatternClaude, + $pathPatternCursor, + $pathPatternVscodeAgentPlugins, + $pathPatternCopilotKustoGraph, + $pathPatternClaudeKustoGraph, + $pathPatternCursorKustoGraph, + $pathPatternVscodeAgentPluginsKustoGraph, + $pathPatternCopilotAzureLocal, + $pathPatternClaudeAzureLocal, + $pathPatternCursorAzureLocal, + $pathPatternVscodeAgentPluginsAzureLocal, + $pathPatternCopilotFoundryIq, + $pathPatternClaudeFoundryIq, + $pathPatternCursorFoundryIq, + $pathPatternVscodeAgentPluginsFoundryIq +) diff --git a/.github/plugins/foundry-iq-skills/hooks/scripts/pluginPathAllowPattern.sh b/.github/plugins/foundry-iq-skills/hooks/scripts/pluginPathAllowPattern.sh new file mode 100644 index 00000000..1acfcc1f --- /dev/null +++ b/.github/plugins/foundry-iq-skills/hooks/scripts/pluginPathAllowPattern.sh @@ -0,0 +1,27 @@ +# --- azure-skills plugin --- +# The Copilot CLI pattern wildcards the catalog/marketplace folder name +# (e.g. "awesome-copilot") since it does not necessarily match the +# plugin's own name ("azure"). +[[ "$p" == *".copilot/installed-plugins/"*"/azure/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/azure-skills/azure/"*"/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/claude-plugins-official/azure/"*"/skills/"* ]] && return 0 +[[ "$p" == *".cursor/plugins/cache/"*"/azure/"*"/skills/"* ]] && return 0 +[[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-skills/skills/"* ]] && return 0 + +# --- azure-kusto-graph-skills plugin --- +[[ "$p" == *".copilot/installed-plugins/"*"/azure-kusto-graph-skills/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/azure-skills/azure-kusto-graph-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *".cursor/plugins/cache/"*"/azure-kusto-graph-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-kusto-graph-skills/skills/"* ]] && return 0 + +# --- azure-local-skills plugin --- +[[ "$p" == *".copilot/installed-plugins/"*"/azure-local-skills/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/azure-skills/azure-local-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *".cursor/plugins/cache/"*"/azure-local-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-local-skills/skills/"* ]] && return 0 + +# --- foundry-iq-skills plugin --- +[[ "$p" == *".copilot/installed-plugins/"*"/foundry-iq-skills/skills/"* ]] && return 0 +[[ "$p" == *".claude/plugins/cache/azure-skills/foundry-iq-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *".cursor/plugins/cache/"*"/foundry-iq-skills/"*"/skills/"* ]] && return 0 +[[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/foundry-iq-skills/skills/"* ]] && return 0 diff --git a/.github/plugins/foundry-iq-skills/hooks/scripts/track-telemetry.js b/.github/plugins/foundry-iq-skills/hooks/scripts/track-telemetry.js new file mode 100644 index 00000000..f16586e0 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/hooks/scripts/track-telemetry.js @@ -0,0 +1,42 @@ +const { spawnSync } = require("node:child_process"); +const os = require("node:os"); +const path = require("node:path"); + +function getHookCommand(platform) { + if (platform === "win32") { + return { + command: "powershell.exe", + args: [ + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + path.join(__dirname, "track-telemetry.ps1"), + ], + }; + } + + return { + command: "bash", + args: [path.join(__dirname, "track-telemetry.sh")], + }; +} + +function run(platform = os.platform(), spawn = spawnSync) { + const { command, args } = getHookCommand(platform); + const result = spawn(command, args, { stdio: "inherit" }); + + if (result.error) { + console.error(`Failed to run telemetry hook: ${result.error.message}`); + return 1; + } + + return result.status ?? 1; +} + +module.exports = { getHookCommand, run }; + +if (require.main === module) { + process.exitCode = run(); +} diff --git a/.github/plugins/foundry-iq-skills/hooks/scripts/track-telemetry.ps1 b/.github/plugins/foundry-iq-skills/hooks/scripts/track-telemetry.ps1 new file mode 100644 index 00000000..30484e84 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/hooks/scripts/track-telemetry.ps1 @@ -0,0 +1,527 @@ +# Telemetry tracking hook for Azure Copilot Skills +# Reads JSON input from stdin, tracks relevant events, and publishes via MCP +# +# === Client Format Reference === +# +# Copilot CLI: +# - Field names: camelCase (toolName, sessionId, toolArgs) +# - Tool names: lowercase (skill, view) +# - MCP prefix: azure- (e.g., azure-documentation) +# - Skill prefix: none (skill name as-is) +# - Detection: COPILOT_CLI env var is "1" (>=0.0.421); fallback: "toolArgs" without "hook_event_name" (<0.0.421) +# +# Claude Code: +# - Field names: snake_case (tool_name, session_id, tool_input, hook_event_name) +# - Tool names: PascalCase (Skill, Read, Edit) +# - MCP prefix: mcp__plugin_azure_azure__ (double underscores) +# - Skill prefix: azure: (e.g., azure:azure-prepare) +# - Detection: has "hook_event_name", tool_use_id does NOT contain "__vscode" +# +# Cursor: +# - Field names: snake_case (tool_name, session_id, tool_input, hook_event_name) +# - Tool names: PascalCase for file reads (Read); raw MCP tool name from afterMCPExecution +# - Skill paths: .cursor/plugins/cache//azure//skills//SKILL.md +# - Detection: has "hook_event_name" and "cursor_version" +# - MCP detection: afterMCPExecution event with mcp_server_name "azure" +# +# VS Code: +# - Field names: snake_case (tool_name, session_id, tool_input, hook_event_name) +# - Tool names: snake_case (read_file, replace_string_in_file) +# - MCP prefix: mcp_azure_mcp_ (e.g., mcp_azure_mcp_documentation) +# - Skill paths: .vscode/agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-skills/skills//SKILL.md (VS Code) +# .vscode-insiders/agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-skills/skills//SKILL.md (VS Code Insiders) +# .agents/skills//SKILL.md +# - Detection: has "hook_event_name", tool_use_id contains "__vscode" +# or transcript_path contains "Code" +# - Client name: "Visual Studio Code" (stable) or "Visual Studio Code - Insiders" +# derived from transcript_path (e.g., .../Code - Insiders/User/...) +# - Note: Skills under .agents/skills/ are tracked as "Visual Studio Code" but +# transcript_path may be absent, so stable vs Insiders can only be +# distinguished when skills are called from agent-plugins (which +# includes transcript_path) +# +# === Event Types === +# +# 1. skill_invocation +# - Triggered when: the "skill"/"Skill" tool is called with a skill name, +# OR a SKILL.md file is read from a recognized azure-skills path +# - Tracked fields: --skill-name , --skill-version +# +# 2. tool_invocation +# - Triggered when: a tool matching an Azure MCP prefix is called +# (azure-*, mcp__plugin_azure_azure__*, mcp_azure_mcp_*), or when Cursor +# sends afterMCPExecution with mcp_server_name "azure" +# - Tracked field: --tool-name +# +# 3. reference_file_read +# - Triggered when: a file read tool (view/Read/read_file) targets a file +# inside a recognized azure-skills path that is NOT a SKILL.md +# - These are the reference/instruction files that skills bundle alongside +# SKILL.md (e.g., recipes, templates, requirement docs) +# - Tracked fields: --file-reference , +# --skill-version +# +# === Skill Version === +# +# The skill version is read from the SKILL.md frontmatter (metadata.version), +# which the build stamps at package time. It is resolved as follows: +# - skill/Skill tool call: locate SKILL.md relative to this script's plugin +# root ("/skills//SKILL.md") +# - SKILL.md read: read the version from the SKILL.md being read +# - reference_file_read: read the version from the sibling SKILL.md at the +# root of the skill folder the reference lives in +# - Example: azure-validate/references/recipes/azd/README.md +# +# === Reference File Detection === +# +# When a file read tool is invoked (Copilot CLI: "view", Claude Code/Cursor: +# "Read", VS Code: "read_file"), the script extracts the file path from the tool input +# and checks if it falls within a recognized azure-skills folder: +# +# Path field lookup order: +# - toolArgs.path / toolArgs.filePath (Copilot CLI) +# - tool_input.filePath / tool_input.file_path / tool_input.path (Claude Code / VS Code) +# +# Recognized install paths (one set per plugin, see $pathPatterns below): +# azure-skills: +# - .copilot/installed-plugins//azure/skills/... +# ( is the marketplace/catalog folder the plugin was +# installed under, e.g. "awesome-copilot" — it does not necessarily +# match the plugin's own name, "azure") +# - .claude/plugins/cache/azure-skills/azure//skills/... +# - .claude/plugins/cache/claude-plugins-official/azure//skills/... +# - .cursor/plugins/cache//azure//skills/... +# - .vscode/agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-skills/skills/... +# azure-kusto-graph-skills: +# - .copilot/installed-plugins//azure-kusto-graph-skills/skills/... +# - .claude/plugins/cache/azure-skills/azure-kusto-graph-skills//skills/... +# - .cursor/plugins/cache//azure-kusto-graph-skills//skills/... +# - .vscode/agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-kusto-graph-skills/skills/... +# azure-local-skills: +# - .copilot/installed-plugins//azure-local-skills/skills/... +# - .claude/plugins/cache/azure-skills/azure-local-skills//skills/... +# - .cursor/plugins/cache//azure-local-skills//skills/... +# - .vscode/agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-local-skills/skills/... +# shared: +# - .agents/skills/... +# +# If the path matches AND is not a SKILL.md file, the relative path after +# "skills/" is extracted and emitted as a reference_file_read event. +# SKILL.md reads are tracked as skill_invocation instead (not double-counted). +# +# === Debugging === +# +# If the AZURE_SKILLS_TELEMETRY_LOG_DIR env var is set, the script will create +# a "raw-input" subdirectory and write each raw JSON input to a timestamped file +# for debugging. It will also append a "telemetry.log" file with MCP args for +# each tracked event. +# +# When using `--plugin-dir` to load a local plugin the AZURE_SKILLS_PLUGIN_ROOT +# env var should be set so that the script can detect local skill paths for +# reference_file_read events. + +$ErrorActionPreference = "SilentlyContinue" + +# Dumps raw input to a file in the AZURE_SKILLS_TELEMETRY_LOG_DIR/raw-input/ +# directory for debugging if the env var is set. +function Write-RawInputToFile { + param([string]$RawInput) + if ($env:AZURE_SKILLS_TELEMETRY_LOG_DIR) { + $logDir = $env:AZURE_SKILLS_TELEMETRY_LOG_DIR + $rawInputDir = Join-Path $logDir 'raw-input' + if (-not (Test-Path -LiteralPath $rawInputDir)) { + New-Item -ItemType Directory -Path $rawInputDir -Force | Out-Null + } + $timestamp = (Get-Date).ToUniversalTime().ToString("yyyyMMddTHHmmssZ") + $rawInputFile = Join-Path $rawInputDir "$timestamp.json" + try { + $utf8WithoutBom = New-Object System.Text.UTF8Encoding($false) + [System.IO.File]::WriteAllText($rawInputFile, $RawInput, $utf8WithoutBom) + } catch { } + } +} + +# Writes a debug log entry to the AZURE_SKILLS_TELEMETRY_LOG_DIR/telemetry.log file +# if the env var is set. +function Write-TelemetryDebugLog { + param([string]$Content) + + if ($env:AZURE_SKILLS_TELEMETRY_LOG_DIR) { + $logDir = $env:AZURE_SKILLS_TELEMETRY_LOG_DIR + $logFile = Join-Path $logDir 'telemetry.log' + $logEntry = "$(Get-Date -Format 'yyyy-MM-ddTHH:mm:ss') | $Content" + try { + Add-Content -Path $logFile -Value $logEntry -ErrorAction SilentlyContinue + } catch { } + } +} + +# Skip telemetry if opted out +if ($env:AZURE_MCP_COLLECT_TELEMETRY -eq "false") { + Write-Output '{"continue":true}' + exit 0 +} + +# Return success and exit +function Write-Success { + Write-Output '{"continue":true}' + exit 0 +} + +# Removes UTF-8 BOM markers and the common Windows mojibake forms that can +# precede Cursor hook JSON after stdin passes through Windows PowerShell. +function Remove-LeadingUtf8BomArtifacts { + param([AllowEmptyString()][string]$Value) + + if ($null -eq $Value) { return $Value } + + $bomArtifacts = @( + [string][char]0xFEFF, + (-join ([char[]]@(0x00EF, 0x00BB, 0x00BF))), + (-join ([char[]]@(0x2229, 0x2557, 0x2510))) + ) + + do { + $removedArtifact = $false + foreach ($artifact in $bomArtifacts) { + if ($Value.StartsWith($artifact, [System.StringComparison]::Ordinal)) { + $Value = $Value.Substring($artifact.Length) + $removedArtifact = $true + break + } + } + } while ($removedArtifact) + + return $Value +} + +# Resolve this script's directory so we can locate bundled skills. In the +# installed plugin, hooks/ and skills/ are siblings under the plugin root, so +# /skills//SKILL.md is the skill definition. +$scriptDir = $PSScriptRoot +if (-not $scriptDir) { $scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path } +$skillsDir = Join-Path (Split-Path -Parent (Split-Path -Parent $scriptDir)) 'skills' +$pluginPathAllowPattern = Join-Path $scriptDir 'pluginPathAllowPattern.ps1' + +# Return true only when a target belongs to this hook's plugin. Since this hook +# is copied into every plugin, comparing through the skills directory prevents +# each installed copy from reporting the same skill or reference event. +function Test-OwnedSkillPath { + # targetPath is either path to the SKILL.md or to a reference file + param([string]$TargetPath) + if ([string]::IsNullOrWhiteSpace($TargetPath)) { return $false } + $skillsRootNorm = (($skillsDir -replace '\\', '/') -replace '/+', '/').TrimEnd('/') + $targetPathNorm = ($TargetPath -replace '\\', '/') -replace '/+', '/' + return $targetPathNorm.StartsWith("$skillsRootNorm/", [System.StringComparison]::OrdinalIgnoreCase) +} + +# Extract the skill version from a SKILL.md frontmatter (metadata.version). +# Returns $null if the file or version cannot be read. +function Get-SkillVersion { + param([string]$SkillMdPath) + if (-not $SkillMdPath) { return $null } + $SkillMdPath = $SkillMdPath -replace '\\', '/' + if (-not (Test-Path -LiteralPath $SkillMdPath)) { return $null } + try { + $lines = Get-Content -LiteralPath $SkillMdPath -ErrorAction SilentlyContinue + } catch { return $null } + $inFrontmatter = $false + foreach ($line in $lines) { + if ($line -match '^---\s*$') { + if (-not $inFrontmatter) { $inFrontmatter = $true; continue } + else { break } + } + if ($inFrontmatter -and $line -match '^\s*version:\s*(.+?)\s*$') { + return $Matches[1].Trim().Trim('"').Trim("'") + } + } + return $null +} + +# Extract the plugin version from the top-level .plugin/plugin.json manifest. +# Returns $null if the file or expected JSON value cannot be read. +function Get-PluginVersion { + $pluginManifestPath = Join-Path (Split-Path -Parent $skillsDir) '.plugin/plugin.json' + if (-not (Test-Path -LiteralPath $pluginManifestPath)) { return $null } + try { + $manifest = Get-Content -LiteralPath $pluginManifestPath -Raw -ErrorAction Stop | ConvertFrom-Json -ErrorAction Stop + if ($manifest.version -is [string] -and -not [string]::IsNullOrWhiteSpace($manifest.version)) { + return $manifest.version + } + } catch { } + return $null +} + +# === Main Processing === + +# Read stdin as bytes and decode it as UTF-8. Reading through Console.In and +# re-encoding with Console.InputEncoding can introduce code-page mojibake. +try { + $stdinStream = [Console]::OpenStandardInput() + $inputBuffer = New-Object System.IO.MemoryStream + $stdinStream.CopyTo($inputBuffer) + $utf8WithoutBom = New-Object System.Text.UTF8Encoding($false) + $rawInput = $utf8WithoutBom.GetString($inputBuffer.ToArray()) +} catch { + Write-Success +} finally { + if ($inputBuffer) { $inputBuffer.Dispose() } +} + +# Some clients prefix the JSON stream with a UTF-8 BOM. Cursor on Windows can +# surface an additional mojibake copy of that marker after stdin decoding. +$rawInput = Remove-LeadingUtf8BomArtifacts -Value $rawInput + +# Return success and exit if no input +if ([string]::IsNullOrWhiteSpace($rawInput)) { + Write-Success +} + +Write-RawInputToFile -RawInput $rawInput + +# === STEP 1: Read and parse input === + +# Parse JSON input +try { + $inputData = $rawInput | ConvertFrom-Json +} catch { + Write-Success +} + +# Extract fields from hook data +# Support Copilot CLI (camelCase), Claude Code (snake_case), and VS Code (snake_case) formats +$toolName = $inputData.toolName +if (-not $toolName) { + $toolName = $inputData.tool_name +} + +$sessionId = $inputData.sessionId +if (-not $sessionId) { + $sessionId = $inputData.session_id +} +$hookEventName = $inputData.hook_event_name +$mcpServerName = $inputData.mcp_server_name + +# Get tool arguments (Copilot CLI: toolArgs, Claude Code / VS Code: tool_input) +$toolInput = $inputData.toolArgs +if (-not $toolInput) { + $toolInput = $inputData.tool_input +} + +$timestamp = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + +# Detect client name based on input format +# Copilot CLI (>=0.0.421): COPILOT_CLI env var is "1" — primary signal, checked first +# Copilot CLI (<0.0.421): has "toolArgs" field without "hook_event_name" — backward compat fallback +# Cursor: has hook_event_name AND a "cursor_version" field +# VS Code: has hook_event_name AND tool_use_id contains "__vscode" or transcript_path contains "Code" +# Claude Code: has hook_event_name, tool_use_id does NOT contain "__vscode" +$hasHookEventName = $inputData.PSObject.Properties.Name -contains "hook_event_name" +$hasToolArgs = $inputData.PSObject.Properties.Name -contains "toolArgs" +$toolUseId = $inputData.tool_use_id +$transcriptPath = $inputData.transcript_path +$cursorVersion = $inputData.cursor_version +$isVscodeToolUseId = $toolUseId -and ($toolUseId -match '__vscode') +# Match path separators around "Code" or "Code - Insiders" to avoid matching "Claude Code" +$isVscodeTranscript = $transcriptPath -and ($transcriptPath -match '[/\\]Code( - Insiders)?[/\\]') + +# Copilot CLI check first — env var available since v0.0.421 +if ($env:COPILOT_CLI -eq "1") { + $clientName = "copilot-cli" +} elseif ($hasHookEventName -and $cursorVersion) { + $clientName = "cursor" +} elseif ($hasHookEventName -and ($isVscodeToolUseId -or $isVscodeTranscript)) { + # Detect VS Code variant from transcript_path + # Insiders: ...AppData\Roaming\Code - Insiders\User\... + # Stable: ...AppData\Roaming\Code\User\... + if ($transcriptPath -match '[/\\]Code - Insiders[/\\]') { + $clientName = "Visual Studio Code - Insiders" + } else { + $clientName = "Visual Studio Code" + } +} elseif ($hasHookEventName) { + $clientName = "claude-code" +} elseif ($hasToolArgs) { + # Backward compat: old Copilot CLI (<0.0.421) sent toolArgs without hook_event_name + # Claude Code never sends toolArgs, so this is unambiguous + $clientName = "copilot-cli" +} else { + $clientName = "unknown" +} + +# Skip if no tool name found in any format +if (-not $toolName) { + Write-Success +} + +# Helper to extract path from tool input (handles 'path', 'filePath', 'file_path') +function Get-ToolInputPath { + if ($toolInput.path) { return $toolInput.path } + if ($toolInput.filePath) { return $toolInput.filePath } + if ($toolInput.file_path) { return $toolInput.file_path } + return $null +} + +# === STEP 2: Determine what to track for azmcp === + +# Path patterns per client, one block per plugin (used for SKILL.md and +# file-reference matching). When onboarding another plugin, add a new block by +# swapping both the catalog/plugin segments (e.g. "azure" and "azure-skills") +# for the new plugin's name. + +. $pluginPathAllowPattern + +# --- shared across all plugins --- +$pathPatternAgentsSkills = '\.agents/skills/' + +# Put the path patterns into an array for easier iteration +$pathPatterns = @($pluginPathPatterns) + @($pathPatternAgentsSkills) + +# If $env:AZURE_SKILLS_PLUGIN_ROOT is set, add it to the path patterns for local skill development +if ($env:AZURE_SKILLS_PLUGIN_ROOT) { + $localSkillsPath = [regex]::Escape($env:AZURE_SKILLS_PLUGIN_ROOT) + '/skills/' + $localSkillsPath = $localSkillsPath -replace '\\', '/' -replace '/+', '/' + $pathPatterns += $localSkillsPath +} + +$shouldTrack = $false +$eventType = $null +$skillName = $null +$skillVersion = $null +$azureToolName = $null +$filePath = $null + +# Check for skill invocation via 'skill'/'Skill' tool +if ($toolName -eq "skill" -or $toolName -eq "Skill") { + $skillName = $toolInput.skill + # Claude Code prefixes skill names with "azure:" (e.g., "azure:azure-prepare") + # Strip it to get the actual skill name for the allowlist + if ($skillName -and $skillName.StartsWith("azure:")) { + $skillName = $skillName.Substring(6) + } + $skillMdPath = Join-Path $skillsDir (Join-Path $skillName 'SKILL.md') + if ($skillName -and (Test-Path -LiteralPath $skillMdPath) -and (Test-OwnedSkillPath $skillMdPath)) { + $eventType = "skill_invocation" + $shouldTrack = $true + $skillVersion = Get-SkillVersion $skillMdPath + } +} + +# Check for skill invocation (reading SKILL.md files) +# Copilot CLI: "view", Claude Code: "Read", VS Code: "read_file" +if ($toolName -eq "view" -or $toolName -eq "Read" -or $toolName -eq "read_file") { + $pathToCheck = Get-ToolInputPath + if ($pathToCheck) { + # Normalize path: convert to lowercase, replace backslashes, and squeeze consecutive slashes + $pathLower = $pathToCheck.ToLower() -replace '\\', '/' -replace '/+', '/' + + # Check for SKILL.md pattern — only match azure-skills paths (see pathPatterns above) + $isAzureSkillMd = $false + foreach ($pattern in $pathPatterns) { + if ($pathLower -match "${pattern}[^/]+/skill\.md") { + $isAzureSkillMd = $true + break + } + } + + if ($isAzureSkillMd -and (Test-OwnedSkillPath $pathToCheck)) { + $pathNormalized = $pathToCheck -replace '\\', '/' -replace '/+', '/' + if ($pathNormalized -match '/skills/([^/]+)/SKILL\.md$') { + $skillName = $Matches[1] + $eventType = "skill_invocation" + $shouldTrack = $true + $skillVersion = Get-SkillVersion $pathToCheck + } + } + } +} + +# Check for Azure MCP tool invocation +# Copilot CLI: "azure-*" prefix (e.g., azure-documentation) +# Claude Code: "mcp__plugin_azure_azure__*" prefix (e.g., mcp__plugin_azure_azure__documentation) +# Cursor: afterMCPExecution with mcp_server_name "azure"; remove Cursor's +# optional display prefix (e.g., MCP:get_azure_bestpractices) +# VS Code: "mcp_azure_mcp_*" prefix (e.g., mcp_azure_mcp_documentation) +if ($toolName) { + if ($clientName -eq "cursor" -and $hookEventName -eq "afterMCPExecution" -and $mcpServerName -eq "azure") { + $azureToolName = $toolName + if ($azureToolName.StartsWith("MCP:", [System.StringComparison]::Ordinal)) { + $azureToolName = $azureToolName.Substring(4) + } + $eventType = "tool_invocation" + $shouldTrack = $true + } elseif ($toolName.StartsWith("azure-") -or $toolName.StartsWith("mcp__plugin_azure_azure__") -or $toolName.StartsWith("mcp_azure_mcp_")) { + $azureToolName = $toolName + $eventType = "tool_invocation" + $shouldTrack = $true + } +} + +# Capture file path from any tool input (only track files in azure skills folder) +# Skip if already matched as SKILL.md skill_invocation — SKILL.md is not a valid file-reference +if (-not $filePath -and -not $skillName) { + $pathToCheck = Get-ToolInputPath + if ($pathToCheck) { + # Normalize path for matching: replace backslashes and squeeze consecutive slashes + $pathLower = $pathToCheck.ToLower() -replace '\\', '/' -replace '/+', '/' + + $matchesPattern = $false + foreach ($pattern in $pathPatterns) { + if ($pathLower -match $pattern) { + $matchesPattern = $true + break + } + } + if ($matchesPattern -and (Test-OwnedSkillPath $pathToCheck)) { + # Extract relative path after 'skills/' + $pathNormalized = $pathToCheck -replace '\\', '/' -replace '/+', '/' + + if ($pathNormalized -match '.*/skills/(.+)$') { + $filePath = $Matches[1] + + if (-not $shouldTrack) { + $shouldTrack = $true + $eventType = "reference_file_read" + # Resolve the version from the sibling SKILL.md at the root + # of the skill folder this reference lives in. + $skillNameSeg = ($filePath -split '/')[0] + $skillRootAbs = $pathNormalized.Substring(0, $pathNormalized.Length - $filePath.Length) + $skillVersion = Get-SkillVersion ("$skillRootAbs$skillNameSeg/SKILL.md") + } + } + } + } +} + +# === STEP 3: Publish event === + +if ($shouldTrack) { + $pluginVersion = Get-PluginVersion + + # Build MCP command arguments + $mcpArgs = @( + "server", "plugin-telemetry", + "--timestamp", $timestamp, + "--client-name", $clientName + ) + + if ($eventType) { $mcpArgs += "--event-type"; $mcpArgs += $eventType } + if ($sessionId) { $mcpArgs += "--session-id"; $mcpArgs += $sessionId } + if ($skillName) { $mcpArgs += "--skill-name"; $mcpArgs += $skillName } + if ($skillVersion) { $mcpArgs += "--skill-version"; $mcpArgs += $skillVersion } + if ($pluginVersion) { $mcpArgs += "--plugin-version"; $mcpArgs += $pluginVersion } + if ($azureToolName) { $mcpArgs += "--tool-name"; $mcpArgs += $azureToolName } + # Convert forward slashes to backslashes for azmcp allowlist compatibility + if ($filePath) { $mcpArgs += "--file-reference"; $mcpArgs += ($filePath -replace '/', '\') } + + # Publish telemetry via npx + try { + & npx -y @azure/mcp@latest @mcpArgs 2>&1 | Out-Null + } catch { } + + # If AZURE_SKILLS_TELEMETRY_LOG_DIR env var is set, append the args to the telemetry.log file in that directory (for debugging) + Write-TelemetryDebugLog -Content "MCP Args: $($mcpArgs -join ' ')" +} + +# Output success to stdout (required by hooks) +Write-Success diff --git a/.github/plugins/foundry-iq-skills/hooks/scripts/track-telemetry.sh b/.github/plugins/foundry-iq-skills/hooks/scripts/track-telemetry.sh new file mode 100755 index 00000000..e7327301 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/hooks/scripts/track-telemetry.sh @@ -0,0 +1,480 @@ +#!/bin/bash + +# Telemetry tracking hook for Azure Copilot Skills +# Reads JSON input from stdin, tracks relevant events, and publishes via MCP +# +# === Client Format Reference === +# +# Copilot CLI: +# - Field names: camelCase (toolName, sessionId, toolArgs) +# - Tool names: lowercase (skill, view) +# - MCP prefix: azure- (e.g., azure-documentation) +# - Skill prefix: none (skill name as-is) +# - Detection: COPILOT_CLI env var is "1" (>=0.0.421); fallback: "toolArgs" without "hook_event_name" (<0.0.421) +# +# Claude Code: +# - Field names: snake_case (tool_name, session_id, tool_input, hook_event_name) +# - Tool names: PascalCase (Skill, Read, Edit) +# - MCP prefix: mcp__plugin_azure_azure__ (double underscores) +# - Skill prefix: azure: (e.g., azure:azure-prepare) +# - Detection: has "hook_event_name", tool_use_id does NOT contain "__vscode" +# +# Cursor: +# - Field names: snake_case (tool_name, session_id, tool_input, hook_event_name) +# - Tool names: PascalCase for file reads (Read); raw MCP tool name from afterMCPExecution +# - Skill paths: .cursor/plugins/cache//azure//skills//SKILL.md +# - Detection: has "hook_event_name" and "cursor_version" +# - MCP detection: afterMCPExecution event with mcp_server_name "azure" +# +# VS Code: +# - Field names: snake_case (tool_name, session_id, tool_input, hook_event_name) +# - Tool names: snake_case (read_file, replace_string_in_file) +# - MCP prefix: mcp_azure_mcp_ (e.g., mcp_azure_mcp_documentation) +# - Skill paths: .vscode/agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-skills/skills//SKILL.md (VS Code) +# .vscode-insiders/agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-skills/skills//SKILL.md (VS Code Insiders) +# .agents/skills//SKILL.md +# - Detection: has "hook_event_name", tool_use_id contains "__vscode" +# or transcript_path contains "Code" +# - Client name: "Visual Studio Code" (stable) or "Visual Studio Code - Insiders" +# derived from transcript_path (e.g., .../Code - Insiders/User/...) +# - Note: Skills under .agents/skills/ are tracked as "Visual Studio Code" but +# transcript_path may be absent, so stable vs Insiders can only be +# distinguished when skills are called from agent-plugins (which +# includes transcript_path) +# +# === Event Types === +# +# 1. skill_invocation +# - Triggered when: the "skill"/"Skill" tool is called with a skill name, +# OR a SKILL.md file is read from a recognized azure-skills path +# - Tracked fields: --skill-name , --skill-version +# +# 2. tool_invocation +# - Triggered when: a tool matching an Azure MCP prefix is called +# (azure-*, mcp__plugin_azure_azure__*, mcp_azure_mcp_*), or when Cursor +# sends afterMCPExecution with mcp_server_name "azure" +# - Tracked field: --tool-name +# +# 3. reference_file_read +# - Triggered when: a file read tool (view/Read/read_file) targets a file +# inside a recognized azure-skills path that is NOT a SKILL.md +# - These are the reference/instruction files that skills bundle alongside +# SKILL.md (e.g., recipes, templates, requirement docs) +# - Tracked fields: --file-reference , +# --skill-version +# +# === Skill Version === +# +# The skill version is read from the SKILL.md frontmatter (metadata.version), +# which the build stamps at package time. It is resolved as follows: +# - skill/Skill tool call: locate SKILL.md relative to this script's plugin +# root ("/skills//SKILL.md") +# - SKILL.md read: read the version from the SKILL.md being read +# - reference_file_read: read the version from the sibling SKILL.md at the +# root of the skill folder the reference lives in +# - Example: azure-validate/references/recipes/azd/README.md +# +# === Reference File Detection === +# +# When a file read tool is invoked (Copilot CLI: "view", Claude Code/Cursor: +# "Read", VS Code: "read_file"), the script extracts the file path from the tool input +# and checks if it falls within a recognized azure-skills folder: +# +# Path field lookup order: +# - toolArgs.path / toolArgs.filePath (Copilot CLI) +# - tool_input.filePath / tool_input.file_path / tool_input.path (Claude Code / VS Code) +# +# Recognized install paths (one set per plugin, see is_azure_skills_path): +# azure-skills: +# - .copilot/installed-plugins//azure/skills/... +# ( is the marketplace/catalog folder the plugin was +# installed under, e.g. "awesome-copilot" — it does not necessarily +# match the plugin's own name, "azure") +# - .claude/plugins/cache/azure-skills/azure//skills/... +# - .claude/plugins/cache/claude-plugins-official/azure//skills/... +# - .cursor/plugins/cache//azure//skills/... +# - .vscode/agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-skills/skills/... +# azure-kusto-graph-skills: +# - .copilot/installed-plugins//azure-kusto-graph-skills/skills/... +# - .claude/plugins/cache/azure-skills/azure-kusto-graph-skills//skills/... +# - .cursor/plugins/cache//azure-kusto-graph-skills//skills/... +# - .vscode/agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-kusto-graph-skills/skills/... +# azure-local-skills: +# - .copilot/installed-plugins//azure-local-skills/skills/... +# - .claude/plugins/cache/azure-skills/azure-local-skills//skills/... +# - .cursor/plugins/cache//azure-local-skills//skills/... +# - .vscode/agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-local-skills/skills/... +# shared: +# - .agents/skills/... +# +# If the path matches AND is not a SKILL.md file, the relative path after +# "skills/" is extracted and emitted as a reference_file_read event. +# SKILL.md reads are tracked as skill_invocation instead (not double-counted). +# +# === Debugging === +# +# If the AZURE_SKILLS_TELEMETRY_LOG_DIR env var is set, the script will create +# a "raw-input" subdirectory and write each raw JSON input to a timestamped file +# for debugging. It will also append a "telemetry.log" file with MCP args for +# each tracked event. +# +# When using `--plugin-dir` to load a local plugin the AZURE_SKILLS_PLUGIN_ROOT +# env var should be set so that the script can detect local skill paths for +# reference_file_read events. + +set +e # Don't exit on errors - fail silently for privacy + +# Skip telemetry if opted out +if [ "${AZURE_MCP_COLLECT_TELEMETRY}" = "false" ]; then + echo '{"continue":true}' + exit 0 +fi + +# Return success and exit +return_success() { + echo '{"continue":true}' + exit 0 +} + +# Dumps raw input to a file in the AZURE_SKILLS_TELEMETRY_LOG_DIR/raw-input/ +# directory for debugging if the env var is set. +write_raw_input_to_file() { + local rawInputValue="$1" + [ -n "$AZURE_SKILLS_TELEMETRY_LOG_DIR" ] || return 0 + local rawInputDir="$AZURE_SKILLS_TELEMETRY_LOG_DIR/raw-input" + mkdir -p "$rawInputDir" 2>/dev/null || return 0 + local ts + ts=$(date -u +"%Y%m%dT%H%M%SZ") + printf '%s\n' "$rawInputValue" > "$rawInputDir/$ts.json" 2>/dev/null || true +} + +# Appends a debug log entry to the AZURE_SKILLS_TELEMETRY_LOG_DIR/telemetry.log +# file if the env var is set. +write_telemetry_debug_log() { + local content="$1" + [ -n "$AZURE_SKILLS_TELEMETRY_LOG_DIR" ] || return 0 + local logFile="$AZURE_SKILLS_TELEMETRY_LOG_DIR/telemetry.log" + echo "$(date +"%Y-%m-%dT%H:%M:%S") | $content" >> "$logFile" 2>/dev/null || true +} + +# Resolve this script's directory so we can locate bundled skills. In the +# installed plugin, hooks/ and skills/ are siblings under the plugin root, so +# /../../skills//SKILL.md is the skill definition. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" 2>/dev/null && pwd)" +SKILLS_DIR="$(cd "$SCRIPT_DIR/../.." 2>/dev/null && pwd)/skills" +PLUGIN_PATH_ALLOW_PATTERN="$SCRIPT_DIR/pluginPathAllowPattern.sh" + +# Return true only when a target belongs to this hook's plugin. Since this hook +# is copied into every plugin, comparing through the skills directory prevents +# each installed copy from reporting the same skill or reference event. +is_owned_skill_path() { + # targetPath is either path to the SKILL.md or to a reference file + local targetPath="$1" + local skillsRootNorm + local targetPathNorm + skillsRootNorm=$(echo "$SKILLS_DIR" | tr '[:upper:]' '[:lower:]' | tr '\\' '/' | sed 's|//*|/|g; s|/$||') + targetPathNorm=$(echo "$targetPath" | tr '[:upper:]' '[:lower:]' | tr '\\' '/' | sed 's|//*|/|g') + [[ "$targetPathNorm" == "$skillsRootNorm/"* ]] +} + +# Extract the skill version from a SKILL.md frontmatter (metadata.version). +# Prints nothing if the file or version cannot be read. +get_skill_version() { + local skillMdPath="$1" + [ -n "$skillMdPath" ] || return 0 + # Normalize backslashes so Windows-style paths are readable + skillMdPath="$(echo "$skillMdPath" | tr '\\' '/')" + [ -f "$skillMdPath" ] || return 0 + # Read the frontmatter block (between the first two --- lines) and pull the + # version value, stripping surrounding quotes and whitespace. + sed -n '/^---[[:space:]]*$/,/^---[[:space:]]*$/p' "$skillMdPath" 2>/dev/null \ + | grep -E '^[[:space:]]*version:[[:space:]]*' \ + | head -1 \ + | sed -E 's/^[[:space:]]*version:[[:space:]]*//; s/^["'"'"']//; s/["'"'"'][[:space:]]*$//; s/[[:space:]]*$//' +} + +# Extract the plugin version from the top-level .plugin/plugin.json manifest. +# Prints nothing if the file or expected JSON value cannot be read. +get_plugin_version() { + local pluginManifestPath + pluginManifestPath="$(dirname "$SKILLS_DIR")/.plugin/plugin.json" + [ -f "$pluginManifestPath" ] || return 0 + node -e ' + try { + const manifest = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); + if (typeof manifest.version === "string" && manifest.version) process.stdout.write(manifest.version); + } catch { } + ' "$pluginManifestPath" 2>/dev/null +} + +# === JSON Parsing Functions (using sed - portable across platforms) === + +# Extract simple string field from JSON +extract_json_field() { + local json="$1" + local field="$2" + echo "$json" | sed -n "s/.*\"$field\":[[:space:]]*\"\([^\"]*\)\".*/\1/p" +} + +# Extract nested field from toolArgs/tool_input (e.g., toolArgs.skill or tool_input.skill) +extract_toolargs_field() { + local json="$1" + local field="$2" + local value="" + # Try Copilot CLI format (toolArgs) first, then Claude Code / VS Code format (tool_input) + value=$(echo "$json" | sed -n "s/.*\"toolArgs\":[[:space:]]*{[^}]*\"$field\":[[:space:]]*\"\([^\"]*\)\".*/\1/p") + if [ -z "$value" ]; then + value=$(echo "$json" | sed -n "s/.*\"tool_input\":[[:space:]]*{[^}]*\"$field\":[[:space:]]*\"\([^\"]*\)\".*/\1/p") + fi + echo "$value" +} + +# Extract path from toolArgs/tool_input (handles 'path', 'filePath', 'file_path') +extract_toolargs_path() { + local json="$1" + local path_value="" + + # Try Copilot CLI format (toolArgs) first + path_value=$(echo "$json" | sed -n 's/.*"toolArgs":[[:space:]]*{[^}]*"path":[[:space:]]*"\([^"]*\)".*/\1/p') + if [ -z "$path_value" ]; then + path_value=$(echo "$json" | sed -n 's/.*"toolArgs":[[:space:]]*{[^}]*"filePath":[[:space:]]*"\([^"]*\)".*/\1/p') + fi + # Fall back to Claude Code / VS Code format (tool_input) + if [ -z "$path_value" ]; then + path_value=$(echo "$json" | sed -n 's/.*"tool_input":[[:space:]]*{[^}]*"filePath":[[:space:]]*"\([^"]*\)".*/\1/p') + fi + if [ -z "$path_value" ]; then + path_value=$(echo "$json" | sed -n 's/.*"tool_input":[[:space:]]*{[^}]*"file_path":[[:space:]]*"\([^"]*\)".*/\1/p') + fi + if [ -z "$path_value" ]; then + path_value=$(echo "$json" | sed -n 's/.*"tool_input":[[:space:]]*{[^}]*"path":[[:space:]]*"\([^"]*\)".*/\1/p') + fi + + echo "$path_value" +} + +# === Main Processing === + +# Check if stdin has data +if [ -t 0 ]; then + return_success +fi + +# Read entire stdin at once - hooks send one complete JSON per invocation +rawInput=$(cat) + +# Return success and exit if no input +if [ -z "$rawInput" ]; then + return_success +fi + +write_raw_input_to_file "$rawInput" + +# === STEP 1: Read and parse input === + +# Extract fields from hook data +# Support Copilot CLI (camelCase), Claude Code (snake_case), and VS Code (snake_case) formats +toolName=$(extract_json_field "$rawInput" "toolName") +sessionId=$(extract_json_field "$rawInput" "sessionId") +hookEventName=$(extract_json_field "$rawInput" "hook_event_name") +mcpServerName=$(extract_json_field "$rawInput" "mcp_server_name") + +# Fall back to Claude Code / VS Code snake_case field names +if [ -z "$toolName" ]; then + toolName=$(extract_json_field "$rawInput" "tool_name") +fi +if [ -z "$sessionId" ]; then + sessionId=$(extract_json_field "$rawInput" "session_id") +fi + +timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + +# Detect client name based on input format +# Copilot CLI (>=0.0.421): COPILOT_CLI env var is "1" — primary signal, checked first +# Copilot CLI (<0.0.421): has "toolArgs" field without "hook_event_name" — backward compat fallback +# Cursor: has hook_event_name AND a "cursor_version" field +# VS Code: has hook_event_name AND tool_use_id contains "__vscode" or transcript_path contains "Code" +# Claude Code: has hook_event_name, tool_use_id does NOT contain "__vscode" +if [ "$COPILOT_CLI" = "1" ]; then + clientName="copilot-cli" +elif echo "$rawInput" | grep -q '"hook_event_name"'; then + toolUseId=$(extract_json_field "$rawInput" "tool_use_id") + transcriptPath=$(extract_json_field "$rawInput" "transcript_path") + cursorVersion=$(extract_json_field "$rawInput" "cursor_version") + # Normalize backslashes to forward slashes for consistent matching + transcriptPathNorm=$(echo "$transcriptPath" | tr '\\' '/') + if [ -n "$cursorVersion" ]; then + clientName="cursor" + # Match path separators around "Code" or "Code - Insiders" to avoid matching "Claude Code" + elif [[ "$toolUseId" == *"__vscode"* ]] || [[ "$transcriptPathNorm" == */Code/* ]] || [[ "$transcriptPathNorm" == */Code\ -\ Insiders/* ]]; then + # Detect VS Code variant from transcript_path + # Insiders: ...AppData/Roaming/Code - Insiders/User/... + # Stable: ...AppData/Roaming/Code/User/... + if [[ "$transcriptPathNorm" == */Code\ -\ Insiders/* ]]; then + clientName="Visual Studio Code - Insiders" + else + clientName="Visual Studio Code" + fi + else + clientName="claude-code" + fi +elif echo "$rawInput" | grep -q '"toolArgs"'; then + # Backward compat: old Copilot CLI (<0.0.421) sent toolArgs without hook_event_name + # Claude Code never sends toolArgs, so this is unambiguous + clientName="copilot-cli" +else + clientName="unknown" +fi + +# Skip if no tool name found in any format +if [ -z "$toolName" ]; then + return_success +fi + +# === STEP 2: Determine what to track for azmcp === + +# Check if a path matches any known plugin skills folder structure. + # Each plugin has its own block below — when onboarding another plugin, add a new + # block by swapping both the catalog/plugin segments (e.g. "azure" and + # "azure-skills") for the new plugin. Returns 0 (true) if matched, 1 (false) otherwise. +is_azure_skills_path() { + local p="$1" + + if . "$PLUGIN_PATH_ALLOW_PATTERN"; then return 0; fi + + # --- shared across all plugins --- + [[ "$p" == *".agents/skills/"* ]] && return 0 + + # Local plugin development: match paths under AZURE_SKILLS_PLUGIN_ROOT/skills/ + # (e.g. when loading a local plugin via `--plugin-dir`) + if [ -n "$AZURE_SKILLS_PLUGIN_ROOT" ]; then + local localRoot + localRoot=$(echo "$AZURE_SKILLS_PLUGIN_ROOT" | tr '[:upper:]' '[:lower:]' | tr '\\' '/' | sed 's|//*|/|g') + [[ "$p" == *"${localRoot}/skills/"* ]] && return 0 + fi + return 1 +} + +shouldTrack=false +eventType="" +skillName="" +skillVersion="" +azureToolName="" +filePath="" + +# Check for skill invocation via 'skill'/'Skill' tool +if [ "$toolName" = "skill" ] || [ "$toolName" = "Skill" ]; then + skillName=$(extract_toolargs_field "$rawInput" "skill") + # Claude Code prefixes skill names with "azure:" (e.g., "azure:azure-prepare") + # Strip it to get the actual skill name for the allowlist + skillName="${skillName#azure:}" + skillMdPath="$SKILLS_DIR/$skillName/SKILL.md" + if [ -n "$skillName" ] && [ -f "$skillMdPath" ] && is_owned_skill_path "$skillMdPath"; then + eventType="skill_invocation" + shouldTrack=true + skillVersion=$(get_skill_version "$skillMdPath") + fi +fi + +# Check for skill invocation (reading SKILL.md files) +# Copilot CLI: "view", Claude Code: "Read", VS Code: "read_file" +if [ "$toolName" = "view" ] || [ "$toolName" = "Read" ] || [ "$toolName" = "read_file" ]; then + pathToCheck=$(extract_toolargs_path "$rawInput") + if [ -n "$pathToCheck" ]; then + # Normalize path: convert to lowercase, replace backslashes, and squeeze consecutive slashes + pathLower=$(echo "$pathToCheck" | tr '[:upper:]' '[:lower:]' | tr '\\' '/' | sed 's|//*|/|g') + + # Check for SKILL.md pattern — only match azure-skills paths + if is_azure_skills_path "$pathLower" && is_owned_skill_path "$pathToCheck" && [[ "$pathLower" == *"/skill.md" ]]; then + pathNormalized=$(echo "$pathToCheck" | tr '\\' '/' | sed 's|//*|/|g') + if [[ "$pathNormalized" =~ /skills/([^/]+)/SKILL\.md$ ]]; then + skillName="${BASH_REMATCH[1]}" + eventType="skill_invocation" + shouldTrack=true + skillVersion=$(get_skill_version "$pathToCheck") + fi + fi + fi +fi + +# Check for Azure MCP tool invocation +# Copilot CLI: "azure-*" prefix (e.g., azure-documentation) +# Claude Code: "mcp__plugin_azure_azure__*" prefix (e.g., mcp__plugin_azure_azure__documentation) +# Cursor: afterMCPExecution with mcp_server_name "azure"; remove Cursor's +# optional display prefix (e.g., MCP:get_azure_bestpractices) +# VS Code: "mcp_azure_mcp_*" prefix (e.g., mcp_azure_mcp_documentation) +if [ -n "$toolName" ]; then + if [ "$clientName" = "cursor" ] && [ "$hookEventName" = "afterMCPExecution" ] && [ "$mcpServerName" = "azure" ]; then + azureToolName="${toolName#MCP:}" + eventType="tool_invocation" + shouldTrack=true + elif [[ "$toolName" == azure-* ]] || [[ "$toolName" == mcp__plugin_azure_azure__* ]] || [[ "$toolName" == mcp_azure_mcp_* ]]; then + azureToolName="$toolName" + eventType="tool_invocation" + shouldTrack=true + fi +fi + +# Capture file path from any tool input (only track files in azure skills folder) +# Skip if already matched as SKILL.md skill_invocation — SKILL.md is not a valid file-reference +if [ -z "$filePath" ] && [ -z "$skillName" ]; then + pathToCheck=$(extract_toolargs_path "$rawInput") + if [ -n "$pathToCheck" ]; then + # Normalize path for matching: replace backslashes and squeeze consecutive slashes + pathLower=$(echo "$pathToCheck" | tr '[:upper:]' '[:lower:]' | tr '\\' '/' | sed 's|//*|/|g') + + # Check if path matches azure skills folder structure + if is_azure_skills_path "$pathLower" && is_owned_skill_path "$pathToCheck"; then + # Extract relative path after 'skills/' + pathNormalized=$(echo "$pathToCheck" | tr '\\' '/' | sed 's|//*|/|g') + + if [[ "$pathNormalized" =~ .*/skills/(.+)$ ]]; then + filePath="${BASH_REMATCH[1]}" + + if [ "$shouldTrack" = false ]; then + shouldTrack=true + eventType="reference_file_read" + # Resolve the version from the sibling SKILL.md at the root + # of the skill folder this reference lives in. Strip the + # relative suffix by length (literal removal) so glob + # metacharacters in the path can't corrupt the result. + skillNameSeg="${filePath%%/*}" + skillRootAbs="${pathNormalized:0:${#pathNormalized}-${#filePath}}" + skillVersion=$(get_skill_version "${skillRootAbs}${skillNameSeg}/SKILL.md") + fi + fi + fi + fi +fi + +# === STEP 3: Publish event via azmcp === + +if [ "$shouldTrack" = true ]; then + pluginVersion=$(get_plugin_version) + + # Build MCP command arguments (using array for proper quoting) + mcpArgs=( + "server" "plugin-telemetry" + "--timestamp" "$timestamp" + "--client-name" "$clientName" + ) + + [ -n "$eventType" ] && mcpArgs+=("--event-type" "$eventType") + [ -n "$sessionId" ] && mcpArgs+=("--session-id" "$sessionId") + [ -n "$skillName" ] && mcpArgs+=("--skill-name" "$skillName") + [ -n "$skillVersion" ] && mcpArgs+=("--skill-version" "$skillVersion") + [ -n "$pluginVersion" ] && mcpArgs+=("--plugin-version" "$pluginVersion") + [ -n "$azureToolName" ] && mcpArgs+=("--tool-name" "$azureToolName") + # Convert forward slashes to backslashes for azmcp allowlist compatibility + [ -n "$filePath" ] && mcpArgs+=("--file-reference" "$(echo "$filePath" | tr '/' '\\')") + + # Publish telemetry via npx + npx -y @azure/mcp@latest "${mcpArgs[@]}" >/dev/null 2>&1 || true + + # If AZURE_SKILLS_TELEMETRY_LOG_DIR env var is set, append the args to the + # telemetry.log file in that directory (for debugging) + write_telemetry_debug_log "MCP Args: ${mcpArgs[*]}" +fi + +# Output success to stdout (required by hooks) +return_success diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/SKILL.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/SKILL.md new file mode 100644 index 00000000..68a20b3d --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/SKILL.md @@ -0,0 +1,59 @@ +--- +name: foundry-iq +description: "Foundry IQ knowledge bases. WHEN: make local or Blob documents searchable; create/diagnose KBs; triage unsupported connectors or multi-source KB creation/reconfiguration; connect existing KB to agents (including multi-source KBs); create/reuse a Search service; retrieve from an existing knowledge base with citations; no brand words needed. Read its procedure before query/target questions. NOT: other KB providers, repository-file search, classic Azure AI Search index/query/app work, generic agent creation." +license: MIT +compatibility: Azure +metadata: + author: Microsoft + version: "0.1.1" +--- + +# Foundry IQ +Read one procedure before questions/actions; invocation is not a read. +Read the owning procedure before blocked/unsupported responses too. +Before mutation plans/approval, successfully read its required pre-action references, selected +branches only. Do not reload successful reads. On failure, try only permitted +bounded exact-path reads with any supported reader; never bypass restrictions +or search broadly. If still unavailable: `blocked: reference-unavailable`, name +missing references and inability to plan. Never invent requirements/plans. +Failures first. + +Cleanup and receipt-backed execution go directly to their lifecycle/producer owner; +do not reopen Search intake, provisioning or hardening. + +Before expensive service discovery, reuse supplied resource/intent; otherwise ask +one early **USE EXISTING / FIND CANDIDATES / CREATE NEW** choice. +Only FIND enumerates; supplied identity uses exact/minimum scoped resolution. +CREATE checks its proposed name, not existing-service inventories. Preserve these +answers across source/KB/CU/model handoffs; selection is not write approval. + +Searchable docs: KB + validated retrieval. Confirm intent once; KS-only must be explicit. +Child success is not KB completion. +Unclear/compound/mode/completion: [read](references/intent-routing.md). + +Route by requested operation, not existing KB source count. +Connecting an existing KB with two or more sources, without changing the KB, uses Connect. +Read-only operations use their owning procedure and actual helper constraints; +this does not add retrieval modes or supported source kinds. +Explicit unsupported provisioning or multi-source KB creation/reconfiguration uses Diagnose +and stops before discovery, even when connecting an agent is also requested. +Do not silently execute only the supported part of a compound request. +If existing-KB connection versus KB creation/reconfiguration is unclear, read +intent-routing and clarify that scope before Azure discovery. Never infer KB mutation. + +|Outcome|Read| +|---|---| +|Cleanup|[Plan cleanup](lifecycle/cleanup.md)| +| Failure/drift | [Diagnose](troubleshooting/diagnose.md) | +| Unsupported connector provisioning / multi-source KB creation or reconfiguration | [Diagnose](troubleshooting/diagnose.md) | +| Connect | [Connect](agents/connect.md) | +| Read KB | [Query](knowledge-bases/retrieve.md) | +| Search only | [Search](search-services/create.md) | +| File KS only | [File](knowledge-sources/create-file.md) | +| Blob/ADLS KS only | [Blob](knowledge-sources/create-azure-blob.md) | +| Searchable/KB | [KB](knowledge-bases/create.md) | + +Generic agents: `microsoft-foundry`. +Reads: no approval; approve unchanged plans before writes. +Hide hashes. No Search/Storage keys, scope widening, guessed identity/boundaries, +drift repair or joint cleanup/creation approval. Acceptance != success. diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-hosted-toolbox.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-hosted-toolbox.md new file mode 100644 index 00000000..72b36544 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-hosted-toolbox.md @@ -0,0 +1,140 @@ +# Hosted toolbox-only connection + +For an existing unversioned toolbox consumer, changing the default needs no source, +`azure.yaml`, environment setter or deployment. Preserve the observed agent version/definition. + +**Execution limit:** service version creation/promotion exists; this helper is +GET-only. Conditional ETag/default-update enforcement is unverified. Changed bindings +return `toolbox-promotion-concurrency-unverified` before connection/version creation. +No arbitrary PATCH headers or unconditional `azd ai toolbox publish` bypass. +Obtain the service owner's conditional-update contract, not Hosted source. + +Disclose upfront: the owner is the **retained connection/toolbox-version owner**. +Hosted cleanup is unsupported. Retain prior immutable versions and legacy +connections; rollback needs its own explicit plan/approval, never automatic action. + +## Resolve and inspect + +Accept project/KB names, resource IDs or endpoints. Exact selections win. +Resolve missing parents only within selected subscription/RG/account. +Ambiguity returns `selection_candidates`; denied/malformed pages block, never widen scope. +Search index lists never prove KB identity or absence. + +Four layers: **project connection** authenticates KB MCP; **toolbox** holds the default; +**immutable toolbox version** references connections; **Hosted runtime binding** consumes +the endpoint. The KB stays unchanged in Search. + +This recipe requires `RemoteTool` + `AgenticIdentityToken` + Search audience +`https://search.azure.com/` + exact KB MCP endpoint. `CustomKeys` and +`ProjectManagedIdentity` are not interchangeable here; other authorized custom +Hosted runtimes may support those auth modes. Preserve legacy names. +Collide by connection NAME, not endpoint. New explicit names may share a KB endpoint; +never overwrite mismatches or retry random names. Opaque tool auth/headers block; +preserve tool approval/filter policies. + +The access principal is the active published Hosted agent's `instance_identity.principal_id`, +not the blueprint or project principal used by the Prompt `ProjectManagedIdentity` +recipe. Verify existing Search Index Data Reader at the Search service and Foundry +User at the project. The helper checks these exact built-in grants; equivalent +custom grants need separate authoritative verification, not broader assignments. + +The consumer endpoint is `/toolboxes//mcp?api-version=v1`. +`/toolboxes//versions//mcp` is a pinned developer endpoint, not +equivalent. A mismatched endpoint/environment requires explaining the actual +runtime/config/deploy change and then handing off/requesting source if needed. +FoundryToolbox prefers present `TOOLBOX_ENDPOINT` (empty is invalid); only when absent does it combine +`FOUNDRY_PROJECT_ENDPOINT` (trailing slash removed) and `TOOLBOX_NAME`. +Explicit endpoint wins over name/project settings; pinned or mismatched results block. +These are supported configuration conventions, not proof arbitrary code uses them: +a constructor URL overrides the resolver. Runtime usage stays unverified until actual acceptance. + +## GET-only assessment + +Save semantic choices as `intent.json`: + +```json +{ + "schema_version": "1.0", + "scope": {"subscription_id": "", "resource_group": "", "account_name": ""}, + "project": "", + "search_service": "", + "knowledge_base": "", + "agent_name": "", + "agent_version": "", + "toolbox_name": "", + "tool_label": "", + "connection_name": "", + "reader_assignment_id": "/providers/Microsoft.Authorization/roleAssignments/", + "project_assignment_id": "/providers/Microsoft.Authorization/roleAssignments/", + "retention_owner": "" +} +``` + +```text +python helpers/hosted_connect.py --plan intent.json +``` + +Omit `account_name` only when scoped account/project discovery is needed. +Optional `known_agents: [{"name":"...","version":"..."}]` adds up to 20 known +binding readbacks, not an account-wide consumer scan. Limits: 1 MiB input/REST +responses; 20 pages/100 items per scoped inventory; at most 20 accounts and 200 tools. + +Output: `approval_summary`, private fingerprint, request IDs, zero writes. +Exact configured reuse is `planned`, without mutation approval. +Changed state is `blocked`; `execution_input` is null and `execution_available` is false; +no consent/apply mode. Fresh agent/default/version/connection or CLI-context drift blocks. + +Present this compact delta, not integrity hashes: + +| Item | Before | Proposed after | +|---|---|---| +| KB tool binding | Observed endpoint/connection | Selected endpoint/new compatible name | +| Toolbox default | Observed immutable version | New immutable version; ID not yet created | +| Agent | Exact observed version/runtime | **UNCHANGED** | + +## Shared-default approval and future write gate + +Promotion affects **every consumer** following the default, including external +clients. Show known bindings and unknown scope; never infer exclusive ownership +from a selected-agent read or partial inventory. Require explicit approval of this +shared-default effect plus the precise binding/auth/tool delta and retention. + +SDK 2.4 `project.toolboxes` exposes `get(name)`, `get_version(name, version)`, +`create_version(name, tools=...)`, `update(name, default_version=...)`. +REST `v1` uses GET `/toolboxes/` and `/toolboxes//versions/`, +POST `/toolboxes//versions`, then PATCH `/toolboxes/` with +`{"default_version":""}`. These are service capabilities, +**not executable approval through this helper**. + +Future writes require conditional promotion semantics, fresh default/ETag/protected state, +conflict blocking and preservation of unrelated tools/skills/metadata/policies. +The first toolbox version becomes default automatically; never create it accidentally. +Verify connection/version before promotion, then default and unchanged agent binding. +Partial failures retain original errors/ownership; no automatic rollback or cleanup. + +## Acceptance and portal help + +If no known-answer question is supplied, first reuse authorized retrieval evidence for this KB. +Otherwise the workflow owner may perform bounded KB retrieval only within an +approved data/model-cost boundary, then propose an evidence-backed question for +customer approval. Read-only retrieval may invoke embedding/chat and incur cost. +Do not fabricate payloads, add a probe when evidence is already known, or query +during this GET-only assessment. Optional `supported_question` and +`unrelated_question` are candidates, not invocation consent. Require a separate +unrelated abstention question and approval. + +Acceptance requires the actual selected agent's `knowledge_base_retrieve` activity, +original-source citations and unsupported-question abstention, not merely KB REST. +Pin its version, keep requested sessions/conversations and usage distinct, and +leave unavailable runtime/tool payload evidence unverified. + +UI help is optional, never a setup gate. Current first-party guidance: +**Manage > Project details > Connected resources**; older layouts/user screenshots +may say **Management center > Connected resources**. For the reported portal layout, +look under **Build > Tools > Toolboxes**; labels vary and this path is not API evidence. +Authorities: failure/conflict/uncertainty only. + +- [FoundryToolbox resolver](https://github.com/microsoft/agent-framework/blob/main/python/packages/foundry_hosting/agent_framework_foundry_hosting/_toolbox.py) +- [Toolbox operations and endpoints](https://learn.microsoft.com/azure/foundry/agents/how-to/tools/toolbox) +- [Connection UI](https://learn.microsoft.com/azure/foundry/how-to/connections-add) +- [Hosted code ownership](https://learn.microsoft.com/azure/foundry/agents/concepts/hosted-agents) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-hosted.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-hosted.md new file mode 100644 index 00000000..28f5c0c7 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-hosted.md @@ -0,0 +1,168 @@ +# Existing Hosted Agent connection + +Branch of [Connect](connect.md), not a creation workflow. +Inspect remotely first; no source path for inspection or exact reuse. +Missing `azure.yaml` in cwd is not a discovery blocker. +Do not reinstall, reinitialize or inspect policy upfront. +Retain resources; Hosted cleanup is unsupported. + +## Inspect + +Use known project endpoint, exact agent name/version, KB MCP identity, runtime +principal and actual connection/toolbox/binding readbacks through authenticated, +observed read operations. Missing read tooling is a tooling blocker, not a source +request. Run these azd reads only with already available project/environment: + +```text +azd ai agent show --output json --no-prompt +azd ai connection show --output json --no-prompt +azd ai toolbox show --output json --no-prompt +``` + +Never use `azd ai project set` as a read; context changes require approval. +Inspect compatible connections/toolboxes before choosing a new name. Never +overwrite a Prompt connection using `ProjectManagedIdentity` to make it Hosted. + +Compare agent identity/version, model, telemetry, ACR, +network, environment, connection target/auth/audience and toolbox/default tools. +For omitted audience or `UnknownConnectionPropertiesV2`, use the exact +[nonsecret connection readback](../references/platform-interfaces.md#hosted-connection-readback). +Projection gaps are warnings only after authoritative equality is proven; +denial, conflicting properties or still-missing required evidence blocks. + +Resolve the actual published agent principal, not its blueprint or project MI. +Check effective runtime access and plan only missing grants: + +| Permission | Grant scope | +|---|---| +| Search Index Data Reader | `` | +| Foundry User or documented equivalent for toolbox access | `` | + +Reuse compatible effective grants. Disclose that Search service scope covers +its indexes, while Foundry User grants broader project data operations, not just +toolbox reads. No automatic grant, scope widening or caller-role change. A newly +observed principal or access requirement needs a separate approved access plan. + +## Select binding before approval + +Unchanged consumer: [toolbox-only branch](connect-hosted-toolbox.md); no deploy. + +Source-free changes require observed runtime support and an authoritative surface. +Otherwise explain the exact code/config/deploy delta and handoff/request source +only then. Hosted code orchestrates tools: existing agents may need source too; +no arbitrary Hosted tool changes through Prompt APIs. + +For the source-backed recipe, inspect its supported mode and emitted environment, +not just local variables. For `FoundryToolbox`, an explicitly +empty `TOOLBOX_ENDPOINT` raises instead of falling back to `TOOLBOX_NAME`. + +| Mode | `TOOLBOX_ENDPOINT` | `TOOLBOX_NAME` | +|---|---|---| +| Endpoint | `` | `""` or absent | +| Name | absent | `` | + +The consumer endpoint is +`/toolboxes//mcp?api-version=v1`. +Do not substitute the version-specific developer endpoint returned by some +toolbox readers. Preserve an already compatible exact binding without +standardizing its mode. For new/repair bindings, select one supported mode: +if the manifest injects an empty endpoint, prefer endpoint mode when supported. +Unsetting a local variable does not remove a manifest's runtime environment key. +If no valid binding is possible without source edits, stop for separate review. + +If installed, validate resolution offline with the SDK; do not install it for +this check. No token/model call or standalone +agentic toolbox `tools/list`: runtime identity requires published-agent context. + +## Approved changes only + +This source-backed recipe requires the approved existing `azd ai agent` source project; +unsupported/image-only source blocks this recipe, not remote inspection. +Retain source/config hashes for source/config changes or source-backed redeployment. +Plan grants/resource/binding deltas, source digest, costs and verification. +Exact state skips every mutation below. +After approved access changes, create only absent exact resources: + +```text +azd ai connection create --kind remote-tool --target --auth-type agentic-identity --audience https://search.azure.com/ --output json --no-prompt +azd ai toolbox create --from-file --output json --no-prompt +``` + +Verify stored `AgenticIdentityToken`, audience, target and project before +toolbox creation. YAML contains only description and the exact connection name; +no credentials or extra tools. Creation auto-publishes the first toolbox version +and writes `TOOLBOX__MCP_ENDPOINT` locally; disclose that write. + +Apply only the selected approved binding delta. Endpoint mode: + +```text +azd env set TOOLBOX_ENDPOINT --no-prompt +``` + +If the source exposes an unused name variable, clear it only as an approved delta: + +```text +azd env set TOOLBOX_NAME "" --no-prompt +``` + +Name mode, only with the endpoint absent from the emitted runtime environment: + +```text +azd env set TOOLBOX_NAME --no-prompt +``` + +Deploy only when remote configuration differs, under the same agent name: + +```text +azd deploy --no-prompt +``` + +Independently read back the returned version, actual principal, binding and +protected state. `active` metadata is not runtime-health proof. Unknown/changed +identity, source drift or failure stops; no blind retry, `--force`, handcrafted +mutation API, replacement agent or automatic cleanup. + +## Verify and reconcile + +Pin the observed version at session creation. A new session does not guarantee a new Responses +conversation. Start both fresh: + +```text +azd ai agent invoke "" --protocol responses --version --new-session --new-conversation --no-prompt +``` + +Capture the first response's session ID as `` and its +conversation ID. Verify both are distinct from prior trial IDs. Missing or reused +first-response IDs block the second call. Do not rely on automatic session selection; +explicitly pin the captured session while starting a fresh conversation: + +```text +azd ai agent invoke "" --protocol responses --session-id --new-conversation --no-prompt +``` + +Sessions bind their version at creation; `--version` and `--session-id` are mutually exclusive. +If the installed CLI lacks `--session-id`, block verification rather than falling +back to a saved session. Require the second response's session ID to equal the +captured ID and its conversation ID to differ from the first and all prior trials. +Missing or mismatched IDs make isolation unverified: stop before monitoring or +further invocations. Do not silently add invocations to repair a bounded test. + +Correlate actual `knowledge_base_retrieve` success with each request and response, +using returned events or supported logs: + +```text +azd ai agent monitor --session-id --tail 300 --no-prompt +``` + +Require supported answers with original citations and unsupported answers exactly +`I don't know.` without citations. Errors are not abstention. Tool success proves +execution, not the contents of an unavailable payload: record missing arguments, +results or usage and leave payload-level faithfulness unverified. Never infer +empty retrieval or change source/telemetry to manufacture proof. + +Compare identities, versions, tools, roles, protected source and binding. +Repeat reconciliation through reads only: no setters, create, publish +or deploy calls on exact state. Zero configuration writes excludes authorized +invocation/conversation/session activity and local test metadata; report both. +Retain resources. Cleanup remains separately approved and +`hosted-cleanup-unsupported` in this skill. diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-prompt-sdk-fallback.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-prompt-sdk-fallback.md new file mode 100644 index 00000000..0931c8d1 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect-prompt-sdk-fallback.md @@ -0,0 +1,139 @@ +# Existing Prompt Agent SDK fallback + +Use this fallback only when authenticated Foundry MCP is unavailable +or lacks connection/version operations, never denial/conflict. +Select before planning; require the parent's approved child fingerprint before ARM creation; +do not read policy unless a creation failure implicates it. Unsupported settings block. +Read `helpers/contracts.md`; verify SDK/CLI identity. Install only missing/incompatible dependencies after approval: + +```text +python -m pip install --pre "azure-ai-projects>=2.4.0,<3" "azure-identity>=1.25.0,<2" +``` + +Run `az login` only if unauthenticated. ARM uses signed-in CLI identity; +SDK and shared cleanup loader use `AzureCliCredential`, not environment/MI fallback. +Missing CLI authentication blocks noninteractively, never falls back to another identity. +Keep CLI context unchanged; never silently switch tenant/identity. +SDK errors: safe code/HTTP status/server ID (`x-ms-request-id`, then `request-id`). +Client request IDs are not server provenance; raw exception text is withheld. +Ambiguous creation stays partial with original status/ID even if recovery fails. +Observed matching versions do not prove creation ownership. CLI output separates +`resources_remaining.unverified` from run-owned resources; unknown versions remain unnamed. +`request.json` (resolved intent, not hand-built bodies/hashes): + +```json +{ + "schema_version": "1.0", + "project_resource_id": "", + "project_endpoint": "https://account.services.ai.azure.com/api/projects/project", + "search_resource_id": "", + "knowledge_base_name": "kb", + "agent_name": "agent", + "agent_version": "1", + "connection_name": "kb-project-mi", + "is_shared_to_all": false, + "binding_action": "ensure", + "role_assignment_id": "/providers/Microsoft.Authorization/roleAssignments/", + "permission_forwarding": {"mode": "not-applicable"}, + "network": {"posture": "public", "evidence": ""}, + "owner": "" +} +``` + +```text +python helpers/prompt_connect.py --plan request.json +``` + +Use observed choices, never latest. Output: `status: planned`, `approval_summary`, +`execution_input`, `approval.confirmed: false`; no writes/installation/inference. +Read CLI context, exact project/KB/role/connection, selected version and all scoped +version pages including drafts; no account/index/connection enumeration. +Bounds: 1 MiB input/readback/definition, 200 KB sources, 200 agent versions/pages. +At 200 versions, new-version apply/plan blocks before writes; exact reuse remains valid. +Unresolved KB profiles block, never transition the KB. Verify the PROJECT principal's +exact Search Index Data Reader grant; missing roles block, never assign. +Search provisioning/degraded warns only with healthy KB GET; +failed/deleting/disabled or unavailable reads block. + +Review names, binding/grounding delta, sharing, principal/scope and preserved state. +Save `execution_input` privately; after consent set `approval.confirmed` true, +without changing plan or fingerprint. Fresh exact reuse: +`execution_required: false`, `mutation_approval_required: false`; stop without approval/apply. +Otherwise: + +```text +python helpers/prompt_connect.py --input +``` + +Bind project ID/endpoint, agent/version/model/digest, absent/exact ARM connection, +KB MCP endpoint, `allowed_tools: ["knowledge_base_retrieve"]`, `require_approval: "never"`, +role ID/principal/scope, permission forwarding, owner, `cleanup_approved: false`. +`grounding_instructions` binds the exact appended evidence-only text; +old envelopes cannot authorize grounding upgrades. + +```text +For every user question, call knowledge_base_retrieve before answering, including questions that seem unrelated to the knowledge base. Answer only from evidence returned for that question and cite the original sources. Do not answer from general knowledge or assume an answer without retrieval. If the retrieved evidence does not support an answer, reply exactly: I don't know. Do not add citations to an unsupported answer. If retrieval fails, report the failure instead of treating it as no evidence or answering from general knowledge. +``` + +Model/type/connection conflicts and duplicate same-label MCP tools block. Same-label MCP tool drift +requires `binding_action: replace-selected`; inline auth/connectors/conflicting headers +still block. Preserve definition/metadata/description/draft/blueprint reference. +Reuse the sole exact version without `create_version`, or create at most one ARM +connection and one same-agent version via `project.agents.create_version`; read both back. +Normalize only selected MCP `allowed_tools` list versus `tool_names` object encoding +(including `read_only: null`); other fields stay strict, including unrelated tools. +Use that comparison for reuse/recovery/verification, but approval/ownership hashes +bind original readback. Same-name non-equivalent connections conflict even with legacy +`update`; choose a new name, retain legacy/key connections on the same KB endpoint. +Legacy schema/hashes unchanged. Refresh version inventory/prerequisites before writes; +portal changes invalidate plans. Replan after execution; snapshots are not concurrency locks. +Permission forwarding: not applicable or named `search_auth_token` structured input, +value per request only. Exit `2`: no-write blocked; `3`: partial/ambiguous, never success. + +Use `Microsoft.CognitiveServices/accounts/projects/connections`. +Disclose `connection.is_shared_to_all` (default `true`) and actual sharing. +Server type metadata or boolean true→false restriction differences warn, not rewrite. +Missing/malformed sharing, broadened access, identity/target/auth/audience/category/ +required-metadata mismatches block; explicit false rejects nonempty `sharedUserList`. +Preserve warnings after agent failure; compare creation, ambiguous-write recovery and reuse +consistently. `completed` verifies configuration only; actual +invocation/citations/unsupported-question checks remain required. + +Connection approval excludes cleanup; use the separate cleanup planner. +After separate approval, invoke: + +```text +python helpers/prompt_cleanup.py --input +``` + +Cleanup binds run-owned identities, digests and connection ETag; deletes version before +connection; verifies absence including drafts. Preserve prior versions/project/model/KB/roles. + +## Creation receipts + +Before create: +`--cleanup-receipt-dir ""` to approved +`--input`; keep `cleanup_receipts` and original input. +Only `verified` receipts seed [separate cleanup](../lifecycle/cleanup.md); +`acknowledged`/reuse/update/recovery are not ownership proof. +Store native IDs/hashes/versions, never credentials/bodies. + +### Initial Prompt creation only + +Verify project/model/identity/network prerequisites before local initial creation. +`python helpers/prompt_connect.py --plan-initial `. +Closed request: `schema_version: "1.0"`, `owner`, `project_resource_id`, `project_endpoint`, +`agent: {name, definition}`, `prerequisites` evidence strings keyed by +`project`, `model`, `identity`, `network`; optional `inventory_limits`: +integer `max_pages`/`max_resources` (default 20/500, max 100/5000). +Closed definition: `kind: "prompt"`, verified deployment `model`, `instructions`, `tools: []`. +Evidence descriptions are not platform verification. + +Review `execution_input`; change only `approval.confirmed`. +Apply `--input ` with receipt flag: +Foundry v1 `POST /agents?api-version=v1`, never create-version on replay. +Complete project inventory/exact GET prove absence; existing names, changes, +denied/partial reads or ambiguous writes block. No POST retry. +HTTP 200 `AgentObject.versions.latest` plus independent version GET bind the receipt. +Post-ACK failure is partial, not ownership recovery. No invocation/provisioning/grants/ +Hosted creation. Cleanup retains container and other versions. diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect.md new file mode 100644 index 00000000..fe95f0a4 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/connect.md @@ -0,0 +1,146 @@ +# Connect a knowledge base to an agent + +## When to use + +Connect one Prompt/Hosted Agent; optionally create it if missing. +KB: content; project connection: auth; agent MCP tool: versioned use. + +## Do not use + +Do not replace agents, attach multiple base tools, or handle base-free lifecycle. +Permission error is not absence; creation approval is separate. + +## Inputs and discovery order + +Resolve prompt, session, workspace, exact Azure readback, default, then one focused question. +Label sources; missing input blocks. Reads need no approval; silence approves nothing. + +| Input | Why needed | Required? | Discovery order | Safe default | If missing or unanswered | Reconfirmation trigger | +|---|---|---|---|---|---|---| +| Base return/MCP endpoint | Grounding | Required | Session, exact proof | None | Block | Base/API/evidence change | +| Project ID/endpoint | Target | Required | Prompt, workspace, readback | None | Block | ID/tenant change | +| Agent name/inventory/type | Identity | Required | Prompt, readback | Never latest/type | Block ambiguity | Inventory/type | +| Definition/model/baseline; Hosted telemetry | Preservation | By branch | Exact readback | Preserve unrelated fields | Block | Protected state | +| Connection/tool; Hosted toolbox/environment | Binding | By branch | Full readback | One retrieve binding | Block conflict | Binding/endpoint | +| Principal/reader role | Retrieval | Before assignment | Readback | Exact Search scope | Block | Assignment | +| Grounding delta | Evidence | Required | Instructions | Base/citations/`I don't know` | Block loss | Delta | +| Acceptance/unrelated questions | Verification | Before invocation | Authorized evidence | None | Propose candidate | Questions | +| Cleanup/retention owner | Ownership | Required | Prompt/session | None | Block | Owner | +| Provider/install or CLI choice | Missing agent | Conditional | Runtime, then choice | No automatic fallback | Block | Availability/choice | + +## Decisions + +Use [policy diagnostics](../references/search-substrate.md#azure-policy-diagnostics-after-creation-failure) +only after a creation failure implicates policy, never as a gate. + +Read exact IDs/all scoped pages; inaccessible is not absent: + +- **Prompt:** refresh the selected version and all its scoped version pages after + portal add/remove; never assume latest. Exact is zero-write. Collision is the + connection NAME, not KB endpoint: an approved new name can share that endpoint, + preserving legacy/key connections. Same-name mismatch blocks. An explicit + selected-tool switch creates a version, preserving all unrelated fields/tools. +- **Hosted:** inspect remote project, exact agent/version, KB MCP, runtime principal + and connection/toolbox/binding first: no source path for inspection or exact reuse. + Bind source/config digest for agent edits/redeployment. + Conflicts block. Read [Hosted connection](connect-hosted.md) + before plans or verification. + +Only for a missing agent, load [missing-agent creation](create-missing-agent.md) +before questions/plans. Prefer `microsoft-foundry`: delegate approved +identity/configuration and known requirements. Otherwise offer installation +or Prompt CLI creation. Installation refusal is not creation approval. +Require a typed return with project, agent, version/deployment, model/status/identity +and definition/source/config/environment. Independently read back; +use fresh discovery and a new fingerprinted connection plan; +never carry creation approval. Incomplete returns block. + +Read the authenticated `knowledgebases` API, never generated indexes +or `foundryextensions_knowledge_index_list`. Existing multi-source KBs are valid. +Preserve valid KB effort/output/models; preview MCP supports reasoning/synthesis. +Use the exact preview API in SDK; native GA minimal/extractive MCP exists. +KB model: low or medium with supported output; preview minimal/answerSynthesis is valid. +Use the approved [KB transition](../knowledge-bases/create.md#agent-compatible-minimal-transition) +only when an explicit model-free normalization is needed. +Never change the KB implicitly or add a model as a workaround. +Only when authenticated Foundry MCP is unavailable or lacks a required operation, +load [typed Prompt SDK fallback](connect-prompt-sdk-fallback.md). +Denial/conflict blocks. Bind the surface; switching requires fresh review, never bypass. + +## Proposed plan + +List delta, API, identity/scope, network/data, cost, verification, owner +and retained IDs. + +## Confirmation + +Approve concrete changes once; `plan_fingerprint` stays internal, with +`cleanup_approved: false`. Delegated creation is separate. A Hosted principal +known only after deploy requires a separate RBAC plan. Material/protected-state +drift invalidates approval; disclose bounded Prompt warnings below. + +## Mutation + +After confirmation, use exact identities; never retry another name. + +**Prompt:** Use the planned MCP or SDK surface; never invent operations. +Reconcile one `2025-10-01-preview` +`RemoteTool` connection with `ProjectManagedIdentity`, Search audience, and the +`2026-08-01-preview` KB MCP endpoint. Require the exact Search-service-scoped +`Search Index Data Reader` assignment to observed PROJECT `identity.principalId`, +not agent identity. Preserve all fields; add/switch at most one +same-label `MCPTool` with exact endpoint/connection, +`allowed_tools: ["knowledge_base_retrieve"]`, and `require_approval: "never"`. +Under the same agent name, require retrieval for every question, including +unrelated questions. Answer from retrieved evidence, not general knowledge; +unsupported answers are exactly `I don't know` without citations; +retrieval errors must remain errors. Approve exact appended instructions; +preserve previous instructions/versions. Duplicate labels block. +Search provisioning alone is not a connection-write blocker when the KB GET is +healthy; warn, never claim retrieval readiness. Failed/auth/deletion states block. + +After exact identity/binding checks, `type` metadata differences and boolean +`isSharedToAll: true`-to-`false` restriction warn without rewriting. Missing/malformed +sharing, broader access, wrong target/auth/audience/category/required metadata +block. Apply the same rules to reuse; require actual agent tests below. + +**Hosted:** Follow the selected [Hosted connection](connect-hosted.md) procedure. +Only approved missing resources, binding differences and required deployments +are writes; exact existing state skips them all. Never replace agents; +source edits need separate review/handoff. + +## Verification + +Verify protected fields, a `knowledge_base_retrieve` call with original +citations, and unrelated `I don't know` without citations. Preserve behavior; +rerun with stable IDs and zero configuration writes. Account separately for +requested invocations, conversations/sessions and their usage. Record actual +tool execution and returned evidence separately from answer behavior; unavailable +payloads leave payload-level faithfulness unverified, not an inferred empty result. + +## Failure and partial completion + +Preserve first status/message/request ID. Denial, conflict, incomplete +delegation, failed citations, unknown principal or drift blocks. +Record writes, evidence, ownership, recovery and warnings; never widen state. + +## Cleanup + +Separate fingerprinted cleanup deletes only run-owned artifacts in reverse. +Prompt uses `helpers/prompt_cleanup.py` with exact digests. Hosted returns +`hosted-cleanup-unsupported`, zero writes, retaining all resources. +Unclear ownership blocks. + +## Return contract + +Return `planned`/`completed`/`blocked`/`partial`: exact actions/resources, +invocation/readback evidence, ownership, failure, warnings and cleanup status. + +## References + +- [Shared audit schemas](../references/platform-contracts.md). +- [Interface versions](../references/platform-interfaces.md). + +Authorities: failure/conflict/uncertainty only. + +- [Connect Agents to Foundry IQ knowledge bases](https://learn.microsoft.com/azure/foundry/agents/how-to/foundry-iq-connect) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/create-missing-agent.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/create-missing-agent.md new file mode 100644 index 00000000..c8ff31a8 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/agents/create-missing-agent.md @@ -0,0 +1,163 @@ +# Missing agent for a knowledge-base connection + +Load only from Connect for a missing agent, not generic creation, replacement +or the existing-agent SDK connection fallback. Preserve existing agents/versions. + +## Resolve and offer the creation capability + +Use prompt/session/workspace and exact Azure readback to recommend project, +type/name and existing model deployment. Ask only unresolved choices. +Inventory all pages; permission denial or incomplete discovery is not absence. +An existing incompatible agent does not authorize a new name: require explicit +new-agent intent and prove the selected new identity absent. + +Check the runtime registry or exact invocation for `microsoft-foundry`. Skill +availability and authenticated creation-tool availability are separate facts. +Unknown stays unknown; never claim unexecuted delegation. +If unavailable, ask one focused choice, unless already answered: + +- Install Microsoft's Azure Skills plugin (recommended). +- Decline installation and review local CLI creation for a Prompt Agent. +- Stop with setup guidance. + +Do not install automatically. Show the source, host/workspace scope and exact +commands. Installation consent is separate from Azure creation approval; +cancellation stops. In Copilot CLI these are interactive host commands, not +PowerShell commands: + +```text +/plugin marketplace add microsoft/azure-skills +/plugin install azure@azure-skills +/mcp show +``` + +With MCP already configured, the skill-only option is: + +```text +npx skills add https://github.com/microsoft/azure-skills --skill microsoft-foundry +``` + +Skill-only installation does not install or authenticate MCP tools. After +installation, reload if required and recheck runtime visibility, authentication +and exact creation operations. If installation/reload cannot run here, give +the commands to the user. If tools remain unavailable, report +`creation-provider-unavailable`, zero Azure writes, and offer the CLI choice; +do not silently switch. Never bypass an authorization +denial through another surface. + +## Creation plan and approval + +Prefer available delegation. Do not scan policy before either creation path. +Use [policy diagnostics](../references/search-substrate.md#azure-policy-diagnostics-after-creation-failure) +only after a creation failure implicates policy. Preserve known required +settings. Present one immutable creation plan with: + +- Exact tenant/subscription/project ID and matching endpoint, absent agent name, + Prompt/Hosted type, model deployment and full initial definition. +- Provider, API/version, exact expanded commands and request bodies or delegated + commands, local files/installations and digests. +- Identity/RBAC, network/data movement, known requirements, + existing and incremental costs, verification, retained resources and owner. +- `plan_fingerprint` and `cleanup_approved: false`. + +Let the user review delegated commands before execution. Changed commands, +model, scope, known requirements or definition require new approval. Approval to install +or select CLI does not approve creation. Creation does not approve the KB +connection, Search roles, invocation charges or cleanup. + +## Local CLI fallback: Prompt only + +Execute locally only after explicit CLI selection and creation-plan approval. +Reuse an accessible existing project and chat model deployment; read back the +deployment identity, model/version and status rather than copying a sample +model. Missing project/model/rights/network prerequisites block this fallback; +do not provision them, change permissions/networking or substitute a Hosted +Agent. Hosted creation remains delegated; if unavailable, return +`creation-provider-unavailable` with official guidance. + +Use signed-in Azure CLI identity with Foundry v1 REST. +Verify `az version` and `az account show`; authenticate only if needed. +`az rest --resource https://ai.azure.com/` obtains tokens internally. Never use +keys, ask for tokens, print tokens or enable verbose/debug credential logging. + +Before approval, review the complete body in a UTF-8 file: + +```json +{ + "name": "", + "definition": { + "kind": "prompt", + "model": "", + "instructions": "", + "tools": [] + } +} +``` + +No initial KB binding. These CLI templates use no shell variables (Bash, +PowerShell, cmd.exe). Replace placeholders before execution with the approved +endpoint (no trailing slash), name, returned version and OS-native absolute body +path. Keep URLs and the `@`-prefixed file argument quoted. + +```text +az rest --method get --url "/agents/?api-version=v1" --resource https://ai.azure.com/ +``` + +Inspect the actual HTTP result, not just a nonzero exit. A 404 is absence only +after successful project access and complete agent inventory; a 401/403, timeout +or malformed response blocks. For a present identity, list every version and +compare the full approved definition including model, instructions and tools. +A sole exact match is zero-write reuse; multiple matches or drift block. +Never choose latest automatically, overwrite, suffix-create or issue a version +POST on replay. Return connected versions to Connect; never reset their tools. + +Before writing, refresh absence/access and recompute the +approved plan/body digests. Changed evidence needs new review. Only after all +gates pass, issue one create-agent POST, not an update or create-version call: + +```text +az rest --method post --url "/agents?api-version=v1" --resource https://ai.azure.com/ --headers "Content-Type=application/json" --body "@" +``` + +Retain exit status and the first error/status/request ID. Never blindly +retry POST, including on timeout/409/5xx. Reconcile an ambiguous write by reading +the same identity and all versions. If the exact result or ownership is +unproven, report `partial` with potentially created resources and stop; never +switch to SDK/delegation or delete to make it pass. + +## Verify and return to Connect + +For local Prompt creation, independently GET the agent and returned version, +and list all versions; command acceptance is not proof: + +```text +az rest --method get --url "/agents/?api-version=v1" --resource https://ai.azure.com/ +az rest --method get --url "/agents//versions?api-version=v1" --resource https://ai.azure.com/ +az rest --method get --url "/agents//versions/?api-version=v1" --resource https://ai.azure.com/ +``` + +Follow pagination. Require matching project/name/version, Prompt kind, full +definition and usable status, with no duplicate versions or unexpected tools. +Record actual project/agent identities; never guess a principal. Unknown +required identity blocks connection, even if agent creation succeeded. + +Return `completed`, `blocked` or `partial`, IDs/version/model/status, definition +digest, identity evidence, first failure, API/provider/commands, approved +fingerprint, writes, ownership and retained resources. Return to Connect: +independently read back, use fresh discovery and a new fingerprinted connection +plan, and never carry creation approval. After connection approval, invoke the +exact version; require KB tool-call evidence, original citations and abstention. +Direct KB retrieval is not agent proof. Repeat Connect and compare +version/tool/connection IDs and counts with zero writes. + +Before local creation, select [receipt capture](connect-prompt-sdk-fallback.md#initial-local-prompt-creation-only). +No automatic cleanup: separate ownership/approval; retain prior versions, +shared resources and container. + +## Authorities + +Authorities: failure/conflict/uncertainty only. + +[Installation guide](https://learn.microsoft.com/azure/foundry/how-to/develop/use-microsoft-foundry-skill), +[Foundry v1 REST API](https://learn.microsoft.com/rest/api/microsoft-foundry/aiproject), +[Prompt quickstart](https://learn.microsoft.com/azure/foundry/agents/quickstarts/prompt-agent). diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_blob_observation.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_blob_observation.py new file mode 100644 index 00000000..b31e1ca9 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_blob_observation.py @@ -0,0 +1,83 @@ +"""Credential-free semantic projections, separate from cleanup integrity.""" +from __future__ import annotations + +try: + from ._common import HelperFailure, digest + from . import _indexer_observation as indexer +except ImportError: + from _common import HelperFailure, digest + import _indexer_observation as indexer + + +FIELDS = frozenset(("name", "type", "container", "identity", "description", + "dataChangeDetectionPolicy", "dataDeletionDetectionPolicy", "encryptionKey", + "fields", "scoringProfiles", "defaultScoringProfile", "corsOptions", + "suggesters", "analyzers", "normalizers", "tokenizers", "tokenFilters", + "charFilters", "similarity", "semantic", "vectorSearch", + "skills", "cognitiveServices", "knowledgeStore", "indexProjections", + "dataSourceName", "targetIndexName", "skillsetName", "parameters", + "fieldMappings", "outputFieldMappings", "cache", "disabled")) +PROJECTION_FIELDS = {"core_digest", "field_digests"} + + +def observe(child, kind): + excluded = {"@odata.etag"} + if kind == "datasource": + excluded.add("credentials") + if kind == "indexer": + excluded.add("schedule") + core = {key: value for key, value in child.items() if key not in excluded} + fields = {key: digest(value) for key, value in core.items() if key in FIELDS} + fields["additionalProperties"] = digest({key: value for key, value in core.items() if key not in FIELDS}) + result = {"core_digest": digest(core), "field_digests": fields} + if kind == "datasource": + result["credential_digest"] = digest({"present": "credentials" in child, "value": child.get("credentials")}) + return result + + +def note(diagnostics, kind, severity, code, field, request_id): + indexer.note(diagnostics, severity, code, f"{kind}.{field}", + f"{kind}.{field}: {code}; values withheld.", request_id) + + +def compare(current, expected, kind, diagnostics, request_id): + if current["digest"] == expected["digest"] and current != expected: + raise HelperFailure(f"{kind}-evidence-inconsistent", "Equal full hashes have inconsistent projection evidence.", + blocked_at="verification", request_id=request_id) + if current["core_digest"] != expected["core_digest"]: + fields = current["field_digests"].keys() | expected["field_digests"].keys() + for field in sorted(fields): + if current["field_digests"].get(field) != expected["field_digests"].get(field): + note(diagnostics, kind, "error", "definition-drift", field, request_id) + raise HelperFailure("indexer-definition-drift" if kind == "indexer" else "definition-drift", + "Meaningful generated configuration changed; inspect child diagnostics.", + blocked_at="verification", request_id=request_id) + if current["etag"] != expected["etag"]: + note(diagnostics, kind, "info", "generated-version-changed", "@odata.etag", request_id) + if kind == "datasource" and current["credential_digest"] != expected["credential_digest"]: + note(diagnostics, kind, "warning", "datasource-credential-projection-changed", "credentials", request_id) + + +def indexer_only(diagnostics): + return [item for item in diagnostics + if not item["field"].startswith(("datasource.", "indexer.", "skillset.", "index."))] + + +def valid(value, kind, sha256): + fields = {"etag", "digest"} | PROJECTION_FIELDS + if kind == "indexer": + fields |= indexer.PROJECTION_FIELDS + if kind == "datasource": + fields |= {"credential_digest", "binding_proof"} + if not isinstance(value, dict) or set(value) != fields: + return False + hashes = fields - {"etag", "field_digests", "binding_proof"} + return (isinstance(value["etag"], str) and bool(value["etag"].strip()) + and all(isinstance(value[key], str) and sha256.fullmatch(value[key]) for key in hashes) + and isinstance(value["field_digests"], dict) + and set(value["field_digests"]) <= FIELDS | {"additionalProperties"} + and "additionalProperties" in value["field_digests"] + and all(isinstance(item, str) and sha256.fullmatch(item) + for item in value["field_digests"].values()) + and (kind != "datasource" or isinstance(value["binding_proof"], str) + and value["binding_proof"] in {"resource-id", "unverified"})) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_bootstrap_io.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_bootstrap_io.py new file mode 100644 index 00000000..3dfbd89f --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_bootstrap_io.py @@ -0,0 +1,589 @@ +"""Private storage and shell-free native CLI execution for the bootstrap owner.""" +from __future__ import annotations + +import ctypes +import json +import os +import re +import shutil +import stat +import subprocess +import uuid +from contextlib import contextmanager +from pathlib import Path + +try: + from ._common import HelperFailure, canonical_bytes +except ImportError: + from _common import HelperFailure, canonical_bytes + +MAX_BYTES = 1024 * 1024 + + +def failure(code, message): + return HelperFailure(code, message, blocked_at="verification") + + +def read_json(path): + try: + with Path(path).open("rb") as handle: + raw = handle.read(MAX_BYTES + 1) + if len(raw) > MAX_BYTES: + raise failure("bootstrap-input-invalid", "JSON exceeds the one MiB input limit.") + value = json.loads(raw.decode("utf-8")) + pending = [(value, 0)] + count = 0 + while pending: + item, depth = pending.pop() + count += 1 + if depth > 20 or count > 10000: + raise ValueError("JSON structure exceeds limits") + if isinstance(item, dict): + pending.extend((v, depth + 1) for v in item.values()) + elif isinstance(item, list): + pending.extend((v, depth + 1) for v in item) + json.dumps(value, ensure_ascii=False, allow_nan=False).encode("utf-8") + return value + except (OSError, UnicodeError, ValueError, RecursionError) as exc: + raise failure("bootstrap-input-invalid", "Select bounded, valid UTF-8 JSON.") from exc + + +def _windows_ancestor_acl(owner_text, entries, current): + # Windows volume roots are commonly owned by this fixed OS servicing principal. + installer = "S-1-5-80-956008885-3418522649-1831038044-1853292631-2271478464" + trusted = (current, "OW", "SY", "BA", installer) + if owner_text not in tuple("O:" + trustee for trustee in trusted if trustee != "OW"): + raise OSError("Ancestor owner can change the access boundary") + rights = { + "FA": 0x1F01FF, "FR": 0x120089, "FW": 0x120116, "FX": 0x1200A0, + "GA": 0x10000000, "GR": 0x80000000, "GW": 0x40000000, "GX": 0x20000000, + "SD": 0x10000, "RC": 0x20000, "WD": 0x40000, "WO": 0x80000, + "CC": 1, "DC": 2, "LC": 4, "SW": 8, "RP": 16, "WP": 32, "DT": 64, + "LO": 128, "CR": 256, + } + for entry in entries: + parts = entry.split(";") + if len(parts) != 6 or parts[0] not in ("A", "D") or parts[3] or parts[4]: + raise OSError("Unsupported ancestor ACL") + if parts[0] == "D" or "IO" in parts[1] or parts[5] in trusted: + continue + value = parts[2] + try: + mask = int(value, 16) if value.startswith("0x") else 0 + if not value.startswith("0x"): + if not value or len(value) % 2: + raise ValueError("Unsupported rights") + for start in range(0, len(value), 2): + mask |= rights[value[start:start + 2]] + except (KeyError, ValueError) as exc: + raise OSError("Unsupported ancestor rights") from exc + # Delete-child, delete, write-DACL, write-owner, or generic-all can replace retained paths. + if mask & 0x100D0040: + raise OSError("Another principal can substitute an ancestor or its children") + + +def _windows_private(path, *, ancestor=False): + # Inspect effective trustees, not chmod: Windows chmod does not establish privacy. + from ctypes import wintypes as w + adv = ctypes.WinDLL("advapi32", use_last_error=True) + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + pointer = ctypes.c_void_p + adv.GetNamedSecurityInfoW.argtypes = [w.LPWSTR, w.DWORD, w.DWORD] + [ctypes.POINTER(pointer)] * 5 + adv.GetNamedSecurityInfoW.restype = w.DWORD + adv.ConvertSecurityDescriptorToStringSecurityDescriptorW.argtypes = [ + pointer, w.DWORD, w.DWORD, ctypes.POINTER(w.LPWSTR), ctypes.POINTER(w.DWORD), + ] + adv.ConvertSidToStringSidW.argtypes = [pointer, ctypes.POINTER(w.LPWSTR)] + adv.OpenProcessToken.argtypes = [w.HANDLE, w.DWORD, ctypes.POINTER(w.HANDLE)] + adv.GetTokenInformation.argtypes = [w.HANDLE, ctypes.c_int, pointer, w.DWORD, ctypes.POINTER(w.DWORD)] + kernel.GetCurrentProcess.restype = w.HANDLE + kernel.CloseHandle.argtypes = [w.HANDLE] + kernel.LocalFree.argtypes = [pointer] + descriptor, owner, group, dacl, sacl = (pointer() for _ in range(5)) + text, sid_text, token, size = w.LPWSTR(), w.LPWSTR(), w.HANDLE(), w.DWORD() + try: + if adv.GetNamedSecurityInfoW(str(path), 1, 5, ctypes.byref(owner), ctypes.byref(group), + ctypes.byref(dacl), ctypes.byref(sacl), ctypes.byref(descriptor)): + raise OSError("Cannot inspect private ACL") + if not adv.ConvertSecurityDescriptorToStringSecurityDescriptorW( + descriptor, 1, 5, ctypes.byref(text), None, + ): + raise OSError("Cannot inspect security descriptor") + if not adv.OpenProcessToken(kernel.GetCurrentProcess(), 8, ctypes.byref(token)): + raise OSError("Cannot inspect current owner") + adv.GetTokenInformation(token, 1, None, 0, ctypes.byref(size)) + buffer = ctypes.create_string_buffer(size.value) + if not adv.GetTokenInformation(token, 1, buffer, size, ctypes.byref(size)): + raise OSError("Cannot inspect current owner") + sid = ctypes.cast(buffer, ctypes.POINTER(pointer))[0] + if not adv.ConvertSidToStringSidW(sid, ctypes.byref(sid_text)): + raise OSError("Cannot inspect current owner") + sddl = text.value + current = sid_text.value + owner_text, separator, acl = sddl.partition("D:") + if not separator or not dacl: + raise OSError("Owner or DACL is not private") + entries = re.findall(r"\(([^()]*)\)", acl) + if not entries or re.sub(r"\([^()]*\)", "", acl) not in ("", "P", "AI", "PAI"): + raise OSError("Unsupported ACL") + if ancestor: + _windows_ancestor_acl(owner_text, entries, current) + return + if owner_text != "O:" + current: + raise OSError("Owner is not the operator") + for entry in entries: + parts = entry.split(";") + if len(parts) != 6 or parts[0] != "A" or parts[5] not in (current, "OW", "SY", "BA"): + raise OSError("ACL grants access to another principal") + finally: + if token: + kernel.CloseHandle(token) + for allocated in (descriptor, text, sid_text): + if allocated: + kernel.LocalFree(ctypes.cast(allocated, pointer)) + + +def _outside_plugin(path): + skill = Path(__file__).resolve().parents[1] + plugin = skill.parent.parent + protected = [skill] + if any((plugin / marker / "plugin.json").is_file() for marker in (".plugin", ".claude-plugin", ".cursor-plugin")): + protected.append(plugin) + if any(root in (path.resolve(), *path.resolve().parents) for root in protected): + raise OSError("Receipts must be outside the installed plugin") + + +def _safe_leaf(name): + return (isinstance(name, str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,199}", name) + and not name.endswith(".") + and not re.fullmatch(r"(?i)(?:con|prn|aux|nul|com[1-9]|lpt[1-9])", name.split(".")[0])) + + +def _validated_directory(value): + if not isinstance(value, str) or not Path(value).is_absolute(): + raise failure("bootstrap-receipt-private", "Select an absolute, existing private receipt directory.") + path = Path(value) + try: + for part in (path, *path.parents): + info = part.lstat() + if part == path: + selected = info + if stat.S_ISLNK(info.st_mode) or getattr(info, "st_file_attributes", 0) & 0x400: + raise OSError("Linked paths are unsupported") + _outside_plugin(path) + if not stat.S_ISDIR(selected.st_mode): + raise OSError("Receipt location must be a directory") + if os.name == "nt": + _windows_private(path) + else: + if selected.st_uid != os.getuid() or stat.S_IMODE(selected.st_mode) & 0o077: + raise OSError("Directory must be private to its owner") + except (OSError, ValueError) as exc: + raise failure("bootstrap-receipt-private", "Receipt location must be user-owned and private; no ACLs were changed.") from exc + return path, selected + + +def private_directory(value): + return _validated_directory(value)[0] + + +@contextmanager +def _windows_security(): + """An explicit protected, inheritable owner/SYSTEM/admin DACL, supplied at creation.""" + from ctypes import wintypes as w + adv = ctypes.WinDLL("advapi32", use_last_error=True) + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + pointer = ctypes.c_void_p + class Attributes(ctypes.Structure): + _fields_ = [("length", w.DWORD), ("descriptor", pointer), ("inherit", w.BOOL)] + adv.OpenProcessToken.argtypes = [w.HANDLE, w.DWORD, ctypes.POINTER(w.HANDLE)] + adv.GetTokenInformation.argtypes = [w.HANDLE, ctypes.c_int, pointer, w.DWORD, ctypes.POINTER(w.DWORD)] + adv.ConvertSidToStringSidW.argtypes = [pointer, ctypes.POINTER(w.LPWSTR)] + adv.ConvertStringSecurityDescriptorToSecurityDescriptorW.argtypes = [ + w.LPCWSTR, w.DWORD, ctypes.POINTER(pointer), ctypes.POINTER(w.DWORD)] + kernel.GetCurrentProcess.restype = w.HANDLE + kernel.CloseHandle.argtypes = [w.HANDLE] + kernel.LocalFree.argtypes = [pointer] + token, size, sid_text, descriptor = w.HANDLE(), w.DWORD(), w.LPWSTR(), pointer() + try: + if not adv.OpenProcessToken(kernel.GetCurrentProcess(), 8, ctypes.byref(token)): + raise OSError("Owner unavailable") + adv.GetTokenInformation(token, 1, None, 0, ctypes.byref(size)) + buffer = ctypes.create_string_buffer(size.value) + if not adv.GetTokenInformation(token, 1, buffer, size, ctypes.byref(size)): + raise OSError("Owner unavailable") + if not adv.ConvertSidToStringSidW(ctypes.cast(buffer, ctypes.POINTER(pointer))[0], ctypes.byref(sid_text)): + raise OSError("Owner unavailable") + sddl = f"O:{sid_text.value}D:P(A;OICI;FA;;;{sid_text.value})(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)" + if not adv.ConvertStringSecurityDescriptorToSecurityDescriptorW(sddl, 1, ctypes.byref(descriptor), None): + raise OSError("Private descriptor unavailable") + yield Attributes(ctypes.sizeof(Attributes), descriptor, False) + finally: + if token: + kernel.CloseHandle(token) + for allocated in (sid_text, descriptor): + if allocated: + kernel.LocalFree(ctypes.cast(allocated, pointer)) + + +@contextmanager +def _pinned_directory(path, *, private=True): + """Pin every ancestor against substitution; POSIX writes remain handle-relative.""" + handles = [] + try: + if os.name == "nt": + from ctypes import wintypes as w + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.CreateFileW.argtypes = [w.LPCWSTR, w.DWORD, w.DWORD, ctypes.c_void_p, + w.DWORD, w.DWORD, w.HANDLE] + kernel.CreateFileW.restype = w.HANDLE + kernel.CloseHandle.argtypes = [w.HANDLE] + for part in reversed((path, *path.parents)): + # No FILE_SHARE_DELETE: an opened ancestor cannot be renamed/replaced. + handle = kernel.CreateFileW(str(part), 0x81, 3, None, 3, 0x02200000, None) + if handle == ctypes.c_void_p(-1).value: + raise OSError("Directory cannot be pinned") + handles.append(handle) + info = part.lstat() + if not stat.S_ISDIR(info.st_mode) or getattr(info, "st_file_attributes", 0) & 0x400: + raise OSError("Linked directory") + _windows_private(part, ancestor=True) + if private: + _validated_directory(str(path)) + yield None + else: + if not hasattr(os, "O_DIRECTORY") or not hasattr(os, "O_NOFOLLOW"): + raise OSError("Handle-relative creation unavailable") + for part in reversed((path, *path.parents)): + fd = os.open(str(part) if not handles else part.name, + os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, + **({"dir_fd": handles[-1]} if handles else {})) + handles.append(fd) + info = os.fstat(fd) + # A foreign writable ancestor can substitute descendants, even if the leaf is 0700. + if info.st_uid not in (0, os.getuid()) or stat.S_IMODE(info.st_mode) & 0o022: + raise OSError("Unsafe ancestor ownership or write access") + if private: + _, selected = _validated_directory(str(path)) + opened = os.fstat(handles[-1]) + if (selected.st_dev, selected.st_ino) != (opened.st_dev, opened.st_ino): + raise OSError("Directory identity changed") + yield handles[-1] + finally: + cleanup_failed = False + for handle in reversed(handles): + try: + if os.name == "nt": + cleanup_failed = not kernel.CloseHandle(handle) or cleanup_failed + else: + os.close(handle) + except OSError: + cleanup_failed = True + if cleanup_failed: + raise OSError("Directory handle cleanup failed") + + +def create_private_directory(value): + """Create one new leaf, never modify an existing directory or its ancestors.""" + if not isinstance(value, str) or not Path(value).is_absolute(): + raise failure("bootstrap-receipt-private", "Select an absolute new private directory.") + path = Path(value) + try: + if not _safe_leaf(path.name): + raise OSError("Unsafe directory leaf") + _outside_plugin(path) + with _pinned_directory(path.parent, private=False) as parent_fd: + if os.name == "nt": + from ctypes import wintypes as w + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.CreateDirectoryW.argtypes = [w.LPCWSTR, ctypes.c_void_p] + with _windows_security() as attributes: + if not kernel.CreateDirectoryW(str(path), ctypes.byref(attributes)): + raise OSError("Private directory creation failed") + else: + os.mkdir(path.name, 0o700, dir_fd=parent_fd) + with _pinned_directory(path): + return private_directory(str(path)) + except (OSError, ValueError, NotImplementedError) as exc: + raise failure("bootstrap-receipt-private", "Private leaf creation failed; no existing directory ACLs were changed.") from exc + + +def validate_private_artifact_directory(value): + path = private_directory(value) + try: + with _pinned_directory(path): + return private_directory(value) + except (OSError, ValueError, NotImplementedError) as exc: + raise failure("bootstrap-receipt-private", "Private directory or ancestor stability could not be verified; no ACLs were changed.") from exc + + +def _windows_private_open(path, *, create=True): + import msvcrt + from ctypes import wintypes as w + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.CreateFileW.argtypes = [w.LPCWSTR, w.DWORD, w.DWORD, ctypes.c_void_p, + w.DWORD, w.DWORD, w.HANDLE] + kernel.CreateFileW.restype = w.HANDLE + kernel.CloseHandle.argtypes = [w.HANDLE] + if create: + with _windows_security() as attributes: + handle = kernel.CreateFileW(str(path), 0xC0010000, 1, ctypes.byref(attributes), 1, 0x80200000, None) + else: + handle = kernel.CreateFileW(str(path), 0x80000000, 1, None, 3, 0x00200000, None) + if handle == ctypes.c_void_p(-1).value: + raise OSError("Private file creation failed") + try: + return msvcrt.open_osfhandle(handle, (os.O_RDWR if create else os.O_RDONLY) | os.O_BINARY) + except BaseException: + kernel.CloseHandle(handle) + raise + + +def _windows_publish(fd, destination): + import msvcrt + from ctypes import wintypes as w + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + target = str(destination) + size = len(target.encode("utf-16-le")) + class Rename(ctypes.Structure): + _fields_ = [("replace", w.BOOL), ("root", w.HANDLE), ("size", w.DWORD), + ("name", w.WCHAR * (size // 2 + 1))] + value = Rename(False, None, size, target) + kernel.SetFileInformationByHandle.argtypes = [w.HANDLE, ctypes.c_int, ctypes.c_void_p, w.DWORD] + # Rename the owned, write-through handle, never reopen a substitutable temporary pathname. + if not kernel.SetFileInformationByHandle(msvcrt.get_osfhandle(fd), 3, ctypes.byref(value), ctypes.sizeof(value)): + raise OSError("Private artifact publication failed") + + +def _cleanup_owned_temporary(directory, name, owned, *, directory_fd=None, primary=None): + path = directory / name if directory_fd is None else name + kwargs = {} if directory_fd is None else {"dir_fd": directory_fd} + try: + current = os.stat(path, follow_symlinks=False, **kwargs) + except FileNotFoundError: + outcome = "already-absent" + except (OSError, NotImplementedError): + outcome = "inspection-unavailable; entry left untouched" + else: + if (current.st_dev, current.st_ino) != (owned.st_dev, owned.st_ino): + outcome = "identity-changed; entry left untouched" + else: + try: + os.unlink(path, **kwargs) + outcome = "removed" + except FileNotFoundError: + outcome = "already-absent" + except (OSError, NotImplementedError): + outcome = "removal-unconfirmed; private temporary may remain" + warning = "Private temporary cleanup: " + outcome + "." + if primary is not None: + primary.warnings.append(warning) + elif outcome not in ("removed", "already-absent"): + error = failure("bootstrap-receipt-failed", "Private temporary cleanup could not be confirmed; no artifact success is reported.") + error.warnings.append(warning) + raise error + return outcome + + +def atomic_private_file(directory, name, value, *, max_bytes=16 * MAX_BYTES): + """Publish complete, verified JSON without replacing any existing filesystem entry.""" + if not _safe_leaf(name): + raise failure("bootstrap-receipt-failed", "Artifact name must be a safe ordinary leaf.") + try: + data = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, + allow_nan=False).encode("utf-8") + b"\n" + except (TypeError, ValueError, UnicodeError, RecursionError) as exc: + raise failure("bootstrap-receipt-failed", "Artifact must contain valid finite UTF-8 JSON.") from exc + if len(data) > max_bytes: + raise failure("bootstrap-receipt-failed", "Artifact exceeds its output byte bound.") + directory = Path(directory) + temporary = ".artifact-" + uuid.uuid4().hex + owned = None + primary = None + verified = False + try: + with _pinned_directory(directory) as directory_fd: + kwargs = {} if directory_fd is None else {"dir_fd": directory_fd} + leaf = lambda name: directory / name if directory_fd is None else name + try: + fd = (_windows_private_open(directory / temporary) if os.name == "nt" else + os.open(leaf(temporary), os.O_RDWR | os.O_CREAT | os.O_EXCL | + os.O_NOFOLLOW, 0o600, **kwargs)) + try: + handle = os.fdopen(fd, "w+b") + except BaseException: + os.close(fd) + raise + with handle: + owned = os.fstat(handle.fileno()) + if os.name == "nt": + _windows_private(directory / temporary) + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + handle.seek(0) + if handle.read() != data or os.fstat(handle.fileno()).st_nlink != 1: + raise OSError("Artifact integrity failed") + if os.name == "nt": + _validated_directory(str(directory)) + _windows_publish(handle.fileno(), directory / name) + published = owned + owned = None + os.fsync(handle.fileno()) + # Revalidate before publication; cleanup is restricted to our original inode. + if os.name != "nt": + current = os.stat(leaf(temporary), follow_symlinks=False, **kwargs) + if (current.st_dev, current.st_ino) != (owned.st_dev, owned.st_ino): + raise OSError("Artifact identity changed") + _validated_directory(str(directory)) + os.link(temporary, name, src_dir_fd=directory_fd, dst_dir_fd=directory_fd, + follow_symlinks=False) + for entry in (temporary, name): + current = os.stat(entry, dir_fd=directory_fd, follow_symlinks=False) + if (current.st_dev, current.st_ino) != (owned.st_dev, owned.st_ino): + raise OSError("Artifact changed during publication") + os.unlink(temporary, dir_fd=directory_fd) + published = owned + owned = None + os.fsync(directory_fd) + fd = (_windows_private_open(directory / name, create=False) if os.name == "nt" else + os.open(name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=directory_fd)) + try: + handle = os.fdopen(fd, "rb") + except BaseException: + os.close(fd) + raise + with handle: + observed = os.fstat(handle.fileno()) + if ((observed.st_dev, observed.st_ino) != (published.st_dev, published.st_ino) + or not stat.S_ISREG(observed.st_mode) or observed.st_nlink != 1 + or handle.read(len(data) + 1) != data): + raise OSError("Published artifact integrity failed") + if os.name == "nt": + _windows_private(directory / name) + _validated_directory(str(directory)) + verified = True + except HelperFailure as exc: + primary = exc + raise + except (OSError, ValueError, NotImplementedError) as exc: + primary = failure("bootstrap-receipt-failed", "Private artifact could not be verified and persisted; no artifact success is reported.") + raise primary from exc + finally: + if owned is not None: + try: + _cleanup_owned_temporary(directory, temporary, owned, + directory_fd=directory_fd, primary=primary) + except HelperFailure as exc: + primary = exc + raise + except (OSError, ValueError, NotImplementedError) as exc: + if primary is not None: + primary.warnings.append("Private directory handle cleanup could not be confirmed.") + raise primary from primary.__cause__ + error = failure("bootstrap-receipt-failed", "Private artifact could not be verified and persisted; no artifact success is reported.") + if verified: + error.warnings.append("Private directory handle cleanup could not be confirmed.") + raise error from exc + return directory / name + + +def private_file(directory, name, value): + return private_bytes(directory, name, canonical_bytes(value) + b"\n") + + +def private_bytes(directory, name, data): + if not _safe_leaf(name): + raise failure("bootstrap-receipt-failed", "Receipt name must be a safe, ordinary leaf name.") + if not isinstance(data, bytes): + raise failure("bootstrap-receipt-failed", "Private content must be bounded bytes.") + if len(data) > MAX_BYTES: + raise failure("bootstrap-receipt-failed", "Sanitized receipt exceeds its bound.") + directory_fd = file_fd = None + primary = None + try: + directory, selected = _validated_directory(str(directory)) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + if os.name == "nt": + file_fd = os.open(directory / name, flags, 0o600) + else: + if not hasattr(os, "O_DIRECTORY") or not hasattr(os, "O_NOFOLLOW"): + raise failure("bootstrap-receipt-private", "POSIX handle-relative private creation is unavailable.") + directory_fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + opened = os.fstat(directory_fd) + if ((opened.st_dev, opened.st_ino) != (selected.st_dev, selected.st_ino) + or not stat.S_ISDIR(opened.st_mode) or opened.st_uid != os.getuid() + or stat.S_IMODE(opened.st_mode) & 0o077): + raise failure("bootstrap-receipt-private", "Opened receipt directory changed identity, ownership or permissions.") + file_fd = os.open(name, flags, 0o600, dir_fd=directory_fd) + handle = os.fdopen(file_fd, "wb") + file_fd = None + with handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + except HelperFailure as exc: + primary = exc + raise + except (OSError, NotImplementedError) as exc: + primary = failure("bootstrap-receipt-failed", "Private receipt could not be persisted; retain the returned ownership handoff.") + raise primary from exc + finally: + cleanup_failed = False + for descriptor in (file_fd, directory_fd): + if descriptor is not None: + try: + os.close(descriptor) + except OSError: + cleanup_failed = True + if cleanup_failed: + if primary is not None: + primary.warnings.append("Receipt descriptor cleanup could not be confirmed.") + else: + raise failure("bootstrap-receipt-failed", "Receipt descriptor cleanup could not be confirmed.") + return directory / name + + +def cli_prefix(): + executable = shutil.which("az") + if executable is None: + raise failure("bootstrap-tool-unavailable", "A signed-in Azure CLI installation is required.") + if os.name == "nt": + if Path(executable).suffix.lower() not in (".cmd", ".bat"): + raise failure("bootstrap-tool-unavailable", "Windows requires the supported bundled CLI Python layout.") + # Use the installed CLI's own interpreter, never cmd.exe or caller commands. + # Azure/azure-cli: build_scripts/windows/scripts/az_msi.cmd (and az_zip.cmd). + python = Path(executable).parent.parent / "python.exe" + if not python.is_file(): + raise failure("bootstrap-tool-unavailable", "This Windows CLI layout has no supported bundled Python launcher.") + return [str(python), "-IBm", "azure.cli"] + return [executable] + + +def run_cli(arguments, timeout): + """Capture native CLI output; size validation is post-capture, not a memory bound.""" + env = dict(os.environ, AZURE_EXTENSION_USE_DYNAMIC_INSTALL="no", + AZURE_CORE_COLLECT_TELEMETRY="no", AZURE_CORE_ONLY_SHOW_ERRORS="true", + AZURE_CORE_NO_COLOR="true", AZURE_LOGGING_ENABLE_LOG_FILE="false", + AZURE_AUTO_UPGRADE_ENABLE="false") + try: + command = cli_prefix() + arguments + ["--output", "json", "--only-show-errors"] + result = subprocess.run(command, stdin=subprocess.DEVNULL, capture_output=True, + shell=False, env=env, timeout=timeout, check=False) + except subprocess.TimeoutExpired as exc: + raise failure("bootstrap-cli-timeout", "Native CLI timed out; remote completion and process-tree cleanup are not established.") from exc + except OSError as exc: + cause = exc + for _ in range(8): + if not isinstance(cause, OSError): + break + cause = cause.__context__ + if isinstance(cause, subprocess.TimeoutExpired): + error = failure("bootstrap-cli-timeout", "Native CLI timed out; remote completion is not established.") + error.warnings.append("Standard subprocess cleanup also failed; process state is unknown.") + else: + error = failure("bootstrap-cli-unavailable", "Native CLI execution failed; process and remote state may be unknown.") + raise error from exc + if len(result.stdout) > MAX_BYTES or len(result.stderr) > MAX_BYTES: + raise failure("bootstrap-cli-output-limit", "Captured CLI output exceeds one MiB per stream; this is not a capture-memory bound.") + return result.returncode, result.stdout, result.stderr diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_cleanup_dependencies.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_cleanup_dependencies.py new file mode 100644 index 00000000..1bb6a766 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_cleanup_dependencies.py @@ -0,0 +1,435 @@ +from __future__ import annotations + +import inspect +import re +from typing import Any +from urllib.parse import parse_qs, urlencode, urljoin, urlsplit + +try: + from ._common import HelperFailure, digest, odata_name, require_allowed_fields, sdk_error_metadata +except ImportError: + from _common import HelperFailure, digest, odata_name, require_allowed_fields, sdk_error_metadata + + +COLLECTIONS = {"index": "indexes", "indexer": "indexers", "skillset": "skillsets", "datasource": "datasources"} +SHA = re.compile(r"sha256:[a-f0-9]{64}\Z") + + +def fail(code, message): + return HelperFailure(code, message, blocked_at="cleanup-dependencies") + + +def limits(value=None): + value = {"max_pages": 20, "max_resources": 500} if value is None else value + if not isinstance(value, dict) or set(value) != {"max_pages", "max_resources"}: + raise fail("input-schema-invalid", "Inventory limits require max_pages and max_resources.") + if any(type(value[key]) is not int or not 1 <= value[key] <= maximum + for key, maximum in (("max_pages", 100), ("max_resources", 5000))): + raise fail("input-schema-invalid", "Inventory limits exceed supported bounded discovery.") + return dict(value) + + +def generated(current): + kind = current.get("kind") + if kind not in ("file", "azureBlob"): + raise fail("source-cleanup-kind-unsupported", "Only File and Blob/ADLS generated ownership is supported.") + parameters = current.get("fileParameters" if kind == "file" else "azureBlobParameters") + raw = parameters.get("createdResources") if isinstance(parameters, dict) else None + required = {"index"} if kind == "file" else set(COLLECTIONS) + if not isinstance(raw, dict): + raise fail("generated-ownership-unproven", "The exact source must expose its generated resource identities.") + names = {("datasource" if key == "dataSourceConnection" else key): value for key, value in raw.items()} + if set(names) != required or len(names) != len(raw): + raise fail("generated-ownership-unproven", "Unexpected, missing or colliding generated child identities.") + for name in names.values(): + odata_name(name) + return names + + +def validate_guard(plan): + guard = plan.get("dependency_guard") + if guard is None: + return + if not isinstance(guard, dict) or plan.get("operation") != "delete": + raise fail("input-schema-invalid", "Dependency guards belong only to cleanup deletion plans.") + kind = guard.get("kind") + if kind == "search-source" and plan.get("resource_type") == "knowledge-source": + fields, records, record_fields = ( + {"kind", "limits", "generated", "inventory_digest"}, + guard.get("generated"), {"type", "name", "etag", "definition_digest"}, + ) + if not isinstance(guard.get("inventory_digest"), str) or not SHA.fullmatch(guard["inventory_digest"]): + raise fail("input-schema-invalid", "A complete scoped inventory digest is required.") + elif kind == "prompt-connection" and isinstance(plan.get("connection"), dict): + fields, records, record_fields = ( + {"kind", "limits", "versions", "toolboxes"}, guard.get("versions"), {"name", "version", "definition_digest"}, + ) + if not isinstance(guard.get("toolboxes"), list): + raise fail("input-schema-invalid", "Complete toolbox version snapshots are required.") + records = records + guard["toolboxes"] if isinstance(records, list) else records + else: + raise fail("input-schema-invalid", "Dependency guard and cleanup target must agree.") + require_allowed_fields(guard, fields, label="Cleanup dependency guard") + if set(guard) != fields or not isinstance(records, list): + raise fail("input-schema-invalid", "The dependency snapshot must be complete.") + bounds = limits(guard["limits"]) + if len(records) > bounds["max_resources"]: + raise fail("input-schema-invalid", "The dependency snapshot exceeds approved limits.") + identities = set() + for record_index, record in enumerate(records): + if not isinstance(record, dict) or set(record) != record_fields: + raise fail("input-schema-invalid", "Dependency snapshot records are closed objects.") + if any(not isinstance(value, str) or not value.strip() for value in record.values()): + raise fail("input-schema-invalid", "Dependency snapshots require exact nonempty identities and versions.") + if not SHA.fullmatch(record["definition_digest"]): + raise fail("input-schema-invalid", "Dependency snapshots require canonical definition digests.") + domain = "toolbox" if kind == "prompt-connection" and record_index >= len(guard["versions"]) else "agent" + identity = (domain, record.get("type", record.get("version")), record["name"]) + if identity in identities: + raise fail("input-schema-invalid", "Duplicate dependency snapshot identity.") + identities.add(identity) + if kind == "search-source": + kinds = {record["type"] for record in records} + if kinds not in ({"index"}, set(COLLECTIONS)) or len(kinds) != len(records): + raise fail("input-schema-invalid", "Generated snapshots must identify one complete supported cascade.") + + +def _list_search(plan, collection, token, transport, bounds): + base = f"{plan['endpoint'].rstrip('/')}/{collection}" + url = base + "?" + urlencode({"api-version": plan["api_version"]}) + seen, names, records = set(), set(), [] + for _ in range(bounds["max_pages"]): + if url in seen: + raise fail("dependency-inventory-partial", "A continuation cycle prevents complete dependency discovery.") + seen.add(url) + response = transport("GET", url, token) + body = response.body + if response.status != 200: + raise HelperFailure("dependency-inventory-failed", "Scoped collection GET failed; consumer absence is unproven.", + blocked_at="cleanup-dependencies", status=response.status, request_id=response.request_id) + if not isinstance(body, dict) or not isinstance(body.get("value"), list): + raise fail("dependency-inventory-invalid", "Scoped collection GET did not return a complete JSON page.") + if set(body) - {"value", "@odata.nextLink", "@odata.context"}: + raise fail("dependency-inventory-partial", "An unsupported pagination contract cannot prove absence.") + for item in body["value"]: + if not isinstance(item, dict): + raise fail("dependency-inventory-invalid", "A dependency inventory item is not an object.") + name = item.get("name") + odata_name(name) + if name in names: + raise fail("dependency-inventory-changed", "Duplicate identities across inventory pages require fresh planning.") + names.add(name) + records.append(item) + if len(records) > bounds["max_resources"]: + raise fail("dependency-inventory-limit", "Complete scoped discovery exceeds the approved object bound.") + next_link = body.get("@odata.nextLink") + if next_link is None: + return sorted(records, key=lambda item: item["name"]) + if len(records) >= bounds["max_resources"]: + raise fail("dependency-inventory-limit", "More inventory pages exceed the approved object bound.") + if not isinstance(next_link, str) or not next_link: + raise fail("dependency-inventory-partial", "Malformed continuation is not an empty final page.") + candidate = urljoin(base, next_link) + parsed, expected = urlsplit(candidate), urlsplit(base) + query = parse_qs(parsed.query, keep_blank_values=True) + if ( + parsed.scheme != expected.scheme or parsed.netloc != expected.netloc or parsed.path != expected.path + or parsed.fragment or parsed.username or parsed.password + or query.get("api-version") != [plan["api_version"]] + or set(query) - {"api-version", "$skiptoken", "service", "pageSize", "search", "searchType"} + or query.get("search", [""]) != [""] + or ("service" in query and query["service"][0].casefold() + not in {expected.hostname.casefold(), expected.hostname.split(".")[0].casefold()}) + or any(len(values) != 1 for values in query.values()) + ): + raise fail("dependency-inventory-scope", "Continuation changed the selected service, collection, API or filter.") + url = candidate + raise fail("dependency-inventory-limit", "Complete dependency discovery exceeds the approved page bound.") + + +def _projections(skillset): + projections = skillset.get("indexProjections") + if projections is None: + return [] + if not isinstance(projections, dict) or not isinstance(projections.get("selectors"), list): + raise fail("dependency-inventory-invalid", "Skillset projection consumers are incomplete.") + names = [] + for selector in projections["selectors"]: + if not isinstance(selector, dict): + raise fail("dependency-inventory-invalid", "A skillset projection selector is incomplete.") + name = selector.get("targetIndexName") + odata_name(name) + names.append(name) + return names + + +def search_snapshot(plan, current, token, *, transport, bounds): + try: + from . import search_reconcile + except ImportError: + import search_reconcile + names = generated(current) + inventories = {} + bases = _list_search(plan, "knowledgebases", token, transport, bounds) + inventories["knowledgebases"] = bases + for base in bases: + sources = base.get("knowledgeSources") + if not isinstance(sources, list) or any(not isinstance(item, dict) or not isinstance(item.get("name"), str) for item in sources): + raise fail("dependency-inventory-invalid", "Every KB definition must expose its complete source references.") + if any(item["name"] == plan["name"] for item in sources): + raise HelperFailure( + "source-in-use", "A retained KB still references this source; separately plan that KB first, never detach it.", + blocked_at="cleanup-dependencies", warnings=["referencing-kb:" + base["name"]], + ) + sources = _list_search(plan, "knowledgesources", token, transport, bounds) + inventories["knowledgesources"] = sources + if sum(item["name"] == plan["name"] for item in sources) != 1: + raise fail("dependency-inventory-changed", "The selected source disappeared from the complete service inventory.") + for source in sources: + if source["name"] == plan["name"]: + if generated(source) != names or source.get("@odata.etag") != current.get("@odata.etag"): + raise fail("definition-drift", "Source inventory and exact readback disagree.") + continue + if source.get("kind") == "searchIndex": + parameters = source.get("searchIndexParameters") + index = parameters.get("searchIndexName") if isinstance(parameters, dict) else None + if not isinstance(index, str): + raise fail("dependency-inventory-invalid", "An existing-index source has unknown target identity.") + shared = index == names["index"] + elif source.get("kind") in ("file", "azureBlob"): + other = generated(source) + shared = any(other.get(kind) == name for kind, name in names.items()) + else: + raise fail("dependency-consumer-opaque", "An unsupported source kind prevents complete generated-index consumer proof.") + if shared: + raise fail("generated-resource-shared", "Another knowledge source references a generated cleanup resource.") + indexers = _list_search(plan, "indexers", token, transport, bounds) + inventories["indexers"] = indexers + for indexer in indexers: + if not all(isinstance(indexer.get(key), str) for key in ("dataSourceName", "targetIndexName")): + raise fail("dependency-inventory-invalid", "Indexer dependency identities are incomplete.") + if indexer.get("skillsetName") is not None and not isinstance(indexer["skillsetName"], str): + raise fail("dependency-inventory-invalid", "Indexer skillset identity is malformed.") + if indexer["name"] == names.get("indexer"): + if any(indexer.get(field) != names[kind] for field, kind in ( + ("dataSourceName", "datasource"), ("targetIndexName", "index"), ("skillsetName", "skillset"), + )): + raise fail("definition-drift", "The generated indexer no longer binds the exact owned pipeline.") + elif any(indexer.get(field) == names[kind] for field, kind in ( + ("dataSourceName", "datasource"), ("targetIndexName", "index"), ("skillsetName", "skillset"), + ) if kind in names): + raise fail("generated-resource-shared", "Another indexer consumes a generated cleanup resource.") + skillsets = _list_search(plan, "skillsets", token, transport, bounds) + inventories["skillsets"] = skillsets + for skillset in skillsets: + if skillset["name"] == names.get("skillset"): + if any(index != names["index"] for index in _projections(skillset)): + raise fail("generated-resource-shared", "The generated skillset also serves an outside-plan index.") + continue + skills = skillset.get("skills") + if not isinstance(skills, list) or any(not isinstance(skill, dict) or not isinstance(skill.get("@odata.type"), str) for skill in skills): + raise fail("dependency-inventory-invalid", "Skillset consumers are incomplete.") + if any(".Custom." in skill["@odata.type"] for skill in skills): + raise fail("dependency-consumer-opaque", "Custom skill code has no complete native generated-resource consumer contract.") + if names["index"] in _projections(skillset): + raise fail("generated-resource-shared", "Another skillset projects into the generated index.") + snapshots = [] + for kind, name in sorted(names.items()): + url = f"{plan['endpoint'].rstrip('/')}/{COLLECTIONS[kind]}('{odata_name(name)}')?api-version={plan['api_version']}" + child, _ = search_reconcile.read_resource(url, token, transport=transport) + if child is None: + raise fail("generated-resource-missing", "A generated child is missing; source absence alone cannot prove the remaining cascade.") + etag = child.get("@odata.etag") + if child.get("name") != name or not isinstance(etag, str) or not etag.strip(): + raise fail("generated-version-unavailable", "Every generated object requires an exact name and fresh ETag.") + snapshots.append({"type": kind, "name": name, "etag": etag, "definition_digest": digest(child)}) + refreshed, _ = search_reconcile.read_resource(search_reconcile.resource_url(plan), token, transport=transport) + if (refreshed is None or search_reconcile._definition(refreshed) != search_reconcile._definition(current) + or refreshed.get("@odata.etag") != current.get("@odata.etag") or generated(refreshed) != names): + raise fail("definition-drift", "The source changed during dependency discovery.") + inventory_state = { + kind: [ + {"name": item["name"], "etag": item.get("@odata.etag"), + "definition": search_reconcile._definition(item), + "generated": generated(item) if kind == "knowledgesources" and item.get("kind") in ("file", "azureBlob") else None} + for item in items + ] + for kind, items in inventories.items() + } + return {"kind": "search-source", "limits": bounds, "generated": snapshots, "inventory_digest": digest(inventory_state)} + + +def verify_search(plan, current, token, *, transport): + guard = plan["dependency_guard"] + if search_snapshot(plan, current, token, transport=transport, bounds=guard["limits"]) != guard: + raise fail("dependency-drift", "A generated ETag/definition or scoped dependency inventory changed after approval.") + + +def _paged(items, bounds): + if not callable(getattr(items, "by_page", None)): + raise fail("connection-consumer-api-unavailable", "SDK ItemPaged.by_page is required to prove complete project inventory.") + pages = iter(items.by_page()) + if not hasattr(pages, "continuation_token"): + raise fail("connection-consumer-api-unavailable", "SDK page continuation state is required for bounded complete discovery.") + count = 0 + for _ in range(bounds["max_pages"]): + try: + page = next(pages) + except StopIteration: + return + for item in page: + count += 1 + if count > bounds["max_resources"]: + raise fail("dependency-inventory-limit", "Project inventory exceeds the approved object bound.") + yield item + if pages.continuation_token is None: + return + if count >= bounds["max_resources"]: + raise fail("dependency-inventory-limit", "More project pages exceed the approved object bound.") + raise fail("dependency-inventory-limit", "Project inventory exceeds the approved page bound.") + + +def _connection_use(definition, plan, connection): + if definition.get("kind") != "prompt": + raise fail("connection-consumer-opaque", "Project inventory contains Hosted/workflow/external consumers; their runtime connection use is not enumerable here.") + if set(definition) - {"rai_config", "kind", "model", "instructions", "temperature", "top_p", "reasoning", + "tools", "tool_choice", "text", "structured_inputs"}: + raise fail("connection-consumer-opaque", "Unknown Prompt definition fields prevent complete consumer interpretation.") + tools = definition.get("tools") + if not isinstance(tools, list): + raise fail("connection-consumer-opaque", "A Prompt version has no complete tool definition.") + expected = f"{plan['project_resource_id']}/connections/{connection['name']}".casefold() + uses = False + for tool in tools: + if not isinstance(tool, dict) or tool.get("type") != "mcp": + raise fail("connection-consumer-opaque", "A non-MCP tool needs a verified native connection-consumer contract.") + if (set(tool) - {"type", "name", "description", "tool_configs", "server_label", "server_url", "connector_id", + "authorization", "server_description", "headers", "allowed_tools", "require_approval", + "defer_loading", "project_connection_id"} or tool.get("connector_id") is not None): + raise fail("connection-consumer-opaque", "Unknown tool fields or connector resolution cannot prove project-connection absence.") + configurations = tool.get("tool_configs") + if configurations is not None and ( + not isinstance(configurations, dict) or any( + not isinstance(value, dict) or set(value) - {"pin", "additional_search_text"} + or (value.get("pin") is not None and type(value["pin"]) is not bool) + or (value.get("additional_search_text") is not None and not isinstance(value["additional_search_text"], str)) + for value in configurations.values() + ) + ): + raise fail("connection-consumer-opaque", "Tool configuration is not the documented visibility/search-only contract.") + reference = tool.get("project_connection_id") + if reference is None: + if not isinstance(tool.get("server_url"), str): + raise fail("connection-consumer-opaque", "An MCP tool lacks an explicit connection or server target.") + continue + if (not isinstance(reference, str) or not reference.strip() or reference != reference.strip() + or any(character in reference for character in ("%", "?", "#", "{", "}", "\\", "\r", "\n", "\t"))): + raise fail("connection-consumer-opaque", "An MCP connection reference is malformed.") + if "/" in reference: + try: + from .prompt_connect import PROJECT_ID + except ImportError: + from prompt_connect import PROJECT_ID + project, _, leaf = reference.rstrip("/").casefold().rpartition("/connections/") + if PROJECT_ID.fullmatch(project) is None or not leaf or "/" in leaf: + raise fail("connection-consumer-opaque", "An MCP reference is neither a project connection name nor an exact ARM ID.") + uses |= reference.rstrip("/").casefold() == expected + else: + uses |= reference.casefold() == connection["name"].casefold() + return uses + + +def prompt_snapshot(plan, current, *, sdk_loader, bounds, allow_selected=True): + properties = current.get("properties") + if not isinstance(properties, dict) or properties.get("isSharedToAll") is not False or properties.get("sharedUserList") not in (None, []): + raise fail("connection-sharing-unverified", "Shared/unknown connection scope requires account/external consumer enumeration, not project-only inventory.") + AIProjectClient, _, _, _, extras = sdk_loader() + AzureCliCredential, AzureError = extras + client = AIProjectClient(endpoint=plan["project_endpoint"], credential=AzureCliCredential()) + selected = plan.get("agent") + selected_id = (selected["name"], selected["version"]) if selected else None + versions, toolboxes, names, identities = [], [], set(), set() + try: + if "include_drafts" not in inspect.signature(client.agents.list_versions).parameters: + raise fail("connection-consumer-api-unavailable", "agents.list_versions(include_drafts=True) is required; this SDK cannot prove draft-consumer absence.") + for agent in _paged(client.agents.list(), bounds): + name = getattr(agent, "name", None) + if not isinstance(name, str) or not name or name in names: + raise fail("dependency-inventory-changed", "Project agent inventory contains missing or duplicate identities.") + names.add(name) + for item in _paged(client.agents.list_versions(agent_name=name, include_drafts=True), bounds): + version = str(getattr(item, "version", "")) + identity = (name, version) + if not re.fullmatch(r"[1-9][0-9]*", version) or identity in identities: + raise fail("dependency-inventory-changed", "Version inventory contains missing or duplicate identities.") + identities.add(identity) + if len(identities) > bounds["max_resources"]: + raise fail("dependency-inventory-limit", "Total project versions exceed the approved object bound.") + observed = client.agents.get_version(agent_name=name, agent_version=version) + definition = observed.definition.as_dict() + if observed.name != name or str(observed.version) != version or not isinstance(definition, dict): + raise fail("dependency-inventory-changed", "Exact version readback disagrees with project inventory.") + uses = _connection_use(definition, plan, plan["connection"]) + if identity == selected_id and digest(definition) != selected["owned_definition_digest"]: + raise fail("definition-drift", "The explicitly selected version changed during consumer discovery.") + if identity == selected_id and not allow_selected: + raise fail("agent-absence-unverified", "The selected version must be absent before connection deletion.") + if uses and identity != selected_id: + raise HelperFailure( + "connection-shared-consumer", "A retained agent version consumes this connection.", + blocked_at="cleanup-dependencies", warnings=[f"consumer:{name}:{version}"], + ) + versions.append({"name": name, "version": version, "definition_digest": digest(definition)}) + names, identities = set(), set() + for toolbox in _paged(client.toolboxes.list(), bounds): + name = getattr(toolbox, "name", None) + if not isinstance(name, str) or not name.strip() or name in names: + raise fail("dependency-inventory-changed", "Toolbox inventory contains missing or duplicate names.") + names.add(name) + for item in _paged(client.toolboxes.list_versions(name=name), bounds): + version = getattr(item, "version", None) + identity = (name, version) + if not isinstance(version, str) or not version.strip() or identity in identities: + raise fail("dependency-inventory-changed", "Toolbox version inventory is incomplete or duplicated.") + identities.add(identity) + if len(versions) + len(identities) > bounds["max_resources"]: + raise fail("dependency-inventory-limit", "Combined agent/toolbox versions exceed approved bounds.") + observed = client.toolboxes.get_version(name=name, version=version) + definition = observed.as_dict() + if (not isinstance(definition, dict) or definition.get("name") != name + or definition.get("version") != version): + raise fail("dependency-inventory-changed", "Exact toolbox version disagrees with inventory.") + if (set(definition) - {"metadata", "id", "name", "version", "description", "created_at", "tools", "skills", "policies"} + or definition.get("skills") not in (None, []) or definition.get("policies") not in (None, {})): + raise fail("connection-consumer-opaque", "Toolbox skills/policies have no complete connection-consumer contract.") + if _connection_use({"kind": "prompt", "tools": definition.get("tools")}, plan, plan["connection"]): + raise HelperFailure( + "connection-shared-consumer", "A retained toolbox version consumes this connection.", + blocked_at="cleanup-dependencies", warnings=[f"toolbox-consumer:{name}:{version}"], + ) + toolboxes.append({"name": name, "version": version, "definition_digest": digest(definition)}) + except AzureError as exc: + raise HelperFailure(message="Complete project agent/version GET inventory failed; absence is unproven.", + blocked_at="cleanup-dependencies", **sdk_error_metadata(exc, "connection-inventory-failed")) from exc + except (AttributeError, TypeError, ValueError) as exc: + raise fail("connection-consumer-api-unavailable", "The installed SDK lacks complete typed agent/version inventory readback.") from exc + finally: + client.close() + return {"kind": "prompt-connection", "limits": bounds, + "versions": sorted(versions, key=lambda item: (item["name"], item["version"])), + "toolboxes": sorted(toolboxes, key=lambda item: (item["name"], item["version"]))} + + +def verify_prompt(plan, current, *, sdk_loader, allow_selected=True): + connection = plan["connection"] + if (digest(current) != connection["owned_definition_digest"] + or (current.get("etag") or current.get("@odata.etag")) != connection["expected_etag"]): + raise fail("definition-drift", "The connection definition or ETag changed after cleanup approval.") + guard = plan["dependency_guard"] + actual = prompt_snapshot(plan, current, sdk_loader=sdk_loader, bounds=guard["limits"], allow_selected=allow_selected) + expected_versions = guard["versions"] + selected = plan.get("agent") + if selected and not any(item["name"] == selected["name"] and item["version"] == selected["version"] for item in actual["versions"]): + expected_versions = [item for item in expected_versions if (item["name"], item["version"]) != (selected["name"], selected["version"])] + if actual["versions"] != expected_versions or actual["toolboxes"] != guard["toolboxes"]: + raise fail("dependency-drift", "Protected project agent/version definitions changed after cleanup approval.") diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_cleanup_receipts.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_cleanup_receipts.py new file mode 100644 index 00000000..5533e5b1 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_cleanup_receipts.py @@ -0,0 +1,239 @@ +"""Filtered, immutable creation evidence; never a credentialized callback surface.""" +from __future__ import annotations + +import copy +from datetime import datetime +from pathlib import Path +import uuid + +try: + from . import _bootstrap_io as private_io + from ._common import HelperFailure, digest, load_approved_input, reject_secrets, odata_name, validate_search_endpoint +except ImportError: + import _bootstrap_io as private_io + from _common import HelperFailure, digest, load_approved_input, reject_secrets, odata_name, validate_search_endpoint + + +def fail(code, message): + return HelperFailure(code, message, blocked_at="cleanup-provenance") + + +def version_identity(value): + get = value.get if isinstance(value, dict) else lambda key: getattr(value, key, None) + identifier, created = get("id"), get("created_at") + if isinstance(created, datetime): + created = int(created.timestamp()) if created.tzinfo is not None else None + if not isinstance(identifier, str) or not identifier.strip() or type(created) is not int or created <= 0: + return None + return {"id": identifier, "created_at": created} + + +def add_argument(parser): + parser.add_argument("--cleanup-receipt-dir", type=Path, + help="Existing absolute private directory for filtered original-create evidence.") + + +class Capture: + def __init__(self, directory, document): + self.directory = private_io.private_directory(str(directory)) + self.plan_digest = digest(document["plan"]) + self.owner = document["plan"].get("owner") + self.records = {} + self.summaries = [] + + def start(self, target, acknowledgement): + acknowledgement.setdefault("version_identity", None) + record = { + "schema_version": "1.0", "kind": "cleanup-creation-receipt", + "plan_digest": self.plan_digest, "owner": self.owner, "target": target, + "state": "acknowledged", "acknowledgement": acknowledgement, "snapshot": None, + } + key = digest(target) + self.records[key] = record + self._persist(record) + + def finish(self, target, snapshot): + record = copy.deepcopy(self.records[digest(target)]) + ack = record["acknowledgement"] + snapshot.setdefault("version_identity", None) + if (snapshot["version_identity"] != ack["version_identity"] + or (target["type"] == "prompt-agent-version" and ack["version_identity"] is None)): + raise fail("creation-version-unproven", "Native version identity/creation time must match the original acknowledgement.") + if not isinstance(ack["request_id"], str) or not ack["request_id"].strip(): + raise fail("creation-request-id-unavailable", "Original native acknowledgement lacks a request ID.") + if snapshot["definition_digest"] != ack["definition_digest"]: + raise fail("definition-drift", "Readback differs from the acknowledged original creation.") + record.update(state="verified", snapshot=snapshot) + self._persist(record) + + def _persist(self, record): + reject_secrets(record) + value = {**record, "integrity": digest(record)} + try: + path = private_io.private_file(self.directory, uuid.uuid4().hex + ".cleanup.json", value) + except OSError as exc: + raise fail("cleanup-receipt-persistence-failed", "Acknowledged creation evidence could not be retained privately.") from exc + self.summaries.append({"target": record["target"], "state": record["state"], + "receipt_file": str(path), "evidence_digest": value["integrity"]}) + + +def search_target(plan): + return {"type": plan["resource_type"], "endpoint": validate_search_endpoint(plan["endpoint"]), + **{key: plan[key] for key in ("name", "api_version")}} + + +def search_ack(capture, plan, response): + try: + from . import search_reconcile as search, _cleanup_dependencies as dependencies + except ImportError: + import search_reconcile as search, _cleanup_dependencies as dependencies + generated = None + if plan["resource_type"] == "knowledge-source": + try: + generated = dependencies.generated(response.body) if isinstance(response.body, dict) else None + except HelperFailure: + pass + capture.start(search_target(plan), { + "operation": "search-create", "status": response.status, "request_id": response.request_id, + "definition_digest": digest(search._definition(plan["desired"])), + "etag_evidence": search.response_etags(response), "generated": generated, "version": None, + }) + + +def search_finish(capture, plan, current, token, transport): + try: + from . import search_reconcile as search, _cleanup_dependencies as dependencies + except ImportError: + import search_reconcile as search, _cleanup_dependencies as dependencies + target = search_target(plan) + ack = capture.records[digest(target)]["acknowledgement"] + etag = search.resolve_etag(ack["etag_evidence"], ack["request_id"]) + if ack["status"] != 201 or not etag or current is None or current.get("@odata.etag") != etag: + raise fail("creation-version-unproven", "Require original HTTP 201 and the unchanged acknowledged ETag, not a later GET version.") + children = [] + if plan["resource_type"] == "knowledge-source": + if ack["generated"] is None or dependencies.generated(current) != ack["generated"]: + raise fail("generated-creation-evidence-unavailable", "The original create response did not identify this exact generated cascade.") + for kind, name in sorted(ack["generated"].items()): + url = f"{plan['endpoint'].rstrip('/')}/{dependencies.COLLECTIONS[kind]}('{odata_name(name)}')?api-version={plan['api_version']}" + child, _ = search.read_resource(url, token, transport=transport) + if child is None or child.get("name") != name or not child.get("@odata.etag"): + raise fail("generated-version-unavailable", "An original generated child lacks exact version readback.") + children.append({"type": kind, "name": name, "etag": child["@odata.etag"], "definition_digest": digest(child)}) + refreshed, _ = search.read_resource(search.resource_url(plan), token, transport=transport) + if (refreshed is None or refreshed.get("@odata.etag") != etag + or search._definition(refreshed) != search._definition(current) + or dependencies.generated(refreshed) != ack["generated"]): + raise fail("definition-drift", "Source changed during original generated-child capture.") + capture.finish(target, {"definition_digest": digest(search._definition(current)), "etag": etag, "generated": children}) + + +def project_target(plan, kind, *, name=None, version=None): + target = {"type": kind, "project_resource_id": plan["project_resource_id"].casefold(), + "project_endpoint": plan["project_endpoint"].rstrip("/"), + "name": name or plan["connection"]["name"]} + if version is not None: + target["version"] = version + return target + + +def connection_ack(capture, plan, response): + body = response.body if isinstance(response.body, dict) else {} + capture.start(project_target(plan, "project-connection"), { + "operation": "project-connection-create", "status": response.status, "request_id": response.request_id, + "definition_digest": digest(body), "version": None, "generated": None, + "etag_evidence": {"body": body.get("etag") or body.get("@odata.etag"), + "headers": [v for k, v in response.headers.items() if k.casefold() == "etag"]}, + }) + + +def connection_finish(capture, plan, body): + try: + from .search_reconcile import resolve_etag + except ImportError: + from search_reconcile import resolve_etag + target = project_target(plan, "project-connection") + ack = capture.records[digest(target)]["acknowledgement"] + etag = resolve_etag(ack["etag_evidence"], ack["request_id"]) + if ack["status"] != 201 or not etag or etag != (body.get("etag") or body.get("@odata.etag")): + raise fail("creation-version-unproven", "Connection readback must retain its original HTTP 201 ETag.") + capture.finish(target, {"definition_digest": digest(body), "etag": etag, "generated": []}) + + +def sdk_response_hook(metadata): + def capture(response): + native = response.http_response + metadata["request_id"] = next((v for k, v in native.headers.items() + if k.casefold() in ("request-id", "x-request-id", "x-ms-request-id", "apim-request-id")), None) + metadata["status"] = native.status_code + return capture + + +def agent_ack(capture, plan, created, metadata): + target = project_target(plan, "prompt-agent-version", name=created.name, version=str(created.version)) + capture.start(target, { + "operation": "agents.create_version", "status": metadata.get("status"), "request_id": metadata.get("request_id"), + "definition_digest": digest(created.definition.as_dict()), "version": str(created.version), + "etag_evidence": None, "generated": None, + "version_identity": version_identity(created), + }) + if (metadata.get("status") not in (200, 201) or created.name != plan["agent"]["name"] + or str(created.version) == plan["agent"]["version"] or version_identity(created) is None): + raise fail("ownership-unproven", "The original SDK return must identify a newly created version in the approved agent.") + return target + + +def load(input_path, receipt_path, target): + try: + from .blob_recheck import read_private + from . import search_reconcile as search + except ImportError: + from blob_recheck import read_private + import search_reconcile as search + _, plan, fingerprint = load_approved_input(input_path) + record = read_private(receipt_path) + reject_secrets(record) + fields = {"schema_version", "kind", "plan_digest", "owner", "target", "state", "acknowledgement", "snapshot", "integrity"} + if (not isinstance(record, dict) or set(record) != fields or record["schema_version"] != "1.0" + or record["kind"] != "cleanup-creation-receipt" or record["state"] != "verified" + or record["plan_digest"] != fingerprint or record["owner"] != plan.get("owner") + or record["target"] != target + or record["integrity"] != digest({k: v for k, v in record.items() if k != "integrity"})): + raise fail("ownership-unproven", "Require the exact original verified producer receipt and original approval; ACK/recovery/reuse cannot substitute.") + ack, snapshot = record["acknowledgement"], record["snapshot"] + if (not isinstance(ack, dict) or set(ack) != {"operation", "status", "request_id", "definition_digest", "etag_evidence", "generated", "version", "version_identity"} + or not isinstance(snapshot, dict) or set(snapshot) != {"definition_digest", "etag", "generated", "version_identity"} + or not isinstance(ack["request_id"], str) or not ack["request_id"].strip() + or snapshot["definition_digest"] != ack["definition_digest"] + or not isinstance(snapshot["definition_digest"], str) or not search.SHA256.fullmatch(snapshot["definition_digest"]) + or not isinstance(snapshot["generated"], list)): + raise fail("ownership-unproven", "Producer receipt lacks complete acknowledged version evidence.") + if (snapshot["version_identity"] != ack["version_identity"] + or (target["type"] == "prompt-agent-version" and ( + not isinstance(snapshot["version_identity"], dict) + or set(snapshot["version_identity"]) != {"id", "created_at"} + or version_identity(snapshot["version_identity"]) != snapshot["version_identity"]))): + raise fail("ownership-unproven", "Native version birth identity is missing or changed.") + if target["type"] != "prompt-agent-version" and ( + not isinstance(ack["etag_evidence"], dict) or set(ack["etag_evidence"]) != {"body", "headers"} + or not isinstance(ack["etag_evidence"]["headers"], list) + ): + raise fail("ownership-unproven", "Native ETag evidence must be complete.") + if target["type"] in ("knowledge-base", "knowledge-source"): + if (ack["operation"] != "search-create" or ack["status"] != 201 + or search.resolve_etag(ack["etag_evidence"], ack["request_id"]) != snapshot["etag"] + or not isinstance(snapshot["etag"], str) or not snapshot["etag"].strip()): + raise fail("creation-version-unproven", "Original Search create acknowledgement and snapshot versions disagree.") + elif target["type"] == "project-connection": + if (ack["operation"] != "project-connection-create" or ack["status"] != 201 + or search.resolve_etag(ack["etag_evidence"], ack["request_id"]) != snapshot["etag"] + or not snapshot["etag"] or snapshot["generated"]): + raise fail("creation-version-unproven", "Original connection acknowledgement and readback versions disagree.") + elif target["type"] == "prompt-agent-version": + if (ack["operation"] not in ("agents.create_version", "agents.create") or ack["status"] not in (200, 201) + or (ack["operation"] == "agents.create" and ack["status"] != 200) + or ack["version"] != target["version"] or snapshot["etag"] is not None or snapshot["generated"]): + raise fail("ownership-unproven", "Require original SDK version-create return and its exact readback.") + else: + raise fail("cleanup-kind-unsupported", "No producer receipt contract supports this target.") + return plan, record diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_common.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_common.py new file mode 100644 index 00000000..21021249 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_common.py @@ -0,0 +1,949 @@ +from __future__ import annotations + +import hashlib +import json +import math +import re +import shutil +import socket +import subprocess +import sys +import threading +import time +import uuid +from collections.abc import Mapping +from contextlib import suppress +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from email.utils import parsedate_to_datetime +from http.client import HTTPException +from pathlib import Path +from typing import Any, Callable, NamedTuple +from urllib.error import HTTPError, URLError +from urllib.parse import quote, urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen + + +SEARCH_AUDIENCE = "https://search.azure.com" +MANAGEMENT_AUDIENCE = "https://management.azure.com" +SEARCH_HOST = re.compile( + r"^[a-z0-9](?:[a-z0-9-]{0,58}[a-z0-9])?\.search\.windows\.net$" +) +SECRET_FIELDS = { + "accesskey", + "accesstoken", + "accountkey", + "apikey", + "applicationsecret", + "authorization", + "clientsecret", + "credential", + "connectionsecret", + "key", + "password", + "privatekey", + "refreshtoken", + "sas", + "sastoken", + "secret", + "sharedaccesskey", + "storageaccountkey", + "token", +} +RESOURCE_ID_CONNECTION = re.compile(r"^ResourceId=/[^;\r\n]+;?$") + + +def normalize_azure_location(value: Any) -> str | None: + """Normalize ASCII case/whitespace only; this does not validate region availability.""" + if not isinstance(value, str) or len(value) > 128 or not value.isascii(): + return None + compact = re.sub(r"\s+", "", value, flags=re.ASCII).lower() + return compact if re.fullmatch(r"[a-z][a-z0-9]{1,40}", compact) else None + + +def _normalize_response_headers(headers: Mapping[str, str]) -> dict[str, str]: + normalized: dict[str, str] = {} + for name, value in headers.items(): + key = name.lower() + if key in {"x-ms-request-id", "request-id"} and normalized.get(key): + continue + if key == "retry-after" and key in normalized: + normalized[key] = "" + continue + normalized[key] = value + return normalized + + +def _request_id(headers: Mapping[str, str]) -> str | None: + normalized = _normalize_response_headers(headers) + return normalized.get("x-ms-request-id") or normalized.get("request-id") or None + + +@dataclass(frozen=True) +class HttpResult: + status: int + body: Any + headers: dict[str, str] + etag_values: tuple[str, ...] | None = None + recovery_deadline: float | None = None + ack_failure: HelperFailure | None = None + + @property + def request_id(self) -> str | None: + return _request_id(self.headers) + + @property + def retry_after(self) -> RetryAfter: + return retry_after_metadata(self.headers) + + +class RetryAfter(NamedTuple): + kind: str + value: float = 0 + + +class RetryAfterTiming(NamedTuple): + received_at_utc: float | None + not_before_utc: float | None + server_delay_seconds: int | None = None + + +def valid_utc_timestamp(value): + return type(value) in (int, float) and -62135596800 <= value < 253402300800 and math.isfinite(value) + + +def retry_after_not_before(metadata, received_at): + """Resolve typed metadata against UTC, without shortening a server interval.""" + if not valid_utc_timestamp(received_at) or not isinstance(metadata, RetryAfter): + return None + value = metadata.value + if not valid_utc_timestamp(value): + return None + if metadata.kind == "seconds": + value = received_at + value if value >= 0 else math.inf + elif metadata.kind == "date-rfc850": + try: + epoch = datetime(1970, 1, 1, tzinfo=timezone.utc) + parsed = epoch + timedelta(seconds=value) + current_year = (epoch + timedelta(seconds=received_at)).year + year = current_year // 100 * 100 + parsed.year % 100 + if year > current_year + 50: + year -= 100 + value = parsed.replace(year=year).timestamp() + except (ValueError, OverflowError): + return None + elif metadata.kind in {"missing", "invalid"}: + value = received_at + 1 + elif metadata.kind != "date": + return None + return value if valid_utc_timestamp(value) else None + + +def retry_after_timing(headers, *, received_at=None): + """Portable timing alongside the unchanged, capped RetryAfter protocol.""" + received_at = time.time() if received_at is None else received_at + if not valid_utc_timestamp(received_at): + return RetryAfterTiming(None, None) + metadata = retry_after_metadata(headers) + deadline = retry_after_not_before(metadata, received_at) + server_delay = metadata.value if metadata.kind == "seconds" else None + if metadata.kind == "overlong": + values = [value for name, value in headers.items() + if isinstance(name, str) and name.lower() == "retry-after"] + if (len(values) == 1 and isinstance(values[0], str) and len(values[0]) <= 128 + and re.fullmatch(r"[0-9]+", values[0].strip(" \t"))): + seconds = int(values[0].strip(" \t")) + if seconds < 253402300800 - received_at: + deadline = received_at + seconds + server_delay = seconds + return RetryAfterTiming(received_at, deadline, server_delay) + + +def retry_after_metadata(headers: Mapping[str, str]) -> RetryAfter: + """Retain only a bounded delay/date, never raw server header text.""" + values = [] + for name, value in headers.items(): + if isinstance(name, str) and name.lower() == "retry-after": + values.append(value) + if len(values) > 1: + return RetryAfter("invalid") + if not values: + return RetryAfter("missing") + value = values[0] + if not isinstance(value, str): + return RetryAfter("invalid") + # An unbounded field cannot safely authorize an early request. + if len(value) > 128: + return RetryAfter("overlong") + if not value.isascii(): + return RetryAfter("invalid") + value = value.strip(" \t") + if re.fullmatch(r"[0-9]+", value): + seconds = int(value) + return RetryAfter("seconds", seconds) if seconds <= 30 else RetryAfter("overlong") + # HTTP-date includes obsolete RFC850/asctime forms, but not arbitrary email dates. + if not re.fullmatch( + r"(?:[A-Z][a-z]{2}, [0-9]{2} [A-Z][a-z]{2} [0-9]{4} [0-9:]{8} GMT" + r"|[A-Z][a-z]+, [0-9]{2}-[A-Z][a-z]{2}-[0-9]{2} [0-9:]{8} GMT" + r"|[A-Z][a-z]{2} [A-Z][a-z]{2} [ 0-9][0-9] [0-9:]{8} [0-9]{4})", value + ): + return RetryAfter("invalid") + try: + parsed = parsedate_to_datetime(value) + stamp = parsed.replace(tzinfo=timezone.utc).timestamp() + return RetryAfter("date-rfc850" if "-" in value else "date", stamp) + except (TypeError, ValueError, OverflowError): + return RetryAfter("invalid") + + +class HelperFailure(RuntimeError): + def __init__( + self, + code: str, + message: str, + *, + blocked_at: str, + writes: list[dict[str, Any]] | None = None, + resources_remaining: list[dict[str, Any]] | None = None, + resources_reused: list[dict[str, Any]] | None = None, + resources_unverified: list[dict[str, Any]] | None = None, + request_id: str | None = None, + status: int | None = None, + partial: bool = False, + warnings: list[str] | None = None, + retry_after: RetryAfter | None = None, + recovery_deadline: float | None = None, + retry_after_timing: RetryAfterTiming | None = None, + ) -> None: + super().__init__(message) + self.code = code + self.message = message + self.blocked_at = blocked_at + self.writes = writes or [] + self.resources_remaining = resources_remaining or [] + self.resources_reused = resources_reused or [] + self.resources_unverified = resources_unverified or [] + self.request_id = request_id + self.http_status = status + self.partial = partial + self.warnings = warnings or [] + self.retry_after = retry_after or RetryAfter("missing") + self.recovery_deadline = recovery_deadline + self.retry_after_timing = retry_after_timing + self.response_close_failed = False + self.file_batch = None + + +class ReadRecovery: + """One 429 delay opportunity and one deadline across an explicit read sequence.""" + + def __init__(self, *, monotonic=None, wall_clock=None, sleeper=None, deadline=None, on_wait=None): + self.monotonic = monotonic or time.monotonic + self.wall_clock = wall_clock or time.time + self.sleeper = sleeper or time.sleep + self.deadline = min(self.monotonic() + 60, deadline if deadline is not None else math.inf) + self.delayed = False + self.request_ids: list[str] = [] + self.warnings: list[str] = [] + self.on_wait = on_wait + + @staticmethod + def safe_id(request_id): + return request_id if isinstance(request_id, str) and re.fullmatch( + r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}", request_id + ) else "[withheld]" + + def record(self, request_id): + if request_id and len(self.request_ids) < 202: + self.request_ids.append(self.safe_id(request_id)) + + def diagnostics(self): + return [*self.warnings, *( + ["Read-only request IDs: " + ", ".join(self.request_ids)] if self.request_ids else [] + )] + + def annotate(self, failure): + for warning in self.diagnostics(): + if warning not in failure.warnings: + failure.warnings.append(warning) + return failure + + def _stop(self, code, message): + self.warnings.append(message) + raise self.annotate(HelperFailure( + code, message + " Stop; resume only read-only verification, never replay the write.", + blocked_at="verification", + request_id=self.request_ids[-1] if self.request_ids else None, + )) + + def delay(self, failure): + if failure.blocked_at == "local-persistence": + self._stop("read-recovery-persistence-failed", "Required receipt persistence failed; recovery is terminal.") + if failure.response_close_failed: + self._stop("read-recovery-response-close-failed", "HTTP error response cleanup failed; delayed recovery is blocked.") + if failure.http_status != 429: + return + if failure.recovery_deadline is not None: + self.deadline = min(self.deadline, failure.recovery_deadline) + if self.delayed: + self._stop("read-recovery-exhausted", "The single HTTP 429 delay opportunity is exhausted.") + metadata = failure.retry_after + delay = metadata.value + if metadata.kind in {"date", "date-rfc850"}: + now = self.wall_clock() + deadline = retry_after_not_before(metadata, now) + if deadline is None: + self._stop("read-recovery-delay-exceeded", "Retry-After date cannot be represented safely.") + delay = max(0, deadline - now) + elif metadata.kind in {"missing", "invalid"}: + delay = 1 + self.warnings.append("Retry-After missing/invalid; using the fixed 1-second fallback.") + if metadata.kind == "overlong" or not math.isfinite(delay) or delay > 30: + self._stop("read-recovery-delay-exceeded", "Retry-After exceeds the 30-second wait allowance; it was not shortened.") + remaining = self.deadline - self.monotonic() + if remaining <= delay: + self._stop("read-recovery-budget-exhausted", "Insufficient recovery read budget for Retry-After.") + self.delayed = True + start = self.monotonic() + if self.on_wait is not None: + self.on_wait(delay) + self.sleeper(delay) + elapsed = self.monotonic() - start + if elapsed < delay: + self._stop("read-recovery-wait-incomplete", "The required Retry-After delay did not elapse.") + if self.monotonic() >= self.deadline: + self._stop("read-recovery-budget-exhausted", "Recovery read deadline elapsed during the wait.") + + def get(self, url, token, *, transport, max_requests=2, **kwargs): + self.deadline = min(self.deadline, kwargs.get("response_deadline", math.inf)) + first = None + for attempt in range(2): + try: + remaining = self.deadline - self.monotonic() + if remaining <= 0 or attempt >= max_requests: + self._stop("read-recovery-budget-exhausted", "The recovery read deadline elapsed.") + options = {**kwargs, "follow_redirects": False, "response_deadline": self.deadline, + "timeout": min(kwargs.get("timeout", 180), remaining), + "max_response_bytes": kwargs.get("max_response_bytes", 1024 * 1024)} + result = transport("GET", url, token, **options) + if result.recovery_deadline is not None: + self.deadline = min(self.deadline, result.recovery_deadline) + self.record(result.request_id) + if self.monotonic() >= self.deadline: + self._stop("read-recovery-budget-exhausted", "The recovery response exceeded its read deadline.") + if result.status != 429: + return result + raise HelperFailure( + "azure-http-error", "Azure request failed with HTTP 429.", + blocked_at="verification", status=429, request_id=result.request_id, + retry_after=result.retry_after, + recovery_deadline=result.recovery_deadline, + ) + except HelperFailure as failure: + if not self.request_ids or self.request_ids[-1] != failure.request_id: + self.record(failure.request_id) + if failure.http_status != 429 or attempt: + if first is not None: + self.warnings.append( + f"Recovery read stopped ({failure.code}); delay opportunity exhausted; " + "initial read failure retained. Resume read-only; never replay the write." + ) + raise self.annotate(first) from failure + raise self.annotate(failure) + first = failure + try: + if attempt + 1 >= max_requests: + self._stop("read-recovery-exhausted", "No requests remain in the recovery read allowance.") + self.delay(failure) + except HelperFailure as stopped: + self.warnings.append(stopped.message) + raise self.annotate(first) from stopped + raise AssertionError("unreachable") + + +def canonical_bytes(value: Any) -> bytes: + return json.dumps( + value, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def digest(value: Any) -> str: + return f"sha256:{hashlib.sha256(canonical_bytes(value)).hexdigest()}" + + +def file_digest(path: Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + hasher.update(chunk) + return f"sha256:{hasher.hexdigest()}" + + +def load_approved_input(path: Path) -> tuple[dict[str, Any], dict[str, Any], str]: + try: + document = json.loads(path.read_text(encoding="utf-8")) + except OSError as exc: + raise HelperFailure( + "input-unreadable", + "Input file cannot be read.", + blocked_at="input-resolution", + ) from exc + except json.JSONDecodeError as exc: + raise HelperFailure( + "input-invalid-json", + f"Input file is not valid JSON: {exc}", + blocked_at="input-resolution", + ) from exc + if not isinstance(document, dict) or document.get("schema_version") != "1.0": + raise HelperFailure( + "input-schema-invalid", + "Input must be an object with schema_version 1.0.", + blocked_at="input-resolution", + ) + reject_secrets(document) + require_allowed_fields( + document, + {"schema_version", "plan", "approval"}, + label="input envelope", + ) + plan = document.get("plan") + approval = document.get("approval") + if not isinstance(plan, dict) or not isinstance(approval, dict): + raise HelperFailure( + "input-schema-invalid", + "Input must contain plan and approval objects.", + blocked_at="input-resolution", + ) + require_allowed_fields( + approval, + {"confirmed", "fingerprint"}, + label="approval", + ) + computed = digest(plan) + if approval.get("confirmed") is not True: + raise HelperFailure( + "approval-missing", + "The exact plan has not been explicitly approved.", + blocked_at="confirmation", + ) + if approval.get("fingerprint") != computed: + raise HelperFailure( + "approval-mismatch", + "The approved fingerprint does not match the canonical plan.", + blocked_at="confirmation", + ) + return document, plan, computed + + +def reject_secrets(value: Any, *, path: str = "") -> None: + if isinstance(value, dict): + for key, child in value.items(): + normalized = re.sub(r"[^a-z0-9]", "", str(key).casefold()) + child_path = f"{path}/{key}" + if normalized in SECRET_FIELDS and child not in (None, "", []): + raise HelperFailure( + "secret-input-forbidden", + f"Secret-bearing field is forbidden at {child_path}.", + blocked_at="input-resolution", + ) + if normalized == "connectionstring" and child not in (None, ""): + if ( + not isinstance(child, str) + or RESOURCE_ID_CONNECTION.fullmatch(child) is None + ): + raise HelperFailure( + "secret-input-forbidden", + "Only one ResourceId storage connectionString component is allowed.", + blocked_at="input-resolution", + ) + reject_secrets(child, path=child_path) + elif isinstance(value, list): + for index, child in enumerate(value): + reject_secrets(child, path=f"{path}/{index}") + + +def require_allowed_fields( + value: dict[str, Any], + allowed: set[str], + *, + label: str, +) -> None: + if set(value) - allowed: + raise HelperFailure( + "input-schema-invalid", + f"{label} contains unsupported fields.", + blocked_at="input-resolution", + ) + + +def is_ambiguous_mutation_failure(failure: HelperFailure) -> bool: + return ( + failure.partial + or failure.code == "azure-response-ambiguous" + or failure.http_status in {408, 429} + or ( + isinstance(failure.http_status, int) + and failure.http_status >= 500 + ) + ) + + +def is_ambiguous_status(status: int) -> bool: + return status in {408, 429} or status >= 500 + + +def sdk_error_status(error: Exception) -> int | None: + for source in (error, getattr(error, "response", None)): + status = getattr(source, "status_code", None) + if type(status) is int and 100 <= status <= 599: + return status + return None + + +def sdk_error_metadata(error: Exception, fallback_code: str | None = None) -> dict[str, Any]: + """Read only bounded identifier fields, never exception text or response bodies.""" + def identifier(value: Any) -> str | None: + if isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}", value): + return value + return None + + response = getattr(error, "response", None) + raw_headers = getattr(response, "headers", None) + headers = {} + if isinstance(raw_headers, Mapping): + headers = _normalize_response_headers({ + key: identifier(value) + for key, value in raw_headers.items() + if isinstance(key, str) and key.lower() in {"x-ms-request-id", "request-id", "x-ms-error-code"} + }) + result = {"status": sdk_error_status(error), "request_id": _request_id(headers)} + if fallback_code is not None: + detail = getattr(error, "error", None) + code = detail.get("code") if isinstance(detail, Mapping) else getattr(detail, "code", None) + result["code"] = ( + identifier(code) or identifier(getattr(error, "code", None)) + or headers.get("x-ms-error-code") or fallback_code + ) + return result + + +def is_ambiguous_sdk_error(error: Exception) -> bool: + status = sdk_error_status(error) + return status is None or is_ambiguous_status(status) + + +def redact_sensitive(value: Any) -> Any: + if isinstance(value, dict): + result: dict[str, Any] = {} + for key, child in value.items(): + normalized = re.sub(r"[^a-z0-9]", "", str(key).casefold()) + if normalized in SECRET_FIELDS or normalized == "connectionstring": + result[str(key)] = "[REDACTED]" + else: + result[str(key)] = redact_sensitive(child) + return result + if isinstance(value, list): + return [redact_sensitive(child) for child in value] + return value + + +def validate_search_endpoint(endpoint: Any) -> str: + if not isinstance(endpoint, str): + raise HelperFailure( + "endpoint-invalid", + "Search endpoint must be a string.", + blocked_at="input-resolution", + ) + try: + endpoint.encode("utf-8") + parsed = urlsplit(endpoint) + port = parsed.port + except (ValueError, UnicodeEncodeError) as exc: + raise HelperFailure( + "endpoint-invalid", + "Search endpoint must be a valid UTF-8 HTTPS service root.", + blocked_at="input-resolution", + ) from exc + if ( + parsed.scheme != "https" + or not parsed.hostname + or SEARCH_HOST.fullmatch(parsed.hostname) is None + or parsed.path not in {"", "/"} + or parsed.query + or parsed.fragment + or parsed.username + or parsed.password + or port not in {None, 443} + ): + raise HelperFailure( + "endpoint-invalid", + "Search endpoint must be an HTTPS search.windows.net service root.", + blocked_at="input-resolution", + ) + return endpoint.rstrip("/") + + +def odata_name(name: Any) -> str: + if not isinstance(name, str) or not name or len(name) > 128: + raise HelperFailure( + "name-invalid", + "Resource name must be a non-empty string no longer than 128 characters.", + blocked_at="input-resolution", + ) + try: + return quote(name.replace("'", "''"), safe="") + except UnicodeEncodeError as exc: + raise HelperFailure( + "name-invalid", + "Resource name must be valid UTF-8 text.", + blocked_at="input-resolution", + ) from exc + + +def azure_cli_token(resource: str) -> str: + try: + executable = shutil.which("az") + if executable is None: + raise FileNotFoundError("Azure CLI executable was not found on PATH.") + completed = subprocess.run( + [ + executable, + "account", + "get-access-token", + "--resource", + resource, + "--query", + "accessToken", + "--output", + "tsv", + ], + check=True, + capture_output=True, + text=True, + timeout=60, + ) + except FileNotFoundError as exc: + raise HelperFailure( + "azure-cli-unavailable", + "Azure CLI is required for keyless authentication.", + blocked_at="execution", + ) from exc + except subprocess.CalledProcessError as exc: + raise HelperFailure( + "azure-authentication-failed", + "Azure CLI could not acquire the required access token.", + blocked_at="execution", + ) from exc + except subprocess.TimeoutExpired as exc: + raise HelperFailure( + "azure-authentication-timeout", + "Azure CLI did not return an access token within 60 seconds.", + blocked_at="execution", + ) from exc + token = completed.stdout.strip() + if not token: + raise HelperFailure( + "azure-authentication-failed", + "Azure CLI returned an empty access token.", + blocked_at="execution", + ) + return token + + +class _NoRedirect(HTTPRedirectHandler): + def redirect_request(self, req: Any, fp: Any, code: int, msg: str, headers: Any, newurl: str) -> None: + return None + + +def _read_response_with_deadline( + response: Any, deadline: float, max_bytes: int, *, method: str +) -> bytes: + def timed_out() -> HelperFailure: + return HelperFailure( + "response-deadline-exceeded", + "HTTP response body exceeded its monotonic deadline.", + blocked_at="verification", + request_id=_request_id(response.headers), + status=response.status, + partial=method not in {"GET", "HEAD"}, + ) + + remaining = deadline - time.monotonic() + if remaining <= 0: + raise timed_out() + connection = getattr(getattr(getattr(response, "fp", None), "raw", None), "_sock", None) + if not isinstance(connection, socket.socket): + raise HelperFailure( + "response-deadline-unsupported", + "Deadline reads require an interruptible urllib HTTP socket.", + blocked_at="verification", + request_id=_request_id(response.headers), + status=response.status, + partial=method not in {"GET", "HEAD"}, + ) + expired = threading.Event() + + def interrupt() -> None: + expired.set() + # BufferedReader.close can wait on the active read lock; interrupt its socket instead. + with suppress(OSError): + connection.shutdown(socket.SHUT_RDWR) + with suppress(OSError): + connection.close() + + watchdog = threading.Timer(max(0, deadline - time.monotonic()), interrupt) + watchdog.start() + try: + if expired.is_set() or time.monotonic() >= deadline: + raise timed_out() + try: + payload = response.read(max_bytes + 1) + except (OSError, HTTPException, ValueError) as exc: + if expired.is_set() or time.monotonic() >= deadline: + raise timed_out() from exc + raise HelperFailure( + "response-read-failed", "HTTP response body could not be read.", + blocked_at="verification", + request_id=_request_id(response.headers), + status=response.status, + partial=method not in {"GET", "HEAD"}, + ) from exc + if expired.is_set() or time.monotonic() >= deadline: + raise timed_out() + return payload + finally: + watchdog.cancel() + watchdog.join() + + +def http_request( + method: str, + url: str, + token: str, + *, + body: bytes | None = None, + headers: dict[str, str] | None = None, + timeout: int = 180, + raw_response: bool = False, + max_response_bytes: int | None = None, + follow_redirects: bool = True, + response_deadline: float | None = None, +) -> HttpResult: + if response_deadline is not None: + if ( + not isinstance(response_deadline, (int, float)) + or not math.isfinite(response_deadline) + or response_deadline - time.monotonic() > threading.TIMEOUT_MAX + ): + raise HelperFailure( + "response-deadline-invalid", "Response deadline must be a supported finite monotonic time.", + blocked_at="input-resolution", + ) + if max_response_bytes is None: + max_response_bytes = 1024 * 1024 + if not isinstance(max_response_bytes, int) or max_response_bytes < 0: + raise HelperFailure( + "response-limit-invalid", "Response byte limit must be a nonnegative integer.", + blocked_at="input-resolution", + ) + if response_deadline <= time.monotonic(): + raise HelperFailure( + "response-deadline-exceeded", "HTTP deadline elapsed before the request.", + blocked_at="verification", + ) + request_headers = { + "Accept": "application/json;odata.metadata=minimal", + "Authorization": f"Bearer {token}", + "x-ms-client-request-id": str(uuid.uuid4()), + } + request_headers.update(headers or {}) + request = Request(url=url, data=body, headers=request_headers, method=method) + try: + open_request = urlopen if follow_redirects else build_opener(_NoRedirect).open + with open_request(request, timeout=timeout) as response: + if response_deadline is not None: + payload = _read_response_with_deadline( + response, response_deadline, max_response_bytes, method=method + ) + else: + payload = ( + response.read(max_response_bytes + 1) + if max_response_bytes is not None + else response.read() + ) + etag_values = tuple(value for name, value in response.headers.items() if name.lower() == "etag") + response_headers = _normalize_response_headers(response.headers) + if max_response_bytes is not None and len(payload) > max_response_bytes: + raise HelperFailure( + "response-too-large", + "Azure response exceeded the bounded read limit.", + blocked_at="verification", + request_id=_request_id(response_headers), + status=response.status, + partial=method not in {"GET", "HEAD"}, + ) + if raw_response: + parsed: Any = payload + elif not payload: + parsed = None + else: + try: + parsed = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise HelperFailure( + "response-invalid-json", + "Azure returned a non-JSON response where JSON was required.", + blocked_at="verification", + request_id=_request_id(response_headers), + status=response.status, + partial=method not in {"GET", "HEAD"}, + ) from exc + return HttpResult(response.status, parsed, response_headers, etag_values, response_deadline) + except HTTPError as exc: + retry_after = retry_after_metadata(exc.headers) + timing = retry_after_timing(exc.headers) + response_headers = _normalize_response_headers(exc.headers) + status = int(exc.code) + close_failed = False + if status == 429: + try: + exc.close() + except (OSError, HTTPException): + close_failed = True + elif response_deadline is not None: + exc.close() + ambiguous = method not in {"GET", "HEAD"} and is_ambiguous_status(status) + failure = HelperFailure( + "azure-http-error", + f"Azure request failed with HTTP {status}.", + blocked_at="execution", + request_id=_request_id(response_headers), + status=status, + partial=ambiguous, + retry_after=retry_after, + recovery_deadline=response_deadline, + retry_after_timing=timing, + warnings=(["response-close-failed: HTTP 429 response cleanup failed; " + "original HTTP failure retained and delayed recovery blocked."] if close_failed else []), + ) + failure.response_close_failed = close_failed + raise failure from exc + except (URLError, TimeoutError) as exc: + raise HelperFailure( + "azure-response-ambiguous", + "Azure request did not return an authoritative response.", + blocked_at="verification", + partial=method not in {"GET", "HEAD"}, + ) from exc + + +def blocked_result( + failure: HelperFailure, + *, + outcome: str, + fingerprint: str | None, + owner: Any = None, +) -> dict[str, Any]: + result = _blocked_result(failure, outcome=outcome, fingerprint=fingerprint, owner=owner) + if failure.file_batch is not None: + result["file_batch"] = failure.file_batch + result["safe_next_decision"] = ( + "Retain the source and original ACK/journal. Do not rerun creation, replay an uncertain upload, " + "or delete/reset resources. Plan upload-only continuation for never-attempted files with " + "file_upload.py --plan; missing original evidence blocks continuation, not retention." + ) + return result + + +def _blocked_result(failure, *, outcome, fingerprint, owner): + if failure.partial or failure.writes: + return { + "status": "partial", + "outcome": outcome, + "approved_plan": { + "fingerprint": fingerprint, + "confirmed": fingerprint is not None, + }, + "first_failure": { + "code": failure.code, + "operation": failure.blocked_at, + "status": failure.http_status, + "message": failure.message, + "request_id": failure.request_id, + }, + "completed_writes": failure.writes, + "failed_or_unverified_postconditions": [failure.code], + "resources_remaining": { + "run_owned": failure.resources_remaining, + "reused": failure.resources_reused, + **({"unverified": failure.resources_unverified} if failure.resources_unverified else {}), + }, + "rollback": {"possible": False, "exact_plan": []}, + "cleanup": {"status": "separate-plan-and-approval-required"}, + "owner": owner, + "warnings": failure.warnings, + } + result = { + "status": "blocked", + "outcome": outcome, + "blocked_at": failure.blocked_at, + "first_blocker": { + "code": failure.code, + "message": failure.message, + "status": failure.http_status, + "request_id": failure.request_id, + }, + "missing_or_conflicting_input": failure.code, + "read_only_evidence": [], + "writes_performed": [], + "safe_next_decision": "Resolve the first blocker, rebuild the plan, and obtain new approval.", + "ownership": {"run_owned": [], "reused_not_owned": []}, + "cleanup": "not applicable", + } + if fingerprint is not None: + result["approved_plan"] = { + "fingerprint": fingerprint, + "confirmed": True, + } + if failure.warnings: + result["warnings"] = failure.warnings + return result + + +def emit_result( + result: dict[str, Any], *, stream: Any = None, preserve_unapproved_input: bool = False +) -> None: + if stream is None: + stream = sys.stdout + safe = redact_sensitive(result) + if preserve_unapproved_input: + document = result.get("execution_input") + if ( + result.get("status") != "planned" or not isinstance(document, dict) + or document.get("schema_version") != "1.0" or not isinstance(document.get("plan"), dict) + or not isinstance(document.get("approval"), dict) + or document["approval"].get("confirmed") is not False + or document.get("approval") != {"confirmed": False, "fingerprint": digest(document["plan"])} + ): + raise HelperFailure( + "planning-output-invalid", "Only an exact unapproved execution input can retain ResourceId bindings.", + blocked_at="verification", + ) + reject_secrets(document) + require_allowed_fields(document, {"schema_version", "plan", "approval"}, label="planning envelope") + safe["execution_input"] = document + stream.write( + json.dumps( + safe, + sort_keys=True, + separators=(",", ":"), + ) + + "\n" + ) + + +TokenProvider = Callable[[str], str] +Transport = Callable[..., HttpResult] diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_document_adapters.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_document_adapters.py new file mode 100644 index 00000000..be1cd49d --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_document_adapters.py @@ -0,0 +1,336 @@ +"""Bounded lexical, OOXML-part and image-header adapters; no conversion.""" +from __future__ import annotations + +import codecs +from html.parser import HTMLParser +import io +import json +from pathlib import PurePosixPath +import re +import struct +import sys +import zipfile +import zlib + +XML_VERSION = "0.7.1" +MAX_TEXT = 2 * 1024 * 1024 +MAX_ENTRIES = 256 +MAX_INFLATED = 16 * 1024 * 1024 +MAX_RATIO = 100 +MAX_NODES = 100_000 +MAX_DEPTH = 64 +MAX_IMAGE_PIXELS = 100_000_000 +CT = "http://schemas.openxmlformats.org/package/2006/content-types" +W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" +P = "http://schemas.openxmlformats.org/presentationml/2006/main" +A = "http://schemas.openxmlformats.org/drawingml/2006/main" +S = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" +OFFICE = { + "docx": ("/word/document.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"), + "pptx": ("/ppt/presentation.xml", "application/vnd.openxmlformats-officedocument.presentationml.presentation.main+xml"), + "xlsx": ("/xl/workbook.xml", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"), +} + + +class AdapterBlocked(ValueError): + """Fixed content-free reason; callers never forward parser exceptions.""" + + +def facts(**values): + return { + "page_count": "not-applicable", + "text_characters": "not-assessed", "text_available": "not-assessed", + "total_images": "not-assessed", "total_drawings": "not-assessed", + "table_structure": "not-assessed", "layout_relationships": "not-assessed", + "input_profile": "unknown", "answerability": "not-assessed", + "ocr_need": "not-assessed", **values, + } + + +def text_facts(count, basis, **values): + return facts(text_characters=count, text_available=count > 0, text_basis=basis, **values) + + +class HTMLCounts(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.characters = 0 + self.tags = {"img": 0, "table": 0, "tr": 0, "td": 0, "th": 0} + self.suppressed = [] + self.nodes = 0 + + def handle_starttag(self, tag, attrs): + self.nodes += 1 + if self.nodes > MAX_NODES: + raise AdapterBlocked("document-node-limit") + if tag in {"script", "style"}: + self.suppressed.append(tag) + if not self.suppressed and tag in self.tags: + self.tags[tag] += 1 + + def handle_endtag(self, tag): + if self.suppressed and self.suppressed[-1] == tag: + self.suppressed.pop() + + def handle_data(self, data): + if not self.suppressed: + self.characters += len(data) + + +def assess_text(data, hint): + if len(data) > MAX_TEXT: + raise AdapterBlocked("text-input-limit") + try: + text = data.decode("utf-8-sig", errors="strict") + except UnicodeError: + raise AdapterBlocked("unsupported-text-encoding-or-format") from None + if any(ord(char) < 32 and char not in "\t\r\n" for char in text): + raise AdapterBlocked("unsupported-binary-format") + kind = "text" if hint == "auto" else hint + if kind not in {"text", "markdown", "html", "json"}: + raise AdapterBlocked("format-hint-mismatch") + if kind == "json": + def reject_constant(value): + raise AdapterBlocked("nonstandard-json") + + def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise AdapterBlocked("duplicate-json-key") + result[key] = value + return result + + try: + value = json.loads(text, parse_constant=reject_constant, object_pairs_hook=unique_object) + except AdapterBlocked: + raise + except (ValueError, UnicodeError): + raise AdapterBlocked("invalid-json") from None + count = nodes = objects = arrays = keys = 0 + pending = [(value, 1)] + while pending: + value, depth = pending.pop() + nodes += 1 + if nodes > MAX_NODES or depth > MAX_DEPTH: + raise AdapterBlocked("document-node-or-depth-limit") + if isinstance(value, str): + count += len(value) + elif isinstance(value, dict): + objects += 1 + keys += len(value) + pending.extend((item, depth + 1) for item in value.values()) + elif isinstance(value, list): + arrays += 1 + pending.extend((item, depth + 1) for item in value) + return kind, "stdlib-json", text_facts(count, "string-values-only", + objects=objects, arrays=arrays, keys=keys) + if kind == "html": + parser = HTMLCounts() + parser.feed(text) + parser.close() + return kind, "stdlib-HTMLParser", text_facts( + parser.characters, "data-events-excluding-script-style", + html_elements=parser.tags, validation="tolerant-syntax-not-rendering") + return kind, "stdlib-utf8", text_facts( + len(text), "decoded-characters-including-whitespace-and-markup", + markdown_structure="not-assessed" if kind == "markdown" else "not-applicable") + + +def xml_parser(): + try: + import defusedxml + from defusedxml.ElementTree import fromstring + from defusedxml.common import DefusedXmlException + from xml.etree.ElementTree import ParseError + except ImportError: + raise AdapterBlocked("office-parser-dependency-missing") from None + if defusedxml.__version__ != XML_VERSION: + raise AdapterBlocked("office-parser-version-unsupported") + return fromstring, (DefusedXmlException, ParseError) + + +def bounded_xml(data, parse, errors): + if len(data) > MAX_TEXT: + raise AdapterBlocked("archive-entry-limit") + try: + root = parse(data, forbid_dtd=True, forbid_entities=True, forbid_external=True) + except errors: + raise AdapterBlocked("unsafe-or-invalid-office-xml") from None + pending, nodes = [(root, 1)], 0 + while pending: + element, depth = pending.pop() + nodes += 1 + if nodes > MAX_NODES or depth > MAX_DEPTH: + raise AdapterBlocked("document-node-or-depth-limit") + pending.extend((child, depth + 1) for child in element) + return root, nodes + + +def assess_office(data, hint, parse, errors): + try: + with zipfile.ZipFile(io.BytesIO(data)) as archive: + entries = archive.infolist() + if len(entries) > MAX_ENTRIES: + raise AdapterBlocked("archive-entry-count-limit") + names, total = set(), 0 + for entry in entries: + path = PurePosixPath(entry.filename) + if (entry.filename in names or path.is_absolute() or ".." in path.parts + or "\\" in entry.filename or ":" in entry.filename): + raise AdapterBlocked("unsafe-or-duplicate-archive-entry") + names.add(entry.filename) + total += entry.file_size + if entry.flag_bits & 1: + raise AdapterBlocked("encrypted-office-archive") + if entry.compress_type not in {zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED}: + raise AdapterBlocked("unsupported-archive-compression") + if (entry.file_size > MAX_TEXT or total > MAX_INFLATED + or entry.file_size > MAX_RATIO * max(entry.compress_size, 1)): + raise AdapterBlocked("archive-inflation-limit") + if entry.filename.lower().endswith("vbaproject.bin"): + raise AdapterBlocked("macro-package-not-assessed") + + def read_xml(part): + if part not in names: + raise AdapterBlocked("incomplete-office-package") + with archive.open(part) as source: + content = source.read(MAX_TEXT + 1) + return bounded_xml(content, parse, errors) + + if "[Content_Types].xml" not in names: + raise AdapterBlocked("unsupported-zip-format") + types, nodes = read_xml("[Content_Types].xml") + if types.tag != f"{{{CT}}}Types": + raise AdapterBlocked("unsupported-office-namespace") + declared = [(child.get("PartName"), child.get("ContentType")) for child in types] + if any("macroenabled" in (kind or "").lower() for _, kind in declared): + raise AdapterBlocked("macro-package-not-assessed") + kinds = [kind for kind, definition in OFFICE.items() if definition in declared] + if len(kinds) != 1: + raise AdapterBlocked("unsupported-office-profile") + kind = kinds[0] + if hint not in {"auto", kind}: + raise AdapterBlocked("format-hint-mismatch") + main = OFFICE[kind][0][1:] + root, used = read_xml(main) + nodes += used + expected = {"docx": f"{{{W}}}document", "pptx": f"{{{P}}}presentation", + "xlsx": f"{{{S}}}workbook"}[kind] + if root.tag != expected: + raise AdapterBlocked("unsupported-office-namespace") + patterns = { + "docx": r"word/document\.xml", + "pptx": r"ppt/slides/slide[0-9]+\.xml", + "xlsx": r"xl/(worksheets/sheet[0-9]+|sharedStrings|tables/table[0-9]+)\.xml", + } + parts = sorted(part for part in names if re.fullmatch(patterns[kind], part)) + if not parts: + raise AdapterBlocked("office-content-parts-unavailable") + characters = tables = image_refs = paragraphs = formulas = cells = 0 + for part in parts: + tree, used = (root, 0) if part == main else read_xml(part) + nodes += used + if nodes > MAX_NODES: + raise AdapterBlocked("document-node-limit") + allowed_roots = { + "docx": {f"{{{W}}}document"}, "pptx": {f"{{{P}}}sld"}, + "xlsx": {f"{{{S}}}worksheet", f"{{{S}}}sst", f"{{{S}}}table"}, + }[kind] + if tree.tag not in allowed_roots: + raise AdapterBlocked("unsupported-office-namespace") + for element in tree.iter(): + if element.tag == {"docx": f"{{{W}}}t", "pptx": f"{{{A}}}t", + "xlsx": f"{{{S}}}t"}[kind]: + characters += len(element.text or "") + tables += element.tag in {f"{{{W}}}tbl", f"{{{A}}}tbl", f"{{{S}}}table"} + image_refs += element.tag == f"{{{A}}}blip" + paragraphs += element.tag in {f"{{{W}}}p", f"{{{A}}}p"} + formulas += element.tag == f"{{{S}}}f" + cells += element.tag == f"{{{S}}}c" + result = text_facts( + characters, "stored-text-elements-not-rendered-or-cell-occurrences", + page_count="not-assessed", inspected_xml_parts=len(parts), + table_elements=tables, image_reference_elements=image_refs, + paragraphs=paragraphs if kind != "xlsx" else "not-applicable", + formula_elements=formulas if kind == "xlsx" else "not-applicable", + cell_elements=cells if kind == "xlsx" else "not-applicable", + slide_parts=len(parts) if kind == "pptx" else "not-applicable", + declared_sheets=sum(e.tag == f"{{{S}}}sheet" for e in root.iter()) + if kind == "xlsx" else "not-applicable", + coverage={"docx": "main-document-only", "pptx": "all-standard-slide-parts", + "xlsx": "standard-worksheet-shared-string-table-parts"}[kind], + relationships="not-resolved", rendering="not-assessed", + ) + return kind, f"stdlib-zipfile+defusedxml=={XML_VERSION}", result + except (zipfile.BadZipFile, zipfile.LargeZipFile, EOFError, zlib.error): + raise AdapterBlocked("invalid-office-archive") from None + + +def image_facts(kind, width, height, **values): + if not width or not height or width * height > MAX_IMAGE_PIXELS: + raise AdapterBlocked("image-dimension-limit") + return kind, "stdlib-struct-header", facts( + width=width, height=height, text_characters="not-assessed", + text_available="not-assessed", validation="header-only-pixels-not-assessed", + frame_count="not-assessed", **values) + + +def assess_png(data): + if len(data) < 33 or data[8:16] != b"\0\0\0\rIHDR": + raise AdapterBlocked("invalid-png-header") + width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", data[16:29]) + if (zlib.crc32(data[12:29]) != struct.unpack(">I", data[29:33])[0] + or compression or filtering or interlace not in (0, 1) + or depth not in {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, + 4: {8, 16}, 6: {8, 16}}.get(color, set())): + raise AdapterBlocked("invalid-png-header") + return image_facts("png", width, height, bit_depth=depth, color_type=color) + + +def assess_jpeg(data): + position = 2 + for _ in range(4096): + if position >= len(data) or data[position] != 255: + raise AdapterBlocked("invalid-jpeg-header") + while position < len(data) and data[position] == 255: + position += 1 + if position >= len(data): + raise AdapterBlocked("invalid-jpeg-header") + marker = data[position] + position += 1 + if marker in (0xDA, 0xD9) or position + 2 > len(data): + raise AdapterBlocked("jpeg-frame-header-unavailable") + length = int.from_bytes(data[position:position + 2], "big") + if length < 2 or position + length > len(data): + raise AdapterBlocked("invalid-jpeg-header") + if marker in (0xC0, 0xC1, 0xC2): + if length < 8: + raise AdapterBlocked("invalid-jpeg-header") + depth, height, width, components = struct.unpack(">BHHB", data[position + 2:position + 8]) + if length != 8 + 3 * components or not 1 <= components <= 4 or depth not in (8, 12): + raise AdapterBlocked("invalid-jpeg-header") + return image_facts("jpeg", width, height, precision=depth, components=components) + position += length + raise AdapterBlocked("image-marker-limit") + + +def assess_document(data, hint, deny_reads): + for encoding in ("utf-8-sig", "cp437", "ascii", "utf-8"): + codecs.lookup(encoding) + office = data.startswith((b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08")) + parse, errors = xml_parser() if office else (None, ()) + sys.addaudithook(deny_reads) + if office: + return assess_office(data, hint, parse, errors) + if data.startswith(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"): + raise AdapterBlocked("legacy-or-encrypted-office-not-assessed") + if data.startswith(b"\x89PNG\r\n\x1a\n"): + if hint not in {"auto", "png"}: + raise AdapterBlocked("format-hint-mismatch") + return assess_png(data) + if data.startswith(b"\xff\xd8"): + if hint not in {"auto", "jpeg"}: + raise AdapterBlocked("format-hint-mismatch") + return assess_jpeg(data) + return assess_text(data, hint) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_document_limits.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_document_limits.py new file mode 100644 index 00000000..38059779 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_document_limits.py @@ -0,0 +1,95 @@ +"""OS-enforced limits installed in the document worker before parser imports.""" +from __future__ import annotations + +import os +import sys + +MEMORY_BYTES = 256 * 1024 * 1024 +CPU_SECONDS = 10 + + +def install_limits(): + """Return a live Windows job handle, or None on Linux; fail closed elsewhere.""" + if sys.platform == "linux": + import resource + + resource.setrlimit(resource.RLIMIT_AS, (MEMORY_BYTES, MEMORY_BYTES)) + resource.setrlimit(resource.RLIMIT_CPU, (CPU_SECONDS, CPU_SECONDS)) + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + resource.setrlimit(resource.RLIMIT_FSIZE, (0, 0)) + return None + if sys.platform != "win32": + raise OSError("unsupported limit platform") + + import ctypes + from ctypes import wintypes as w + + class Basic(ctypes.Structure): + _fields_ = [ + ("process_time", ctypes.c_longlong), ("job_time", ctypes.c_longlong), + ("flags", w.DWORD), ("min_ws", ctypes.c_size_t), + ("max_ws", ctypes.c_size_t), ("active", w.DWORD), + ("affinity", ctypes.c_size_t), ("priority", w.DWORD), + ("scheduling", w.DWORD), + ] + + class IO(ctypes.Structure): + _fields_ = [(name, ctypes.c_ulonglong) for name in + ("read_ops", "write_ops", "other_ops", "read_bytes", + "write_bytes", "other_bytes")] + + class Extended(ctypes.Structure): + _fields_ = [ + ("basic", Basic), ("io", IO), ("process_memory", ctypes.c_size_t), + ("job_memory", ctypes.c_size_t), ("peak_process", ctypes.c_size_t), + ("peak_job", ctypes.c_size_t), + ] + + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.CreateJobObjectW.argtypes = [ctypes.c_void_p, w.LPCWSTR] + kernel.CreateJobObjectW.restype = w.HANDLE + kernel.SetInformationJobObject.argtypes = [w.HANDLE, ctypes.c_int, + ctypes.c_void_p, w.DWORD] + kernel.SetInformationJobObject.restype = w.BOOL + kernel.GetCurrentProcess.restype = w.HANDLE + kernel.AssignProcessToJobObject.argtypes = [w.HANDLE, w.HANDLE] + kernel.AssignProcessToJobObject.restype = w.BOOL + kernel.CloseHandle.argtypes = [w.HANDLE] + kernel.CloseHandle.restype = w.BOOL + job = kernel.CreateJobObjectW(None, None) + if not job: + raise ctypes.WinError(ctypes.get_last_error()) + limits = Extended() + # PROCESS_TIME | ACTIVE_PROCESS | PROCESS_MEMORY | KILL_ON_JOB_CLOSE. + limits.basic.flags = 0x2 | 0x8 | 0x100 | 0x2000 + limits.basic.process_time = CPU_SECONDS * 10_000_000 + limits.basic.active = 1 + limits.process_memory = MEMORY_BYTES + if not kernel.SetInformationJobObject(job, 9, ctypes.byref(limits), + ctypes.sizeof(limits)): + error = ctypes.get_last_error() + kernel.CloseHandle(job) + raise ctypes.WinError(error) + if not kernel.AssignProcessToJobObject(job, kernel.GetCurrentProcess()): + error = ctypes.get_last_error() + kernel.CloseHandle(job) + raise ctypes.WinError(error) + # Keep the handle until process exit: closing it terminates this worker. + return job + + +def deny_side_effects(event, args): + """Defense in depth, not a sandbox for arbitrary Python/native code.""" + if event.startswith("socket.") or event in { + "subprocess.Popen", "os.system", "os.posix_spawn", "os.fork", + "os.remove", "os.unlink", "os.rename", "os.replace", "os.mkdir", "os.rmdir", + "os.truncate", "os.chmod", "os.chown", "os.lchown", "os.utime", + "os.link", "os.symlink", "os.chflags", "os.setxattr", "os.removexattr", + }: + raise PermissionError("Document side effect denied") + if event == "open": + _, mode, flags = args + if (mode and any(char in mode for char in "wax+")) or ( + flags & (os.O_WRONLY | os.O_RDWR | os.O_CREAT | os.O_TRUNC) + ): + raise PermissionError("Document write denied") diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_indexer_observation.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_indexer_observation.py new file mode 100644 index 00000000..0e20ee22 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_indexer_observation.py @@ -0,0 +1,135 @@ +"""Private, versioned evidence for generated indexers; no service operations.""" +from __future__ import annotations + +import re +from datetime import datetime, timedelta, timezone + +try: + from ._common import HelperFailure, digest +except ImportError: + from _common import HelperFailure, digest + + +PROJECTION_FIELDS = {"non_schedule_digest", "schedule_digest", "schedule_raw_digest"} +TICKS = 10_000_000 +DURATION = re.compile(r"P(?:(?P[0-9]{1,9})D)?(?:T(?:(?P[0-9]{1,9})H)?" + r"(?:(?P[0-9]{1,9})M)?(?:(?P[0-9]{1,9})(?:\.(?P[0-9]{1,7}))?S)?)?") +INSTANT = re.compile(r"([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})" + r"(?:\.([0-9]{1,7}))?(Z|[+-][0-9]{2}:[0-9]{2})") + + +def note(diagnostics, severity, code, field, message, request_id): + entry = dict(severity=severity, code=code, field=field, message=message, request_id=request_id) + if entry not in diagnostics: + diagnostics.append(entry) + + +def failure(diagnostics, code, field, message, request_id): + note(diagnostics, "error", code, field, message, request_id) + return HelperFailure(code, message, blocked_at="indexer-verification", request_id=request_id) + + +def interval(value): + match = DURATION.fullmatch(value) if isinstance(value, str) and len(value) <= 96 else None + if not match or not any(match.group(k) for k in ("days", "hours", "minutes", "seconds")): + raise ValueError("Unsupported duration") + if "T" in value and not any(match.group(k) for k in ("hours", "minutes", "seconds")): + raise ValueError("Empty time component") + parts = match.groupdict() + seconds = sum(int(parts[k] or 0) * scale for k, scale in + (("days", 86400), ("hours", 3600), ("minutes", 60), ("seconds", 1))) + ticks = seconds * TICKS + int((parts["fraction"] or "").ljust(7, "0")) + if not 300 * TICKS <= ticks <= 86400 * TICKS: + raise ValueError("Indexer interval outside documented bounds") + return ticks + + +def instant(value): + match = INSTANT.fullmatch(value) if isinstance(value, str) and len(value) <= 40 else None + if not match or match[8] == "-00:00": + raise ValueError("Unknown or unsupported timezone") + year, month, day, hour, minute, second = map(int, match.groups()[:6]) + zone = match[8] + offset = 0 + if zone != "Z": + hours, minutes = int(zone[1:3]), int(zone[4:6]) + if hours > 14 or minutes > 59 or hours == 14 and minutes: + raise ValueError("Invalid offset") + offset = (hours * 60 + minutes) * (-1 if zone[0] == "-" else 1) + stamp = datetime(year, month, day, hour, minute, second, + tzinfo=timezone(timedelta(minutes=offset))).astimezone(timezone.utc) + elapsed = stamp - datetime(1, 1, 1, tzinfo=timezone.utc) + return (elapsed.days * 86400 + elapsed.seconds) * TICKS + int((match[7] or "").ljust(7, "0")) + + +def schedule(value, *, present=True, diagnostics=None, request_id=None): + diagnostics = diagnostics if diagnostics is not None else [] + if value is None: + return {"state": "null" if present else "missing"} + field = "schedule" + try: + if not isinstance(value, dict) or set(value) - {"interval", "startTime"}: + raise ValueError("Unknown shape") + field = "schedule.interval" + period = interval(value.get("interval")) + field = "schedule.startTime" + start = instant(value["startTime"]) if value.get("startTime") is not None else None + return {"state": "object", "interval": period, "startTime": start} + except (ValueError, OverflowError) as exc: + raise failure(diagnostics, "indexer-schedule-evidence-invalid", field, + f"Generated indexer {field} is malformed or unsupported; no default is inferred.", + request_id) from exc + + +def observe(child, approved, diagnostics, request_id): + observed = schedule(child.get("schedule"), present="schedule" in child, + diagnostics=diagnostics, request_id=request_id) + expected = schedule(approved, diagnostics=diagnostics, request_id=request_id) + if approved is not None: + for field in ("interval", "startTime"): + if field == "startTime" and approved.get(field) is None: + continue + if observed.get(field) != expected[field]: + raise failure(diagnostics, "indexer-schedule-constraint-violation", f"schedule.{field}", + f"Generated indexer schedule.{field} violates the approved constraint; this is not an ingestion failure.", + request_id) + if child["schedule"].get(field) != approved.get(field): + note(diagnostics, "info", "indexer-schedule-format-equivalent", f"schedule.{field}", + "Schedule formats represent the same duration or instant.", request_id) + if observed != expected: + note(diagnostics, "warning", "indexer-schedule-unconstrained", "schedule", + "Generated schedule differs on unconstrained timing; recurring work/cost is possible. No schedule was changed or default assumed.", + request_id) + return { + "etag": child["@odata.etag"], "digest": digest(child), + "non_schedule_digest": digest({k: v for k, v in child.items() if k not in {"schedule", "@odata.etag"}}), + "schedule_digest": digest(observed), + "schedule_raw_digest": digest({"present": "schedule" in child, "value": child.get("schedule")}), + } + + +def compare(current, expected, diagnostics, request_id): + if current == expected: + return + if not PROJECTION_FIELDS <= set(expected): + raise failure(diagnostics, "indexer-legacy-evidence-insufficient", "indexer", + "Legacy full-hash evidence differs; no schedule preimage/projection exists. Retain it; do not auto-upgrade.", + request_id) + if current["digest"] == expected["digest"]: + raise failure(diagnostics, "indexer-evidence-inconsistent", "indexer", + "Equal full hashes have inconsistent version/projection evidence.", request_id) + if current["non_schedule_digest"] != expected["non_schedule_digest"]: + raise failure(diagnostics, "indexer-definition-drift", "indexer", + "A non-schedule indexer field changed; schedule policy cannot admit it.", request_id) + if current["schedule_raw_digest"] == expected["schedule_raw_digest"]: + raise failure(diagnostics, "indexer-version-unexplained", "@odata.etag", + "Indexer version/full hash changed without an observed schedule change; the revision is unproven.", request_id) + equivalent = current["schedule_digest"] == expected["schedule_digest"] + note(diagnostics, "info" if equivalent else "warning", + "indexer-schedule-format-equivalent" if equivalent else "indexer-schedule-unconstrained", + "schedule", "Only schedule changed; every other observed field matches the retained projection and approved timing constraints hold.", + request_id) + + +def warnings(diagnostics): + return list(dict.fromkeys(item["message"] for item in diagnostics if item["severity"] == "warning")) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_initial_prompt.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_initial_prompt.py new file mode 100644 index 00000000..5db316b1 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_initial_prompt.py @@ -0,0 +1,181 @@ +"""Receipt-producing form of the existing local Foundry v1 create-agent fallback.""" +from __future__ import annotations + +import re +from urllib.parse import quote + +try: + from . import prompt_connect, _cleanup_dependencies as dependencies, _cleanup_receipts as receipts + from ._common import HelperFailure, canonical_bytes, digest, reject_secrets, require_allowed_fields, sdk_error_status, sdk_error_metadata +except ImportError: + import prompt_connect, _cleanup_dependencies as dependencies, _cleanup_receipts as receipts + from _common import HelperFailure, canonical_bytes, digest, reject_secrets, require_allowed_fields, sdk_error_status, sdk_error_metadata + + +def fail(code, message): + return HelperFailure(code, message, blocked_at="initial-prompt-creation") + + +def validate(plan): + reject_secrets(plan) + require_allowed_fields(plan, {"operation", "sdk_major", "project_resource_id", "project_endpoint", + "agent", "owner", "cleanup_approved", "inventory_limits", "absence_inventory_digest", + "prerequisites"}, label="Initial Prompt creation") + prompt_connect._project_identity(plan) + if (plan.get("operation") != "create-initial-prompt-agent" or plan.get("sdk_major") != 2 + or plan.get("cleanup_approved") is not False or not isinstance(plan.get("owner"), str) or not plan["owner"].strip()): + raise fail("input-schema-invalid", "Initial Prompt creation needs separate approved creation intent and owner.") + agent = plan.get("agent") + if not isinstance(agent, dict) or set(agent) != {"name", "definition"}: + raise fail("input-schema-invalid", "Select one exact initial agent name and complete definition.") + if not isinstance(agent["name"], str) or not re.fullmatch(r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?", agent["name"]): + raise fail("input-schema-invalid", "Select a v1 agent name, never a suffix or latest.") + definition = agent["definition"] + if (not isinstance(definition, dict) or set(definition) != {"kind", "model", "instructions", "tools"} + or definition["kind"] != "prompt" or not isinstance(definition["model"], str) or not definition["model"].strip() + or not isinstance(definition["instructions"], str) or definition["tools"] != []): + raise fail("input-schema-invalid", "Initial creation permits only an explicit Prompt model/instructions with no tools.") + prerequisites = plan.get("prerequisites") + if (not isinstance(prerequisites, dict) or set(prerequisites) != {"project", "model", "identity", "network"} + or any(not isinstance(v, str) or not v.strip() for v in prerequisites.values())): + raise fail("initial-prerequisites-unverified", "Retain the existing local-creation procedure's project/model/identity/network evidence.") + dependencies.limits(plan.get("inventory_limits")) + if not isinstance(plan.get("absence_inventory_digest"), str) or not dependencies.SHA.fullmatch(plan["absence_inventory_digest"]): + raise fail("input-schema-invalid", "The complete scoped absence inventory must be fingerprinted.") + + +def absence(plan, sdk_loader): + Client, _, _, _, extras = sdk_loader() + Credential, AzureError = extras + client = Client(endpoint=plan["project_endpoint"], credential=Credential()) + try: + names = [] + for item in dependencies._paged(client.agents.list(), dependencies.limits(plan["inventory_limits"])): + name = getattr(item, "name", None) + if not isinstance(name, str) or not name.strip() or name in names: + raise fail("initial-inventory-incomplete", "Agent absence needs a complete, unambiguous project inventory.") + names.append(name) + if plan["agent"]["name"] in names: + raise fail("initial-agent-exists", "Retain the existing agent; return to Connect rather than creating a version.") + try: + client.agents.get(agent_name=plan["agent"]["name"]) + except AzureError as exc: + if sdk_error_status(exc) == 404: + return digest(sorted(names)) + raise + raise fail("initial-agent-exists", "Exact agent GET found an existing identity.") + except AzureError as exc: + raise HelperFailure(message="Complete initial-agent absence discovery failed.", blocked_at="initial-prompt-creation", + **sdk_error_metadata(exc, "initial-inventory-failed")) from exc + except (AttributeError, TypeError, ValueError) as exc: + raise fail("initial-api-unavailable", "SDK complete agent listing and exact agent GET are required.") from exc + finally: + client.close() + + +def plan(request, sdk_loader): + if not isinstance(request, dict): + raise fail("input-schema-invalid", "Initial creation request must be a closed object.") + require_allowed_fields(request, {"schema_version", "project_resource_id", "project_endpoint", "agent", + "owner", "inventory_limits", "prerequisites"}, label="Initial Prompt request") + if request.get("schema_version") != "1.0": + raise fail("input-schema-invalid", "Initial request requires schema_version 1.0.") + proposed = {k: v for k, v in request.items() if k != "schema_version"} + proposed.update(operation="create-initial-prompt-agent", sdk_major=2, cleanup_approved=False, + inventory_limits=dependencies.limits(request.get("inventory_limits")), + absence_inventory_digest="sha256:" + "0" * 64) + validate(proposed) + proposed["absence_inventory_digest"] = absence(proposed, sdk_loader) + fingerprint = digest(proposed) + return {"status": "planned", "outcome": "create-initial-prompt-agent", "plan_fingerprint": fingerprint, + "execution_required": True, "mutation_approval_required": True, + "execution_input": {"schema_version": "1.0", "plan": proposed, + "approval": {"confirmed": False, "fingerprint": fingerprint}}, + "approval_summary": {"create": [proposed["agent"]["name"]], "retain": ["existing agents", "projects", "models", "grants"], + "model_readiness": "Provided prerequisite evidence must be independently verified before approval.", + "invocation": "not performed", "cleanup": "separate owned-version approval; container retained"}} + + +def verify_created(plan, target, birth, sdk_loader): + Client, _, _, _, extras = sdk_loader() + Credential, AzureError = extras + client = Client(endpoint=plan["project_endpoint"], credential=Credential()) + try: + agent = client.agents.get(agent_name=target["name"]) + if agent.name != target["name"]: + raise fail("initial-readback-invalid", "Initial agent identity readback changed.") + versions = [ + str(item.version) for item in dependencies._paged( + client.agents.list_versions(agent_name=target["name"], include_drafts=True), + dependencies.limits(plan["inventory_limits"])) + ] + if versions != [target["version"]]: + raise fail("initial-version-inventory-changed", "Initial creation must retain exactly its acknowledged version, with no duplicates or unexpected versions.") + current = client.agents.get_version(agent_name=target["name"], agent_version=target["version"]) + if (current.name != target["name"] or str(current.version) != target["version"] + or current.definition.as_dict() != plan["agent"]["definition"] + or receipts.version_identity(current) != birth): + raise fail("definition-drift", "Exact initial version differs from its native birth snapshot.") + except AzureError as exc: + raise HelperFailure(message="Initial agent/version verification failed.", blocked_at="initial-prompt-creation", + **sdk_error_metadata(exc, "initial-readback-failed")) from exc + except (AttributeError, TypeError, ValueError) as exc: + raise fail("initial-api-unavailable", "Complete draft-inclusive agent/version readback is required.") from exc + finally: + client.close() + + +def execute(document, *, capture, token_provider, transport, sdk_loader): + plan = document["plan"] + validate(plan) + if capture is None or capture.plan_digest != document["_computed_fingerprint"] or capture.owner != plan["owner"]: + raise fail("cleanup-receipt-required", "Initial creation requires its original approved input and protected receipt directory.") + if absence(plan, sdk_loader) != plan["absence_inventory_digest"]: + raise fail("initial-inventory-drift", "Project inventory changed; replan rather than weakening absence protection.") + token = token_provider("https://ai.azure.com/") + url = plan["project_endpoint"].rstrip("/") + "/agents?api-version=v1" + target = receipts.project_target(plan, "prompt-agent-version", name=plan["agent"]["name"], version="unverified") + try: + result = transport("POST", url, token, body=canonical_bytes(plan["agent"]), headers={"Content-Type": "application/json"}) + except HelperFailure as failure: + if failure.http_status in (None, 408, 429) or (failure.http_status or 0) >= 500: + failure.partial = True + failure.warnings.append("Initial creation outcome unproven; no replay, ownership adoption or automatic cleanup.") + raise + write = {"action": "created", "type": "prompt-agent-version", "name": plan["agent"]["name"]} + if result.status != 200: + raise HelperFailure("initial-create-failed", "Create-agent returned an unexpected status; no replay or ownership adoption.", + blocked_at="initial-prompt-creation", status=result.status, + request_id=result.request_id, partial=(200 <= result.status < 300 or result.status in (408, 429) or result.status >= 500)) + try: + if not isinstance(result.body, dict): + raise fail("initial-create-response-invalid", "Foundry v1 create-agent did not return the documented HTTP 200 AgentObject.") + versions = result.body.get("versions") + latest = versions.get("latest") if isinstance(versions, dict) else None + latest = latest if isinstance(latest, dict) else {} + target["version"] = latest.get("version", "unverified") + capture.start(target, {"operation": "agents.create", "status": result.status, "request_id": result.request_id, + "definition_digest": digest(latest.get("definition")), "version": target["version"], + "etag_evidence": None, "generated": None, "version_identity": receipts.version_identity(latest)}) + if (result.body.get("name") != plan["agent"]["name"] or latest.get("name") != plan["agent"]["name"] + or not isinstance(target["version"], str) or not re.fullmatch(r"[1-9][0-9]*", target["version"]) + or latest.get("definition") != plan["agent"]["definition"] or receipts.version_identity(latest) is None): + raise fail("initial-create-response-invalid", "Native created agent/version differs from approved initial intent.") + write["version"] = target["version"] + readback = transport("GET", plan["project_endpoint"].rstrip("/") + "/agents/" + quote(target["name"], safe="") + + "/versions/" + quote(target["version"], safe="") + "?api-version=v1", token) + if (readback.status != 200 or not isinstance(readback.body, dict) + or readback.body.get("name") != target["name"] or readback.body.get("version") != target["version"] + or readback.body.get("definition") != latest["definition"]): + raise fail("definition-drift", "Initial version readback differs from the original successful create-agent response.") + verify_created(plan, target, receipts.version_identity(latest), sdk_loader) + capture.finish(target, {"definition_digest": digest(latest["definition"]), "etag": None, "generated": [], + "version_identity": receipts.version_identity(readback.body)}) + except HelperFailure as failure: + failure.partial = True + failure.writes.insert(0, write) + raise + return {"status": "completed", "outcome": "create-initial-prompt-agent", + "approved_plan": document["approval"], "resources": {"created": [target], "reused": []}, + "ownership": {"owner": plan["owner"], "run_owned": [target], "reused_not_owned": []}, + "agent_invocation": "not-run", "cleanup": "separate version-only approval; never delete the container"} diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_progress.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_progress.py new file mode 100644 index 00000000..f9e08c92 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_progress.py @@ -0,0 +1,259 @@ +"""Bounded, content-free observations; never a service execution controller.""" +from __future__ import annotations + +from functools import wraps +from contextlib import contextmanager +import copy +from datetime import datetime +import json +import math +import os +import sys +import time +import threading + +try: + from ._common import HelperFailure +except ImportError: + from _common import HelperFailure + + +STAGES = { + "file-source": ("validation", "source-reconciliation", "file-inventory", "file-upload", "file-readback"), + "file-upload": ("file-inventory", "file-upload", "file-readback"), + "blob-source": ("validation", "blob-inventory", "source-reconciliation", "ingestion-cycle", + "blob-readback", "source-readback"), + "blob-monitor": ("ingestion-cycle",), + "blob-capture": ("evidence-validation", "context-check", "source-binding", "blob-inventory", "checkpoint"), + "blob-recheck": ("evidence-validation", "context-check", "source-binding", "blob-inventory", + "ingestion-cycle", "blob-readback", "source-readback", "context-readback"), + "search-bootstrap": ("validation", "context-check", "region-check", "absence-check", "search-submit", + "arm-wait", "arm-readback"), +} +COUNTS = frozenset(("uploads_acknowledged", "files_reused", "files_verified", + "status_checks", "cycle_updates_processed", "cycle_items_skipped", + "files_failed", "files_unverified", "files_not_attempted", "files_pending", "files_ingested")) +FILE_HEARTBEAT_SECONDS = 5 +IO_WARNING = "progress-output-failed: stderr progress could not be written; execution result is authoritative." +SHUTDOWN_WARNING = "progress-shutdown-flush-unresolved: failed stderr could not be redirected; shutdown may override the exit code." + + +def validate_blob_progress(value): + fields = {"schema_version", "phase", "run_start", "processed", "failed", "skipped", + "unit", "total", "remaining", "denominator", "synchronization_status", + "elapsed_seconds", "next_check_seconds"} + if not isinstance(value, dict) or set(value) != fields: + raise ValueError("Invalid Blob progress shape.") + if (value["schema_version"] != "1.0" or value["unit"] != "item-updates" + or value["denominator"] != "not-comparable-to-files" + or value["total"] is not None or value["remaining"] is not None + or value["phase"] not in ("ingesting", "waiting", "throttled", "paused", "completed", "failed") + or value["synchronization_status"] not in ("not-reported", "active", "creating", "deleting")): + raise ValueError("Invalid Blob progress labels.") + for field in ("processed", "failed", "skipped"): + count = value[field] + if count is not None and (type(count) is not int or count < 0): + raise ValueError("Invalid Blob progress count.") + for field in ("elapsed_seconds", "next_check_seconds"): + number = value[field] + if field == "next_check_seconds" and number is None: + continue + if type(number) not in (int, float) or not math.isfinite(number) or number < 0: + raise ValueError("Invalid Blob progress timing.") + if value["next_check_seconds"] is not None and value["next_check_seconds"] > 60: + raise ValueError("Invalid Blob progress wait.") + start = value["run_start"] + if start is not None: + if not isinstance(start, str) or len(start) > 40: + raise ValueError("Invalid Blob progress run.") + parsed = datetime.fromisoformat(start.replace("Z", "+00:00")) + if parsed.tzinfo is None or parsed.isoformat() != start: + raise ValueError("Blob progress run must be a canonical timestamp.") + + +class Progress: + def __init__(self, workflow, *, enabled=True, stream=None, clock=time.monotonic): + self.stages = STAGES[workflow] + self.workflow = workflow + self.enabled = enabled + self.stream = stream + self.clock = clock + self.depth = 0 + self.stage = self.stages[0] + self.counts = {} + self.started = None + self.elapsed = 0.0 + self.last_sent = None + self.last_stage = None + self.output_failed = False + self.shutdown_flush_unresolved = False + self.blob_progress = None + self.last_blob_phase = None + + def blob_update(self, value): + if not self.workflow.startswith("blob-"): + raise ValueError("Blob progress cannot describe a File or ARM workflow.") + validate_blob_progress(value) + self.blob_progress = copy.deepcopy(value) + for field, target in (("processed", "cycle_updates_processed"), ("skipped", "cycle_items_skipped")): + self.counts.pop(target, None) + if value[field] is not None: + self.counts[target] = value[field] + self.update("ingestion-cycle") + + def update(self, stage, **counts): + if stage not in self.stages or self.stages.index(stage) < self.stages.index(self.stage): + raise ValueError("Invalid progress stage transition.") + if any(key not in COUNTS or type(value) is not int or value < 0 for key, value in counts.items()): + raise ValueError("Invalid progress count.") + self.stage = stage + self.counts.update(counts) + self._emit("running") + + def waiting(self, seconds): + if not isinstance(seconds, (int, float)) or not math.isfinite(seconds) or not 0 <= seconds <= 30: + raise ValueError("Invalid recovery wait.") + self._emit("waiting", wait={"reason": "http-429", "seconds": seconds}) + + @contextmanager + def processing_file(self, ordinal, total, attempt): + if (self.workflow not in {"file-source", "file-upload"} or type(ordinal) is not int + or type(total) is not int or not 1 <= ordinal <= total <= 200 or attempt not in (1, 2)): + raise ValueError("Invalid active file.") + active = {"ordinal": ordinal, "total": total, "attempt": attempt} + if not self.enabled or self.output_failed: + yield + return + started = self.clock() + stop = threading.Event() + + def pulse(): + while not stop.wait(FILE_HEARTBEAT_SECONDS): + if self.output_failed: + return + self._emit("processing", active=active, active_started=started) + + self._emit("processing", active=active, active_started=started) + worker = threading.Thread(target=pulse, name="foundry-file-progress", daemon=True) + worker.start() + try: + yield + finally: + stop.set() + worker.join() + + def _emit(self, state, *, wait=None, active=None, active_started=None): + if not self.enabled or self.output_failed: + return + observed = self.clock() + # Clock regressions/nonfinite observations must not create negative time + # or bypass throttling. Progress never shares the service deadline clock. + if math.isfinite(observed): + if self.started is None: + self.started = observed + delta = observed - self.started + if math.isfinite(delta): + self.elapsed = max(self.elapsed, delta) + phase = self.blob_progress["phase"] if self.blob_progress is not None else None + if (state == "running" and self.stage == self.last_stage and phase == self.last_blob_phase + and self.last_sent is not None and self.elapsed - self.last_sent < 1.0): + return + event = { + "event": "progress", "workflow": self.workflow, "activity": self.stage, + "state": state, "elapsed_seconds": round(self.elapsed, 3), + "remaining_checks": [] if state == "completed" else list(self.stages[self.stages.index(self.stage) + 1:]), + } + if self.counts: + event["completed_counts"] = dict(self.counts) + if wait is not None: + event["wait"] = wait + if active is not None: + duration = observed - active_started + event["processing_file"] = {**active, "elapsed_seconds": round(max(0, duration), 3) if math.isfinite(duration) else 0} + if self.blob_progress is not None: + value = self.blob_progress + event["blob_progress"] = copy.deepcopy(value) + counts = "; ".join(f"{field}={value[field] if value[field] is not None else 'unknown'}" + for field in ("processed", "failed", "skipped")) + next_check = ("paused/resumable" if value["phase"] == "paused" else "none") if value["next_check_seconds"] is None else f"{value['next_check_seconds']:g}s" + event["message"] = ( + f"Blob ingestion observation {value['phase']}: item updates {counts}; file total/remaining unknown " + f"(not comparable); elapsed={value['elapsed_seconds']:g}s; next check={next_check}." + ) + stream = self.stream if self.stream is not None else sys.stderr + try: + text = json.dumps(event, sort_keys=True, separators=(",", ":")) + "\n" + if stream.write(text) != len(text): + raise OSError("Short progress write.") + stream.flush() + except (OSError, UnicodeError, ValueError): + # Continue the approved operation, retaining a fixed secondary warning + # even when stderr itself is broken. Never replace a primary failure. + self.output_failed = True + self._disable_failed_default_stderr(stream) + self.last_sent, self.last_stage = self.elapsed, self.stage + self.last_blob_phase = phase + + def _disable_failed_default_stderr(self, stream): + if self.stream is not None or stream is not sys.__stderr__ or stream.closed: + return + try: + if stream.fileno() == 2: + # A failed TextIOWrapper flush can retain pending bytes. Redirect + # only the failed native stderr so shutdown can drain that buffer + # without replacing the authoritative process exit status. + sink = os.open(os.devnull, os.O_WRONLY) + try: + os.dup2(sink, 2) + finally: + if sink != 2: + os.close(sink) + except (OSError, ValueError): + self.shutdown_flush_unresolved = True + + def finish(self, result=None, failure=None): + if self.blob_progress is not None and failure is not None: + self.blob_progress.update(phase="failed", next_check_seconds=None) + if failure is not None: + state = "partial" if failure.partial or failure.writes else "blocked" + else: + state = {"verified": "completed", "unverified": "blocked"}.get(result["status"], result["status"]) + if state not in {"completed", "blocked", "partial"}: + raise ValueError("Invalid progress terminal state.") + self._emit(state) + if self.output_failed: + warnings = failure.warnings if failure is not None else result.setdefault("warnings", []) + if IO_WARNING not in warnings: + warnings.append(IO_WARNING) + if self.shutdown_flush_unresolved and SHUTDOWN_WARNING not in warnings: + warnings.append(SHUTDOWN_WARNING) + + +def reporting(workflow): + """One reporter and terminal event across nested public entrypoints.""" + def decorate(function): + @wraps(function) + def wrapped(*args, progress=None, **kwargs): + if progress is None: + progress = Progress(workflow, enabled=False) + outer = progress.depth == 0 + progress.depth += 1 + try: + result = function(*args, progress=progress, **kwargs) + except HelperFailure as failure: + if outer: + progress.finish(failure=failure) + raise + else: + if outer: + progress.finish(result=result) + return result + finally: + progress.depth -= 1 + return wrapped + return decorate + + +def add_progress_argument(parser): + parser.add_argument("--no-progress", dest="progress", action="store_false", + help="Suppress content-free execution progress on stderr.") diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_prompt_read.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_prompt_read.py new file mode 100644 index 00000000..897f3e08 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/_prompt_read.py @@ -0,0 +1,230 @@ +"""Read-only, resource-specific prerequisites for Prompt connection planning/apply.""" +from __future__ import annotations + +import copy +import json +import re +from typing import Any +from urllib.parse import parse_qs, quote, unquote, urlsplit + +try: + from ._bootstrap_io import run_cli + from ._common import ( + MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, HelperFailure, digest, + require_allowed_fields, + ) +except ImportError: + from _bootstrap_io import run_cli + from _common import ( + MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, HelperFailure, digest, + require_allowed_fields, + ) + +PROJECT_API = "2025-10-01-preview" +SEARCH_API = "2025-05-01" +ROLE_API = "2022-04-01" +READER_ROLE = "1407120a-92aa-4202-b7e9-c0e197c71c8f" +NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}") +GUID = re.compile(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}") +SEARCH_ID = re.compile( + r"/subscriptions/[^/?#\s]+/resourceGroups/[^/?#\s]+/" + r"providers/Microsoft\.Search/searchServices/(?P[a-z0-9-]{2,60})", + re.IGNORECASE, +) + + +def fail(code: str, message: str, response=None, *, status=None, request_id=None) -> HelperFailure: + return HelperFailure( + code, message, blocked_at="reconciliation", + status=response.status if response is not None else status, + request_id=response.request_id if response is not None else request_id, + ) + + +def binding(plan: dict[str, Any]) -> tuple[str, str, str]: + scope = plan["rbac_verified"].get("scope") + match = SEARCH_ID.fullmatch(scope) if isinstance(scope, str) else None + target = urlsplit(plan["connection"]["target"]) + parts = target.path.split("/") + if ( + match is None or target.hostname != match["name"].lower() + ".search.windows.net" + or len(parts) != 4 or parts[1] != "knowledgebases" or parts[3] != "mcp" + or parse_qs(target.query) != {"api-version": ["2026-08-01-preview"]} + or not NAME.fullmatch(unquote(parts[2])) + ): + raise fail("kb-binding-invalid", "Bind an exact Search service ID and preview KB MCP name, not a generated index.") + assignment = plan["rbac_verified"].get("assignment_id") + prefix = scope + "/providers/Microsoft.Authorization/roleAssignments/" + if ( + not isinstance(assignment, str) or not assignment.casefold().startswith(prefix.casefold()) + or not GUID.fullmatch(assignment[len(prefix):]) + ): + raise fail("rbac-scope-invalid", "Select one exact Search-service-scoped role assignment resource ID.") + return scope, "https://" + target.hostname, unquote(parts[2]) + + +def get_object(url, token, transport, *, absent=False, label="resource"): + try: + result = transport( + "GET", url, token, follow_redirects=False, max_response_bytes=1024 * 1024, + ) + except HelperFailure as error: + if absent and error.http_status == 404 and not error.partial: + return None, [error.request_id] if error.request_id else [] + raise + if result.status == 404 and absent: + return None, [result.request_id] if result.request_id else [] + if result.status != 200: + raise fail(label + "-unavailable", "Exact " + label + " read failed; no fallback or inferred absence.", result) + if not isinstance(result.body, dict) or not result.body: + raise fail(label + "-readback-invalid", "Exact " + label + " readback must be a nonempty object.", result) + return result.body, [result.request_id] if result.request_id else [] + + +def cli_context(project_id, *, cli=run_cli): + rc, out, _ = cli(["account", "show"], 60) + if rc: + raise fail("cli-context-unavailable", "The selected signed-in Azure CLI context could not be read.") + try: + value = json.loads(out) + except (UnicodeError, ValueError) as error: + raise fail("cli-context-invalid", "Azure CLI context is malformed.") from error + user = value.get("user") if isinstance(value, dict) else None + if ( + not isinstance(user, dict) or user.get("type") not in {"user", "servicePrincipal"} + or not isinstance(user.get("name"), str) or not 1 <= len(user["name"]) <= 256 + or value.get("environmentName") != "AzureCloud" + or str(value.get("state", "")).casefold() != "enabled" + or str(value.get("id", "")).casefold() != project_id.split("/")[2].casefold() + or not isinstance(value.get("tenantId"), str) or not GUID.fullmatch(value["tenantId"]) + ): + raise fail("cli-context-conflict", "Select the project's enabled subscription and tenant without changing identity.") + return {"subscription_id": value["id"].lower(), "tenant_id": value["tenantId"].lower(), + "principal": user["name"], "principal_type": user["type"]} + + +def kb_state(value, name): + if value.get("name") != name: + raise fail("kb-identity-mismatch", "The exact knowledgebases API returned another KB identity.") + sources, models = value.get("knowledgeSources"), value.get("models", []) + if models is None: + models = [] + effort = value.get("retrievalReasoningEffort") + mode = value.get("outputMode") + if ( + not isinstance(sources, list) or not 1 <= len(sources) <= 200 + or any(not isinstance(item, dict) or not isinstance(item.get("name"), str) + or not NAME.fullmatch(item["name"]) for item in sources) + or len({item["name"] for item in sources}) != len(sources) + or not isinstance(models, list) or any( + not isinstance(item, dict) or not isinstance(item.get("@odata.type"), str) + or not item["@odata.type"].strip() + for item in models + ) + or not isinstance(effort, dict) or effort.get("kind") not in {"minimal", "low", "medium"} + or mode not in {"extractiveData", "answerSynthesis"} + ): + raise fail("kb-configuration-unresolved", "Read complete KB sources, model configuration, reasoning and output; never infer them from generated indexes.") + if not models and (effort["kind"] != "minimal" or mode != "extractiveData"): + raise fail("kb-model-required", "This existing KB configuration requires a KB chat model; connection planning never changes its mode or models.") + material = copy.deepcopy(value) + for field in ("@odata.etag", "description", "tags"): + material.pop(field, None) + return {"name": name, "definition_digest": digest(material)}, { + "source_count": len(sources), "model_configured": bool(models), + "reasoning": effort["kind"], "output": mode, + } + + +def read_dependencies(plan, *, token_provider, transport, cli=run_cli, capture_context=False): + scope, endpoint, kb_name = binding(plan) + project_id = plan["project_resource_id"] + context = cli_context(project_id, cli=cli) if capture_context else None + token = token_provider(MANAGEMENT_AUDIENCE) + request_ids = [] + project, ids = get_object(MANAGEMENT_AUDIENCE + project_id + "?api-version=" + PROJECT_API, + token, transport, label="project") + request_ids.extend(ids) + identity, props = project.get("identity"), project.get("properties") + endpoints = props.get("endpoints") if isinstance(props, dict) else None + identity_types = identity.get("type") if isinstance(identity, dict) else None + identity_types = {item.strip() for item in identity_types.split(",")} if isinstance(identity_types, str) else set() + if ( + str(project.get("id", "")).casefold() != project_id.casefold() + or not isinstance(identity, dict) or identity_types not in ({"SystemAssigned"}, {"SystemAssigned", "UserAssigned"}) + or not isinstance(identity.get("principalId"), str) or not GUID.fullmatch(identity["principalId"]) + or not isinstance(identity.get("tenantId"), str) or not GUID.fullmatch(identity["tenantId"]) + or not isinstance(props, dict) or str(props.get("provisioningState", "")).casefold() != "succeeded" + or not isinstance(endpoints, dict) or plan["project_endpoint"].rstrip("/") not in { + value.rstrip("/") for value in endpoints.values() if isinstance(value, str) + } + ): + raise fail("project-identity-unverified", "Require the selected ready Foundry PROJECT endpoint and its system-assigned principalId/tenantId.", + request_id=ids[-1] if ids else None) + if context is not None and context["tenant_id"] != identity["tenantId"].lower(): + raise fail("cli-context-conflict", "CLI tenant differs from the observed project identity tenant.") + project_state = {"id": project_id, "endpoint": plan["project_endpoint"].rstrip("/"), + "principal_id": identity["principalId"].lower(), "tenant_id": identity["tenantId"].lower()} + + search, ids = get_object(MANAGEMENT_AUDIENCE + scope + "?api-version=" + SEARCH_API, + token, transport, label="search") + request_ids.extend(ids) + props = search.get("properties") + if str(search.get("id", "")).casefold() != scope.casefold() or not isinstance(props, dict): + raise fail("search-identity-unverified", "The selected Search resource identity is unresolved.", request_id=ids[-1] if ids else None) + status, provisioning = str(props.get("status", "")).lower(), str(props.get("provisioningState", "")).lower() + if status not in {"running", "provisioning", "degraded"} or provisioning not in {"succeeded", "provisioning"}: + raise fail("search-operation-blocked", "Search is failed, disabled, deleting or unresolved; no connection write is allowed.", + request_id=ids[-1] if ids else None) + warnings = [] if status == "running" and provisioning == "succeeded" else [ + "Search is provisioning/degraded; healthy KB GET permits connection configuration only, not readiness or retrieval proof." + ] + search_state = {"id": scope, "endpoint": endpoint, "access_digest": digest({ + key: props.get(key) for key in ("disableLocalAuth", "authOptions", "publicNetworkAccess", "networkRuleSet", "privateEndpointConnections") + })} + + assignment_id = plan["rbac_verified"]["assignment_id"] + assignment, ids = get_object(MANAGEMENT_AUDIENCE + assignment_id + "?api-version=" + ROLE_API, + token, transport, label="role-assignment") + request_ids.extend(ids) + role = assignment.get("properties") + if ( + str(assignment.get("id", "")).casefold() != assignment_id.casefold() + or not isinstance(role, dict) + or str(role.get("principalId", "")).casefold() != project_state["principal_id"] + or str(role.get("scope", "")).casefold() != scope.casefold() + or str(role.get("roleDefinitionId", "")).casefold() not in { + "/providers/microsoft.authorization/roledefinitions/" + READER_ROLE, + "/subscriptions/" + scope.split("/")[2].lower() + "/providers/microsoft.authorization/roledefinitions/" + READER_ROLE, + } + or role.get("principalType", "ServicePrincipal") != "ServicePrincipal" + or role.get("condition") not in (None, "") + ): + raise fail("project-reader-role-unverified", "Search Index Data Reader must be an unconditional exact-scope grant to the observed PROJECT principal, not the agent identity.", + request_id=ids[-1] if ids else None) + rbac = {"assignment_id": assignment_id, "principal_id": project_state["principal_id"], + "scope": scope, "role_definition_id": READER_ROLE} + url = endpoint + "/knowledgebases('" + quote(kb_name.replace("'", "''"), safe="") + "')?api-version=2026-08-01-preview" + kb, ids = get_object(url, token_provider(SEARCH_AUDIENCE), transport, absent=True, label="knowledge-base") + request_ids.extend(ids) + if kb is None: + raise fail("knowledge-base-absent", "The exact KB is absent; a generated index is not a knowledge base.", + status=404, request_id=ids[-1] if ids else None) + try: + knowledge_base, profile = kb_state(kb, kb_name) + except HelperFailure as error: + error.request_id = ids[-1] if ids else None + raise + state = {"project": project_state, "search": search_state, "rbac": rbac, "knowledge_base": knowledge_base} + if context is not None: + state["cli_context"] = context + expected = plan.get("verified_dependencies") + if expected is not None: + require_allowed_fields(expected, set(state), label="Verified Prompt dependencies") + if expected != state: + raise fail("connection-prerequisite-drift", "Project principal, CLI context, KB binding or exact role changed since planning; refresh before approval.") + elif plan["rbac_verified"].get("verified") is True and ( + str(plan["rbac_verified"]["principal_id"]).lower() != project_state["principal_id"] + ): + raise fail("project-reader-role-unverified", "The approved principal is not the observed Foundry PROJECT identity.") + return state, profile, warnings, request_ids diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob-contracts.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob-contracts.md new file mode 100644 index 00000000..29865368 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob-contracts.md @@ -0,0 +1,142 @@ +# Blob/ADLS contract + +[Blob progress](../references/blob-ingestion-progress.md): `--no-progress` disables stderr. + +## Planning + +```text +python helpers/blob_source.py --plan intent.json +``` + +Intent: + +```json +{"schema_version":"1.0","endpoint":"https://svc.search.windows.net","name":"manuals","owner":"owner@example.com","storage_id":"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/Example/providers/Microsoft.Storage/storageAccounts/example","container":"documents","prefix":"Reports/","is_adls":false,"api_version":"2026-04-01","processing":"minimal-lexical","network_access":"public","identity":"system-assigned","permission_options":[],"ingestion_schedule":null,"description":null,"rbac":{"assignments":[{"id":"","principalId":""}]},"network":{"posture":"public","evidence":""},"inventory_limits":{"max_pages":100,"max_objects":10000,"max_requests":25000,"deadline_seconds":600},"poll":{"deadline_seconds":600,"max_requests":120,"interval_seconds":5}} +``` + +Content fit (no default): `minimal-lexical`=minimal/no vectors, +`minimal-vector`=minimal/[vectors](vector-contracts.md), +`standard-cu`=standard/optional vectors; not KB reasoning. +Public/system-assigned; no schedule settings/permission ingestion. +Blob: `2026-04-01` or `2026-08-01-preview`; ADLS: preview, `is_adls: true`, +directory prefix without trailing `/`. Root: explicit `prefix: ""`, no widening. +Description: text/null; other modes: `planning-processing-unsupported`. + +No writes/auth changes/installs/polling/cleanup. Verify access/cost/data preapproval. + +Two exact Search GETs bracket two-pass Storage observation. Reuse +requires matching definition/ETag, Storage binding and four generated identities. +`reuse_input_file`/`reuse_result_file`: retained approved input/successful creation +result. Validate pairing/types before auth; load only for redacted binding. +Required fingerprints/boundaries/source ETags/definitions/generated identities +must match fresh readback. Unproven binding: `source-binding-unverified`. +Old consent/readiness never authorizes new work. + +Exit `0`: `status: planned`, `plan_fingerprint`, `execution_input`, +`approval_summary`, request IDs and `writes_performed: []`. Save only +`execution_input`; its approval is false and the executor rejects it until +actual unchanged-plan approval. Private artifacts preserve ResourceId connection +strings/fingerprints; ordinary result redaction is unchanged. +Summaries omit paths/ACLs/hashes; no readiness. Exact reuse sets `execution_required` and +`mutation_approval_required` false: no executor or approval needed. +Drift: rerun `--plan`, replace the artifact, discard consent; never hand-edit +nested plans/hashes. Cleanup stays separate and run-owned-only. + +For `standard-cu`, read [CU choices/wire](blob-cu-contracts.md). +`content_understanding` binds CU; optional `embedding` selects source vectors. + +## Discovery + +`python helpers/blob_source.py --discover ` accepts this closed +JSON (no approval): + +```json +{"boundary":{"storage_id":"","container":"","prefix":"Reports/","is_adls":false},"inventory_limits":{"max_pages":100,"max_objects":10000,"max_requests":25000,"deadline_seconds":600}} +``` + +Arbitrary partial-name prefixes: `boundary-ambiguous`. +Public cloud/private DNS. + +Read all pages twice including empty continuations. Exit `0` returns +`status: discovered`, `mutation: none`, `writes_performed: []`, boundary/account, +sorted objects (URL/path/size/ETag/version), ADLS path/property digests, +request IDs, `inventory_digest`; not approval/ingestion. +Incomplete/duplicate/out-of-scope/unreadable/unstable evidence blocks. +ADLS HEADs path/ancestor ACLs; no raw ACL/content. Operator access is not Search access. +Every path has an ACL HEAD; every non-root path also has a properties HEAD for +type and matching ETag. Root has no type header. Preserve ancestors and normalize +quoted/unquoted Blob/DFS ETags only for comparison; never skip required reads. + +Inventory bounds apply per discovery: pages/objects per pass; requests/time +across both passes, including empty pages and all HEADs. Maximum accepted limits +are 1000 pages, 100000 objects, 200000 requests and 3600 seconds; exceeding any +bound blocks, never truncates. Planning adds at most two Search GETs and shares +its time budget with discovery. Bodies are capped at 8 MiB plus one overflow +probe byte. Socket watchdog interrupts body reads; DNS/connect/header +acquisition and scheduling are not hard wall-clock bounded. Redirects and +unsupported deadline readers block. Limits are safety bounds, not spending consent. + +## Apply + +`python helpers/blob_source.py --input ` uses: + +```json +{"schema_version":"1.0","plan":{},"approval":{"confirmed":true,"fingerprint":"sha256:"}} +``` + +Canonical JSON: UTF-8, sorted keys, ASCII-escaped/compact. +`operation: reconcile-and-monitor`, `owner`, +`cleanup_approved: false`, discovered `boundary`, `inventory_digest`, +`inventory_limits`, `source`, +`poll: {"deadline_seconds":600,"max_requests":120,"interval_seconds":5}`. +Planner reuse: `expected_generated`, four typed/name/service-managed identities, +checked before writes; legacy artifacts remain valid. +Planner creation sets `expected_source_absent: true`; an intervening source +blocks. Redacted GET requires the PUT ETag (body/header); conflicts block. +ambiguous unproven ownership remains partial, never adopted or cleaned up. + +`source`: `operation: reconcile`, `resource_type: +knowledge-source`, exact `endpoint`/`name`/`api_version`, `action: create` +or `reuse`, matching `owner`, `cleanup_approved: false`, approved +`desired`, and `source_evidence` with `verified: true` and the same +`inventory_digest`. ADLS also requires `path_verified: true`, `acl_verified: +true` attest metadata, not effective Search permissions. + +`desired`: matching `name`, `kind: azureBlob`; `azureBlobParameters`: +`connectionString: ResourceId=`, `containerName`, +exact `folderPath` (`null` for explicit root), boolean `isADLSGen2`, +`ingestionParameters`. Optional [shared controls](contracts.md). +No keys/SAS/updates/credential environments or supplied `createdResources`/`assetStore`. + +Recompute approval/evidence; block drift. Create conditionally/reuse zero-write; +supplied ETag must match. +Poll GET `knowledgesources('{name}')/status` within bounds. +Only 408/429/500/502/503/504/transport-ambiguous reads retry; 403/404 block. +Watch expiry pauses; it does not fail ingestion. Backoff and explicit GET-only +continuation/compact output: [recheck](../references/blob-readiness-recheck.md). + +Require completed start/end times at or after the pre-write cutoff, +nonnegative integer `itemsUpdatesProcessed`, `itemsUpdatesFailed`, and +`itemsSkipped`, zero failures, no observed relevant errors or active cycle. +Missing counts are not zero. +Completed `status` may be absent; reject `partialSuccess`/`failure`/malformed/unknown. +Never trigger indexers/edit schedules; stale success cannot pass. +Reuse also excludes the first observed completion. + +## Results + +`--receipt-dir`: PUT acknowledgement before GET. +[Binding/recovery](../references/blob-binding-evidence.md): redaction, recovery, +reuse; never reconstruct cutoffs. +Timing differences: [indexer checks](../references/blob-indexer-observation.md). + +Exit `0`: `completed`; unverified zero-write reuse is exit `2`, +`blocked` with `reconciliation: completed`. After a write, exit `3` / `partial` +retains approval/first failure/`resources_remaining.run_owned`. +Conflicting children aren't owned. Return generated identities/readiness/digests/ +warnings/cleanup; errors retain status/request ID, not sensitive text. +No rollback/source writes/child edits. + +Owner handles RBAC/private links. CLI Entra: ARM/Storage/Search audiences. +Probes: ARM GET `2025-06-01`, Storage `2025-05-05` listing, +ADLS HEAD `action=getAccessControl&upn=false`. diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob-cu-contracts.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob-cu-contracts.md new file mode 100644 index 00000000..160e9f8a --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob-cu-contracts.md @@ -0,0 +1,132 @@ +# Optional Blob Content Understanding branch + +Read only for `processing: standard-cu` in the [Blob procedure](../knowledge-sources/create-azure-blob.md). +Selected Search/Storage/AI Services only; no provisioning, uploads, asset store, +permission changes or KB creation. Resolve processing before planning; never +downgrade CU or add it to minimal-vector. + +## Availability and dependencies + +Read [shared CU ingestion](../references/cu-ingestion.md) for independent model +purposes, necessary checks and non-executable pending-dependency drafts. +For portal/required-field questions, +use the [API/service/helper settings matrix](../references/standard-cu.md#source-settings-api-versus-helper). + +Blob requires CU-capable `AIServices` in a supported region with an embedding deployment, +not Search `embeddingModel` or source vectors. +Image verbalization/source chat are disabled. +Ingestion does not choose KB reasoning, reranking, query mode or synthesis. + +Use the existing Search system-assigned identity with Cognitive Services User +on that account; Storage stays ResourceId/keyless with Storage +Blob Data Reader. Do not copy File CU credential channels or enable local auth. +The Blob documentation's `aiServices` keyless configuration omits `apiKey`. +Keys, user-assigned identities, private networking, schedules, asset stores and +permission ingestion are outside this standard branch. + +Verify CU capability/region/deployments, effective Search identity/RBAC and +reachability through exact readbacks. +`configuration` references selected processing/required deployment evidence. +Existing dependencies permit planning; no setup-owner or `azd` prerequisite. +Do not ask for a separate account-configuration confirmation. +Default mappings need contract/observed-error evidence; never change them. +`cu-prerequisite-missing`: name missing evidence, not a generic setup gate. +No provisioning/model tests or default extraction downgrade. + +## Closed intent + +Use the ordinary [Blob intent](blob-contracts.md#planning), set +`processing: standard-cu` and add this required object: + +```json +{"content_understanding":{"endpoint":"https://models.services.ai.azure.com","auth":"system-assigned","prerequisites":{"resource":"","configuration":"","identity":"","network":""}}} +``` + +Fields required; references: 1–4096 characters. +Omit `embedding` (or use null in the intent) to leave source vectors off. +Only when selected, add the closed [embedding choice](vector-contracts.md). +CU and source-vector endpoints are independently validated; they need not match. +Combine CU/auth/source/vector costs and data consent; no manual credential/MI +confirmation. Missing setup uses approved native bootstrap. +A single trailing slash is immaterial. +Custom CU endpoints, URL paths/queries/ports and credentials block. +Minimal modes omit `content_understanding`. +Private artifacts retain choices; summaries omit references, which are attestations, not CU calls. + +New CU plans emit fingerprint-bound `cu_plan_version: "1.0"`. This CU-specific +marker requires the CU choice and exact presence/value binding of optional embeddings. +Legacy approved wire-only standard artifacts, including `expected_source_absent` +and `expected_generated`, retain their original admission, guards and fingerprints. +Those generic guards predate CU planning; they do not select the new contract. +Never rewrite historical approved artifacts/receipts. Removing choices from a +marked CU artifact blocks if CU is missing or the wire still requests removed +embeddings; changing/removing its marker invalidates existing +approval. No new field is sent in the Search request. + +## Wire and approval + +Both GA `2026-04-01` and preview `2026-08-01-preview` +expose `azureBlobParameters.ingestionParameters` with +`contentExtractionMode: standard`, `aiServices: {"uri": ""}`, +`disableImageVerbalization: true`, null identity/schedule. +Set `embeddingModel` only for selected source vectorization; otherwise omit it. +Preview also sets public networking and empty permission options. +ADLS remains preview-only in this helper with unchanged path/ACL controls. + +Use the same `blob_source.py --plan` → private unapproved `execution_input` → +one explicit unchanged-plan approval → `--input` flow. Execution uses conditional +PUT `/knowledgesources('')?api-version=` with +`If-None-Match: *`, exact definition GET and status GET; no direct CU API call. +Never overwrite or suffix around a collision. + +CU creation also retains approved input and uses `--receipt-dir`. +Timeout/interruption: [read-only recheck](../references/blob-readiness-recheck.md), +not another PUT or direct CU call. Fresh reuse capture carries no creation ownership. + +Show combined source/CU/embedding cost, access and processing changes before +approval. CU is billable with no free document allowance; selected documents +move to CU and, only if selected, source embeddings, possibly across regions. +Search retains outputs. +The CU skill uses underlying CU `2025-11-01`; Search manages that call, not a user-selectable +version. Both GA and preview Swagger +declare `embeddingModel` optional/nullable, with no required or conditional +standard-extraction vectorizer constraint. CU deployment prerequisites +must not be substituted for Search wire-field requirements. +Search and CU limits apply, including five-minute analysis timeout with possible charges. +No universal quality claim. Acceptance does not prove CU support; see +[format limits](../references/platform-interfaces.md#source-formats). + +## Verification and failure + +Require exact CU mode/endpoint/auth readback. Allow only absent/null/empty-string +`aiServices.apiKey` and null/absent ingestion identity. Masked/nonempty/malformed +keys or another identity return `cu-auth-conflict`, without echoing values. +Known null metadata, a single endpoint trailing slash and the documented +`resultsProcessing: rerank` default do not require manual plan edits. +Different endpoints, models, extraction, auth, scope, ETags or generated identities +block; rerun planning and discard old consent. + +Acceptance proves configuration only. The owning executor requires a relevant +completed zero-failure ingestion cycle, not `active` or old success. +[Vector verification](vector-contracts.md#read-only-verification-plan), only when configured, additionally +observes generated dimensions/profile/vectorizer and nonempty zero-skip readiness. +Retrieval needs its separately approved query; neither proves extraction quality +or corpus-wide relevance. CU-only vector probes return `embedding-not-configured`, +not “CU unsupported.” Fresh exact reuse remains mutation-approval-free, +not ingestion/retrieval proof. Tests/mocks are not live evidence. + +Keep the first failure/request ID. A failed post-write check is partial with +run-owned source/children retained; never automatically delete anything. +Use existing separately approved source cleanup only, never Storage, +dependencies, shared resources or role assignments. + +## Authorities + +Authorities: failure/conflict/uncertainty only. + +[Blob prerequisites](https://learn.microsoft.com/azure/search/agentic-knowledge-source-how-to-blob#prerequisites), +[GA create](https://learn.microsoft.com/rest/api/searchservice/knowledge-sources/create?view=rest-searchservice-2026-04-01), +[preview create](https://learn.microsoft.com/rest/api/searchservice/knowledge-sources/create?view=rest-searchservice-2026-08-01-preview), +[CU skill](https://learn.microsoft.com/azure/search/cognitive-search-skill-content-understanding), +[GA Swagger](https://github.com/Azure/azure-rest-api-specs/blob/main/specification/search/data-plane/Search/stable/2026-04-01/search.json), +[preview Swagger](https://github.com/Azure/azure-rest-api-specs/blob/main/specification/search/data-plane/Search/preview/2026-08-01-preview/search.json). diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_inventory.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_inventory.py new file mode 100644 index 00000000..9814c864 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_inventory.py @@ -0,0 +1,330 @@ +from __future__ import annotations + +import re +import time +from typing import Any, Callable +from urllib.parse import quote, unquote, urlencode +from xml.etree import ElementTree + +try: + from ._common import ( + MANAGEMENT_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, + digest, http_request, reject_secrets, require_allowed_fields, + ) +except ImportError: + from _common import ( # type: ignore[no-redef] + MANAGEMENT_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, + digest, http_request, reject_secrets, require_allowed_fields, + ) + + +STORAGE_AUDIENCE = "https://storage.azure.com/" +ARM_VERSION = "2025-06-01" +STORAGE_VERSION = "2025-05-05" +ACCOUNT_ID = re.compile( + r"^/subscriptions/[0-9a-fA-F-]{36}/resourceGroups/[^/;?#\r\n]+/" + r"providers/Microsoft\.Storage/storageAccounts/([a-z0-9]{3,24})$" +) +CONTAINER = re.compile(r"^[a-z0-9](?:[a-z0-9-]{1,61}[a-z0-9])$") + + +def _fail(code: str, message: str, request_id: str | None = None) -> HelperFailure: + return HelperFailure(code, message, blocked_at="source-preflight", request_id=request_id) + + +def _strip_etag_quotes(value: str) -> str: + # Blob List XML returns unquoted ETags; DFS HTTP responses return + # RFC 7232-quoted ETags for the identical underlying value. + return value[1:-1] if len(value) >= 2 and value[0] == '"' and value[-1] == '"' else value + + +def validate_boundary(value: Any) -> dict[str, Any]: + if not isinstance(value, dict): + raise _fail("boundary-invalid", "An explicitly selected Storage boundary is required.") + reject_secrets(value) + require_allowed_fields(value, {"storage_id", "container", "prefix", "is_adls"}, label="boundary") + if ( + not isinstance(value.get("storage_id"), str) + or ACCOUNT_ID.fullmatch(value["storage_id"]) is None + or not isinstance(value.get("container"), str) + or CONTAINER.fullmatch(value["container"]) is None + or "--" in value["container"] + or not isinstance(value.get("prefix"), str) + or type(value.get("is_adls")) is not bool + ): + raise _fail("boundary-invalid", "Exact Storage ID, container, explicit prefix (including empty root), and subtype are required.") + prefix = value["prefix"] + try: + value["storage_id"].encode("utf-8") + prefix.encode("utf-8") + except UnicodeError as exc: + raise _fail("boundary-invalid", "Storage ID and prefix must be valid UTF-8.") from exc + if ( + prefix.startswith("/") + or "\\" in prefix + or any(ord(c) < 32 for c in prefix) + or (any(part in {".", "..", ""} for part in prefix.rstrip("/").split("/")) and prefix != "") + or "//" in prefix + or value["is_adls"] and prefix.endswith("/") + or not value["is_adls"] and prefix and not prefix.endswith("/") + ): + raise _fail( + "boundary-ambiguous", + "Use explicit empty root, a Blob folder prefix ending in '/', or an ADLS directory path without a trailing '/'. Arbitrary partial-name prefixes are not verified.", + ) + return dict(value) + + +def validate_limits(value: Any) -> dict[str, int]: + if not isinstance(value, dict): + raise _fail("input-schema-invalid", "inventory_limits must be an object.") + require_allowed_fields(value, {"max_pages", "max_objects", "max_requests", "deadline_seconds"}, + label="inventory limits") + for field, maximum in ( + ("max_pages", 1000), ("max_objects", 100000), ("max_requests", 200000), + ("deadline_seconds", 3600), + ): + if type(value.get(field)) is not int or not 1 <= value[field] <= maximum: + raise _fail("input-schema-invalid", "Inventory limits must be bounded positive integers.") + return value + + +class _Reader: + def __init__( + self, limits: dict[str, int], token_provider: TokenProvider, + transport: Transport, monotonic: Callable[[], float], + deadline: float | None = None, + ) -> None: + self.limits = limits + self.transport = transport + self.monotonic = monotonic + self.deadline = monotonic() + limits["deadline_seconds"] + if deadline is not None: + self.deadline = min(self.deadline, deadline) + self.token_provider = token_provider + self.tokens: dict[str, str] = {} + self.request_ids: list[str] = [] + self.requests = 0 + + def read(self, method: str, url: str, *, arm: bool = False, raw: bool = False) -> Any: + if method not in {"GET", "HEAD"}: + raise _fail("source-write-forbidden", "Storage probes may only read.") + remaining = self.deadline - self.monotonic() + if self.requests >= self.limits["max_requests"] or remaining <= 0: + raise _fail("inventory-incomplete", "Inventory exceeded the approved request or time bound.") + audience = MANAGEMENT_AUDIENCE if arm else STORAGE_AUDIENCE + if audience not in self.tokens: + self.tokens[audience] = self.token_provider(audience) + remaining = self.deadline - self.monotonic() + if remaining <= 0: + raise _fail("inventory-incomplete", "Inventory deadline elapsed during authentication.") + self.requests += 1 + try: + result = self.transport( + method, url, self.tokens[audience], timeout=min(30, remaining), + headers={} if arm else {"x-ms-version": STORAGE_VERSION, "Accept": "application/xml"}, + raw_response=raw, max_response_bytes=8 * 1024 * 1024, follow_redirects=False, + response_deadline=time.monotonic() + remaining, + ) + except HelperFailure as failure: + if failure.code in {"response-too-large", "response-deadline-exceeded"}: + raise HelperFailure( + "inventory-incomplete", "Source evidence exceeded the body or time bound.", + blocked_at="source-preflight", request_id=failure.request_id, status=failure.http_status, + ) from failure + raise HelperFailure( + "source-inaccessible", "Source evidence could not be read; inaccessible is not absent.", + blocked_at="source-preflight", request_id=failure.request_id, status=failure.http_status, + ) from failure + if result.request_id: + self.request_ids.append(result.request_id) + if result.status != 200: + raise HelperFailure( + "source-inaccessible", "Source evidence could not be read; inaccessible is not absent.", + blocked_at="source-preflight", request_id=result.request_id, status=result.status, + ) + if self.monotonic() >= self.deadline: + raise _fail("inventory-incomplete", "Inventory deadline elapsed during a read.", result.request_id) + return result + + +def _account(boundary: dict[str, Any], reader: _Reader) -> dict[str, Any]: + response = reader.read( + "GET", f"https://management.azure.com{quote(boundary['storage_id'], safe='/')}?api-version={ARM_VERSION}", arm=True, + ) + body = response.body + if not isinstance(body, dict) or body.get("id") != boundary["storage_id"]: + raise _fail("storage-identity-mismatch", "Storage readback does not match the selected resource.", response.request_id) + properties = body.get("properties") + if not isinstance(properties, dict): + raise _fail("hns-unverified", "Storage readback must explicitly establish HNS.", response.request_id) + # isHnsEnabled is set only at account creation and is immutable thereafter; ARM omits it + # from readback whenever it was never explicitly enabled, which documented behavior treats + # as a permanent, unambiguous false (Blob, not ADLS) rather than an unknown/undetermined state. + raw_hns = properties.get("isHnsEnabled") + if type(raw_hns) is not bool: + if raw_hns is not None: + raise _fail("hns-unverified", "Storage readback must explicitly establish HNS.", response.request_id) + hns_value = False + else: + hns_value = raw_hns + if hns_value != boundary["is_adls"]: + raise _fail("source-drift", "Storage HNS differs from the selected subtype.", response.request_id) + if properties.get("provisioningState") != "Succeeded": + raise _fail("storage-not-ready", "Storage account provisioning is not complete.", response.request_id) + endpoints = properties.get("primaryEndpoints") + account = boundary["storage_id"].rsplit("/", 1)[1] + selected_endpoints = {} + for kind in ("blob", "dfs") if boundary["is_adls"] else ("blob",): + expected = f"https://{account}.{kind}.core.windows.net/" + if not isinstance(endpoints, dict) or endpoints.get(kind) != expected: + raise _fail("storage-endpoint-unverified", "Storage service endpoint is not the selected public-cloud account endpoint.", response.request_id) + selected_endpoints[kind] = expected + return { + "id": body["id"], "hns": hns_value, "endpoints": selected_endpoints, + "network_digest": digest({k: properties.get(k) for k in + ("publicNetworkAccess", "networkAcls", "privateEndpointConnections")}), + } + + +def _object_name(element: Any) -> str: + if element is None or not isinstance(element.text, str) or not element.text: + raise _fail("inventory-invalid", "Listed object has no exact name.") + name = element.text + if element.get("Encoded") == "true": + if re.search(r"%(?![0-9A-Fa-f]{2})", name): + raise _fail("inventory-invalid", "Listed encoded name is ambiguous.") + try: + name = unquote(name, errors="strict") + except UnicodeError as exc: + raise _fail("inventory-invalid", "Listed encoded name is invalid.") from exc + elif element.get("Encoded") not in {None, "false"}: + raise _fail("inventory-invalid", "Unknown listed-name encoding.") + if any(ord(c) < 32 for c in name): + raise _fail("inventory-invalid", "Control characters in object names are unsupported.") + return name + + +def _objects( + boundary: dict[str, Any], account: dict[str, Any], reader: _Reader, +) -> list[dict[str, Any]]: + prefix = boundary["prefix"] + if boundary["is_adls"] and prefix: + prefix += "/" + base = account["endpoints"]["blob"] + boundary["container"] + marker = "" + markers: set[str] = set() + objects: dict[str, dict[str, Any]] = {} + for _ in range(reader.limits["max_pages"]): + query = {"restype": "container", "comp": "list", "prefix": prefix, + "maxresults": str(min(5000, reader.limits["max_objects"])), "marker": marker} + response = reader.read("GET", base + "?" + urlencode(query), raw=True) + payload = response.body + if not isinstance(payload, bytes) or b" reader.limits["max_objects"]: + raise _fail("inventory-incomplete", "Inventory exceeds the approved object bound.", response.request_id) + marker = root.findtext("NextMarker") or "" + if not marker: + return [objects[name] for name in sorted(objects)] + if marker in markers: + raise _fail("inventory-incomplete", "Storage repeated a continuation token.", response.request_id) + markers.add(marker) + raise _fail("inventory-incomplete", "Unconsumed pages exceed the approved page bound.") + + +def _adls_paths( + boundary: dict[str, Any], account: dict[str, Any], objects: list[dict[str, Any]], reader: _Reader, +) -> list[dict[str, Any]]: + paths: dict[str, str] = {"": "directory", boundary["prefix"]: "directory"} + identities = {item["path"]: item for item in objects} + for name, kind in [(boundary["prefix"], "directory")] + [(item["path"], item["kind"]) for item in objects]: + paths[name] = kind + parts = name.split("/") + for index in range(1, len(parts)): + paths["/".join(parts[:index])] = "directory" + records = [] + for path in sorted(paths): + base_url = account["endpoints"]["dfs"] + boundary["container"] + "/" + quote(path, safe="/") + acl_response = reader.read("HEAD", base_url + "?action=getAccessControl&upn=false") + headers = {key.lower(): value for key, value in acl_response.headers.items()} + fields = ("x-ms-owner", "x-ms-group", "x-ms-permissions", "x-ms-acl", "etag") + if any(not headers.get(field) for field in fields): + raise _fail("adls-evidence-unverified", "Exact ADLS path type and ACL/property readback are required.", acl_response.request_id) + # getAccessControl never returns x-ms-resource-type; the filesystem root itself has + # no resource type either, so only non-root paths can be, and must be, type-verified + # via a separate plain getProperties HEAD. + if path: + props_response = reader.read("HEAD", base_url) + properties = {key.lower(): value for key, value in props_response.headers.items()} + resource_type = properties.get("x-ms-resource-type") + if resource_type != paths[path]: + raise _fail("adls-evidence-unverified", "Exact ADLS path type and ACL/property readback are required.", props_response.request_id) + if ( + not properties.get("etag") + or _strip_etag_quotes(properties["etag"]) != _strip_etag_quotes(headers["etag"]) + ): + raise _fail("source-drift", "ADLS access-control and properties ETags disagree.", props_response.request_id) + if path in identities and _strip_etag_quotes(headers["etag"]) != _strip_etag_quotes(identities[path]["etag"]): + raise _fail("source-drift", "Blob and DFS path ETags disagree.", acl_response.request_id) + records.append({"path": path, "kind": paths[path], + "properties_digest": digest({field: headers[field] for field in fields})}) + return records + + +def _snapshot(boundary: dict[str, Any], reader: _Reader) -> dict[str, Any]: + account = _account(boundary, reader) + objects = _objects(boundary, account, reader) + adls = _adls_paths(boundary, account, objects, reader) if boundary["is_adls"] else [] + return {"boundary": boundary, "account": account, "objects": objects, "adls_paths": adls} + + +def discover( + boundary: Any, limits: Any, *, + token_provider: TokenProvider = azure_cli_token, transport: Transport = http_request, + monotonic: Callable[[], float] = time.monotonic, + deadline: float | None = None, +) -> dict[str, Any]: + boundary = validate_boundary(boundary) + reader = _Reader(validate_limits(limits), token_provider, transport, monotonic, deadline) + first = _snapshot(boundary, reader) + second = _snapshot(boundary, reader) + if digest(first) != digest(second): + raise _fail("source-drift", "Consecutive complete source observations differ; no stable evidence is available.") + return { + "status": "discovered", **second, "inventory_digest": digest(second), + "mutation": "none", "writes_performed": [], "request_ids": reader.request_ids, + "operator_reachability": "verified", "managed_ingestion_reachability": "not-proven", + "warnings": [ + "Observations are not an atomic Storage snapshot or a lock; source objects can change afterward.", + "ACL readback establishes observed metadata, not effective Search principal permissions.", + ], + } diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_recheck.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_recheck.py new file mode 100644 index 00000000..2b474bde --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_recheck.py @@ -0,0 +1,1131 @@ +"""Retained Blob creation/reuse observations and read-only readiness follow-up.""" +from __future__ import annotations + +import argparse +import copy +import json +import os +import re +import stat +import sys +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlsplit + +try: + from . import _bootstrap_io as private_io + from ._progress import Progress, add_progress_argument, reporting + from . import blob_inventory, blob_source, search_reconcile, source_vector, _indexer_observation as indexer + from . import _blob_observation as semantic + from ._common import ( + HelperFailure, SEARCH_AUDIENCE, azure_cli_token, blocked_result, digest, + emit_result, http_request, reject_secrets, + ) +except ImportError: + import _bootstrap_io as private_io + from _progress import Progress, add_progress_argument, reporting + import blob_inventory, blob_source, search_reconcile, source_vector + import _indexer_observation as indexer + import _blob_observation as semantic + from _common import ( + HelperFailure, SEARCH_AUDIENCE, azure_cli_token, blocked_result, digest, + emit_result, http_request, reject_secrets, + ) + + +fail = blob_source._failure +COLLECTIONS = {"datasource": "datasources", "indexer": "indexers", + "skillset": "skillsets", "index": "indexes"} + + +def _json(raw): + def unique(pairs): + value = {} + for key, child in pairs: + if key in value: + raise ValueError("Duplicate field") + value[key] = child + return value + try: + value = json.loads(raw, object_pairs_hook=unique) + json.dumps(value, allow_nan=False, ensure_ascii=False).encode("utf-8") + if not isinstance(value, dict): + raise ValueError("Expected object") + return value + except (ValueError, UnicodeError, RecursionError) as exc: + raise fail("recheck-evidence-invalid", "Retain bounded, unmodified UTF-8 object evidence.") from exc + + +def read_private(path): + path = Path(path) + private_io.private_directory(str(path.parent)) + try: + selected = path.lstat() + if (not stat.S_ISREG(selected.st_mode) or selected.st_nlink != 1 + or getattr(selected, "st_file_attributes", 0) & 0x400): + raise OSError("Not an ordinary private file") + if os.name == "nt": + private_io._windows_private(path) + elif selected.st_uid != os.getuid() or stat.S_IMODE(selected.st_mode) & 0o077: + raise OSError("Not private") + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + with os.fdopen(descriptor, "rb") as handle: + opened = os.fstat(handle.fileno()) + if (selected.st_dev, selected.st_ino) != (opened.st_dev, opened.st_ino): + raise OSError("Evidence changed identity") + raw = handle.read(private_io.MAX_BYTES + 1) + if len(raw) > private_io.MAX_BYTES: + raise OSError("Evidence exceeds bound") + return _json(raw.decode("utf-8")) + except (OSError, UnicodeError) as exc: + raise fail("recheck-evidence-unreadable", "Select existing private, unlinked evidence files; no permissions were changed.") from exc + + +def account_context(): + code, stdout, _ = private_io.run_cli(["account", "show"], 30) + if code: + raise fail("recheck-auth-context-unavailable", "Current signed-in CLI context is inaccessible; details withheld.") + account = _json(stdout) + user = account.get("user") + if (account.get("environmentName") != "AzureCloud" or account.get("state") != "Enabled" + or not isinstance(user, dict) or user.get("type") not in ("user", "servicePrincipal") + or any(not isinstance(value, str) or not value.strip() for value in + (account.get("id"), account.get("tenantId"), user.get("name")))): + raise fail("recheck-auth-context-unavailable", "An enabled public-cloud tenant/subscription/principal context is required.") + return digest({key: account[key] for key in ("id", "tenantId", "environmentName")} + | {"principal": {"name": user["name"], "type": user["type"]}}) + + +def _supported(plan): + try: + source, _ = blob_source._validate_plan(plan) + ingestion = source["desired"]["azureBlobParameters"]["ingestionParameters"] + except (KeyError, TypeError, AttributeError, RecursionError) as exc: + raise fail("recheck-evidence-invalid", "Original Blob creation plan is malformed.") from exc + if (ingestion.get("contentExtractionMode") not in ("minimal", "standard") or ingestion.get("identity") is not None + or source["api_version"] not in {"2026-04-01", "2026-08-01-preview"} + or plan["boundary"]["is_adls"] and source["api_version"] != "2026-08-01-preview"): + raise fail("recheck-scope-unsupported", "Checkpointing requires supported Blob extraction and provable system-assigned authentication.") + schedule = ingestion.get("ingestionSchedule") + indexer.schedule(schedule) + + +def _etag(value): + etag = value.get("@odata.etag") if isinstance(value, dict) else None + if not isinstance(etag, str) or not etag.strip(): + raise fail("recheck-evidence-missing", "Every source/generated readback requires a nonempty string ETag.") + return etag + + +def _processing_readback(plan, current): + desired = plan["source"]["desired"]["azureBlobParameters"]["ingestionParameters"] + if desired.get("contentExtractionMode") == "standard": + ai = desired.get("aiServices") + if not isinstance(ai, dict) or not isinstance(ai.get("uri"), str) or not ai["uri"].strip(): + raise fail("recheck-processing-unverified", "Standard extraction needs its original CU endpoint evidence.") + blob_source.verify_content_understanding_readback({"endpoint": ai["uri"]}, current) + source_vector.verify_source_readback(plan.get("embedding"), current) + model = desired.get("embeddingModel") + if model is not None and "embedding" not in plan: + parameters = model.get("azureOpenAIParameters") if isinstance(model, dict) else None + if (not isinstance(parameters, dict) or model.get("kind") != "azureOpenAI" + or any(not isinstance(parameters.get(key), str) or not parameters[key].strip() + for key in ("resourceUri", "deploymentId", "modelName"))): + raise fail("recheck-processing-unverified", "Legacy embedding configuration cannot be verified by this client.") + source_vector.verify_source_readback({ + "endpoint": parameters["resourceUri"], "deployment": parameters["deploymentId"], + "model": parameters["modelName"], + }, current) + + +def _binding(plan, record, current, generated, binding_receipts=None): + source = plan["source"] + observed = {"type": "knowledge-source", "name": source["name"], "etag": _etag(current), + "definition_digest": digest(search_reconcile._definition(current))} + if source["action"] == "create": + blob_source._verify_creation_binding(source, plan["boundary"], current, generated, (plan, record)) + elif (record["verification"]["readback"] != observed + or not search_reconcile.definitions_match(source["desired"], current) + or source.get("expected_etag", observed["etag"]) != observed["etag"]): + raise fail("source-binding-unverified", "Reused source does not match its retained read-only identity/configuration.") + if (blob_source.generated_resources(current, strict=True) != generated + or plan.get("expected_generated", generated) != generated): + raise fail("definition-drift", "Generated source identities changed.") + if binding_receipts is not None and ( + binding_receipts[0]["owner"] != plan["owner"] + or binding_receipts[0]["inventory_digest"] != plan["inventory_digest"] + ): + raise fail("source-binding-unverified", "Retained creation must bind the same owner and Storage inventory.") + blob_source._verify_storage_binding( + source, plan["boundary"], current, generated, + lambda: (plan, record) if source["action"] == "create" else binding_receipts, + ) + _processing_readback(plan, current) + + +def _sanitized_datasource_url(url): + parsed = urlsplit(url) + search_reconcile.validate_search_endpoint(f"{parsed.scheme}://{parsed.netloc}") + if (parsed.fragment or parsed.query not in { + f"api-version={version}" for version in search_reconcile.SUPPORTED_API_VERSIONS} + or re.fullmatch(r"/datasources\('[a-zA-Z0-9][a-zA-Z0-9_-]{0,127}'\)", parsed.path) is None): + raise fail("recheck-read-url-invalid", "Sanitized binding requires one exact datasource identity and the bound supported API version.") + return url + "&includeConnectionString=true" + + +def _reader(transport, token_provider, *, warnings=None): + recovery = None + recovery_warnings = warnings if warnings is not None else [] + + def tracked(method, url, token, **kwargs): + try: + parsed = urlsplit(url) + search_reconcile.validate_search_endpoint(f"{parsed.scheme}://{parsed.netloc}") + if parsed.fragment: + raise fail("recheck-read-url-invalid", "Read-only Search URLs cannot contain fragments.") + if parsed.query not in { + f"api-version={version}" for version in search_reconcile.SUPPORTED_API_VERSIONS}: + suffix = "&includeConnectionString=true" + if not url.endswith(suffix) or _sanitized_datasource_url(url[:-len(suffix)]) != url: + raise fail("recheck-read-url-invalid", "Only the exact sanitized datasource GET option is permitted.") + kwargs["timeout"] = min(60, kwargs.get("timeout", 60)) + if recovery is None: + response = transport(method, url, token, **kwargs) + else: + response = recovery.get( + url, token, transport=transport, + max_requests=1 if url.endswith("&includeConnectionString=true") else 2, + **kwargs, + ) + ids.extend(recovery.request_ids[:-1]) + if method == "GET" and response.status == 200 and isinstance(response.body, dict): + etag = search_reconcile.resolve_etag(search_reconcile.response_etags(response), response.request_id) + if etag is not None: + response = search_reconcile.HttpResult( + response.status, {**response.body, "@odata.etag": etag}, response.headers, response.etag_values, + ) + return response + except HelperFailure as failure: + if recovery is not None: + ids.extend(recovery.request_ids) + elif failure.request_id: + ids.append(failure.request_id) + raise + read, original_get, ids = source_vector._reader(tracked, token_provider) + + def get(url): + nonlocal recovery + recovery = search_reconcile.ReadRecovery() + try: + result = original_get(url) + get.request_id = ids[-1] if ids else None + return result + except HelperFailure as failure: + if failure.code in {"vector-resource-missing", "vector-evidence-missing", "etag-invalid"}: + missing = failure.code == "vector-resource-missing" + raise HelperFailure( + "recheck-resource-missing" if missing else "recheck-evidence-missing", + "An exact original source/generated definition or ETag is unavailable.", + blocked_at="verification", status=404 if missing else failure.http_status, + request_id=ids[-1] if ids else failure.request_id, + ) from failure + raise + finally: + recovery_warnings.extend(recovery.warnings) + recovery = None + get.recovery_warnings = recovery_warnings + return read, get, ids + + +def _configuration(plan, creation, generated, get, *, binding_receipts=None, expected=None, + diagnostics=None, observations=None, binding_observations=None): + diagnostics = diagnostics if diagnostics is not None else [] + source = plan["source"] + current = get(search_reconcile.resource_url(source)) + _binding(plan, creation, current, generated, binding_receipts) + names = {item["type"]: item["name"] for item in generated} + snapshots = {} + failures = [] + for kind, collection in COLLECTIONS.items(): + name = names[kind] + url = f"{source['endpoint'].rstrip('/')}/{collection}('{name}')?api-version={source['api_version']}" + try: + snapshots[kind] = _child_configuration( + plan, kind, name, names, get, url, expected, diagnostics, observations, binding_observations, + ) + except HelperFailure as failure: + if failure.request_id is None: + failure.request_id = getattr(get, "request_id", None) + semantic.note(diagnostics, kind, "error", failure.code, "definition", failure.request_id) + failures.append(failure) + else: + semantic.note(diagnostics, kind, "info", "generated-configuration-verified", "definition", + getattr(get, "request_id", None)) + refreshed = get(search_reconcile.resource_url(source)) + _binding(plan, creation, refreshed, generated, binding_receipts) + if failures: + raise failures[0] + return snapshots + + +def _child_configuration(plan, kind, name, names, get, url, expected, diagnostics, observations, + binding_observations=None): + source = plan["source"] + child = get(url) + etag = _etag(child) + if child.get("name") != name: + raise fail("definition-drift", "Generated configuration returned an unrelated identity.") + if kind == "datasource": + boundary = plan["boundary"] + container, credentials = child.get("container"), child.get("credentials") + invariants = { + "type": child.get("type") == ("adlsgen2" if boundary["is_adls"] else "azureblob"), + "container.name": isinstance(container, dict) and container.get("name") == boundary["container"], + "container.query": isinstance(container, dict) and container.get("query") in + (boundary["prefix"], None if not boundary["prefix"] else boundary["prefix"]), + "credentials": credentials is None or + isinstance(credentials, dict) and not set(credentials) - {"connectionString"}, + "identity": child.get("identity") is None, + } + for field, valid in invariants.items(): + if not valid: + semantic.note(diagnostics, kind, "error", "source-binding-unverified", field, + getattr(get, "request_id", None)) + if not all(invariants.values()): + raise fail("source-binding-unverified", "Generated datasource type/container/prefix/identity or credential shape conflicts with the bound source.", + request_id=getattr(get, "request_id", None)) + try: + binding = blob_source._connection_binding( + credentials.get("connectionString") if isinstance(credentials, dict) else None, boundary, + ) + except HelperFailure as failure: + failure.request_id = getattr(get, "request_id", None) + semantic.note(diagnostics, kind, "error", failure.code, "credentials.connectionString", failure.request_id) + raise + if kind == "indexer": + request_id = getattr(get, "request_id", None) + for field, target in (("dataSourceName", "datasource"), ("targetIndexName", "index"), ("skillsetName", "skillset")): + if child.get(field) != names[target]: + raise indexer.failure(diagnostics, "indexer-binding-mismatch", field, + f"Generated indexer {field} does not match the bound resource; details withheld.", request_id) + observed = indexer.observe(child, source["desired"]["azureBlobParameters"]["ingestionParameters"].get("ingestionSchedule"), + diagnostics, request_id) + if observations is not None: + observations.append({"request_id": request_id, "etag_digest": digest(etag), + **{key: value for key, value in observed.items() if key != "etag"}}) + snapshot = observed if expected is None or indexer.PROJECTION_FIELDS <= set(expected[kind]) else { + "etag": etag, "digest": digest(child), + } + if expected is None or semantic.PROJECTION_FIELDS <= set(expected[kind]): + snapshot.update(semantic.observe(child, kind)) + if expected is not None: + if semantic.PROJECTION_FIELDS <= set(expected[kind]): + semantic.compare(snapshot, expected[kind], kind, diagnostics, request_id) + # A core-equal ETag-only revision is not schedule or configuration drift. + if snapshot["schedule_raw_digest"] != expected[kind]["schedule_raw_digest"]: + indexer.compare(snapshot, expected[kind], diagnostics, request_id) + else: + indexer.compare(snapshot, expected[kind], diagnostics, request_id) + return snapshot + # Keep only integrity observations, never generated credentials or skill text. + snapshot = {"etag": etag, "digest": digest(child)} + if expected is None or semantic.PROJECTION_FIELDS <= set(expected[kind]): + snapshot.update(semantic.observe(child, kind)) + if kind == "datasource": + snapshot["binding_proof"] = "resource-id" if binding == "visible" else "unverified" + if expected is not None: + semantic.compare(snapshot, expected[kind], kind, diagnostics, getattr(get, "request_id", None)) + if kind == "datasource" and binding == "concealed": + _current_datasource_binding(get, url, snapshot, plan["boundary"], diagnostics, + binding_observations) + elif kind == "datasource" and binding == "concealed": + semantic.note(diagnostics, kind, "info", "datasource-credential-projection-concealed", + "credentials.connectionString", getattr(get, "request_id", None)) + return snapshot + visible_preimage = False + if kind == "datasource" and expected and expected[kind]["etag"] == etag: + # Only concealed/verified credentials may vary; every other byte and the ETag stay bound. + for credentials in ( + None, {}, {"connectionString": None}, {"connectionString": ""}, + {"connectionString": ""}, {"connectionString": ""}, + {"connectionString": f"ResourceId={plan['boundary']['storage_id']}"}, + {"connectionString": f"ResourceId={plan['boundary']['storage_id']};"}, + ): + candidate = {**child, "credentials": credentials} + if digest(candidate) == expected[kind]["digest"]: + snapshot["digest"] = expected[kind]["digest"] + if isinstance(credentials, dict) and isinstance(credentials.get("connectionString"), str): + visible_preimage = credentials["connectionString"].startswith("ResourceId=") + candidate = {key: value for key, value in child.items() if key != "credentials"} + if digest(candidate) == expected[kind]["digest"]: + snapshot["digest"] = expected[kind]["digest"] + if snapshot != expected[kind]: + raise fail("generated-legacy-evidence-insufficient", + "Legacy generated full-hash evidence differs; no core projection was retained.", + request_id=getattr(get, "request_id", None)) + if kind == "datasource" and binding == "concealed": + if visible_preimage: + semantic.note(diagnostics, kind, "warning", "datasource-credential-projection-changed", + "credentials", getattr(get, "request_id", None)) + else: + _current_datasource_binding(get, url, {"etag": etag, "digest": digest(child), **semantic.observe(child, kind)}, + plan["boundary"], diagnostics, binding_observations) + return snapshot + + +def _current_datasource_binding(get, url, snapshot, boundary, diagnostics, observations=None): + observation = { + "schema_version": "1.0", "status": "unverified", "request_id": None, + "plain": {"etag_digest": digest(snapshot["etag"]), "definition_digest": snapshot["digest"], + "core_digest": snapshot["core_digest"]}, + "sanitized": None, + } + if observations is not None: + observations.append(observation) + # The service sanitizes internally; this option never requests raw keys or SAS. + try: + current = get(_sanitized_datasource_url(url)) + except HelperFailure as failure: + observation["request_id"] = failure.request_id + raise + credentials = current.get("credentials") + request_id = getattr(get, "request_id", None) + observation["request_id"] = request_id + projection = semantic.observe(current, "datasource") + observation["sanitized"] = { + "etag_digest": digest(_etag(current)), "definition_digest": digest(current), + "core_digest": projection["core_digest"], + } + if (_etag(current) != snapshot["etag"] + or observation["sanitized"]["core_digest"] != snapshot["core_digest"]): + fields = projection["field_digests"].keys() | snapshot["field_digests"].keys() + changed = [field for field in sorted(fields) + if projection["field_digests"].get(field) != snapshot["field_digests"].get(field)] + if _etag(current) != snapshot["etag"]: + changed.append("@odata.etag") + for field in changed: + semantic.note(diagnostics, "datasource", "error", "datasource-binding-unverified", field, request_id) + raise fail("datasource-binding-unverified", + "Independent datasource binding readback changed revision or configuration; no stable current proof is available.", + request_id=request_id) + try: + visible = (isinstance(credentials, dict) and set(credentials) == {"connectionString"} + and blob_source._connection_binding(credentials["connectionString"], boundary) == "visible") + except HelperFailure: + visible = False + if not visible: + semantic.note(diagnostics, "datasource", "error", "datasource-binding-unverified", + "credentials.connectionString", request_id) + raise fail("datasource-binding-unverified", + "Sanitized current datasource readback does not prove the exact selected keyless ResourceId; values withheld. Retain resources and investigate this evidence gap read-only.", + request_id=request_id) + observation["status"] = "verified" + semantic.note(diagnostics, "datasource", "info", "datasource-current-binding-verified", + "credentials.connectionString", request_id) + + +def _baseline_cycle(source, read, token_provider): + url = search_reconcile.resource_url(source).replace(")?", ")/status?") + response = read("GET", url, token_provider(SEARCH_AUDIENCE)) + body = response.body + if response.status != 200 or not isinstance(body, dict) or body.get("kind") != "azureBlob": + raise HelperFailure("ingestion-inaccessible", "Initial reuse status is unavailable.", + blocked_at="verification", status=response.status, request_id=response.request_id) + last = body.get("lastSynchronizationState") + if last is None: + return None + if not isinstance(last, dict): + raise fail("ingestion-status-invalid", "Initial reuse synchronization must be an object.") + if last.get("endTime") is None: + blob_source._timestamp(last.get("startTime")) + return None + cycle = [last.get("startTime"), last.get("endTime")] + _validate_cycle(cycle) + return cycle + + +def _validate_cycle(cycle): + if cycle is None: + return + if not isinstance(cycle, list) or len(cycle) != 2: + raise fail("recheck-evidence-invalid", "Retain the original observed reuse cycle, not a reconstructed bound.") + if blob_source._timestamp(cycle[1]) < blob_source._timestamp(cycle[0]): + raise fail("ingestion-status-invalid", "Initial synchronization interval is invalid.") + + +def _binding_receipts(paths): + if paths is None: + return None + if (not isinstance(paths, (tuple, list)) or len(paths) != 2 + or any(not isinstance(path, (str, Path)) or not str(path).strip() for path in paths)): + raise fail("reuse-evidence-invalid", "Select both existing private creation evidence files.") + _, prior, fingerprint = _document(paths[0]) + result = read_private(paths[1]) + blob_source._validate_reuse_receipts(prior, fingerprint, result) + return prior, result + + +class Checkpoint: + def __init__(self, directory, plan, *, context_provider=account_context, binding_receipts=None): + _supported(plan) + self.directory = private_io.private_directory(str(directory)) + self.context_provider = context_provider + self.context = context_provider() + self.plan_digest = digest(plan) + self.operation_id = uuid.uuid4().hex + self.summary = None + self.excluded_cycle = None + self.request_ids = [] + self.recovery_warnings = [] + self.acknowledgement = None + self.write_acknowledgement = None + self.write_observation = None + self.binding_receipts = binding_receipts + self.diagnostics = [] + self.observations = [] + self.binding_observations = [] + self.configuration = None + self.creation = None + + def acknowledge(self, plan, response, not_before, *, url, body, headers): + if (digest(plan) != self.plan_digest or plan["source"]["action"] != "create" + or response.status not in {200, 201} + or url != search_reconcile.resource_url(plan["source"]) + or headers.get("If-None-Match") != "*" or "If-Match" in headers + or body != search_reconcile.canonical_bytes(plan["source"]["desired"])): + raise fail("recheck-ownership-unproven", "Only the exact successful conditional create can retain a write acknowledgement.") + self.write_observation = { + "schema_version": "1.0", "state": "acknowledged", "type": "knowledge-source", + "name": plan["source"]["name"], "http_status": response.status, "request_id": response.request_id, + } + if self.context_provider() != self.context: + raise fail("recheck-auth-context-drift", "CLI context changed before write acknowledgement retention.") + receipt = { + "schema_version": "1.0", "kind": "blob-write-acknowledgement", + "operation_id": self.operation_id, "plan_digest": self.plan_digest, + "not_before": not_before.isoformat(), "context_digest": self.context, + "write": { + "method": "PUT", "url": url, "if_none_match": headers["If-None-Match"], + "body_digest": digest(plan["source"]["desired"]), "status": response.status, + "request_id": response.request_id, "response_etags": search_reconcile.response_etags(response), + "generated": blob_source.generated_resources(response.body) if isinstance(response.body, dict) else [], + }, + } + reject_secrets(receipt) + receipt["integrity"] = digest(receipt) + path = private_io.private_file(self.directory, self.operation_id + ".blob-write.json", receipt) + self.write_acknowledgement = {"operation_id": self.operation_id, "receipt_file": path.name, + "evidence_digest": receipt["integrity"], "status": "retained"} + + def persist(self, plan, result, generated, not_before, *, token_provider, transport): + reused = plan["source"]["action"] == "reuse" + selected, other = ("reused", "created") if reused else ("created", "reused") + owned = "reused_not_owned" if reused else "run_owned" + if (digest(plan) != self.plan_digest or len(result["resources"][selected]) != 1 + or result["resources"][other] or result["resources"][selected] != result["ownership"][owned]): + raise fail("recheck-ownership-unproven", "Retain exact acknowledged creation or read-only reuse; never adopt shared resources.") + generated = copy.deepcopy(generated) + creation = {key: copy.deepcopy(result[key]) for key in + ("approved_plan", "resources", "verification", "ownership")} + creation["source"] = {"generated": generated} + seed = { + "schema_version": "1.0", "kind": "blob-source-acknowledgement", + "operation_id": self.operation_id, "plan_digest": digest(plan), + "not_before": not_before.isoformat(), "context_digest": self.context, + "creation": creation, "request_ids": [], + } + if not reused: + if self.context_provider() != self.context: + raise fail("recheck-auth-context-drift", "CLI context changed before acknowledgement retention.") + seed["integrity"] = digest(seed) + path = private_io.private_file(self.directory, self.operation_id + ".blob-ack.json", seed) + self.acknowledgement = {"operation_id": self.operation_id, "receipt_file": path.name, + "evidence_digest": seed["integrity"], "status": "retained"} + read, get, ids = _reader(transport, token_provider, warnings=self.recovery_warnings) + self.request_ids = ids + configuration = _configuration(plan, creation, generated, get, binding_receipts=self.binding_receipts, + diagnostics=self.diagnostics, observations=self.observations) + if reused: + self.excluded_cycle = _baseline_cycle(plan["source"], read, token_provider) + if self.context_provider() != self.context: + raise fail("recheck-auth-context-drift", "CLI context changed during checkpoint capture; no checkpoint was retained.") + receipt = { + "schema_version": "3.1" if reused else "3.0", "kind": "blob-readiness-checkpoint", + "operation_id": self.operation_id, "plan_digest": digest(plan), + "not_before": not_before.isoformat(), "context_digest": self.context, + "creation": creation, "configuration": configuration, "request_ids": ids, + } + if reused: + receipt["excluded_cycle"] = self.excluded_cycle + if self.binding_receipts is not None: + receipt["schema_version"] = "3.2" + receipt["binding_digest"] = digest(self.binding_receipts) + receipt["integrity"] = digest(receipt) + path = private_io.private_file(self.directory, self.operation_id + ".blob-readiness.json", receipt) + self.request_ids = ids + self.summary = {"operation_id": self.operation_id, "receipt_file": path.name, + "evidence_digest": receipt["integrity"], "status": "retained"} + self.configuration, self.creation = configuration, creation + + def verify(self, plan, *, token_provider, transport): + _, get, ids = _reader(transport, token_provider, warnings=self.recovery_warnings) + try: + _configuration(plan, self.creation, self.creation["source"]["generated"], get, + expected=self.configuration, binding_receipts=self.binding_receipts, + diagnostics=self.diagnostics, observations=self.observations, + binding_observations=self.binding_observations) + finally: + self.request_ids.extend(ids) + if self.context_provider() != self.context: + raise fail("recheck-auth-context-drift", "CLI context changed during generated readback.") + + def finish(self, result): + record = {"schema_version": "1.0", "kind": "blob-operation-result", + "operation_id": self.operation_id, "plan_digest": self.plan_digest, + "result": copy.deepcopy(result)} + reject_secrets(record) + record["integrity"] = digest(record) + path = private_io.private_file(self.directory, self.operation_id + ".blob-result.json", record) + result["result_evidence"] = {"receipt_file": path.name, "status": "retained"} + + +def _document(input_path): + document = read_private(input_path) + reject_secrets(document) + if set(document) != {"schema_version", "plan", "approval"} or document["schema_version"] != "1.0": + raise fail("recheck-evidence-invalid", "Retain the original source envelope.") + plan = document["plan"] + if not isinstance(plan, dict): + raise fail("recheck-evidence-invalid", "The original plan is unavailable.") + _supported(plan) + fingerprint = digest(plan) + approval = document["approval"] + if (not isinstance(approval, dict) or set(approval) != {"confirmed", "fingerprint"} + or type(approval["confirmed"]) is not bool or approval["fingerprint"] != fingerprint + or plan["source"]["action"] == "create" and approval["confirmed"] is not True): + raise fail("approval-mismatch", "Retain unchanged creation consent or the fingerprinted read-only reuse plan.") + return document, plan, fingerprint + + +def _load(input_path, receipt_path, *, acknowledgement=False, binding_receipts=None): + document, plan, fingerprint = _document(input_path) + receipt = read_private(receipt_path) + reject_secrets(receipt) + if acknowledgement and receipt.get("kind") == "blob-write-acknowledgement": + _validate_write(plan, receipt) + return plan, receipt + reused = plan["source"]["action"] == "reuse" + version = receipt.get("schema_version") + if not isinstance(version, str): + raise fail("recheck-evidence-invalid", "Checkpoint schema version must be a supported string.") + core_projected = version in {"3.0", "3.1", "3.2"} and not acknowledgement + projected = (version in {"2.0", "2.1", "2.2"} or core_projected) and not acknowledgement + bound = reused and version in {"1.2", "2.2", "3.2"} + fields = {"schema_version", "kind", "operation_id", "plan_digest", "not_before", + "context_digest", "creation", "configuration", "request_ids", "integrity"} + if reused: + fields.add("excluded_cycle") + if bound: + fields.add("binding_digest") + if acknowledgement: + fields.remove("configuration") + expected_version = ("3" if core_projected else "2" if projected else "1") + (".2" if bound else ".1" if reused else ".0") + if (set(receipt) != fields or version != expected_version + or receipt["kind"] != ("blob-source-acknowledgement" if acknowledgement else "blob-readiness-checkpoint") + or acknowledgement and reused + or bound and (binding_receipts is None or receipt["binding_digest"] != digest(binding_receipts)) + or receipt["plan_digest"] != fingerprint + or not isinstance(receipt["operation_id"], str) + or re.fullmatch("[0-9a-f]{32}", receipt["operation_id"]) is None + or receipt["integrity"] != digest({k: v for k, v in receipt.items() if k != "integrity"})): + raise fail("recheck-evidence-invalid", "Original operation/cutoff/checkpoint integrity is missing or changed; never reconstruct it.") + blob_source._timestamp(receipt["not_before"]) + if reused: + _validate_cycle(receipt["excluded_cycle"]) + creation = receipt["creation"] + try: + observed = creation["verification"]["readback"] + generated = creation["source"]["generated"] + expected = {"type": "knowledge-source", "name": plan["source"]["name"], + "etag": observed["etag"], + "definition_digest": digest(search_reconcile._definition(plan["source"]["desired"]))} + valid = ( + set(creation) == {"approved_plan", "resources", "verification", "ownership", "source"} + and set(creation["source"]) == {"generated"} + and set(creation["verification"]) == {"readback", "absence", "request_ids", "idempotency"} + and creation["verification"]["absence"] is False + and creation["verification"]["idempotency"] == "exact readback is zero-write" + and isinstance(creation["verification"]["request_ids"], list) + and all(isinstance(item, str) for item in creation["verification"]["request_ids"]) + and creation["approved_plan"] == document["approval"] + and creation["resources"] == {"created": [] if reused else [expected], + "reused": [expected] if reused else [], "updated": [], "skipped": []} + and creation["ownership"] == {"run_owned": [] if reused else [expected], + "reused_not_owned": [expected] if reused else [], "owner": plan["owner"]} + and observed == expected and isinstance(expected["etag"], str) and bool(expected["etag"].strip()) + and generated == blob_source.generated_resources( + {"azureBlobParameters": {"createdResources": {item["type"]: item["name"] for item in generated}}}, + strict=True, + ) + and (acknowledgement or set(receipt["configuration"]) == set(COLLECTIONS)) + and isinstance(receipt["request_ids"], list) + and all(isinstance(item, str) for item in receipt["request_ids"]) + and isinstance(receipt["context_digest"], str) + and search_reconcile.SHA256.fullmatch(receipt["context_digest"]) + ) + for kind, item in ({} if acknowledgement else receipt["configuration"]).items(): + if core_projected: + valid = valid and semantic.valid(item, kind, search_reconcile.SHA256) + if kind == "indexer": + valid = valid and item.get("core_digest") == item.get("non_schedule_digest") + if kind == "datasource" and item.get("binding_proof") == "resource-id": + valid = valid and item["credential_digest"] in { + digest({"present": True, "value": {"connectionString": f"ResourceId={plan['boundary']['storage_id']}{suffix}"}}) + for suffix in ("", ";") + } + continue + hashes = {"digest"} | (indexer.PROJECTION_FIELDS if projected and kind == "indexer" else set()) + valid = valid and set(item) == {"etag"} | hashes and isinstance(item["etag"], str) and bool(item["etag"].strip()) + valid = valid and all(isinstance(item[key], str) and search_reconcile.SHA256.fullmatch(item[key]) for key in hashes) + except (KeyError, TypeError, AttributeError): + valid = False + if not valid: + raise fail("recheck-ownership-unproven", "Checkpoint must retain exact acknowledged ownership, generated configuration and provenance.") + return plan, receipt + + +def _historical_result(receipt_path, receipt): + path = Path(receipt_path).parent / (receipt["operation_id"] + ".blob-result.json") + if not path.exists(): + return "not-recorded-by-pre-monitor-checkpoint" + record = read_private(path) + reject_secrets(record) + if (set(record) != {"schema_version", "kind", "operation_id", "plan_digest", "result", "integrity"} + or record["schema_version"] != "1.0" or record["kind"] != "blob-operation-result" + or record["operation_id"] != receipt["operation_id"] or record["plan_digest"] != receipt["plan_digest"] + or record["integrity"] != digest({key: value for key, value in record.items() if key != "integrity"}) + or not isinstance(record["result"], dict)): + raise fail("recheck-evidence-invalid", "Historical operation result is malformed or changed; preserve original evidence.") + result = record["result"] + readiness = result.get("readiness", {}) + if not isinstance(readiness, dict): + raise fail("recheck-evidence-invalid", "Historical readiness must be an object.") + watch = readiness.get("watch") + if watch is not None and ( + not isinstance(watch, dict) or watch.get("schema_version") != "1.0" + or not isinstance(watch.get("state"), str) + or watch.get("state") not in {"paused", "completed", "blocked"}): + raise fail("recheck-evidence-invalid", "Historical watch metadata is malformed or unsupported.") + return {"receipt_file": path.name, "status": result.get("status"), + "first_failure": result.get("first_failure", result.get("first_blocker")), + "watch": copy.deepcopy(watch), + "writes_performed": result.get("writes_performed", result.get("completed_writes", []))} + + +def _validate_write(plan, receipt): + try: + write = receipt["write"] + valid = ( + set(receipt) == {"schema_version", "kind", "operation_id", "plan_digest", "not_before", + "context_digest", "write", "integrity"} + and receipt["schema_version"] == "1.0" and plan["source"]["action"] == "create" + and receipt["plan_digest"] == digest(plan) + and isinstance(receipt["operation_id"], str) + and re.fullmatch("[0-9a-f]{32}", receipt["operation_id"]) is not None + and isinstance(receipt["context_digest"], str) + and search_reconcile.SHA256.fullmatch(receipt["context_digest"]) is not None + and receipt["integrity"] == digest({k: v for k, v in receipt.items() if k != "integrity"}) + and set(write) == {"method", "url", "if_none_match", "body_digest", "status", + "request_id", "response_etags", "generated"} + and write["method"] == "PUT" and write["if_none_match"] == "*" + and type(write["status"]) is int and write["status"] in {200, 201} + and write["url"] == search_reconcile.resource_url(plan["source"]) + and write["body_digest"] == digest(plan["source"]["desired"]) + and isinstance(write["request_id"], str) and bool(write["request_id"].strip()) + and set(write["response_etags"]) == {"body", "headers"} + and isinstance(write["response_etags"]["headers"], list) + and isinstance(write["generated"], list) + ) + if write["generated"]: + valid = valid and write["generated"] == blob_source.generated_resources( + {"azureBlobParameters": {"createdResources": { + item["type"]: item["name"] for item in write["generated"]}}}, strict=True, + ) + except (KeyError, TypeError, AttributeError): + valid = False + if not valid: + raise fail("recheck-ownership-unproven", "Retain the private authenticated conditional-write receipt; input booleans or observed existence cannot replace it.") + blob_source._timestamp(receipt["not_before"]) + if search_reconcile.resolve_etag(write["response_etags"], write["request_id"]) is None: + raise fail("creation-version-unproven", "Write acknowledgement has no response ETag; never borrow a later GET version.", + request_id=write["request_id"]) + + +@reporting("blob-capture") +def capture(input_path, directory, *, token_provider=azure_cli_token, transport=http_request, + storage_transport=http_request, context_provider=account_context, + now=lambda: datetime.now(timezone.utc), progress: Progress | None = None, + binding_paths=None): + progress.update("evidence-validation") + document, plan, fingerprint = _document(input_path) + if plan["source"]["action"] != "reuse": + raise fail("recheck-scope-unsupported", "Fresh capture accepts only a reuse plan; it cannot recover missing original creation proof.") + binding_receipts = _binding_receipts(binding_paths) + progress.update("context-check") + checkpoint = Checkpoint(directory, plan, context_provider=context_provider, binding_receipts=binding_receipts) + cutoff = now() + progress.update("source-binding") + _, get, ids = _reader(transport, token_provider) + current = get(search_reconcile.resource_url(plan["source"])) + generated = blob_source.generated_resources(current, strict=True) + result = search_reconcile._completed( + "blob-readiness-capture", fingerprint, plan["source"], action="reused", + readback=current, request_ids=ids, absence=False, + ) + result["approved_plan"] = document["approval"] + progress.update("blob-inventory") + inventory = blob_inventory.discover(plan["boundary"], plan["inventory_limits"], + token_provider=token_provider, transport=storage_transport) + if inventory["inventory_digest"] != plan["inventory_digest"]: + raise fail("source-drift", "Selected Storage/ACL evidence differs from the reuse plan.") + result["verification"]["request_ids"].extend(inventory["request_ids"]) + progress.update("checkpoint") + checkpoint.persist(plan, result, generated, cutoff, token_provider=token_provider, transport=transport) + return { + "status": "completed", "outcome": "blob-readiness-capture", + "recheck_checkpoint": checkpoint.summary, "writes_performed": [], + "readiness": {"status": "unverified"}, "retrieval": "unverified", "knowledge_base": "not-verified", + "ownership": result["ownership"], "cleanup": {"separate_confirmation_required": True}, + "read_only_evidence": {"request_ids": result["verification"]["request_ids"] + checkpoint.request_ids}, + "indexer_diagnostics": semantic.indexer_only(checkpoint.diagnostics), + "generated_diagnostics": checkpoint.diagnostics, "indexer_observations": checkpoint.observations, + "warnings": [blob_source.SNAPSHOT_WARNING, "Fresh reuse observation is not recovered creation ownership or ingestion proof.", + *get.recovery_warnings, *checkpoint.recovery_warnings, + *indexer.warnings(checkpoint.diagnostics)], + } + + +@reporting("blob-capture") +def recover(input_path, acknowledgement_path, directory, *, token_provider=azure_cli_token, + transport=http_request, storage_transport=http_request, context_provider=account_context, + progress: Progress | None = None): + progress.update("evidence-validation") + plan, acknowledgement = _load(input_path, acknowledgement_path, acknowledgement=True) + historical = _historical_result(acknowledgement_path, acknowledgement) + directory = private_io.private_directory(str(directory)) + progress.update("context-check") + if context_provider() != acknowledgement["context_digest"]: + raise fail("recheck-auth-context-drift", "Current context differs from the acknowledged original run.") + progress.update("source-binding") + _, get, ids = _reader(transport, token_provider) + diagnostics, observations, binding_observations = [], [], [] + if acknowledgement["kind"] == "blob-write-acknowledgement": + write = acknowledgement["write"] + current = get(search_reconcile.resource_url(plan["source"])) + if (_etag(current) != search_reconcile.resolve_etag(write["response_etags"], write["request_id"]) + or search_reconcile._definition(current) != search_reconcile._definition(plan["source"]["desired"])): + raise fail("definition-drift", "Current source differs from the acknowledged conditional-write version/definition.", + request_id=getattr(get, "request_id", None)) + generated = blob_source.generated_resources(current, strict=True) + if write["generated"] and generated != write["generated"]: + raise fail("definition-drift", "Generated identities differ from the create response.") + verified = search_reconcile._completed( + "create-blob-knowledge-source", acknowledgement["plan_digest"], plan["source"], + action="created", readback=current, request_ids=[write["request_id"], *ids], absence=False, + ) + creation = {key: verified[key] for key in ("approved_plan", "resources", "verification", "ownership")} + creation["source"] = {"generated": generated} + else: + creation = acknowledgement["creation"] + configuration = _configuration(plan, creation, creation["source"]["generated"], get, + diagnostics=diagnostics, observations=observations) + progress.update("blob-inventory") + inventory = blob_inventory.discover(plan["boundary"], plan["inventory_limits"], + token_provider=token_provider, transport=storage_transport) + ids.extend(inventory["request_ids"]) + if inventory["inventory_digest"] != plan["inventory_digest"]: + raise fail("source-drift", "Storage/ACL inventory differs from the acknowledged original run.") + progress.update("checkpoint") + _configuration(plan, creation, creation["source"]["generated"], get, expected=configuration, + diagnostics=diagnostics, observations=observations, binding_observations=binding_observations) + if context_provider() != acknowledgement["context_digest"]: + raise fail("recheck-auth-context-drift", "CLI context changed during recovery observation.") + receipt = {**acknowledgement, "schema_version": "3.0", "kind": "blob-readiness-checkpoint", "configuration": configuration, + "creation": creation, "request_ids": ids} + receipt.pop("write", None) + receipt.pop("integrity") + receipt["integrity"] = digest(receipt) + path = private_io.private_file(directory, receipt["operation_id"] + ".blob-readiness.json", receipt) + return { + "status": "completed", "outcome": "blob-readiness-recovery-capture", "writes_performed": [], + "recheck_checkpoint": {"operation_id": receipt["operation_id"], "receipt_file": path.name, + "evidence_digest": receipt["integrity"], "status": "retained"}, + "readiness": {"status": "unverified"}, "retrieval": "unverified", "knowledge_base": "not-verified", + "ownership": {"run_owned": [], "reused_not_owned": []}, + "original_run": {"ownership": creation["ownership"], "not_before": receipt["not_before"], + "historical_created": copy.deepcopy(creation["resources"]["created"]), + "original_failure": historical, + "acknowledgement_digest": acknowledgement["integrity"]}, + "read_only_evidence": {"request_ids": ids}, + "indexer_diagnostics": semantic.indexer_only(diagnostics), + "generated_diagnostics": diagnostics, "indexer_observations": observations, + "datasource_binding_observations": binding_observations, + "safe_next_decision": "Run blob_recheck.py --input with the unchanged original input and --receipt with this checkpoint; then return to the KB/retrieval owner. Preserve the original first failure separately.", + "warnings": [blob_source.SNAPSHOT_WARNING, "Configuration was observed during recovery; no earlier generated revision or new ownership is asserted.", + *get.recovery_warnings, + *indexer.warnings(diagnostics)], + } + + +@reporting("blob-recheck") +def recheck(input_path, receipt_path, *, token_provider=azure_cli_token, + transport=http_request, storage_transport=http_request, + context_provider=account_context, monotonic=time.monotonic, sleep=time.sleep, + now=lambda: datetime.now(timezone.utc), progress: Progress | None = None, + binding_paths=None, watch_limits=None, cancelled=lambda: False): + progress.update("evidence-validation") + binding_receipts = _binding_receipts(binding_paths) + plan, receipt = _load(input_path, receipt_path, binding_receipts=binding_receipts) + limits = blob_source._poll_limits(watch_limits if watch_limits is not None else plan["poll"]) + creation = receipt["creation"] + generated = creation["source"]["generated"] + readiness = {"status": "unverified"} + historical = "not-recorded-by-pre-monitor-checkpoint" + ids = [] + recovery_warnings = [] + diagnostics, observations, binding_observations = [], [], [] + try: + historical = _historical_result(receipt_path, receipt) + progress.update("context-check") + if context_provider() != receipt["context_digest"]: + raise fail("recheck-auth-context-drift", "Current CLI tenant/subscription/principal differs from the original run; no auth changes were made.") + read, get, ids = _reader(transport, token_provider) + recovery_warnings = get.recovery_warnings + + def inventory(): + inventory = blob_inventory.discover( + plan["boundary"], plan["inventory_limits"], token_provider=token_provider, + transport=storage_transport, + ) + ids.extend(inventory["request_ids"]) + if inventory["inventory_digest"] != plan["inventory_digest"]: + raise fail("source-drift", "Selected Storage/ACL evidence differs from original creation.") + + for stage in ("before", "after"): + if stage == "after": + progress.update("blob-readback") + inventory() + progress.update("source-binding" if stage == "before" else "source-readback") + _configuration(plan, creation, generated, get, binding_receipts=binding_receipts, + expected=receipt["configuration"], diagnostics=diagnostics, observations=observations, + binding_observations=binding_observations) + if stage == "before": + progress.update("blob-inventory") + inventory() + readiness = blob_source.monitor( + plan["source"], not_before=blob_source._timestamp(receipt["not_before"]), + limits=limits, token_provider=token_provider, transport=read, + monotonic=monotonic, sleep=sleep, progress=progress, + excluded_cycle=receipt.get("excluded_cycle"), + cancelled=cancelled, + indexer_name=next(item["name"] for item in generated if item["type"] == "indexer"), + ) + if readiness["status"] != "verified": + raise HelperFailure( + readiness["code"], "Original-run ingestion remains unverified.", + blocked_at="verification", request_id=readiness.get("request_id"), + status=readiness.get("http_status"), + ) + cycle = readiness["synchronization"] + if (cycle["itemsUpdatesProcessed"] == 0 or cycle["itemsSkipped"] + or blob_source._timestamp(cycle["endTime"]) > now()): + raise fail("ingestion-unverified", "A checkpoint is not prior ingestion proof; nonempty zero-skip completion is required.") + progress.update("context-readback") + if context_provider() != receipt["context_digest"]: + raise fail("recheck-auth-context-drift", "CLI context changed during recheck.") + except HelperFailure as failure: + recovery_warnings.extend(failure.warnings) + if failure.request_id and failure.request_id not in ids: + ids.append(failure.request_id) + safe_failure = HelperFailure( + failure.code, failure.message if failure.blocked_at == "indexer-verification" else + "Read-only evidence could not verify readiness; service details withheld.", + blocked_at=failure.blocked_at, status=failure.http_status, request_id=failure.request_id, + ) + result = blocked_result(safe_failure, outcome="blob-readiness-recheck", fingerprint=None) + result["safe_next_decision"] = "Preserve the first failure. GET the exact source/generated definitions; for a concealed source binding supply --reuse-input-file/--reuse-result-file from its successful creation. Fix access or investigate actual drift read-only; do not replay creation, run/reset an indexer or default to cleanup." + if failure.code == "datasource-binding-unverified": + result["first_blocker"]["message"] = ( + "The datasource revision lacks independently observed current Storage binding; " + "this is unverified, not evidence of misconfiguration." + ) + result["safe_next_decision"] = ( + "Retain source and checkpoint. One exact datasource GET with includeConnectionString=true did not " + "provide stable sanitized ResourceId proof. " + "Ask the service owner for authoritative current child binding evidence; root identity and historical " + "receipts are insufficient. Resume the same GET-only recheck if that readback becomes available. " + "Do not retrieve keys, patch/recreate the source, run/reset an indexer, or delete resources." + ) + if failure.code == "indexer-legacy-evidence-insufficient": + result["safe_next_decision"] = "Retain the legacy checkpoint unchanged; GET current definitions and compare any legitimately retained preimage. This helper cannot infer its missing projection. A separately planned --capture reuse operation may observe fresh readiness, not recover historical configuration/ownership. Do not replay writes." + readiness = {**readiness, "status": "unverified"} + if readiness.get("watch", {}).get("state") == "paused": + result["safe_next_decision"] = readiness["safe_next_decision"] + else: + result = {"status": "completed", "outcome": "blob-readiness-recheck", "writes_performed": []} + result.update( + readiness=readiness, retrieval="unverified", knowledge_base="not-verified", + original_run={"operation_id": receipt["operation_id"], "plan_digest": receipt["plan_digest"], + "source_action": plan["source"]["action"], + "evidence_digest": receipt["integrity"], "not_before": receipt["not_before"], + "request_ids": creation["verification"]["request_ids"], + "checkpoint_request_ids": receipt["request_ids"], + "original_failure": historical, + "historical_created": copy.deepcopy(creation["resources"]["created"]), + "ownership": copy.deepcopy(creation["ownership"]), "generated": generated}, + ownership={"run_owned": [], "reused_not_owned": []}, + read_only_evidence={"request_ids": ids}, + indexer_diagnostics=semantic.indexer_only(diagnostics), + generated_diagnostics=diagnostics, indexer_observations=observations, + datasource_binding_observations=binding_observations, + cleanup={"status": "not-requested", "separate_confirmation_required": True}, + recheck_checkpoint={"operation_id": receipt["operation_id"], "receipt_file": Path(receipt_path).name, + "evidence_digest": receipt["integrity"], "status": "retained"}, + warnings=[blob_source.SNAPSHOT_WARNING, + *recovery_warnings, + "Local checkpoint integrity is not a service signature or new ownership/cleanup authorization.", + *indexer.warnings(diagnostics)], + ) + return result + + +def compact_result(result, directory): + """Opt-in versioned presentation; retain the native result privately first.""" + reject_secrets(result) + directory = private_io.private_directory(str(directory)) + path = private_io.private_file(directory, uuid.uuid4().hex + ".blob-result.json", result) + ids = [] + + def collect(value, key=None): + if isinstance(value, dict): + for field, child in value.items(): + collect(child, field) + elif isinstance(value, list): + for child in value: + collect(child, key) + elif (key in {"request_id", "request_ids"} or isinstance(key, str) and key.endswith("_request_ids")) and isinstance(value, str): + ids.append(value) + + collect(result) + failure = result.get("first_failure", result.get("first_blocker")) + original = result.get("original_run", {}) + writes = result.get("writes_performed", result.get("completed_writes", [])) + readiness = result.get("readiness", {"status": "unverified"}) + item_error = readiness.get("first_error") + if readiness.get("code") == "ingestion-timeout" and readiness.get("watch", {}).get("state") == "paused": + failure = None + elif failure and isinstance(item_error, dict) and item_error.get("request_id"): + failure = {**failure, "request_id": item_error["request_id"]} + historical = original.get("original_failure") + historical_failure = historical.get("first_failure") if isinstance(historical, dict) else None + historical_watch = historical.get("watch") if isinstance(historical, dict) else None + if (historical_failure and historical_failure.get("code") == "ingestion-timeout" + and isinstance(historical_watch, dict) and historical_watch.get("schema_version") == "1.0" + and historical_watch.get("state") == "paused"): + historical_failure = None + generated = original.get("generated", result.get("source", {}).get("generated", [])) + diagnostic_keys = ("severity", "code", "field") + diagnostics = dict.fromkeys( + tuple(item[key] for key in diagnostic_keys) + for item in result.get("generated_diagnostics", result.get("indexer_diagnostics", [])) + ) + return { + "schema_version": "1.1" if "progress" in readiness else "1.0", "kind": "blob-operation-summary", + "status": result["status"], "outcome": result["outcome"], + "writes_performed": writes, + "creation_acknowledgement": result.get("creation_acknowledgement"), + "historical_created": [ + {"type": item["type"], "name": item["name"]} + for item in original.get("historical_created", []) + ], + "generated_resources": [{"type": item["type"], "name": item["name"]} for item in generated], + "readiness": {"status": readiness["status"], "watch": readiness.get("watch"), + **({"progress": readiness["progress"]} if "progress" in readiness else {})}, + "first_failure": ({key: failure.get(key) for key in ("code", "status", "request_id")} + if failure else None), + "historical_failure": ({key: historical_failure.get(key) for key in ("code", "status", "request_id")} + if historical_failure else None), + "first_retry": readiness.get("first_retry"), + "diagnostics": [dict(zip(diagnostic_keys, item)) for item in diagnostics], + "request_id_count": len(set(ids)), "evidence_file": path.name, + "checkpoint_file": result.get("recheck_checkpoint", {}).get("receipt_file"), + "retrieval": "unverified", "knowledge_base": "not-verified", + "safe_next_decision": result.get("safe_next_decision", + "Return to the KB/retrieval owner; no cleanup or new writes are authorized." + if readiness["status"] == "verified" else + "Retain resources and inspect the named source/indexer and private evidence. " + "Use receipt-backed GET-only recheck when evidence is available; no write replay or cleanup."), + } + + +def main(argv=None): + parser = argparse.ArgumentParser() + modes = parser.add_mutually_exclusive_group(required=True) + modes.add_argument("--input", type=Path) + modes.add_argument("--capture", type=Path) + modes.add_argument("--recover", type=Path) + parser.add_argument("--receipt", type=Path) + parser.add_argument("--receipt-dir", type=Path) + parser.add_argument("--reuse-input-file", type=Path) + parser.add_argument("--reuse-result-file", type=Path) + parser.add_argument("--watch-seconds", type=int) + parser.add_argument("--watch-max-requests", type=int) + parser.add_argument("--watch-interval", type=int) + parser.add_argument("--compact", action="store_true") + add_progress_argument(parser) + args = parser.parse_args(argv) + if (args.capture and (not args.receipt_dir or args.receipt) + or args.input and (not args.receipt or args.receipt_dir) + or args.recover and (not args.receipt or not args.receipt_dir)): + parser.error("--capture needs --receipt-dir; --input needs --receipt; --recover needs both.") + if bool(args.reuse_input_file) != bool(args.reuse_result_file) or args.recover and args.reuse_input_file: + parser.error("Select both reuse evidence files, only with --capture or --input.") + options = {"binding_paths": (args.reuse_input_file, args.reuse_result_file)} if args.reuse_input_file else {} + watch = (args.watch_seconds, args.watch_max_requests, args.watch_interval) + if any(value is not None for value in watch): + if not args.input or any(value is None for value in watch): + parser.error("Explicit GET-only watch needs --input and all three --watch-* limits.") + options["watch_limits"] = dict(zip(("deadline_seconds", "max_requests", "interval_seconds"), watch)) + try: + if args.capture: + result = capture(args.capture, args.receipt_dir, progress=Progress("blob-capture", enabled=args.progress), **options) + elif args.recover: + result = recover(args.recover, args.receipt, args.receipt_dir, + progress=Progress("blob-capture", enabled=args.progress)) + else: + result = recheck(args.input, args.receipt, progress=Progress("blob-recheck", enabled=args.progress), **options) + except HelperFailure as failure: + outcome = "blob-readiness-recovery-capture" if args.recover else "blob-readiness-capture" if args.capture else "blob-readiness-recheck" + result = blocked_result(failure, outcome=outcome, fingerprint=None) + result["safe_next_decision"] = ( + "Preserve original error/resources. Inspect the exact source and genuine private evidence: " + "--recover needs its retained acknowledgement; --capture needs a reuse plan and any required " + "creation proof. If a checkpoint already exists, inspect it and use --input/--receipt. " + "No write replay, new ownership or cleanup is authorized." + ) + if args.compact: + try: + result = compact_result(result, args.receipt_dir or args.receipt.parent) + except HelperFailure as failure: + result.setdefault("warnings", []).append("compact-evidence-persistence-failed: full native result retained in output.") + result["presentation_failure"] = {"code": failure.code} + emit_result(result) + return 2 + emit_result(result) + return 0 if result["status"] == "completed" else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_source.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_source.py new file mode 100644 index 00000000..cd4ba166 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/blob_source.py @@ -0,0 +1,1346 @@ +from __future__ import annotations + +import argparse +import copy +import json +import math +import re +import sys +import time +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from pathlib import Path +from typing import Any, Callable +from urllib.parse import urlencode + +try: + from ._progress import Progress, add_progress_argument, reporting + from . import blob_inventory, search_reconcile, source_vector, cu_ingestion_auth + from ._common import ( + SEARCH_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, + blocked_result, canonical_bytes, digest, emit_result, http_request, load_approved_input, HttpResult, + reject_secrets, require_allowed_fields, + ) +except ImportError: + from _progress import Progress, add_progress_argument, reporting + import blob_inventory # type: ignore[no-redef] + import search_reconcile # type: ignore[no-redef] + import source_vector # type: ignore[no-redef] + import cu_ingestion_auth + from _common import ( # type: ignore[no-redef] + SEARCH_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, + blocked_result, canonical_bytes, digest, emit_result, http_request, load_approved_input, HttpResult, + reject_secrets, require_allowed_fields, + ) + + +SNAPSHOT_WARNING = ( + "Before/after inventories detect observed drift, not an atomic Storage snapshot " + "or a source lock. Scheduled sources can change after this run." +) +RETRY_STATUS = {408, 429, 500, 502, 503, 504} + + +def validate_content_understanding(value: Any, *, enabled: bool) -> dict[str, Any] | None: + if not enabled: + if value is not None: + raise _failure("cu-choice-conflict", "Minimal extraction must omit Content Understanding choices.") + return None + if not isinstance(value, dict): + raise _failure("cu-prerequisite-missing", "Standard extraction requires an existing CU-capable AIServices account.") + require_allowed_fields(value, {"endpoint", "auth", "prerequisites"}, label="Content Understanding choices") + endpoint = value.get("endpoint") + if ( + not isinstance(endpoint, str) + or re.fullmatch(r"https://[a-z0-9][a-z0-9-]{0,62}\.services\.ai\.azure\.com/?", endpoint) is None + or value.get("auth") != "system-assigned" + ): + raise _failure("cu-auth-unsupported", "Select an exact AIServices services.ai.azure.com endpoint and existing Search system-assigned identity; no keys.") + prerequisites = value.get("prerequisites") + if not isinstance(prerequisites, dict): + raise _failure("cu-prerequisite-missing", "Supply current CU resource, configuration, identity and network evidence references.") + fields = {"resource", "configuration", "identity", "network"} + require_allowed_fields(prerequisites, fields, label="Content Understanding prerequisites") + if any(not isinstance(prerequisites.get(key), str) or not prerequisites[key].strip() + or len(prerequisites[key]) > 4096 for key in fields): + raise _failure("cu-prerequisite-missing", "Owner-verified CU capability/region, configuration, Search role and reachability evidence is required.") + return copy.deepcopy(value) + + +def verify_content_understanding_readback(choice: dict[str, Any] | None, current: Any) -> None: + if choice is None or current is None: + return + parameters = current.get("azureBlobParameters") if isinstance(current, dict) else None + ingestion = parameters.get("ingestionParameters") if isinstance(parameters, dict) else None + ai = ingestion.get("aiServices") if isinstance(ingestion, dict) else None + if ( + not isinstance(ai, dict) or ingestion.get("contentExtractionMode") != "standard" + or not isinstance(ai.get("uri"), str) + or ai["uri"].rstrip("/") != choice["endpoint"].rstrip("/") + ): + raise _failure("cu-readback-mismatch", "Observed standard extraction and CU endpoint must match the selected configuration.") + key = ai.get("apiKey") + if (key is not None and not (isinstance(key, str) and key == "") + or ingestion.get("identity") is not None): + raise _failure("cu-auth-conflict", "Observed CU authentication does not prove system-assigned mode; credential details withheld.") + + +def generated_resources(current: dict[str, Any], *, strict: bool = False) -> list[dict[str, Any]]: + parameters = current.get("azureBlobParameters") + created = parameters.get("createdResources") if isinstance(parameters, dict) else None + generated = [] + if isinstance(created, dict): + for kind, name in created.items(): + if ( + kind in {"datasource", "dataSourceConnection", "indexer", "skillset", "index"} + and isinstance(name, str) and re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_-]{0,127}", name) + ): + generated.append({ + "type": "datasource" if kind == "dataSourceConnection" else kind, + "name": name, "service_managed": True, + }) + generated.sort(key=lambda item: (item["type"], item["name"])) + if strict and ( + not isinstance(created, dict) or len(created) != 4 or len(generated) != 4 + or {item["type"] for item in generated} != {"datasource", "indexer", "skillset", "index"} + ): + raise _failure("generated-resources-unverified", "Exact service-generated identities are required for reuse.") + return generated + + +def _read_intent(path: Path) -> dict[str, Any]: + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise _failure("input-unreadable", "Input must be readable UTF-8 JSON.") from exc + if not isinstance(document, dict): + raise _failure("input-schema-invalid", "Input must be an object.") + return document + + +def _receipt_paths(request: dict[str, Any]) -> tuple[str, str] | None: + paths = [request.get(field) for field in ("reuse_input_file", "reuse_result_file")] + if paths == [None, None]: + return None + if not all(isinstance(path, str) and path.strip() for path in paths): + raise _failure("reuse-evidence-invalid", "Select both retained approved input and creation result files.") + return paths[0], paths[1] + + +def _reuse_receipts(paths: tuple[str, str] | None) -> tuple[dict[str, Any], dict[str, Any]] | None: + if paths is None: + return None + try: + _, prior, fingerprint = load_approved_input(Path(paths[0])) + except UnicodeError as exc: + raise _failure("input-unreadable", "Retained input must be readable UTF-8 JSON.") from exc + _validate_plan(prior) + result = _read_intent(Path(paths[1])) + _validate_reuse_receipts(prior, fingerprint, result) + return prior, result + + +def _validate_reuse_receipts(prior, fingerprint, result): + reject_secrets(result) + if not all(isinstance(result.get(field), dict) for field in ( + "approved_plan", "source_evidence", "resources", "verification", "source" + )): + raise _failure("reuse-evidence-invalid", "Retained creation result sections must be objects.") + if ( + prior["source"]["action"] != "create" or result.get("status") != "completed" + or result["approved_plan"].get("confirmed") is not True + or result.get("approved_plan") != {"confirmed": True, "fingerprint": fingerprint} + or result.get("source_evidence", {}).get("boundary") != prior["boundary"] + or result.get("source_evidence", {}).get("inventory_digest") != prior["inventory_digest"] + ): + raise _failure("reuse-evidence-invalid", "Retained records must prove the exact approved source creation.") + + +def _connection_binding(connection, boundary): + if connection is None or isinstance(connection, str) and connection.lower() in ("", ""): + return "concealed" + if isinstance(connection, str) and connection.startswith("ResourceId="): + if connection.removesuffix(";") != f"ResourceId={boundary['storage_id']}": + raise _failure("boundary-mismatch", "Observed connection targets a different Storage account.") + return "visible" + raise _failure("source-binding-conflict", "Observed credentials are not the selected keyless binding or a recognized concealed value; details withheld.") + + +def _verify_storage_binding( + source: dict[str, Any], boundary: dict[str, Any], current: dict[str, Any], + generated: list[dict[str, Any]], + load_receipts: Callable[[], tuple[dict[str, Any], dict[str, Any]] | None], +) -> None: + connection = current["azureBlobParameters"].get("connectionString") + if _connection_binding(connection, boundary) == "visible": + return + receipts = load_receipts() + if receipts is None: + raise _failure( + "source-binding-unverified", + "Source account binding is unknown: GET the exact source definition or select its retained approved input/successful creation result pair. Redaction is not a mismatch; snippets are not binding proof.", + ) + _verify_creation_binding(source, boundary, current, generated, receipts) + + +def _verify_creation_binding( + source: dict[str, Any], boundary: dict[str, Any], current: dict[str, Any], + generated: list[dict[str, Any]], receipts: tuple[dict[str, Any], dict[str, Any]], +) -> None: + prior, result = receipts + observed = { + "type": "knowledge-source", "name": source["name"], "etag": current["@odata.etag"], + "definition_digest": digest(search_reconcile._definition(current)), + } + if ( + prior["boundary"] != boundary + or any(prior["source"].get(field) != source[field] for field in ("endpoint", "name", "api_version")) + or not search_reconcile.definitions_match(prior["source"]["desired"], current) + or result.get("resources", {}).get("created") != [observed] + or result.get("verification", {}).get("readback") != observed + or result.get("source", {}).get("generated") != generated + ): + raise _failure("source-binding-unverified", "Retained creation evidence does not match the fresh source identity/ETag.") + + +def plan_source( + request: dict[str, Any], *, + token_provider: TokenProvider = azure_cli_token, transport: Transport = http_request, + storage_transport: Transport = http_request, monotonic: Callable[[], float] = time.monotonic, +) -> dict[str, Any]: + if not isinstance(request, dict): + raise _failure("input-schema-invalid", "Planning input must be an object.") + reject_secrets(request) + require_allowed_fields( + request, + {"schema_version", "endpoint", "name", "owner", "storage_id", "container", "prefix", "is_adls", + "api_version", "processing", "network_access", "identity", "permission_options", + "ingestion_schedule", "description", "rbac", "network", "inventory_limits", "poll", + "reuse_input_file", "reuse_result_file", "embedding", "content_understanding"}, + label="Blob planning input", + ) + try: + json.dumps(request, ensure_ascii=False, allow_nan=False).encode("utf-8") + except (UnicodeError, TypeError, ValueError) as exc: + raise _failure("input-schema-invalid", "Planning choices must be valid UTF-8 JSON.") from exc + if request.get("schema_version") != "1.0" or not isinstance(request.get("owner"), str) or not request["owner"].strip(): + raise _failure("input-schema-invalid", "schema_version 1.0 and an explicit owner are required.") + boundary = blob_inventory.validate_boundary({ + field: request.get(field) for field in ("storage_id", "container", "prefix", "is_adls") + }) + limits = copy.deepcopy(blob_inventory.validate_limits(request.get("inventory_limits"))) + poll = copy.deepcopy(_poll_limits(request.get("poll"))) + if ( + request.get("processing") not in ("minimal-lexical", "minimal-vector", "standard-cu") or request.get("network_access") != "public" + or request.get("identity") != "system-assigned" or request.get("permission_options") != [] + or "ingestion_schedule" not in request or request["ingestion_schedule"] is not None + or not isinstance(request.get("api_version"), str) + or request["api_version"] not in {"2026-04-01", "2026-08-01-preview"} + or boundary["is_adls"] and request["api_version"] != "2026-08-01-preview" + ): + raise _failure( + "planning-processing-unsupported", + "This planner supports Blob 2026-04-01/2026-08-01-preview and ADLS 2026-08-01-preview: " + "Internal presets: minimal-lexical = minimal extraction without vectors; " + "minimal-vector = minimal extraction with embeddings; standard-cu = standard CU with optional embeddings. " + "These are not API enums or KB reasoning modes. Requires public, system-assigned, no permissions/schedule. " + "Other requested versions/features need a compatible owner; never change them silently.", + ) + embedding = source_vector.validate_choice( + request.get("embedding"), enabled=( + request["processing"] == "minimal-vector" + or request["processing"] == "standard-cu" and request.get("embedding") is not None + ), + api_version=request["api_version"], + ) + cu = validate_content_understanding( + request.get("content_understanding"), enabled=request["processing"] == "standard-cu", + ) + if "description" not in request or not ( + request["description"] is None or isinstance(request["description"], str) + ): + raise _failure("input-schema-invalid", "description must be explicit text or null.") + rbac, network = request.get("rbac"), request.get("network") + if ( + not isinstance(rbac, dict) or not isinstance(rbac.get("assignments"), list) + or not rbac["assignments"] or not all(isinstance(item, dict) and item for item in rbac["assignments"]) + or not isinstance(network, dict) or network.get("posture") != "public" + or not isinstance(network.get("evidence"), str) or not network["evidence"].strip() + ): + raise _failure("planning-evidence-missing", "Supply observed RBAC assignments and public network evidence; the owner verifies effective access and policy.") + ingestion = { + "contentExtractionMode": "minimal", "disableImageVerbalization": True, + "identity": None, "ingestionSchedule": None, + } + if request["api_version"].endswith("-preview"): + ingestion.update(networkAccessMode="public", ingestionPermissionOptions=[]) + if embedding is not None: + ingestion["embeddingModel"] = source_vector.model_definition(embedding) + if cu is not None: + ingestion.update(contentExtractionMode="standard", aiServices={"uri": cu["endpoint"].rstrip("/")}) + source = { + "operation": "reconcile", "resource_type": "knowledge-source", + "outcome": "create-blob-knowledge-source", + "endpoint": request.get("endpoint"), "name": request.get("name"), + "api_version": request["api_version"], "action": "create", + "owner": request["owner"], "cleanup_approved": False, + "rbac": copy.deepcopy(rbac), "network": copy.deepcopy(network), + "desired": { + "name": request.get("name"), "kind": "azureBlob", "description": request["description"], + "azureBlobParameters": { + "connectionString": f"ResourceId={boundary['storage_id']}", + "containerName": boundary["container"], "folderPath": boundary["prefix"] or None, + "isADLSGen2": boundary["is_adls"], "ingestionParameters": ingestion, + }, + }, + } + # Validate the shared control sections and exact address before any authentication. + for section, fields in ((rbac, {"assignments"}), (network, {"posture", "evidence"})): + require_allowed_fields(section, fields, label="planning controls") + url = search_reconcile.resource_url(source) + receipt_paths = _receipt_paths(request) + receipts = None + + def load_receipts() -> tuple[dict[str, Any], dict[str, Any]] | None: + nonlocal receipts + if receipts is None: + receipts = _reuse_receipts(receipt_paths) + return receipts + + deadline = monotonic() + limits["deadline_seconds"] + + def read_search() -> tuple[dict[str, Any] | None, str | None]: + remaining = deadline - monotonic() + if remaining <= 0: + raise _failure("planning-deadline-exceeded", "Planning read deadline elapsed.") + + def bounded(method: str, target: str, token: str, **kwargs: Any) -> Any: + if method != "GET" or target != url: + raise _failure("planning-write-forbidden", "Planning reads only the exact Search source.") + remaining = deadline - monotonic() + if remaining <= 0: + raise _failure("planning-deadline-exceeded", "Planning read deadline elapsed during authentication.") + return transport( + method, target, token, timeout=min(30, remaining), follow_redirects=False, + max_response_bytes=8 * 1024 * 1024, response_deadline=time.monotonic() + remaining, + ) + + current, request_id = search_reconcile.read_resource(url, token_provider(SEARCH_AUDIENCE), transport=bounded) + try: + source_vector.verify_source_readback(embedding, current) + verify_content_understanding_readback(cu, current) + except HelperFailure as failure: + failure.request_id = request_id + raise + return current, request_id + + current, first_id = read_search() + generated = [] + if current is not None: + if not search_reconcile.definitions_match(source["desired"], current): + raise _failure("definition-conflict", "The exact source has a different definition; do not overwrite, suffix or repair it.") + if not isinstance(current.get("@odata.etag"), str) or not current["@odata.etag"]: + raise _failure("definition-evidence-missing", "Exact reuse requires the current source ETag.") + generated = generated_resources(current, strict=True) + _verify_storage_binding(source, boundary, current, generated, load_receipts) + snapshot = blob_inventory.discover( + boundary, limits, token_provider=token_provider, transport=storage_transport, + monotonic=monotonic, deadline=deadline, + ) + refreshed, last_id = read_search() + if (current is None) != (refreshed is None) or current is not None and ( + refreshed.get("@odata.etag") != current["@odata.etag"] + or not search_reconcile.definitions_match(source["desired"], refreshed) + or generated_resources(refreshed, strict=True) != generated + ): + raise _failure("definition-drift", "Source identity/ETag or generated resources changed during Storage observation.") + if refreshed is not None: + _verify_storage_binding(source, boundary, refreshed, generated, load_receipts) + source.update(action="reuse", expected_etag=refreshed["@odata.etag"]) + if monotonic() >= deadline: + raise _failure("planning-deadline-exceeded", "Planning read deadline elapsed during Search readback.") + evidence = {"verified": True, "inventory_digest": snapshot["inventory_digest"]} + if boundary["is_adls"]: + evidence.update(path_verified=True, acl_verified=True) + source["source_evidence"] = evidence + plan = { + "operation": "reconcile-and-monitor", "owner": request["owner"], "cleanup_approved": False, + "boundary": boundary, "inventory_digest": snapshot["inventory_digest"], + "inventory_limits": limits, "poll": poll, "source": source, + } + if embedding is not None: + plan["embedding"] = embedding + if cu is not None: + plan["content_understanding"] = cu + plan["cu_plan_version"] = "1.0" + if current is not None: + plan["expected_generated"] = generated + else: + plan["expected_source_absent"] = True + _validate_plan(plan) + fingerprint = digest(plan) + mutation = source["action"] == "create" + return { + "status": "planned", "outcome": "create-blob-knowledge-source", + "plan_fingerprint": fingerprint, + "execution_input": { + "schema_version": "1.0", "plan": plan, + "approval": {"confirmed": False, "fingerprint": fingerprint}, + }, + "approval_summary": { + "target": {"endpoint": source["endpoint"], "name": source["name"], "api_version": source["api_version"]}, + "storage_account": boundary["storage_id"].rsplit("/", 1)[1], + "source_kind": "adls-gen2" if boundary["is_adls"] else "azure-blob", + "scope": "selected folder/directory" if boundary["prefix"] else "explicit container/filesystem root", + "object_count": len(snapshot["objects"]), "total_bytes": sum(item["size"] for item in snapshot["objects"]), + "adls_paths_observed": len(snapshot["adls_paths"]), + "processing": ( + f"standard Content Understanding extraction {'with selected embeddings' if embedding else 'without source vectors'}; no image verbalization, chat, permission ingestion or schedule" + if cu else + "minimal extraction with embeddings; no image verbalization, permission ingestion or schedule" + if embedding else "minimal lexical; image verbalization disabled; no models, permission ingestion or schedule" + ), + **({"embedding": source_vector.summary(embedding)} if embedding else {}), + **({"content_understanding": { + "purpose": "Standard document extraction only; not source vectorization or KB answer synthesis.", + "endpoint": cu["endpoint"].rstrip("/"), "auth": cu["auth"], + "authentication": cu_ingestion_auth.approval_summary( + "adlsGen2" if request["is_adls"] else "azureBlob", cu["auth"], creating=mutation, + ), + "kb_reasoning": "Unchanged; KB chat requires separate selection, access and approval.", + "cost_and_data": "Creation sends selected documents to billable CU processing (no free document allowance); generated Search content is retained. Cross-region processing may apply. Selected source embeddings have separate costs.", + "source_vectorization": "azureOpenAI" if embedding else "none", + "prerequisites": "Owner-verified references only, not effective access or successful processing proof. No local auth, role, model or defaults changes.", + }} if cu else {}), + "network": "public; supplied access evidence remains owner-verified", + "supplied_role_assignments": len(rbac["assignments"]), + "source_action": source["action"], "execution_required": mutation, + "mutation_approval_required": mutation, + "ownership": "Only a newly created source and its service-generated children become run-owned; existing Search/Storage/objects/roles remain shared.", + "verification": "Fresh identity/inventory and ADLS owner/ACL metadata only; effective Search access, ingestion readiness and retrieval are unverified.", + "cost_and_retention": "Creation starts indexing and retains generated Search resources; existing charges/schedules continue on reuse.", + "cleanup": "Separate run-owned source cleanup only; never delete Storage objects or shared resources.", + "next_step": ( + "Owner refreshes identity, RBAC, network, source state and cost/data consent, then approves these changes before applying the unchanged private artifact." + if mutation else "Reuse without mutation approval or executor invocation. Refresh discovery before later use; evidence is not future consent." + ), + }, + "read_only_evidence": {"request_ids": [item for item in [first_id, *snapshot["request_ids"], last_id] if item]}, + "writes_performed": [], "warnings": [SNAPSHOT_WARNING], + } + + +def _failure(code: str, message: str, *, request_id: str | None = None) -> HelperFailure: + return HelperFailure(code, message, blocked_at="verification", request_id=request_id) + + +def _timestamp(value: Any) -> datetime: + if not isinstance(value, str): + raise _failure("ingestion-status-invalid", "Synchronization timestamp is missing.") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise _failure("ingestion-status-invalid", "Synchronization timestamp is invalid.") from exc + if parsed.tzinfo is None: + raise _failure("ingestion-status-invalid", "Synchronization timestamp needs a time zone.") + return parsed + + +def _poll_limits(value: Any) -> dict[str, int]: + if not isinstance(value, dict): + raise _failure("input-schema-invalid", "poll must be an object.") + require_allowed_fields( + value, {"deadline_seconds", "max_requests", "interval_seconds"}, label="poll" + ) + for field, maximum in ( + ("deadline_seconds", 3600), ("max_requests", 1000), ("interval_seconds", 60) + ): + if type(value.get(field)) is not int or not 1 <= value[field] <= maximum: + raise _failure("input-schema-invalid", "Polling limits must be bounded positive integers.") + return value + + +def _validate_plan(plan: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: + reject_secrets(plan) + require_allowed_fields( + plan, + {"operation", "owner", "cleanup_approved", "source", "boundary", "inventory_digest", + "inventory_limits", "poll", "expected_generated", "expected_source_absent", "embedding", + "content_understanding", "cu_plan_version"}, + label="Blob source plan", + ) + if plan.get("operation") != "reconcile-and-monitor" or plan.get("cleanup_approved") is not False: + raise _failure("operation-invalid", "Blob application requires reconcile-and-monitor without cleanup.") + source = plan.get("source") + if not isinstance(source, dict) or not isinstance(plan.get("owner"), str) or not plan["owner"]: + raise _failure("input-schema-invalid", "A source plan and owner are required.") + if ( + source.get("operation") != "reconcile" + or source.get("resource_type") != "knowledge-source" + or source.get("action") not in ("create", "reuse") + or not isinstance(source.get("api_version"), str) + or source.get("owner") != plan["owner"] + or not isinstance(source.get("desired"), dict) + or source["desired"].get("kind") != "azureBlob" + or source.get("ai_services_api_key_environment") is not None + or source.get("ai_services_key_acquisition") is not None + ): + raise _failure("step-contract-mismatch", "Only same-owner Blob creation or exact reuse is supported.") + search_reconcile._validate_plan(source) + search_reconcile._resource_url(source) + boundary = blob_inventory.validate_boundary(plan.get("boundary")) + blob_inventory.validate_limits(plan.get("inventory_limits")) + _poll_limits(plan.get("poll")) + if "expected_source_absent" in plan and ( + plan["expected_source_absent"] is not True or source["action"] != "create" + or source.get("expected_etag") is not None + ): + raise _failure("step-contract-mismatch", "Expected absence is only valid for a creation plan.") + if "expected_generated" in plan and ( + source["action"] != "reuse" + or not isinstance(plan["expected_generated"], list) + or len(plan["expected_generated"]) != 4 + or any(not isinstance(item, dict) or set(item) != {"type", "name", "service_managed"} + or item["service_managed"] is not True + or not isinstance(item["type"], str) + or not isinstance(item["name"], str) + or re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_-]{0,127}", item["name"]) is None + for item in plan["expected_generated"]) + or {item["type"] for item in plan["expected_generated"]} != {"datasource", "indexer", "skillset", "index"} + ): + raise _failure("generated-resources-unverified", "Expected generated identities require a complete reuse plan.") + parameters = source["desired"]["azureBlobParameters"] + if ( + parameters["connectionString"].removesuffix(";") != f"ResourceId={boundary['storage_id']}" + or parameters["containerName"] != boundary["container"] + or parameters["folderPath"] != (boundary["prefix"] or None) + or parameters["isADLSGen2"] != boundary["is_adls"] + or "createdResources" in parameters + or source["source_evidence"]["inventory_digest"] != plan.get("inventory_digest") + ): + raise _failure("boundary-mismatch", "Source definition must bind the exact discovered boundary and inventory.") + if ( + not isinstance(plan.get("inventory_digest"), str) + or search_reconcile.SHA256.fullmatch(plan["inventory_digest"]) is None + ): + raise _failure("source-evidence-invalid", "The approved inventory digest is required.") + ingestion = parameters.get("ingestionParameters") + if not isinstance(ingestion, dict) or ingestion.get("assetStore") is not None: + raise _failure( + "source-write-forbidden", + "Ingestion parameters are required; asset-store writes are outside this read-only-source workflow.", + ) + source_vector.validate_plan_choice(plan) + standard = ingestion.get("contentExtractionMode") == "standard" + if "cu_plan_version" in plan and (plan["cu_plan_version"] != "1.0" or not standard): + raise _failure("cu-plan-version-invalid", "CU planner artifacts require version 1.0 and standard extraction.") + # Absence/generated guards predate CU planning and remain valid on legacy + # wire-only artifacts. Only CU-specific provenance selects the new contract. + planner_cu = standard and any(field in plan for field in ( + "cu_plan_version", "embedding", "content_understanding", + )) + cu = validate_content_understanding( + plan.get("content_understanding"), enabled=planner_cu, + ) + if cu is not None: + expected = { + "contentExtractionMode": "standard", "disableImageVerbalization": True, + "identity": None, "ingestionSchedule": None, + "aiServices": {"uri": cu["endpoint"].rstrip("/")}, + } + if "embedding" in plan: + expected["embeddingModel"] = source_vector.model_definition(plan["embedding"]) + if source["api_version"].endswith("-preview"): + expected.update(networkAccessMode="public", ingestionPermissionOptions=[]) + if ingestion != expected or boundary["is_adls"] and source["api_version"] != "2026-08-01-preview": + raise _failure("cu-plan-mismatch", "Standard CU requires the unchanged keyless public definition, matching optional embeddings and no chat, asset store, permissions or schedule.") + return source, boundary + + +def _first_error(state: dict[str, Any]) -> dict[str, Any] | None: + errors = state.get("errors") + if errors is None or errors == []: + return None + if not isinstance(errors, list) or any(not isinstance(error, dict) for error in errors): + raise _failure("ingestion-status-invalid", "Ingestion errors have an invalid shape.") + error = errors[0] + # Document errors can contain content, SAS URLs, and credentials in free text. + return { + "status": error.get("statusCode") if type(error.get("statusCode")) is int else None, + "message": "Document-level ingestion error; sensitive service text withheld.", + "diagnostic_digest": digest(error), + } + + +def _synchronization_state(state, request_id=None): + if state is None: + return None + if not isinstance(state, dict): + raise _failure("ingestion-status-invalid", "Synchronization state must be an object.", request_id=request_id) + return dict(state) + + +def _retry_after(value, *, now=None): + now = now or datetime.now(timezone.utc) + kind = getattr(value, "kind", None) + try: + if kind is not None: + if kind == "overlong": + return math.inf + payload = getattr(value, "value", None) + if (not isinstance(kind, str) or kind not in {"seconds", "date", "date-rfc850"} + or type(payload) not in (int, float) + or (type(payload) is float and not math.isfinite(payload))): + return None + if kind == "seconds": + return payload if payload >= 0 and payload == int(payload) else None + parsed = datetime.fromtimestamp(payload, timezone.utc) + obsolete_date = kind == "date-rfc850" + else: + if not isinstance(value, str) or len(value) > 128: + return None + if value.isascii() and value.isdigit() and len(value) <= 10: + return int(value) + parsed = parsedate_to_datetime(value) + obsolete_date = bool(re.fullmatch(r"[A-Za-z]+, \d{2}-[A-Za-z]{3}-\d{2} \d{2}:\d{2}:\d{2} GMT", value)) + if obsolete_date: + year = now.year // 100 * 100 + parsed.year % 100 + if year > now.year + 50: + year -= 100 + parsed = parsed.replace(year=year) + if parsed.tzinfo is not None: + return max(0, (parsed - now).total_seconds()) + except (ValueError, OverflowError, TypeError, OSError): + pass + return None + + +def _execution_progress(body, not_before): + if (not isinstance(body, dict) or not isinstance(body.get("status"), str) + or body["status"] not in {"running", "error", "unknown"}): + raise _failure("indexer-status-invalid", "Indexer status must be an object.") + if body["status"] == "error": + raise _failure("indexer-status-error", "Indexer availability reports an error; execution success is not inferred.") + run = body.get("lastResult") + if run is None: + return None + if not isinstance(run, dict): + raise _failure("indexer-status-invalid", "Latest indexer execution must be an object.") + start = _timestamp(run.get("startTime")) + if start < not_before: + return None + status = run.get("status") + if not isinstance(status, str) or status not in {"inProgress", "success", "transientFailure", "persistentFailure", "reset"}: + raise _failure("indexer-status-invalid", "Unknown latest execution status; top-level running is not execution proof.") + result = {"startTime": run["startTime"], "run_status": status} + if run.get("endTime") is not None: + if _timestamp(run["endTime"]) < start: + raise _failure("indexer-status-invalid", "Indexer execution interval is invalid.") + result["endTime"] = run["endTime"] + for field, target in (("itemsProcessed", "items_attempted"), ("itemsFailed", "items_failed")): + if field in run: + if type(run[field]) is not int or run[field] < 0: + raise _failure("indexer-status-invalid", "Indexer execution counters are invalid.") + result[target] = run[field] + if result.get("items_failed", 0) > result.get("items_attempted", result.get("items_failed", 0)): + raise _failure("indexer-status-invalid", "Failed item count exceeds attempted items.") + errors = run.get("errors", []) + if errors is not None and (not isinstance(errors, list) or any(not isinstance(item, dict) for item in errors)): + raise _failure("indexer-status-invalid", "Indexer execution errors are malformed.") + result["error_count"] = len(errors or []) + return result + + +def _execution_completed_by(execution, end, not_before): + required = {"startTime", "endTime", "run_status", "error_count"} + if (not isinstance(execution, dict) or not required <= execution.keys() + or execution.keys() - required - {"items_attempted", "items_failed"} + or execution["run_status"] != "success" + or type(execution["error_count"]) is not int or execution["error_count"] != 0 + or any(type(execution[key]) is not int or execution[key] < 0 + for key in ("items_attempted", "items_failed") if key in execution) + or execution.get("items_failed", 0)): + return False + try: + return not_before <= _timestamp(execution["startTime"]) <= _timestamp(execution["endTime"]) <= end + except HelperFailure: + return False + + +@reporting("blob-monitor") +def monitor( + source: dict[str, Any], + *, + not_before: datetime, + limits: dict[str, int], + token_provider: TokenProvider, + transport: Transport, + require_new_cycle: bool = False, + excluded_cycle: list[str] | None = None, + monotonic: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, + progress: Progress | None = None, + cancelled: Callable[[], bool] = lambda: False, + indexer_name: str | None = None, +) -> dict[str, Any]: + _poll_limits(limits) + progress.update("ingestion-cycle") + excluded = tuple(_timestamp(value) for value in excluded_cycle) if excluded_cycle is not None else None + url = ( + f"{source['endpoint'].rstrip('/')}/knowledgesources('{search_reconcile._odata_name(source['name'])}')/status?" + + urlencode({"api-version": source["api_version"]}) + ) + token = token_provider(SEARCH_AUDIENCE) + started = monotonic() + deadline = started + limits["deadline_seconds"] + request_ids: list[str] = [] + first_error: dict[str, Any] | None = None + first_retry: dict[str, Any] | None = None + observed = "no-completed-cycle" + initial_cycle: tuple[Any, Any] | None = None + first_response = True + checks = 0 + delay = limits["interval_seconds"] + latest = {} + previous = None + previous_execution = None + execution_guard = None + pause_reason = "request-limit" + newest_run = None + synchronization_status = "not-reported" + + def report(phase, next_check=None): + value = { + "schema_version": "1.0", "phase": phase, + "run_start": _timestamp(latest["startTime"]).isoformat() if latest.get("startTime") else None, + "processed": latest.get("itemsUpdatesProcessed"), "failed": latest.get("itemsUpdatesFailed"), + "skipped": latest.get("itemsSkipped"), "unit": "item-updates", + "total": None, "remaining": None, "denominator": "not-comparable-to-files", + "synchronization_status": synchronization_status, + "elapsed_seconds": max(0, monotonic() - started), "next_check_seconds": next_check, + } + progress.counts["status_checks"] = checks + progress.blob_update(value) + return value + + def failure_result(*args): + result = _readiness_failure(*args) + result["progress"] = report("failed") + result["watch"] = {"schema_version": "1.0", "state": "blocked", "reason": "evidence-failure", + "elapsed_seconds": max(0, monotonic() - started), + "status_checks": checks, "latest": copy.deepcopy(latest)} + return result + + def read_status(target): + nonlocal checks, retry_after + remaining = deadline - monotonic() + if remaining <= 0: + raise _failure("ingestion-watch-expired", "The client watch window elapsed before this read.") + checks += 1 + response = transport( + "GET", target, token, timeout=min(30, remaining), + max_response_bytes=8 * 1024 * 1024, follow_redirects=False, + response_deadline=monotonic() + min(30, remaining), + ) + if response.request_id: + request_ids.append(response.request_id) + if response.status != 200: + value = getattr(response, "retry_after", None) + if value is None: + value = next((v for k, v in response.headers.items() if k.lower() == "retry-after"), None) + retry_after = _retry_after(value) + raise HelperFailure("ingestion-inaccessible", "Status read is inaccessible.", + blocked_at="verification", status=response.status, request_id=response.request_id) + return response + + while checks < limits["max_requests"]: + remaining = deadline - monotonic() + if cancelled(): + pause_reason = "cancelled" + break + if remaining <= 0: + pause_reason = "deadline" + break + retry_after = None + throttled = False + try: + response = read_status(url) + except KeyboardInterrupt: + pause_reason = "cancelled" + break + except HelperFailure as failure: + if failure.request_id and failure.request_id not in request_ids: + request_ids.append(failure.request_id) + if failure.code == "response-deadline-exceeded": + first_retry = first_retry or {"code": failure.code, "status": failure.http_status, + "request_id": failure.request_id} + if monotonic() >= deadline: + pause_reason = "deadline" + break + if failure.code == "ingestion-watch-expired": + pause_reason = "deadline" + break + if retry_after is None: + retry_after = _retry_after(failure.retry_after) + if (failure.http_status not in RETRY_STATUS + and failure.code not in {"azure-response-ambiguous", "response-deadline-exceeded"}): + return failure_result("ingestion-inaccessible", failure.request_id, request_ids, + first_error, first_retry, failure.http_status) + if first_retry is None: + first_retry = {"code": failure.code, "status": failure.http_status, + "request_id": failure.request_id} + throttled = failure.http_status == 429 + delay = min(60, max(limits["interval_seconds"], delay * 2)) + else: + if monotonic() >= deadline: + pause_reason = "deadline" + break + body = response.body + if not isinstance(body, dict) or body.get("kind") != "azureBlob": + raise _failure("ingestion-status-invalid", "Expected azureBlob status.", request_id=response.request_id) + synchronization_status = body.get("synchronizationStatus", "not-reported") + if synchronization_status not in ("not-reported", "active", "creating", "deleting"): + raise _failure("ingestion-status-invalid", "Unknown knowledge-source synchronization availability.", + request_id=response.request_id) + current = _synchronization_state(body.get("currentSynchronizationState"), response.request_id) + last = _synchronization_state(body.get("lastSynchronizationState"), response.request_id) + for state in (current, last): + if state is not None: + _timestamp(state.get("startTime")) + _first_error(state) + state = current or last + start = _timestamp(state["startTime"]) if state else None + if start is not None and start >= not_before and (newest_run is None or start >= newest_run): + newest_run = start + error = _first_error(state) + if first_error is None and error: + first_error = {**error, "request_id": response.request_id} + latest = {"startTime": state["startTime"], + "state": "in-progress" if current else "completed-cycle-observed"} + for field in ("itemsUpdatesProcessed", "itemsUpdatesFailed", "itemsSkipped"): + if field in state: + if type(state[field]) is not int or state[field] < 0: + raise _failure("ingestion-status-invalid", "Observed synchronization counters are invalid.", + request_id=response.request_id) + latest[field] = state[field] + if current: + observed = "in-progress" + signature = digest(latest) + delay = (limits["interval_seconds"] if signature != previous + else min(60, delay * 2)) + previous = signature + else: + latest = {} + observed = "stale-success-or-unrelated-cycle" if state else "no-completed-cycle" + signature = digest({}) + delay = limits["interval_seconds"] if signature != previous else min(60, delay * 2) + previous = signature + if execution_guard is not None: + latest["indexer_execution"] = copy.deepcopy(execution_guard) + if first_response and require_new_cycle and last: + initial_cycle = (last.get("startTime"), last.get("endTime")) + first_response = False + if ( + last and last.get("endTime") is not None + and _timestamp(last.get("startTime")) >= max(not_before, newest_run or not_before) + and (last.get("startTime"), last.get("endTime")) != initial_cycle + and (excluded is None or (_timestamp(last.get("startTime")), _timestamp(last.get("endTime"))) != excluded) + ): + status = last.get("status") + if status is not None and not isinstance(status, str): + raise _failure("ingestion-status-invalid", "Synchronization status must be a string when present.") + if status in {"failure", "partialSuccess"}: + return failure_result(f"ingestion-{status}", response.request_id, + request_ids, first_error, first_retry) + if status in {None, "success"}: + start = _timestamp(last.get("startTime")) + end = _timestamp(last.get("endTime")) + failed = last.get("itemsUpdatesFailed") + processed = last.get("itemsUpdatesProcessed") + skipped = last.get("itemsSkipped") + if end < start or any(type(count) is not int or count < 0 for count in (failed, processed, skipped)): + raise _failure("ingestion-status-invalid", "Completed synchronization counters or interval are invalid.") + if failed or first_error: + return failure_result("ingestion-partialSuccess", response.request_id, + request_ids, first_error, first_retry) + if (not current and synchronization_status in ("active", "not-reported") + and (execution_guard is None + or _execution_completed_by(execution_guard, end, not_before))): + return { + "status": "verified", + "progress": report("completed"), + "synchronization": { + field: last[field] for field in + ("startTime", "endTime", "itemsUpdatesProcessed", "itemsUpdatesFailed", "itemsSkipped") + }, + "not_before": not_before.isoformat(), + "request_ids": request_ids, + "first_retry": first_retry, + "watch": {"schema_version": "1.0", "state": "completed", + "elapsed_seconds": max(0, monotonic() - started), + "status_checks": checks, "latest": latest}, + } + else: + raise _failure("ingestion-status-invalid", "Unknown synchronization completion status.") + elif last and not current: + observed = "stale-success-or-unrelated-cycle" + if indexer_name and checks < limits["max_requests"] and monotonic() < deadline and not cancelled(): + target = (f"{source['endpoint'].rstrip('/')}/indexers('{search_reconcile._odata_name(indexer_name)}')/status?" + + urlencode({"api-version": source["api_version"]})) + run_response = None + try: + run_response = read_status(target) + cutoff = max(not_before, newest_run or not_before) + if execution_guard is not None: + cutoff = min(cutoff, _timestamp(execution_guard["startTime"])) + execution = _execution_progress(run_response.body, cutoff) + if (execution is not None and execution_guard is not None + and _timestamp(execution["startTime"]) < _timestamp(execution_guard["startTime"])): + execution = None + if execution is not None: + if execution_guard is not None or execution["run_status"] == "inProgress": + execution_guard = copy.deepcopy(execution) + execution_signature = digest(execution) + if execution_signature != previous_execution: + delay = limits["interval_seconds"] + previous_execution = execution_signature + latest["indexer_execution"] = execution + if (execution.get("items_failed", 0) or execution["error_count"] + or execution["run_status"] in {"transientFailure", "persistentFailure"}): + first_error = first_error or { + "status": None, "message": "Latest relevant indexer execution reports failures; details withheld.", + "request_id": run_response.request_id, + } + except KeyboardInterrupt: + pause_reason = "cancelled" + break + except HelperFailure as failure: + if failure.request_id is None and run_response is not None: + failure.request_id = run_response.request_id + if failure.request_id and failure.request_id not in request_ids: + request_ids.append(failure.request_id) + if failure.code == "response-deadline-exceeded": + first_retry = first_retry or {"code": failure.code, "status": failure.http_status, + "request_id": failure.request_id} + if monotonic() >= deadline: + pause_reason = "deadline" + break + if failure.code == "ingestion-watch-expired": + pause_reason = "deadline" + break + if (failure.http_status not in RETRY_STATUS + and failure.code not in {"azure-response-ambiguous", "response-deadline-exceeded"}): + code = failure.code if failure.code in {"indexer-status-invalid", "indexer-status-error"} else "indexer-status-unverified" + return failure_result(code, failure.request_id, request_ids, + first_error, first_retry, failure.http_status) + first_retry = first_retry or {"code": failure.code, "status": failure.http_status, + "request_id": failure.request_id} + throttled = failure.http_status == 429 + if retry_after is None: + retry_after = _retry_after(failure.retry_after) + delay = min(60, delay * 2) + remaining = deadline - monotonic() + if cancelled(): + pause_reason = "cancelled" + break + if retry_after is not None and retry_after > min(60, remaining): + report("throttled" if throttled else "waiting") + pause_reason = "retry-after" + break + if remaining > 0 and checks < limits["max_requests"]: + try: + wait = min(max(delay, retry_after or 0), remaining) + report("throttled" if throttled else + "ingesting" if observed == "in-progress" else "waiting", wait) + sleep(wait) + except KeyboardInterrupt: + pause_reason = "cancelled" + break + if monotonic() >= deadline: + pause_reason = "deadline" + result = _readiness_failure( + "ingestion-timeout", request_ids[-1] if request_ids else None, + request_ids, first_error, first_retry, + ) + result["observed"] = observed + result["watch"] = {"schema_version": "1.0", "state": "paused", "reason": pause_reason, + "elapsed_seconds": max(0, monotonic() - started), + "status_checks": checks, "latest": latest, + "service_execution": "not-modified"} + if first_error or latest.get("itemsUpdatesFailed"): + result["code"] = "ingestion-partialSuccess" + result["progress"] = report("failed" if result["code"] == "ingestion-partialSuccess" else "paused") + result["safe_next_decision"] = ( + "Service reports item failures; retain the partial work and inspect exact private error evidence read-only. " + "A new watch does not repair failures; do not replay writes or default to cleanup." + ) if result["code"] == "ingestion-partialSuccess" else ( + "Client watch paused; this is not an ingestion failure. Resume GET-only with the unchanged input " + "and readiness receipt using blob_recheck.py --input --receipt . " + "Do not recreate, run/reset, delete, or infer corpus completion." + ) + return result + + +def _readiness_failure( + code: str, request_id: str | None, request_ids: list[str], + first_error: dict[str, Any] | None, first_retry: dict[str, Any] | None, + status: int | None = None, +) -> dict[str, Any]: + return { + "status": "unverified", "code": code, "request_id": request_id, + "http_status": status, "request_ids": request_ids, + "first_error": first_error, "first_retry": first_retry, + } + + +@reporting("blob-source") +def execute( + document: dict[str, Any], + *, + token_provider: TokenProvider = azure_cli_token, + transport: Transport = http_request, + storage_transport: Transport = http_request, + now: Callable[[], datetime] = lambda: datetime.now(timezone.utc), + monotonic: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, + checkpoint: Any = None, + progress: Progress | None = None, +) -> dict[str, Any]: + progress.update("validation") + reject_secrets(document) + require_allowed_fields(document, {"schema_version", "plan", "approval", "_computed_fingerprint"}, + label="input envelope") + plan = document.get("plan") + approval = document.get("approval") + if document.get("schema_version") != "1.0" or not isinstance(plan, dict) or not isinstance(approval, dict): + raise _failure("input-schema-invalid", "A typed user-approved envelope is required.") + require_allowed_fields(approval, {"confirmed", "fingerprint"}, label="approval") + fingerprint = digest(plan) + if approval.get("confirmed") is not True or approval.get("fingerprint") != fingerprint: + raise _failure("approval-mismatch", "The exact recomputed plan fingerprint must be approved.") + source, boundary = _validate_plan(plan) + progress.update("blob-inventory") + before = blob_inventory.discover( + boundary, plan["inventory_limits"], token_provider=token_provider, transport=storage_transport, + ) + if before["inventory_digest"] != plan["inventory_digest"]: + raise _failure("source-drift", "Current source evidence differs from approval; reconfirmation is required.") + not_before = now() + generated: list[dict[str, Any]] = [] + write_generated: list[dict[str, Any]] = [] + initial_read = True + creation_etag = None + embedding_transport = source_vector.guard_readback_transport(plan, transport) + + def reconcile_transport(method: str, url: str, token: str, **kwargs: Any) -> Any: + nonlocal initial_read + first_read = method == "GET" and initial_read + if first_read: + initial_read = False + kwargs.update(max_response_bytes=8 * 1024 * 1024, follow_redirects=False) + try: + response = embedding_transport(method, url, token, **kwargs) + except HelperFailure as failure: + if ( + first_read and failure.http_status == 404 and source["action"] == "create" + and source.get("expected_etag") is not None + ): + raise _failure( + "definition-drift", "The source bound by the approved ETag is absent; rebuild the plan.", + request_id=failure.request_id, + ) from failure + raise + if method == "GET" and response.status == 200 and isinstance(response.body, dict): + etag = search_reconcile.resolve_etag(search_reconcile.response_etags(response), response.request_id) + if etag is not None: + response = HttpResult(response.status, {**response.body, "@odata.etag": etag}, + response.headers, response.etag_values) + if creation_etag is not None and etag != creation_etag: + raise _failure("definition-drift", "Source version differs from the acknowledged create response.", + request_id=response.request_id) + if isinstance(response.body, dict): + if method == "GET" and response.status == 200: + parameters = response.body.get("azureBlobParameters") + if isinstance(parameters, dict): + try: + _connection_binding(parameters.get("connectionString"), boundary) + except HelperFailure as failure: + failure.request_id = response.request_id + raise + if first_read and response.status == 200: + if plan.get("expected_source_absent"): + raise _failure("definition-drift", "A source appeared after planning; rebuild the plan.", request_id=response.request_id) + if source.get("expected_etag") is not None and response.body.get("@odata.etag") != source["expected_etag"]: + raise _failure("definition-drift", "Source ETag differs from the approved readback.", request_id=response.request_id) + if method == "GET" and response.status == 200 and ( + plan.get("expected_source_absent") or "expected_generated" in plan + ): + if not isinstance(response.body.get("@odata.etag"), str) or not response.body["@odata.etag"]: + raise _failure("definition-evidence-missing", "Source readback must include its ETag.", request_id=response.request_id) + parameters = response.body.get("azureBlobParameters") + connection = parameters.get("connectionString") if isinstance(parameters, dict) else None + if isinstance(connection, str) and connection.startswith("ResourceId="): + if connection.removesuffix(";") != f"ResourceId={boundary['storage_id']}": + raise _failure("boundary-mismatch", "Readback targets a different Storage account.", request_id=response.request_id) + elif plan.get("expected_source_absent") and ( + not creation_etag or response.body.get("@odata.etag") != creation_etag + ): + raise _failure("source-binding-unverified", "Redacted creation readback lacks a matching acknowledged PUT ETag.", request_id=response.request_id) + generated.clear() + generated.extend(generated_resources(response.body)) + if method == "GET" and "expected_generated" in plan and generated != plan["expected_generated"]: + raise _failure("definition-drift", "Generated identities differ from the approved reuse plan.", request_id=response.request_id) + if method == "PUT" and response.status in {200, 201}: + write_generated[:] = generated + return response + + def created(*, response, url, body, headers): + nonlocal creation_etag + if (source["action"] != "create" or url != search_reconcile.resource_url(source) + or headers.get("If-None-Match") != "*" or "If-Match" in headers + or body != canonical_bytes(source["desired"])): + raise _failure("recheck-ownership-unproven", "Creation acknowledgement must bind the exact approved conditional wire.") + if checkpoint is not None: + checkpoint.acknowledge(plan, response, not_before, url=url, body=body, headers=headers) + creation_etag = search_reconcile.resolve_etag(search_reconcile.response_etags(response), response.request_id) + if creation_etag is None: + raise _failure("creation-version-unproven", "Successful create returned no ETag in body or HTTP headers; a later GET cannot prove its version.", + request_id=response.request_id) + + progress.update("source-reconciliation") + try: + result = search_reconcile.execute( + {"plan": source, "_computed_fingerprint": fingerprint}, + token_provider=token_provider, transport=reconcile_transport, on_created=created, + ) + except HelperFailure as failure: + if failure.writes: + failure.resources_remaining.extend(write_generated or generated) + elif failure.partial: + failure.resources_reused.extend(generated) + if checkpoint is not None and failure.writes: + result = blocked_result(failure, outcome="create-blob-knowledge-source", + fingerprint=fingerprint, owner=plan["owner"]) + result.update(readiness={"status": "unverified"}, knowledge_base="not-verified", retrieval="unverified") + return _checkpoint_result(result, checkpoint) + raise + writes = [ + {"action": "created", "type": item["type"], "name": item["name"]} + for item in result["resources"]["created"] + ] + readiness: dict[str, Any] = {"status": "unverified"} + after = None + owned_generated = list(generated or write_generated) if writes else [] + try: + expected_generated = list(generated) + if checkpoint is not None: + if source["action"] == "create" and ( + not creation_etag or creation_etag != result["verification"]["readback"]["etag"] + or write_generated and generated != write_generated + ): + raise _failure("recheck-ownership-unproven", "Checkpoint requires an acknowledged PUT ETag and unchanged generated identities.") + checkpoint.persist(plan, result, generated, not_before, + token_provider=token_provider, transport=transport) + readiness = monitor( + source, not_before=not_before, limits=plan["poll"], token_provider=token_provider, + transport=transport, require_new_cycle=not bool(writes) and checkpoint is None, + excluded_cycle=checkpoint.excluded_cycle if checkpoint is not None else None, + monotonic=monotonic, sleep=sleep, + progress=progress, + indexer_name=next((item["name"] for item in generated if item["type"] == "indexer"), None), + ) + if readiness["status"] != "verified": + raise HelperFailure( + readiness["code"], "Source reconciliation completed but ingestion readiness is unverified.", + blocked_at="verification", request_id=readiness["request_id"], + status=readiness["http_status"], + ) + progress.update("blob-readback") + after = blob_inventory.discover( + boundary, plan["inventory_limits"], token_provider=token_provider, transport=storage_transport, + ) + if after["inventory_digest"] != before["inventory_digest"]: + raise _failure("source-drift", "Source changed during ingestion; reconfirmation is required.") + progress.update("source-readback") + current, request_id = search_reconcile._get( + search_reconcile._resource_url(source), token_provider(SEARCH_AUDIENCE), transport=reconcile_transport, + ) + if ( + current is None + or search_reconcile._definition(current) != search_reconcile._definition(source["desired"]) + or current.get("@odata.etag") != result["verification"]["readback"]["etag"] + or generated != expected_generated + ): + raise _failure("definition-drift", "Source definition or ETag changed while monitoring.", request_id=request_id) + if request_id: + result["verification"]["request_ids"].append(request_id) + if len(generated) != 4 or {item["type"] for item in generated} != {"datasource", "indexer", "skillset", "index"}: + raise _failure("generated-resources-unverified", "Exact service-generated resource identities could not be read back.") + if checkpoint is not None: + checkpoint.verify(plan, token_provider=token_provider, transport=transport) + except HelperFailure as failure: + confirmed = copy.deepcopy(result) + result = blocked_result( + HelperFailure( + failure.code, failure.message, blocked_at=failure.blocked_at, writes=writes, + resources_remaining=result["ownership"]["run_owned"] + owned_generated, + request_id=failure.request_id, status=failure.http_status, partial=bool(writes), + ), + outcome="create-blob-knowledge-source", fingerprint=fingerprint, owner=plan["owner"], + ) + result["reconciliation"] = "completed" + result["writes_performed"] = writes + result["read_only_evidence"] = { + "inventory_request_ids": before["request_ids"] + (after["request_ids"] if after else []), + "configuration_request_ids": checkpoint.request_ids if checkpoint is not None else [], + } + result["confirmed_reconciliation"] = { + key: confirmed[key] for key in ("resources", "verification", "ownership") + } + result["readiness"] = {**readiness, "status": "unverified"} + if readiness.get("watch", {}).get("state") == "paused": + result["safe_next_decision"] = readiness["safe_next_decision"] + if not writes: + result["ownership"]["reused_not_owned"] = [ + {"type": "knowledge-source", "name": source["name"]}, *generated + ] + else: + result["outcome"] = "create-blob-knowledge-source" + result["readiness"] = readiness + result["verification"]["source_digest"] = after["inventory_digest"] + result["source"] = { + "type": "knowledge-source", "name": source["name"], + "generated": owned_generated if writes else generated, + "observed_generated": generated, + } + result["writes_performed"] = writes + result["source_evidence"] = { + "inventory_digest": before["inventory_digest"], "boundary": boundary, + "operator_reachability": "verified", "managed_ingestion_reachability": result["readiness"]["status"], + } + result.setdefault("warnings", []).append(SNAPSHOT_WARNING) + return _checkpoint_result(result, checkpoint) + + +def _checkpoint_result(result, checkpoint): + if "completed_writes" in result: + result["writes_performed"] = copy.deepcopy(result["completed_writes"]) + if checkpoint is not None: + result["warnings"].extend(checkpoint.recovery_warnings) + result["creation_acknowledgement"] = checkpoint.write_observation or { + "schema_version": "1.0", "state": "unavailable", + } + result["indexer_diagnostics"] = [ + item for item in checkpoint.diagnostics + if not item["field"].startswith(("datasource.", "indexer.", "skillset.", "index.")) + ] + result["generated_diagnostics"] = checkpoint.diagnostics + result["indexer_observations"] = checkpoint.observations + result["datasource_binding_observations"] = checkpoint.binding_observations + for item in checkpoint.diagnostics: + if item["severity"] == "warning" and item["message"] not in result.setdefault("warnings", []): + result["warnings"].append(item["message"]) + result["recheck_checkpoint"] = checkpoint.summary or {"status": "unavailable"} + result["recheck_write_acknowledgement"] = checkpoint.write_acknowledgement or {"status": "unavailable"} + result["recheck_acknowledgement"] = checkpoint.acknowledgement or checkpoint.write_acknowledgement or {"status": "unavailable"} + if checkpoint.summary is None: + result["safe_next_decision"] = "Preserve the first failure and resources. Use blob_recheck.py --recover with unchanged approved input and the retained acknowledgement, then --input/--receipt. Missing/conflicting write ETags or absent private evidence remain blockers; never borrow a GET version, replay creation or default to cleanup." + try: + checkpoint.finish(result) + except HelperFailure as failure: + if result["status"] == "completed": + original = result + result = blocked_result( + HelperFailure(failure.code, failure.message, blocked_at="evidence-retention", + writes=original["writes_performed"], + resources_remaining=original["ownership"]["run_owned"], + partial=bool(original["writes_performed"])), + outcome=original["outcome"], fingerprint=checkpoint.plan_digest, + ) + result["confirmed_reconciliation"] = original + result["writes_performed"] = original["writes_performed"] + result["result_evidence"] = {"status": "unavailable", "code": failure.code} + result.setdefault("warnings", []).append("Final private evidence retention failed; preserve the native result and first failure. No retry was performed.") + return result + + +def main(argv: list[str] | None = None) -> int: + try: + from .private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode + except ImportError: + from private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode + parser = argparse.ArgumentParser() + modes = parser.add_mutually_exclusive_group(required=True) + modes.add_argument("--discover", type=Path) + modes.add_argument("--plan", type=Path) + add_execution_output_argument(parser) + modes.add_argument("--input", type=Path) + parser.add_argument("--receipt-dir", type=Path) + parser.add_argument("--compact", action="store_true") + add_progress_argument(parser) + args = parser.parse_args(argv) + if args.compact and (not args.input or not args.receipt_dir): + parser.error("--compact requires --input and --receipt-dir to retain full private evidence.") + fingerprint = None + owner = None + progress = Progress("blob-source", enabled=args.progress) if args.input else None + execution_started = False + try: + validate_execution_output_mode(args) + if progress is not None: + progress.update("validation") + if args.receipt_dir and not args.input: + raise _failure("input-schema-invalid", "--receipt-dir requires --input; read-only reuse capture uses blob_recheck.py.") + if args.plan: + result = plan_source(_read_intent(args.plan)) + emit_plan_result(result, args.execution_output, preserve_unapproved_input=result["status"] == "planned") + return 0 if result["status"] == "planned" else 2 + elif args.discover: + try: + document = json.loads(args.discover.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise _failure("input-unreadable", "Discovery input must be readable UTF-8 JSON.") from exc + if not isinstance(document, dict): + raise _failure("input-schema-invalid", "Discovery input must be an object.") + reject_secrets(document) + require_allowed_fields(document, {"boundary", "inventory_limits"}, label="discovery input") + result = blob_inventory.discover(document.get("boundary"), document.get("inventory_limits")) + else: + document, plan, fingerprint = load_approved_input(args.input) + owner = plan.get("owner") + checkpoint = None + if args.receipt_dir: + try: + from . import blob_recheck + except ImportError: + import blob_recheck + checkpoint = blob_recheck.Checkpoint(args.receipt_dir, plan) + execution_started = True + if checkpoint is not None: + result = execute(document, checkpoint=checkpoint, progress=progress) + else: + result = execute(document, progress=progress) + except HelperFailure as failure: + if progress is not None and not execution_started: + progress.finish(failure=failure) + result = blocked_result(failure, outcome="blob-source-lifecycle", fingerprint=fingerprint, owner=owner) + if failure.partial and not failure.writes: + result["safe_next_decision"] = "Original create outcome is unproven. Preserve the mutation error and inspect observed resources read-only; no creation ownership, write replay or cleanup is authorized." + if args.compact: + try: + try: + from . import blob_recheck + except ImportError: + import blob_recheck + result = blob_recheck.compact_result(result, args.receipt_dir) + except HelperFailure as failure: + result.setdefault("warnings", []).append("compact-evidence-persistence-failed: full native result retained in output.") + result["presentation_failure"] = {"code": failure.code} + emit_result(result) + return 3 if result["status"] == "partial" else 2 + emit_result(result, preserve_unapproved_input=result["status"] == "planned") + return {"completed": 0, "discovered": 0, "planned": 0, "blocked": 2, "partial": 3}[result["status"]] + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/bootstrap-contracts.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/bootstrap-contracts.md new file mode 100644 index 00000000..c4ca6041 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/bootstrap-contracts.md @@ -0,0 +1,150 @@ +# Search bootstrap + +**New Basic/standard Search in an existing group**. +Other prerequisites/cleanup: [native bootstrap](../references/bootstrap-azure.md). + +Progress: stderr; `--no-progress` disables. + +## Choices + +Python 3.12+, signed-in CLI on Windows/POSIX; no login/install/context switch. +No preflight policy reads. +Prerequisites: **owner-verified references**, not helper-verified. +1–256 printable characters; punctuation allowed. +Errors omit values. + +Creation: `search`/`create`. Existing operations: [intake](search-intake-contracts.md); +legacy `reuse` stays strict, never hardens. +New locations fold ASCII case/whitespace (`East US` → `eastus`); +Not availability proof; bodies/hashes stay unchanged. +Basic: 1–3 replicas/one partition; standard: 1–12 replicas/1/2/3/4/6/12 partitions. +Fixed `SystemAssigned`, keyless auth and public networking; no role changes. +Non-secret tags; creation adds operation/owner tags. + +```json +{"schema_version":"2.0","resource_kind":"search","action":"create", +"subscription_id":"00000000-0000-0000-0000-000000000001","tenant_id":"00000000-0000-0000-0000-000000000002", +"resource_group":"rg1","name":"mysearchservice","location":"westus","sku":"basic","replicas":1,"partitions":1, +"public_network_access":"Enabled","tags":{"project":"knowledge"},"owner":"operator", +"prerequisites":{"quota":"checked; enough","pricing":"approved","network":"approved public","requirements":"settings","exclusive_name_authority":"exclusive name"}, +"limits":{"command_seconds":5,"wait_seconds":10,"wait_interval_seconds":1},"receipt_dir":""} +``` + +Replace examples/limits. Reuse: `exclusive_name_authority: null`; +required tags match; no ownership transfer. + +Region scope (closed): + +```json +{"schema_version":"2.0","subscription_id":"","tenant_id":"","receipt_dir":"","limits":{"command_seconds":5}} +``` + +No RG/name/SKU: account check plus one `az rest` GET: +`/subscriptions//providers/Microsoft.Search?api-version=2021-04-01`. +`searchServices.locations` supplies canonical `available_locations` (1–256). +Exit 0: `discovered`, no approval/artifact. +Create plan/apply: one fresh region GET each, including retained artifacts; no cache. +Unknown: `bootstrap-region-unsupported` plus choices; +denied/malformed: blocked. Reuse skips catalogs. Not SKU/quota/capacity/model/residency proof. + +```text +python "/helpers/bootstrap_azure.py" --regions scope.json +python "/helpers/bootstrap_azure.py" --plan choices.json +python "/helpers/bootstrap_azure.py" --apply "/.plan.json" --approve +``` + +`--plan` uses Azure reads/**local private writes**: `.plan.json` +body, before-state, choices, time, unapproved envelope. +Stdout: `artifact_id`/`approval_summary`, no paths/errors/hashes. +Labels: caller attestation, not helper verification; exact references stay private. +Show resource/network/capacity/tags/cost/limits; `--approve` consents once. +No cached consent or user checksums. + +15-minute expiry; submission markers prevent retries. +On drift/failure/expiry/changed choices, rerun `--plan`; discard consent. +Reconcile uncertainty; no suffix-create/body/hash edits. +Schema 1.0: regenerate (`bootstrap-contract-version`). Removed inputs +`max_polls`/`poll_seconds`/`poll_interval_seconds` are rejected. + +## Execution + +Argv: `az account show --subscription`, `az group show --name --subscription`, +Search ARM GET/PUT API `2025-05-01`. PUT uses `--body` then **one** +`"@" + str(body_path)` argument, Content-Type/client-request-ID headers. +POSIX runs `az` directly; Windows MSI/ZIP uses bundled `python.exe -IBm azure.cli`. +Other Windows layouts block. + +Only normally returned target `ResourceNotFound` proves absence. +Fresh principal/tenant/subscription/group/exact-name absence precede one PUT. +GET/PUT is **not atomic**: require exclusive name authority. +Mandatory atomicity: `bootstrap-concurrency-unresolved`, no invented conditionals. + +Reuse: two fresh material readbacks, required configuration/tags, +keyless identity/principal/tenant, endpoint, provisioning succeeded/status running. +`reused`: no mutation approval/executor or ownership adoption. +Wait uses the same identity; existence is not readiness. + +After PUT: `az resource wait --ids --api-version 2025-05-01 +--custom --interval --timeout `. +Fixed `properties`: succeeded/running or terminal states, lower/title/uppercase. +No caller expressions/custom polling. +Final raw GET establishes material match/ownership **before** readiness. +Wait failure remains primary even with ready GET; provider evidence is secondary. + +Limits: command 1–60 seconds, wait 1–900, interval 1–30. +Wait/final GET: separate `wait_seconds`/`command_seconds`; +sum is not an end-to-end deadline. +`subprocess.run`: argv, `shell=False`, best-effort timeout/cleanup. +CLI/OS/filesystems prevent hard time/process-tree/request-count guarantees. +No PID or confirmed-cleanup claim. Captured output is rejected **after capture** +above 1 MiB per stream: not a streaming cap or memory bound. Input/body/receipt +limits remain 1 MiB. Timeout/execution/receipt failure after attempted PUT is partial. + +Checks: target, principal/tenant, capacity/SKU, auth/network, identity, +required tags, ARM readiness. Metadata/ordering/unrelated tags, ID/enum casing, +equivalent optional defaults do not need another approval. +Never default missing material fields or return extra tags. +Material conflict, unknown ownership or failed readiness blocks/returns partial. + +## Evidence and failure + +Existing absolute private directory **outside the plugin**. +No symlinks/reparse paths. POSIX: directory-FD creation, identity/owner/private-mode +rechecks; exclusive safe leaves, mode 0600. +POSIX semantics required; later pathname uses are not pinned. +Windows checks owner/DACL (owner, SYSTEM, Administrators), not handle-relative +creation; `chmod` does not establish Windows privacy. No ACL changes. +Same-user/admin tampering is possible; unprovable privacy blocks. +Mocked/DrvFS is not native-filesystem or Azure evidence. + +Flushed receipts: operation/target/body/argv, client-request ID, readbacks/digests, +first error code/status/request ID, ownership. No raw errors/output/policy values/tokens/documents. +Message digest/`message_withheld`; secret-free labels/tags. +CLI file logging/dynamic installation: process-disabled. + +Creation-only policy diagnostics: +`RequestDisallowedByPolicy`, supported policy IDs or policy-violation details. +Provider error/statusDetails, never tags/configuration. +A bare 403/RBAC/network/quota failure triggers no policy calls. +Preserve first failure; diagnostic/receipt errors are secondary warnings. +At most eight referenced assignments/selected-subscription definitions +within 60 seconds: `az policy assignment show --name --scope` +and `az policy definition show --name --subscription`. No lists or global scans. +Built-ins/management groups/initiatives/exemptions/missing references need +the policy owner. No compliance/applicability proof, policy changes, write retries +or correction approval. + +Exit 0: `planned`/fresh `reused`/applied `completed` (ARM only). +Data-plane access/ingestion/retrieval stay **unverified**. +Exit 2: `blocked`, no confirmed/ambiguous Azure writes. +Exit 3: `partial`, `first_failure`, `attempted_writes`, `resources_remaining`; +unproven ownership is `unverified`, never run-owned. +Receipt failure: block before submission, partial handoff afterward. +No automatic rollback/deletion. Cleanup always needs separate consent and fresh +ownership/configuration/children/scoped-role evidence. + +Authorities: failure/conflict/uncertainty only. + +[Native wait](https://learn.microsoft.com/en-us/cli/azure/resource#az-resource-wait); +[location metadata](https://learn.microsoft.com/rest/api/resources/subscriptions/list-locations); +[provider GET](https://learn.microsoft.com/rest/api/resources/providers/get?view=rest-resources-2021-04-01). diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/bootstrap_azure.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/bootstrap_azure.py new file mode 100644 index 00000000..4f75017a --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/bootstrap_azure.py @@ -0,0 +1,716 @@ +"""Closed Search bootstrap workflow. Never a caller-programmable command runner.""" +from __future__ import annotations + +import argparse +import copy +import json +import re +import sys +import time +import uuid +from pathlib import Path + +try: + from ._progress import Progress, add_progress_argument, reporting + from ._common import HelperFailure, blocked_result, digest, emit_result, normalize_azure_location, reject_secrets + from ._bootstrap_io import MAX_BYTES, failure, private_directory, private_file, read_json, run_cli +except ImportError: + from _progress import Progress, add_progress_argument, reporting + from _common import HelperFailure, blocked_result, digest, emit_result, normalize_azure_location, reject_secrets + from _bootstrap_io import MAX_BYTES, failure, private_directory, private_file, read_json, run_cli + +API = "2025-05-01" +PROVIDER_API = "2021-04-01" +SCHEMA = "2.0" +WAIT_CONDITION = ( + "(contains(['succeeded','Succeeded','SUCCEEDED'], properties.provisioningState) && " + "contains(['running','Running','RUNNING'], properties.status)) || " + "contains(['failed','Failed','FAILED','canceled','Canceled','CANCELED','cancelled','Cancelled','CANCELLED'," + "'deleting','Deleting','DELETING','deleted','Deleted','DELETED'], properties.provisioningState) || " + "contains(['error','Error','ERROR','degraded','Degraded','DEGRADED'], properties.status)" +) +GUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") +HASH = re.compile(r"sha256:[0-9a-f]{64}") +TEXT = re.compile(r"[A-Za-z0-9 ._@:/()-]{1,256}") +POLICY_ID = re.compile( + r"(?:/subscriptions/[0-9a-f-]{36}(?:/resourceGroups/[A-Za-z0-9_.()-]+" + r"(?:/providers/Microsoft.Search/searchServices/[a-z0-9-]+)?)?" + r"|/providers/Microsoft.Management/managementGroups/[A-Za-z0-9_.-]+)?" + r"/providers/Microsoft.Authorization/(?:policyAssignments|policyDefinitions|policySetDefinitions|policyExemptions)/[A-Za-z0-9_.-]+", + re.IGNORECASE, +) + + +def closed(value, keys, label): + if not isinstance(value, dict) or set(value) != set(keys): + raise failure("bootstrap-input-invalid", label + " needs exactly the documented fields.") + + +def text(value, pattern=TEXT): + return isinstance(value, str) and pattern.fullmatch(value) is not None + + +def validate(request): + closed(request, { + "schema_version", "resource_kind", "action", "subscription_id", "tenant_id", + "resource_group", "name", "location", "sku", "replicas", "partitions", + "public_network_access", "tags", "owner", "prerequisites", "limits", "receipt_dir", + }, "Bootstrap choices") + if request["schema_version"] != SCHEMA: + raise failure("bootstrap-contract-version", "Regenerate choices and artifacts using bootstrap schema 2.0 and native wait limits.") + reject_secrets(request) + if ( + request["resource_kind"] != "search" + or request["action"] not in ("create", "reuse") + or not text(request["subscription_id"], GUID) or not text(request["tenant_id"], GUID) + or not text(request["resource_group"], re.compile(r"[A-Za-z0-9_()-][A-Za-z0-9_.()-]{0,89}")) + or request["resource_group"].endswith(".") + or not text(request["name"], re.compile(r"(?=.{2,60}$)[a-z0-9][a-z0-9]+(?:-[a-z0-9]+)*")) + or normalize_azure_location(request["location"]) is None + or request["sku"] not in ("basic", "standard") + or type(request["replicas"]) is not int or not 1 <= request["replicas"] <= (3 if request["sku"] == "basic" else 12) + or type(request["partitions"]) is not int or request["partitions"] not in ( + (1,) if request["sku"] == "basic" else (1, 2, 3, 4, 6, 12)) + or request["public_network_access"] != "Enabled" or not text(request["owner"]) + ): + raise failure("bootstrap-input-invalid", "Select supported, explicit Search identity, keyless public networking and capacity.") + tags = request["tags"] + if not isinstance(tags, dict) or len(tags) > 20 or any( + not text(k, re.compile(r"[A-Za-z0-9_.-]{1,128}")) or not text(v) for k, v in tags.items() + ): + raise failure("bootstrap-input-invalid", "Tags must be bounded non-secret labels.") + if request["action"] == "create" and any(k in tags for k in ("foundry-iq-owner", "foundry-iq-operation")): + raise failure("bootstrap-input-invalid", "Creation ownership tags are generated, not caller overrides.") + closed(request["prerequisites"], {"quota", "pricing", "network", "requirements", "exclusive_name_authority"}, "Prerequisites") + for key, value in request["prerequisites"].items(): + if key == "exclusive_name_authority" and request["action"] == "reuse" and value is None: + continue + field = f"prerequisites.{key}" + if value is None or isinstance(value, str) and not value.strip(" "): + raise failure("bootstrap-prerequisite-missing", f"{field} requires nonempty owner-verified evidence.") + if not isinstance(value, str) or len(value) > 256: + raise failure("bootstrap-prerequisite-invalid", f"{field} must be a string of 1-256 printable characters.") + for position, character in enumerate(value, 1): + if not character.isprintable(): + raise failure( + "bootstrap-prerequisite-invalid", + f"{field} contains non-printable U+{ord(character):04X} at character {position}; evidence values are withheld.", + ) + closed(request["limits"], {"command_seconds", "wait_seconds", "wait_interval_seconds"}, "Limits") + for key, ceiling in (("command_seconds", 60), ("wait_seconds", 900), ("wait_interval_seconds", 30)): + if type(request["limits"][key]) is not int or not 1 <= request["limits"][key] <= ceiling: + raise failure("bootstrap-input-invalid", "Select explicit bounded command and readiness limits.") + private_directory(request["receipt_dir"]) + return request + + +def ids(request): + group = "/subscriptions/" + request["subscription_id"] + "/resourceGroups/" + request["resource_group"] + return group, group + "/providers/Microsoft.Search/searchServices/" + request["name"] + + +def provider_id(request): + return "/subscriptions/" + request["subscription_id"] + "/providers/Microsoft.Search" + + +def search_locations(value, expected_id): + if ( + not isinstance(value, dict) or folded(value.get("id")) != expected_id.casefold() + or folded(value.get("namespace")) != "microsoft.search" + or not isinstance(value.get("resourceTypes"), list) + or any(not isinstance(item, dict) or not isinstance(item.get("resourceType"), str) + for item in value["resourceTypes"]) + ): + raise failure("bootstrap-region-metadata-invalid", "Search provider metadata is incomplete or belongs to another subscription/provider.") + services = [item for item in value["resourceTypes"] if folded(item["resourceType"]) == "searchservices"] + if len(services) != 1: + raise failure("bootstrap-region-metadata-invalid", "Exactly one Search searchServices resource type is required.") + locations = services[0].get("locations") + if ( + not isinstance(locations, list) or not 1 <= len(locations) <= 256 + or any(normalize_azure_location(location) is None for location in locations) + ): + raise failure("bootstrap-region-metadata-invalid", "Search supported locations are missing, malformed or exceed the bounded list.") + return sorted({normalize_azure_location(location) for location in locations}) + + +def desired(request, operation_id): + # Only new planning normalizes choices; retained approved bodies keep their original representation. + tags = dict(request["tags"]) + if request["action"] == "create": + tags.update({"foundry-iq-owner": request["owner"], "foundry-iq-operation": operation_id}) + return { + "location": request["location"], "tags": tags, "sku": {"name": request["sku"]}, + "identity": {"type": "SystemAssigned"}, + "properties": {"replicaCount": request["replicas"], "partitionCount": request["partitions"], + "publicNetworkAccess": "Enabled", "disableLocalAuth": True}, + } + + +def _json(raw): + if len(raw) > MAX_BYTES: + raise failure("bootstrap-cli-output-limit", "CLI response exceeds its bound.") + try: + value = json.loads(raw.decode("utf-8")) + json.dumps(value, allow_nan=False) + if not isinstance(value, dict) or value.get("nextLink") or value.get("nextToken"): + raise ValueError("Not a complete object") + return value + except (ValueError, UnicodeError, RecursionError) as exc: + raise failure("bootstrap-response-invalid", "CLI must return one complete JSON object; empty or truncated output is not evidence.") from exc + + +def _error(raw): + """Only codes/UUIDs/reference IDs survive; arbitrary Azure messages are not safe receipts.""" + message = raw.decode("utf-8", errors="replace") + code_match = re.search(r'"code"\s*:\s*"([A-Za-z][A-Za-z0-9_.-]{0,100})"|(?:ERROR:\s*)?\(([A-Za-z][A-Za-z0-9_.-]{0,100})\)', message) + code = next((g for g in code_match.groups() if g), None) if code_match else "bootstrap-cli-failed" + request_match = re.search(r"request[\s-]?id[\"'\s:]+([0-9a-f-]{36})", message, re.I) + request_id = request_match.group(1).lower() if request_match and GUID.fullmatch(request_match.group(1).lower()) else None + status_match = re.search(r'"status(?:Code)?"\s*:\s*(4\d\d|5\d\d)', message) + status = int(status_match.group(1)) if status_match else None + reason = re.match(r"(?:ERROR:\s*)?([A-Za-z ]+)\(", message.strip()) + if status is None and reason: + status = {"Bad Request": 400, "Unauthorized": 401, "Forbidden": 403, "Not Found": 404, + "Conflict": 409, "Too Many Requests": 429, "Internal Server Error": 500, + "Service Unavailable": 503, "Gateway Timeout": 504}.get(reason.group(1)) + references = sorted(set(POLICY_ID.findall(message))) + implicated = code == "RequestDisallowedByPolicy" or bool(references) or '"PolicyViolation"' in message + result = failure(code, "Azure CLI failed; untrusted original message text is withheld from output and receipts.") + result.request_id, result.http_status = request_id, status + result.policy_ids = references[:8] + result.policy_implicated = implicated + result.message_digest = digest(message) + result.policy_overflow = len(references) > 8 + return result + + +def projection(resource, required_tags=()): + """Store selected configuration, not arbitrary provider fields/error text.""" + if resource is None: + return None + props, identity = resource.get("properties", {}), resource.get("identity", {}) + if not isinstance(props, dict) or not isinstance(identity, dict): + raise failure("bootstrap-response-invalid", "Search properties and identity must be objects.") + tags = resource.get("tags") + selected = {k.casefold() for k in required_tags} + return { + "id": resource.get("id"), "location": resource.get("location"), + "tags": {k: v for k, v in tags.items() if folded(k) in selected} if isinstance(tags, dict) else tags, + "sku": {"name": resource["sku"].get("name")} if isinstance(resource.get("sku"), dict) else resource.get("sku"), + "identity": {k: identity.get(k) for k in ("type", "principalId", "tenantId")}, + "properties": {k: props.get(k) for k in ( + "replicaCount", "partitionCount", "publicNetworkAccess", "disableLocalAuth", + "provisioningState", "status", "endpoint", + )}, + } + + +def matching(resource, request, body): + value = projection(resource, body["tags"]) + tags = resource.get("tags") if resource is not None else None + if tags is None and not body["tags"]: + tags = {} + group, target = ids(request) + location = normalize_azure_location(body["location"]) + if ( + value is None or not isinstance(value["id"], str) or value["id"].casefold() != target.casefold() + or location is None or normalize_azure_location(value["location"]) != location + or not isinstance(tags, dict) + or any([v for k, v in tags.items() if folded(k) == key.casefold()] != [expected] + for key, expected in body["tags"].items()) + or not isinstance(value["sku"], dict) or folded(value["sku"].get("name")) != body["sku"]["name"] + or folded(value["identity"]["type"]) != "systemassigned" + or any(type(value["properties"][k]) is not type(v) or ( + folded(value["properties"][k]) != v.casefold() if isinstance(v, str) else value["properties"][k] != v + ) for k, v in body["properties"].items()) + ): + raise failure("bootstrap-state-conflict", "Selected Search definition differs; shared or foreign state will not be modified.") + props = resource["properties"] + if props.get("authOptions") not in (None, {}) or resource["identity"].get("userAssignedIdentities") not in (None, {}): + raise failure("bootstrap-state-conflict", "Observed authentication configuration is not the selected keyless system identity.") + rules = props.get("networkRuleSet") + if rules is not None and (not isinstance(rules, dict) or rules.get("ipRules") not in (None, []) + or (rules.get("bypass") is not None and folded(rules["bypass"]) != "none")): + raise failure("bootstrap-state-conflict", "Additional network restrictions require the owning procedure.") + if props.get("privateEndpointConnections") not in (None, []) or props.get("sharedPrivateLinkResources") not in (None, []): + raise failure("bootstrap-state-conflict", "Private/shared network resources are outside this bootstrap slice.") + return value + + +def folded(value): + return value.casefold() if isinstance(value, str) else None + + +def reuse_binding(resource, request, body): + if resource is None: + return None + value = ready(resource, request, body) + if value is None: + raise failure("bootstrap-not-ready", "Fresh Search readback does not establish ARM readiness.") + return {"id": folded(value["id"]), "principal": folded(value["identity"]["principalId"]), + "tenant": folded(value["identity"]["tenantId"])} + + +def ready(resource, request, body): + return _readiness(resource, request, matching(resource, request, body)) + + +def _readiness(resource, request, value): + props = value["properties"] + if folded(props["provisioningState"]) in ("failed", "canceled", "cancelled", "deleting", "deleted") or folded(props["status"]) in ("error", "degraded"): + evidence = [resource.get("error"), resource["properties"].get("error"), + resource["properties"].get("statusDetails")] + raw = "\n".join(value if isinstance(value, str) else json.dumps(value) + for value in evidence if isinstance(value, (str, dict, list))) + error = _error(raw.encode("utf-8")) + if error.code == "bootstrap-cli-failed": + error.code = "bootstrap-provisioning-failed" + error.message = "Search entered a failed, deleting or degraded ARM state; provider message text is withheld." + error.args = (error.message,) + error.creation_failure = True + raise error + if ( + folded(props["provisioningState"]) != "succeeded" or folded(props["status"]) != "running" + or not text(folded(value["identity"]["principalId"]), GUID) + or folded(value["identity"]["tenantId"]) != request["tenant_id"] + or folded(props["endpoint"]) not in ( + "https://" + request["name"] + ".search.windows.net", + "https://" + request["name"] + ".search.windows.net/", + ) + ): + return None + return value + + +class Session: + def __init__(self, request, operation_id, cli=run_cli, clock=time.monotonic, *, target=None): + self.request, self.operation_id = request, operation_id + self.cli, self.clock = cli, clock + self.directory = private_directory(request["receipt_dir"]) + self.sequence = 0 + self.attempted = False + self.approved = False + self.owned = False + self.warnings = [] + self.diagnosed = False + self.policy_references = [] + self.target = target if target is not None else ids(request)[1] + + def record(self, event, value): + self.sequence += 1 + private_file(self.directory, f"{self.operation_id}.{self.sequence:03d}.{uuid.uuid4().hex}.receipt.json", { + "schema_version": "1.0", "operation_id": self.operation_id, "operation": "bootstrap-search", + "target": self.target, "event": event, "evidence": value, + }) + + def call(self, args, timeout=None, mutation=False, waiting=False): + self.record("command", {"argv": args}) + if mutation: + self.attempted = True + try: + response = self.cli( + args + ["--subscription", self.request["subscription_id"]], + self.request["limits"]["wait_seconds"] if waiting else ( + min(self.request["limits"]["command_seconds"], timeout) if timeout is not None else self.request["limits"]["command_seconds"]), + ) + except HelperFailure as error: + if mutation and error.code in ("bootstrap-tool-unavailable", "bootstrap-cli-start-failed"): + self.attempted = False + raise + rc, stdout, stderr = response + if len(stdout) > MAX_BYTES or len(stderr) > MAX_BYTES: + raise failure("bootstrap-cli-output-limit", "CLI output exceeded its bound.") + if rc: + error = _error(stderr or stdout) + try: + self.record("failure", {"code": error.code, "status": error.http_status, "request_id": error.request_id, + "message_digest": error.message_digest, "message_withheld": True, + "policy_ids": error.policy_ids, "policy_references_limited": error.policy_overflow, + "warnings": error.warnings}) + except HelperFailure: + self.warnings.append("Original CLI failure could not be persisted; retain the returned failure and ownership.") + raise error + if waiting: + if stdout.strip() not in (b"", b"null"): + raise failure("bootstrap-wait-result-invalid", "Native wait returned an unexpected result; readiness is unresolved.") + return None + return _json(stdout) + + def get(self, timeout=None): + try: + result = self.call(["rest", "--method", "get", "--url", + "https://management.azure.com" + self.target + "?api-version=" + API], timeout) + except HelperFailure as exc: + if (exc.code == "ResourceNotFound" and exc.http_status in (None, 404) + and not getattr(exc, "cleanup_unconfirmed", False)): + self.record("readback", {"id": self.target, "absence": "ResourceNotFound"}) + return None + raise + if not isinstance(result.get("id"), str) or result["id"].casefold() != self.target.casefold(): + raise failure("bootstrap-response-invalid", "ARM returned another identity.") + self.record("readback", {"id": self.target, "digest": digest(result)}) + return result + + def account_context(self): + account = self.call(["account", "show"]) + if (folded(account.get("id")) != self.request["subscription_id"] or folded(account.get("tenantId")) != self.request["tenant_id"] + or folded(account.get("state")) != "enabled" or account.get("environmentName") != "AzureCloud"): + raise failure("bootstrap-context-conflict", "Signed-in subscription/tenant is not the selected enabled context.") + user = account.get("user") + if not isinstance(user, dict) or not text(user.get("name")) or folded(user.get("type")) not in ("user", "serviceprincipal"): + raise failure("bootstrap-context-conflict", "Signed-in principal evidence is missing or unsupported.") + principal = folded(user["name"]) if text(folded(user["name"]), GUID) else user["name"] + return {"id": folded(account["id"]), "tenantId": folded(account["tenantId"]), "environmentName": account["environmentName"], + "user": {"name": principal, "type": folded(user["type"])}} + + def context(self): + account = self.account_context() + group = self.call(["group", "show", "--name", self.request["resource_group"]]) + props = group.get("properties") + if ( + not isinstance(group.get("id"), str) or group["id"].casefold() != ids(self.request)[0].casefold() + or not isinstance(props, dict) or folded(props.get("provisioningState")) != "succeeded" + ): + raise failure("bootstrap-group-not-ready", "The explicitly selected existing resource group must be ready.") + context = {"account": digest(account), "group": digest({"id": folded(group["id"]), "ready": True})} + self.record("context", {"account": account, "group_id": ids(self.request)[0], "group_digest": context["group"]}) + return context + + def regions(self, *, selected=None): + identity = provider_id(self.request) + value = self.call(["rest", "--method", "get", "--url", + "https://management.azure.com" + identity + "?api-version=" + PROVIDER_API]) + locations = search_locations(value, identity) + self.record("search-regions", {"provider_id": identity, "resource_type": "searchServices", + "api_version": PROVIDER_API, "digest": digest(value), "locations": locations}) + if selected is not None and normalize_azure_location(selected) not in locations: + error = failure("bootstrap-region-unsupported", "Select an advertised Search region; no typo correction or deployment-capacity inference.") + error.available_locations = locations + raise error + return locations + + def policy_evidence(self, original): + if self.diagnosed or not getattr(original, "policy_implicated", False): + return + self.diagnosed = True + end = self.clock() + 60 + queue = list(original.policy_ids) + seen = set() + self.warnings.append("Policy evidence is diagnostic only; cause/compliance remains unresolved for the policy owner.") + scopes = {self.target.casefold(), ids(self.request)[0].casefold(), ("/subscriptions/" + self.request["subscription_id"]).casefold()} + while queue and len(seen) < 8: + identity = queue.pop(0) + if identity.casefold() in seen: + continue + seen.add(identity.casefold()) + self.policy_references.append(identity) + remaining = end - self.clock() + if remaining <= 0: + break + scope, tail = re.split(r"/providers/Microsoft.Authorization/", identity, flags=re.I) + if scope.casefold() not in scopes or "/" not in tail: + self.warnings.append("Referenced policy scope is unsupported; hand off to the policy owner.") + continue + kind, name = tail.split("/", 1) + if kind.casefold() == "policyassignments": + args = ["policy", "assignment", "show", "--name", name, "--scope", scope] + elif kind.casefold() == "policydefinitions" and scope.casefold() == ("/subscriptions/" + self.request["subscription_id"]).casefold(): + args = ["policy", "definition", "show", "--name", name] + else: + self.warnings.append("Referenced policy kind requires owner investigation.") + continue + try: + value = self.call(args, remaining) + if self.clock() >= end: + raise failure("bootstrap-policy-timeout", "Policy evidence arrived after the diagnostic budget.") + if not isinstance(value.get("id"), str) or value["id"].casefold() != identity.casefold(): + raise failure("bootstrap-policy-evidence-mismatch", "Policy read returned another identity.") + # Values/rules can contain sensitive literals: retain identity, digest and + # classification only; the policy owner reads the referenced object. + properties = value.get("properties", value) + if not isinstance(properties, dict): + raise failure("bootstrap-response-invalid", "Policy properties are malformed.") + reference = properties.get("policyDefinitionId") + if isinstance(reference, str) and POLICY_ID.fullmatch(reference): + queue.append(reference) + self.record("policy-evidence", {"id": identity, "digest": digest(value), + "referenced_definition_id": reference if isinstance(reference, str) and POLICY_ID.fullmatch(reference) else None, + "enforcementMode": properties.get("enforcementMode") if properties.get("enforcementMode") in ("Default", "DoNotEnforce", "Enroll") else None, + "interpretation": "unresolved"}) + except HelperFailure as secondary: + self.warnings.append("Secondary policy diagnostic failed: " + secondary.code) + self.warnings.extend(secondary.warnings) + if queue or getattr(original, "policy_overflow", False) or not original.policy_ids: + self.warnings.append("Policy references are missing or exceed this bounded collector; no truncated completeness claim.") + + def blocked(self, error): + if self.attempted: + error.partial = True + error.writes = [{"operation": "PUT", "resource_id": self.target, "submission": "attempted-unverified"}] + error.resources_remaining = [{"resource_id": self.target, "run_owned": self.owned, + "cleanup": "separate consent and fresh ownership/children/roles required"}] + error.warnings.extend(self.warnings) + result = blocked_result(error, outcome="bootstrap-search", fingerprint=None, owner=self.request.get("owner")) + if error.code == "bootstrap-region-unsupported" and hasattr(error, "available_locations"): + result["available_locations"] = error.available_locations + if result["status"] == "partial": + result["approved_plan"] = {"confirmed": self.approved, "artifact_id": self.operation_id} + result["attempted_writes"] = result.pop("completed_writes") + result["completed_writes"] = [] + if not self.owned: + result["resources_remaining"]["unverified"] = result["resources_remaining"]["run_owned"] + result["resources_remaining"]["run_owned"] = [] + result["receipt_id"] = self.operation_id + if self.diagnosed: + result["policy_handoff"] = {"referenced_ids": self.policy_references, "complete": False, + "next_step": "Policy owner investigates exact references; no correction or retry is approved."} + return result + + +def discover_regions(request, *, cli=run_cli): + closed(request, {"schema_version", "subscription_id", "tenant_id", "receipt_dir", "limits"}, "Region discovery") + reject_secrets(request) + closed(request["limits"], {"command_seconds"}, "Region discovery limits") + if ( + request["schema_version"] != SCHEMA + or not text(request["subscription_id"], GUID) or not text(request["tenant_id"], GUID) + or type(request["limits"]["command_seconds"]) is not int + or not 1 <= request["limits"]["command_seconds"] <= 60 + ): + raise failure("bootstrap-input-invalid", "Select subscription/tenant and a 1-60 second region-discovery command limit.") + session = Session(request, str(uuid.uuid4()), cli, target=provider_id(request)) + try: + session.record("context", {"account": session.account_context()}) + locations = session.regions() + return {"status": "discovered", "resource_type": "Microsoft.Search/searchServices", + "available_locations": locations, "receipt_id": session.operation_id, + "mutation_approval_required": False, "execution_required": False, "writes_performed": [], + "verification": "Advertised region support only; not SKU, quota, capacity, models or residency approval."} + except HelperFailure as error: + return session.blocked(error) + + +def plan_request(request, *, cli=run_cli, clock=time.monotonic, execution_output=None): + validate(request) + request = copy.deepcopy(request) + request["location"] = normalize_azure_location(request["location"]) + operation_id = str(uuid.uuid4()) + session = Session(request, operation_id, cli, clock) + try: + context = session.context() + observed = session.get() + body = desired(request, operation_id) + if request["action"] == "create": + if observed is not None: + raise failure("bootstrap-state-conflict", "New intent requires exact-name absence; never overwrite or silently reuse.") + session.regions(selected=request["location"]) + refreshed = session.get() + if refreshed is not None: + raise failure("bootstrap-state-drift", "Exact-name absence changed; creation is blocked.") + else: + binding = reuse_binding(observed, request, body) + if binding is None: + raise failure("bootstrap-not-ready", "Reuse requires a running, keyless Search service with identity readback.") + refreshed = session.get() + if reuse_binding(refreshed, request, body) != binding: + raise failure("bootstrap-state-drift", "Selected Search identity changed; obtain fresh evidence.") + artifact = { + "operation": "bootstrap-search", "operation_id": operation_id, "created_at": int(time.time()), + "choices": copy.deepcopy(request), + "body": body, "before": {**context, "search": digest(observed)}, + } + envelope = {"schema_version": SCHEMA, "plan": artifact, + "approval": {"confirmed": False, "fingerprint": digest(artifact)}} + if execution_output is None: + private_file(session.directory, operation_id + ".plan.json", envelope) + else: + try: + from .private_artifacts import retain_execution_input + except ImportError: + from private_artifacts import retain_execution_input + execution_artifact = retain_execution_input(envelope, execution_output) + session.record("planned", {"body": body, "before": artifact["before"], + "readback": projection(refreshed, body["tags"]), "run_owned": False}) + create = request["action"] == "create" + if execution_output is not None: + return { + "status": "planned" if create else "reused", "artifact_id": operation_id, + "execution_artifact": execution_artifact, "execution_required": create, + "mutation_approval_required": create, "azure_mutation_performed": False, + "local_filesystem": {"artifact_created": True, "receipts_created": True, + "existing_files_changed": False}, + "summary": "Unapproved bootstrap artifact retained privately; review before separate approval.", + } + return { + "status": "planned" if create else "reused", "artifact_id": operation_id, + "execution_required": create, "mutation_approval_required": create, + "approval_summary": {"action": request["action"], "resource_id": session.target, + "location": request["location"], "sku": request["sku"], + "replicas": request["replicas"], "partitions": request["partitions"], + "network": "public; local authentication disabled; SystemAssigned", + "tags": body["tags"], "prerequisites": { + key: {"evidence_present": value is not None, + "verification": "caller-attested; not helper-verified" if value is not None else "not required for reuse"} + for key, value in request["prerequisites"].items() + }, + "limits": request["limits"], "cleanup": "separate approval"}, + "verification": {"arm_readiness": "verified" if not create else "unverified", + "data_plane_access": "unverified", "ingestion": "unverified", "retrieval": "unverified"}, + "writes_performed": [], "run_owned": False, + "observed_dependency": projection(refreshed, body["tags"]), + } + except HelperFailure as error: + return session.blocked(error) + + +def validate_artifact(envelope): + closed(envelope, {"schema_version", "plan", "approval"}, "Execution artifact") + if envelope["schema_version"] != SCHEMA: + raise failure("bootstrap-contract-version", "Regenerate the retained artifact with bootstrap schema 2.0; do not edit its checksum.") + plan, approval = envelope["plan"], envelope["approval"] + closed(plan, {"operation", "operation_id", "created_at", "choices", "body", "before"}, "Execution plan") + closed(approval, {"confirmed", "fingerprint"}, "Approval") + if ( + plan["operation"] != "bootstrap-search" + or not text(plan["operation_id"], GUID) or type(approval["confirmed"]) is not bool + or approval["fingerprint"] != digest(plan) + ): + raise failure("approval-mismatch", "Retain the unchanged planner artifact; integrity is checked internally.") + validate(plan["choices"]) + if type(plan["created_at"]) is not int or not plan["created_at"] <= time.time() <= plan["created_at"] + 900: + raise failure("bootstrap-artifact-expired", "Rerun planning; retained artifacts expire after fifteen minutes.") + closed(plan["before"], {"account", "group", "search"}, "Before-state") + if any(not text(v, HASH) for v in plan["before"].values()) or plan["body"] != desired(plan["choices"], plan["operation_id"]): + raise failure("bootstrap-artifact-invalid", "Artifact body or before-state differs from generated choices.") + return plan + + +@reporting("search-bootstrap") +def apply_artifact(envelope, *, approve=False, cli=run_cli, clock=time.monotonic, progress=None): + progress.update("validation") + plan = validate_artifact(envelope) + if approve is not True: + raise failure("approval-missing", "Explicit approval of the unchanged artifact is required before any execution reads or writes.") + request = plan["choices"] + if request["action"] != "create": + raise failure("bootstrap-execution-unnecessary", "Read-only reuse needs fresh planning, not mutation approval or execution.") + session = Session(request, plan["operation_id"], cli, clock) + session.approved = True + body = plan["body"] + try: + session.record("approved", {"plan": plan, "approval": {"confirmed": True, "fingerprint": digest(plan)}}) + progress.update("context-check") + context = session.context() + if context != {k: plan["before"][k] for k in ("account", "group")}: + raise failure("bootstrap-state-drift", "Account/group changed; rerun planning and discard old consent.") + progress.update("region-check") + session.regions(selected=request["location"]) + progress.update("absence-check") + if plan["before"]["search"] != digest(None) or session.get() is not None: + raise failure("bootstrap-state-drift", "Exact-name absence changed; never overwrite or retry through another name.") + body_path = private_file(session.directory, plan["operation_id"] + "." + uuid.uuid4().hex + ".body.json", body) + command = ["rest", "--method", "put", "--url", + "https://management.azure.com" + session.target + "?api-version=" + API, + "--headers", "Content-Type=application/json", "x-ms-client-request-id=" + str(uuid.uuid4()), + "--body", "@" + str(body_path)] + private_file(session.directory, plan["operation_id"] + ".submission.json", + {"operation_id": plan["operation_id"], "target": session.target, "body": body, "command": command}) + progress.update("search-submit") + try: + session.call(command, mutation=True) + except HelperFailure as original: + if not session.attempted: + raise + # Never retry PUT. Neither diagnostics nor a readback replaces the first error. + progress.update("arm-readback") + try: + observed = session.get() + if observed is not None: + matching(observed, request, body) + session.owned = True + session.record("failure-reconciliation", {"readback": projection(observed, body["tags"]), "run_owned": session.owned}) + if (original.code in ("RequestDisallowedByPolicy", "AuthorizationFailed", "InvalidSkuName") + and observed is None and not getattr(original, "cleanup_unconfirmed", False)): + session.attempted = False + except HelperFailure as secondary: + session.warnings.append("Read-only reconciliation failed: " + secondary.code) + session.warnings.extend(secondary.warnings) + session.policy_evidence(original) + raise original + progress.update("arm-wait") + try: + session.call(["resource", "wait", "--ids", session.target, "--api-version", API, + "--custom", WAIT_CONDITION, "--interval", str(request["limits"]["wait_interval_seconds"]), + "--timeout", str(request["limits"]["wait_seconds"])], waiting=True) + except HelperFailure as original: + try: + progress.update("arm-readback") + observed = session.get() + if observed is not None: + value = matching(observed, request, body) + session.owned = True + try: + _readiness(observed, request, value) + except HelperFailure as provider: + session.warnings.append("Terminal provider readback: " + provider.code) + session.record("terminal-provider-failure", { + "code": provider.code, "status": provider.http_status, "request_id": provider.request_id, + "message_digest": provider.message_digest, "message_withheld": True}) + session.policy_evidence(provider) + session.record("wait-failure-readback", {"readback": projection(observed, body["tags"]), "run_owned": session.owned}) + except HelperFailure as secondary: + session.warnings.append("Wait failure readback failed: " + secondary.code) + session.warnings.extend(secondary.warnings) + raise original + progress.update("arm-readback") + observed = session.get() + value = matching(observed, request, body) if observed is not None else None + if value is not None: + session.owned = True + value = _readiness(observed, request, value) + if value is None: + raise failure("bootstrap-readiness-timeout", "Final ARM readiness was not established; preserve the resource and operation receipts.") + session.record("arm-ready", {"readback": value, "run_owned": True}) + return {"status": "completed", "receipt_id": session.operation_id, "resource_id": session.target, + "run_owned": True, "writes_performed": [{"operation": "PUT", "resource_id": session.target}], + "observed_dependency": value, + "verification": {"arm_readiness": "verified", "data_plane_access": "unverified", + "ingestion": "unverified", "retrieval": "unverified"}, + "cleanup": "separate approval with fresh ownership/children/roles"} + except HelperFailure as error: + if session.attempted and getattr(error, "creation_failure", False): + session.policy_evidence(error) + return session.blocked(error) + + +def main(argv=None): + try: + from .private_artifacts import add_execution_output_argument, validate_execution_output_mode + except ImportError: + from private_artifacts import add_execution_output_argument, validate_execution_output_mode + parser = argparse.ArgumentParser(description="Discover Search regions or plan/apply one selected service; no models, roles or cleanup.") + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--plan", type=Path) + mode.add_argument("--apply", type=Path) + mode.add_argument("--regions", type=Path) + parser.add_argument("--approve", action="store_true") + add_execution_output_argument(parser) + add_progress_argument(parser) + args = parser.parse_args(argv) + try: + validate_execution_output_mode(args) + if (args.plan or args.regions) and args.approve: + raise failure("bootstrap-input-invalid", "Planning cannot approve mutations.") + document = read_json(args.plan or args.apply or args.regions) + result = (discover_regions(document) if args.regions else + plan_request(document, **({"execution_output": args.execution_output} if args.execution_output else {})) if args.plan else apply_artifact( + document, approve=args.approve, progress=Progress("search-bootstrap", enabled=args.progress))) + except HelperFailure as error: + result = blocked_result(error, outcome="bootstrap-search", fingerprint=None) + emit_result(result) + return 3 if result["status"] == "partial" else 2 if result["status"] == "blocked" else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/cleanup_plan.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/cleanup_plan.py new file mode 100644 index 00000000..4c365787 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/cleanup_plan.py @@ -0,0 +1,735 @@ +from __future__ import annotations + +import argparse +import copy +import json +import re +import sys +from pathlib import Path +from typing import Any, Callable + +try: + from . import prompt_cleanup, prompt_connect, search_reconcile, _cleanup_dependencies as dependencies, _cleanup_receipts as receipts + from ._common import ( + SEARCH_AUDIENCE, MANAGEMENT_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, + blocked_result, digest, emit_result, http_request, load_approved_input, + reject_secrets, require_allowed_fields, sdk_error_metadata, sdk_error_status, + validate_search_endpoint, + ) +except ImportError: + import prompt_cleanup + import prompt_connect + import search_reconcile + import _cleanup_dependencies as dependencies + import _cleanup_receipts as receipts + from _common import ( + SEARCH_AUDIENCE, MANAGEMENT_AUDIENCE, HelperFailure, TokenProvider, Transport, azure_cli_token, + blocked_result, digest, emit_result, http_request, load_approved_input, + reject_secrets, require_allowed_fields, sdk_error_metadata, sdk_error_status, + validate_search_endpoint, + ) + + +RETAIN = [ + "outside-plan resources and consumers", "original local and Storage documents", + "Search service", "accounts", "projects", "models", "role assignments", +] +REQUEST_FIELDS = { + "schema_version", "owner", "target", "creation_input_file", + "creation_result_file", "creation_response_file", + "creation_receipt_file", "inventory_limits", "agent_version", "agent_creation_response_file", "agent_creation_receipt_file", +} +SEARCH_FIELDS = {"type", "endpoint", "api_version", "name"} +PROMPT_FIELDS = {"type", "project_resource_id", "project_endpoint", "name", "version"} +RESPONSE_FIELDS = {"schema_version", "target", "operation", "status", "request_id", "body", "generated_resources"} + + +def _failure(code: str, message: str) -> HelperFailure: + return HelperFailure(code, message, blocked_at="cleanup-planning") + + +def _object(value: Any, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise _failure("input-schema-invalid", f"{label} must be an object.") + return value + + +def _text(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _read_json(path: Path) -> dict[str, Any]: + try: + result = json.loads(path.read_text(encoding="utf-8")) + json.dumps(result, allow_nan=False) + except (OSError, UnicodeError, ValueError) as exc: + raise _failure("input-unreadable", "Select readable retained UTF-8 JSON records.") from exc + return _object(result, "Record") + + +def _target(value: Any) -> dict[str, Any]: + target = copy.deepcopy(_object(value, "Cleanup target")) + kind = target.get("type") + if not isinstance(kind, str): + raise _failure("target-selection-required", "Select one typed cleanup target.") + if kind == "hosted": + require_allowed_fields(target, {"type"}, label="Hosted cleanup target") + raise _failure("hosted-cleanup-unsupported", "Hosted teardown remains unsupported; retain the deployment and toolbox.") + if kind in {"knowledge-base", "knowledge-source"}: + require_allowed_fields(target, SEARCH_FIELDS, label="Search cleanup target") + search_reconcile.resource_url({**target, "resource_type": kind}) + target["endpoint"] = validate_search_endpoint(target["endpoint"]) + elif kind in {"prompt-agent-version", "project-connection"}: + fields = PROMPT_FIELDS if kind == "prompt-agent-version" else PROMPT_FIELDS - {"version"} + require_allowed_fields(target, fields, label="Prompt cleanup target") + project_id, endpoint = prompt_connect._project_identity(target) + target["project_resource_id"] = project_id.casefold() + target["project_endpoint"] = endpoint + if not _text(target.get("name")): + raise _failure("target-selection-required", "Select one exact agent or connection name.") + if kind == "prompt-agent-version" and ( + not isinstance(target.get("version"), str) + or re.fullmatch(r"[1-9][0-9]*", target["version"]) is None + ): + raise _failure("target-selection-required", "Select one exact numeric Prompt version, not latest or a list.") + else: + raise _failure("target-selection-required", "Select one supported exact cleanup target.") + return target + + +def _path(request: dict[str, Any], key: str, base_dir: Path) -> Path: + value = request.get(key) + if not _text(value): + raise _failure("ownership-unproven", "Retained approval, result and definitive create response files are required.") + path = Path(value) + return path if path.is_absolute() else base_dir / path + + +def _records( + request: dict[str, Any], target: dict[str, Any], base_dir: Path, +) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: + if "creation_receipt_file" in request: + if target["type"] != "knowledge-source" or "creation_response_file" in request: + raise _failure("input-schema-invalid", "Select either an original Blob checkpoint or native result/response records.") + try: + from . import blob_recheck + except ImportError: + import blob_recheck + original = _read_json(_path(request, "creation_result_file", base_dir)) + reject_secrets(original) + retained = original.get("recheck_checkpoint") + if (original.get("outcome") != "create-blob-knowledge-source" or original.get("status") not in ("completed", "partial") + or not isinstance(retained, dict) or retained.get("status") != "retained"): + raise _failure( + "generated-creation-evidence-unavailable", + "Require the original creation result retaining its generated checkpoint; ACK/recovery GETs cannot prove original child versions.", + ) + prior, receipt = blob_recheck._load( + _path(request, "creation_input_file", base_dir), _path(request, "creation_receipt_file", base_dir), + ) + if prior["source"]["action"] != "create" or prior["owner"] != request["owner"]: + raise _failure("ownership-unproven", "Only an original run-owned creation checkpoint is eligible, never captured reuse.") + original_ownership = _object( + original.get("resources_remaining" if original["status"] == "partial" else "ownership"), + "Original creation ownership", + ) + original_owner = original.get("owner") if original["status"] == "partial" else original_ownership.get("owner") + original_owned = original_ownership.get("run_owned") + if (retained.get("evidence_digest") != receipt["integrity"] + or retained.get("operation_id") != receipt["operation_id"] + or original.get("approved_plan") != {"confirmed": True, "fingerprint": receipt["plan_digest"]} + or original_owner != prior["owner"] or not isinstance(original_owned, list) + or receipt["creation"]["ownership"]["run_owned"][0] not in original_owned): + raise _failure("generated-ownership-unproven", "The original creation result must bind this exact checkpoint and approval; never substitute recovery snapshots.") + result = {"status": "completed", **copy.deepcopy(receipt["creation"])} + generated = receipt["creation"]["source"]["generated"] + body = copy.deepcopy(prior["source"]["desired"]) + body["@odata.etag"] = receipt["creation"]["verification"]["readback"]["etag"] + body["azureBlobParameters"]["createdResources"] = {item["type"]: item["name"] for item in generated} + snapshots = [ + {"type": item["type"], "name": item["name"], "etag": receipt["configuration"][item["type"]]["etag"], + "definition_digest": receipt["configuration"][item["type"]]["digest"]} + for item in generated + ] + return prior, result, {"body": body, "_checkpoint_validated": True, "_generated_snapshots": snapshots} + paths = [_path(request, field, base_dir) for field in ( + "creation_input_file", "creation_result_file", "creation_response_file", + )] + try: + _, prior, fingerprint = load_approved_input(paths[0]) + except UnicodeError as exc: + raise _failure("input-unreadable", "Retained approval must be UTF-8 JSON.") from exc + result, response = _read_json(paths[1]), _read_json(paths[2]) + reject_secrets(result) + reject_secrets(response) + require_allowed_fields(response, RESPONSE_FIELDS, label="Definitive create response record") + if ( + response.get("schema_version") != "1.0" + or not _text(response.get("request_id")) + or _target(response.get("target")) != target + ): + raise _failure("ownership-unproven", "Original create response must bind the exact scope and native request ID.") + if ( + result.get("status") != "completed" + or result.get("approved_plan") != {"confirmed": True, "fingerprint": fingerprint} + or _object(result.get("ownership"), "Creation ownership").get("owner") != prior.get("owner") + or prior.get("owner") != request["owner"] + ): + raise _failure("ownership-unproven", "Retained completed result, approval and accountable owner must agree.") + _object(response.get("body"), "Create response body") + return prior, result, response + + +def _owned_entry(result: dict[str, Any], target: dict[str, Any]) -> dict[str, Any]: + identity = {key: target[key] for key in ("type", "name", "version") if key in target} + + def matches(item: Any) -> bool: + return isinstance(item, dict) and all(item.get(key) == value for key, value in identity.items()) + + resources = _object(result.get("resources"), "Creation resources") + ownership = _object(result.get("ownership"), "Creation ownership") + for container, keys in ( + (resources, ("created", "reused", "updated", "skipped")), + (ownership, ("run_owned", "reused_not_owned")), + ): + for key in keys: + if not isinstance(container.get(key, []), list): + raise _failure("ownership-unproven", "Creation resource records must be complete lists.") + created = [item for item in resources.get("created", []) if matches(item)] + owned = [item for item in ownership.get("run_owned", []) if matches(item)] + if ( + len(created) != 1 or len(owned) != 1 or created[0] != owned[0] + or any(matches(item) for key in ("reused", "updated", "skipped") for item in resources.get(key, [])) + or any(matches(item) for item in ownership.get("reused_not_owned", [])) + ): + raise _failure("ownership-unproven", "Only one definitively created run-owned resource is eligible; never reused or updated.") + return created[0] + + +def _search_prior(prior: dict[str, Any], target: dict[str, Any]) -> dict[str, Any]: + source = prior + if prior.get("operation") in ("reconcile-and-ingest", "reconcile-and-monitor"): + source = _object(prior.get("source"), "Prior source") + if _object(source.get("desired"), "Prior source definition").get("kind") == "file": + try: + from . import file_source + except ImportError: + import file_source + file_source._validate_plan(prior) + else: + try: + from . import blob_source + except ImportError: + import blob_source + blob_source._validate_plan(prior) + search_reconcile._validate_plan(source) + if ( + source.get("operation") != "reconcile" or source.get("action") != "create" + or search_reconcile.resource_url(source) != search_reconcile.resource_url( + {**target, "resource_type": target["type"]} + ) + ): + raise _failure("ownership-unproven", "Creation approval must select this exact source/base with action create.") + return source + + +def _summary(target: dict[str, Any]) -> dict[str, Any]: + is_agent = target["type"] == "prompt-agent-version" + summary = { + "delete": [copy.deepcopy(target)], + "retain": RETAIN + ( + ["agent container", "prior and other agent versions", "project connection", "KB", "KS"] + if is_agent else ["all KS and generated indexes/pipelines", "agent versions", "project connections"] + ), + "blocked": [], + "order": ["Only this exact target; stop after failure. No dependent cleanup is chained."], + "impact": ( + "The selected version and its tool binding become unavailable; the project connection remains." + if is_agent else "The selected KB retrieval endpoint becomes unavailable; retained consumers are not detached." + ), + "ownership": "Original successful-create evidence required; owner metadata and equivalent GET are not proof or RBAC.", + "approval": "Separate cleanup consent: change only approval.confirmed after review.", + "hosted_cleanup": "unsupported", + } + if target["type"] == "knowledge-source": + summary.update( + retain=RETAIN + ["all KBs", "other KS/pipelines", "agent versions", "project connections"], + impact="The selected source, its uploaded File copies/indexed content and exact generated objects are deleted; originals remain.", + order=["Referencing KBs must already be absent or no longer reference this source; no detach is performed.", + "Delete only the source through Search; the service owns the exact approved cascade."], + ) + elif target["type"] == "project-connection": + summary.update( + retain=RETAIN + ["agent containers", "prior/other agent versions", "KBs", "KS/pipelines"], + impact="The selected project connection is removed; no retained agent version or tool is edited.", + order=["Verify all consumers; delete and verify the explicitly selected owned version first, if any.", + "Rescan protected consumers and conditionally delete only this connection; stop on failure."], + ) + return summary + + +def _original_generated(response): + names = dependencies.generated(response["body"]) + if response.get("_checkpoint_validated") is True: + snapshots = response["_generated_snapshots"] + else: + records = response.get("generated_resources") + if not isinstance(records, list): + raise _failure( + "generated-creation-evidence-unavailable", + "Original generated-object GET snapshots with ETags are required: source ETag and createdResources names " + "cannot distinguish a replaced child. Use the retained Blob checkpoint or original generated-object audit records.", + ) + snapshots = [] + for record in records: + if not isinstance(record, dict) or set(record) != {"type", "body"}: + raise _failure("input-schema-invalid", "Original generated records require type and complete native body.") + if not isinstance(record["type"], str) or record["type"] not in names: + raise _failure("generated-ownership-unproven", "Unexpected original generated resource type.") + body = _object(record["body"], "Original generated body") + snapshots.append({ + "type": record["type"], "name": body.get("name"), "etag": body.get("@odata.etag"), + "definition_digest": digest(body), + }) + if ( + len(snapshots) != len(names) + or any(item.get("type") not in names or item.get("name") != names[item["type"]] + or not _text(item.get("etag")) for item in snapshots) + or len({item["type"] for item in snapshots}) != len(names) + ): + raise _failure("generated-ownership-unproven", "Original child snapshots must match the exact complete acknowledged cascade.") + return sorted(snapshots, key=lambda item: item["type"]) + + +def _absent(target: dict[str, Any], request_id: str | None = None) -> dict[str, Any]: + summary = _summary(target) + summary["delete"] = [] + if target["type"] == "knowledge-source": + summary["impact"] = "Source already absent; generated-object absence is unverified. No independent child deletion is authorized." + summary["retain"].append("unverified generated objects") + return { + "status": "already-absent", "outcome": "plan-cleanup", + "execution_required": False, "mutation_approval_required": False, + "approval_summary": summary, + "verification": {"absence": True, "request_ids": [request_id] if request_id else []}, + } + + +def _planned( + target: dict[str, Any], plan: dict[str, Any], executor: str, + retained_targets: list[dict[str, Any]], +) -> dict[str, Any]: + fingerprint = digest(plan) + summary = _summary(target) + summary["retained_targets"] = retained_targets + return { + "status": "planned", "outcome": "plan-cleanup", "executor": executor, + "execution_required": True, "mutation_approval_required": True, + "plan_fingerprint": fingerprint, + "execution_input": { + "schema_version": "1.0", "plan": plan, + "approval": {"confirmed": False, "fingerprint": fingerprint}, + }, + "approval_summary": summary, + } + + +def _plan_search( + request: dict[str, Any], target: dict[str, Any], prior: dict[str, Any], + result: dict[str, Any], response: dict[str, Any], *, + token_provider: TokenProvider, transport: Transport, +) -> dict[str, Any]: + source = _search_prior(prior, target) + owned = _owned_entry(result, target) + body = response["body"] + owned_digest = digest(search_reconcile._definition(body)) + etag = body.get("@odata.etag") + if ( + (response.get("_checkpoint_validated") is not True and ( + response.get("operation") != "search-create" or type(response.get("status")) is not int or response["status"] != 201 + )) or not _text(etag) + or body.get("name") != target["name"] + or not search_reconcile.definitions_match(source["desired"], body) + or owned.get("definition_digest") != owned_digest or owned.get("etag") != etag + ): + raise _failure("ownership-unproven", "Require the original HTTP 201 create body and matching retained owned readback, not GET recovery.") + plan = { + "operation": "delete", "outcome": "cleanup-search-resource", "plan_kind": "cleanup", + "cleanup_approved": True, "owner": request["owner"], + "resource_type": target["type"], "endpoint": target["endpoint"], + "name": target["name"], "api_version": target["api_version"], + "owned_definition_digest": owned_digest, "expected_etag": etag, + } + token = token_provider(SEARCH_AUDIENCE) + current, request_id = search_reconcile.read_resource(search_reconcile.resource_url(plan), token, transport=transport) + if current is None: + return _absent(target, request_id) + if ( + digest(search_reconcile._definition(current)) != owned_digest + or current.get("@odata.etag") != etag + ): + raise _failure("definition-drift", "Current definition or creation ETag changed; same-name replacement is not owned.") + if target["type"] == "knowledge-source": + original = _original_generated(response) + if dependencies.generated(current) != dependencies.generated(body): + raise _failure("generated-ownership-unproven", "Generated identities differ from the acknowledged source creation.") + snapshot = dependencies.search_snapshot( + plan, current, token, transport=transport, bounds=dependencies.limits(request.get("inventory_limits")), + ) + if snapshot["generated"] != original: + raise _failure("generated-incarnation-drift", "Generated definitions/ETags differ from original retained ownership evidence.") + plan["dependency_guard"] = snapshot + search_reconcile._validate_plan(plan) + planned = _planned(target, plan, "helpers/search_reconcile.py", []) + planned["approval_summary"]["delete"].extend([ + {"type": item["type"], "name": item["name"], "endpoint": target["endpoint"], "service_managed": True} + for item in original + ]) + return planned + reject_secrets(current) + plan["desired"] = copy.deepcopy(current) + search_reconcile._validate_plan(plan) + retained = [ + {**target, "type": "knowledge-source", "name": source["name"]} + for source in current["knowledgeSources"] + ] + return _planned(target, plan, "helpers/search_reconcile.py", retained) + + +def _plan_prompt( + request: dict[str, Any], target: dict[str, Any], prior: dict[str, Any], + result: dict[str, Any], response: dict[str, Any], *, + sdk_loader: Callable[[], tuple[Any, Any, Any, Any, Any]], +) -> dict[str, Any]: + prompt_connect._validate_plan(prior) + owned = _owned_entry(result, target) + prior_project, prior_endpoint = prompt_connect._project_identity(prior) + body = response["body"] + require_allowed_fields(body, {"name", "version", "definition", "id", "created_at"}, label="SDK create-version response") + definition = _object(body.get("definition"), "Created Prompt definition") + if ( + response.get("operation") != "agents.create_version" or response.get("status") != "succeeded" + or prior_project.casefold() != target["project_resource_id"] + or prior_endpoint != target["project_endpoint"] + or prior["agent"]["name"] != target["name"] or prior["agent"]["version"] == target["version"] + or body.get("name") != target["name"] or body.get("version") != target["version"] + or definition.get("kind") != "prompt" + or owned.get("definition_digest") != digest(definition) + or receipts.version_identity(body) is None + ): + raise _failure("ownership-unproven", "Require the original successful SDK new-version return, never the baseline or recovered equivalent.") + AIProjectClient, _, _, _, extras = sdk_loader() + AzureCliCredential, AzureError = extras + client = AIProjectClient(endpoint=target["project_endpoint"], credential=AzureCliCredential()) + try: + try: + current = client.agents.get_version(agent_name=target["name"], agent_version=target["version"]) + except AzureError as exc: + if sdk_error_status(exc) == 404: + return _absent(target, sdk_error_metadata(exc).get("request_id")) + raise HelperFailure( + message="Exact Prompt version readback failed; no inventory or deletion attempted.", + blocked_at="cleanup-planning", **sdk_error_metadata(exc, "agent-readback-failed"), + ) from exc + try: + actual = current.definition.as_dict() + identity_matches = current.name == target["name"] and str(current.version) == target["version"] + except (AttributeError, TypeError, ValueError) as exc: + raise _failure("agent-readback-invalid", "Exact version readback is incomplete.") from exc + if (not identity_matches or not isinstance(actual, dict) or digest(actual) != owned["definition_digest"] + or receipts.version_identity(current) != receipts.version_identity(body)): + raise _failure("definition-drift", "Exact Prompt version identity or definition changed.") + finally: + client.close() + plan = { + "operation": "delete", "outcome": "cleanup-prompt-version", "plan_kind": "cleanup", + "cleanup_approved": True, "sdk_major": 2, "owner": request["owner"], + "project_resource_id": target["project_resource_id"], "project_endpoint": target["project_endpoint"], + "agent": { + "name": target["name"], "version": target["version"], "run_owned": True, + "owned_definition_digest": owned["definition_digest"], + "owned_version_identity": receipts.version_identity(body), + }, + } + prompt_cleanup._validate_plan(plan) + return _planned(target, plan, "helpers/prompt_cleanup.py", [ + {**target, "version": prior["agent"]["version"]}, + { + "type": "project-connection", "project_resource_id": target["project_resource_id"], + "project_endpoint": target["project_endpoint"], "name": prior["connection"]["name"], + }, + ]) + + +def _plan_connection(request, target, prior, result, response, *, base_dir, token_provider, transport, sdk_loader): + prompt_connect._validate_plan(prior) + project, endpoint = prompt_connect._project_identity(prior) + body = response["body"] + created = {"type": "project-connection", "name": target["name"]} + owned_write = {"action": "created", "connection": target["name"]} + resources = _object(result.get("resources"), "Creation resources") + ownership = _object(result.get("ownership"), "Creation ownership") + verification = _object(result.get("verification"), "Creation verification") + readback = _object(verification.get("connection_readback"), "Creation connection readback") + def matches(item): + return isinstance(item, dict) and ( + (item.get("type") == "project-connection" and item.get("name") == target["name"]) + or item.get("connection") == target["name"] + ) + + if any(not isinstance(container.get(key, []), list) for container, keys in ( + (resources, ("created", "reused", "updated", "skipped")), (ownership, ("run_owned", "reused_not_owned")), + ) for key in keys): + raise _failure("ownership-unproven", "Creation resource and ownership lists must be complete.") + etag = body.get("etag") or body.get("@odata.etag") + if ( + project.casefold() != target["project_resource_id"] or endpoint != target["project_endpoint"] + or prior["connection"]["name"] != target["name"] or prior["connection"]["action"] != "create" + or response.get("operation") != "project-connection-create" or type(response.get("status")) is not int + or response["status"] != 201 or not _text(etag) + or [item for item in resources.get("created", []) if matches(item)] != [created] + or any(matches(item) for key in ("reused", "updated", "skipped") for item in resources.get(key, [])) + or [item for item in ownership.get("run_owned", []) if matches(item)] != [owned_write] + or any(matches(item) for item in ownership.get("reused_not_owned", [])) + or readback.get("name") != target["name"] or readback.get("definition_digest") != digest(body) + ): + raise _failure("ownership-unproven", "Retain the exact acknowledged created project connection; updated/reused/recovered GETs are not create evidence.") + if not prompt_connect._connection_readback(prior, body)[0]: + raise _failure("ownership-unproven", "The original connection body must match its approved project and configuration.") + plan = { + "operation": "delete", "outcome": "cleanup-prompt-connection", "plan_kind": "cleanup", + "cleanup_approved": True, "sdk_major": 2, "owner": request["owner"], + "project_resource_id": target["project_resource_id"], "project_endpoint": target["project_endpoint"], + "connection": {"name": target["name"], "run_owned": True, "expected_etag": etag, "owned_definition_digest": digest(body)}, + } + url = prompt_connect._connection_url(plan) + token = token_provider(MANAGEMENT_AUDIENCE) + current, request_id = prompt_cleanup._get_connection(url, token, transport=transport) + if current is not None and current != body: + raise _failure("definition-drift", "The project connection changed since acknowledged creation.") + selected = None + if "agent_version" in request: + selected_target = { + **target, "type": "prompt-agent-version", "name": prior["agent"]["name"], "version": request["agent_version"], + } + selected_request = { + "schema_version": "1.0", "owner": request["owner"], "target": selected_target, + "creation_input_file": request["creation_input_file"], "creation_result_file": request["creation_result_file"], + "creation_response_file": request["agent_creation_response_file"], + } + selected = plan_cleanup(selected_request, base_dir=base_dir, token_provider=token_provider, transport=transport, sdk_loader=sdk_loader) + if selected["status"] == "planned": + plan["agent"] = selected["execution_input"]["plan"]["agent"] + if current is None: + if selected is not None and selected["status"] == "planned": + selected["approval_summary"]["already_absent"] = [target] + return selected + return _absent(target, request_id) + plan["dependency_guard"] = dependencies.prompt_snapshot( + plan, current, sdk_loader=sdk_loader, bounds=dependencies.limits(request.get("inventory_limits")), + ) + refreshed, _ = prompt_cleanup._get_connection(url, token, transport=transport) + if refreshed != current: + raise _failure("definition-drift", "Connection changed during project consumer discovery.") + prompt_cleanup._validate_plan(plan) + retained = [ + {**target, "type": "prompt-agent-version", "name": item["name"], "version": item["version"]} + for item in plan["dependency_guard"]["versions"] + if not plan.get("agent") or (item["name"], item["version"]) != (plan["agent"]["name"], plan["agent"]["version"]) + ] + planned = _planned(target, plan, "helpers/prompt_cleanup.py", retained) + if selected is not None and selected["status"] == "planned": + planned["approval_summary"]["delete"].insert(0, selected["approval_summary"]["delete"][0]) + return planned + + +def _plan_protected(request, target, prior, record, *, base_dir, token_provider, transport, sdk_loader): + snapshot = record["snapshot"] + outcome = ("cleanup-search-resource" if target["type"] in ("knowledge-base", "knowledge-source") + else "cleanup-prompt-version" if target["type"] == "prompt-agent-version" else "cleanup-prompt-connection") + plan = {"operation": "delete", "outcome": outcome, "plan_kind": "cleanup", + "cleanup_approved": True, "owner": request["owner"]} + if record["owner"] != request["owner"]: + raise _failure("ownership-unproven", "Original producer owner and cleanup accountable owner differ.") + if target["type"] in ("knowledge-base", "knowledge-source"): + source = _search_prior(prior, target) + if snapshot["definition_digest"] != digest(search_reconcile._definition(source["desired"])): + raise _failure("ownership-unproven", "Receipt does not bind the original approved Search definition.") + plan.update(resource_type=target["type"], **{k: target[k] for k in ("endpoint", "name", "api_version")}, + owned_definition_digest=snapshot["definition_digest"], expected_etag=snapshot["etag"]) + token = token_provider(SEARCH_AUDIENCE) + current, request_id = search_reconcile.read_resource(search_reconcile.resource_url(plan), token, transport=transport) + if current is None: + return _absent(target, request_id) + if digest(search_reconcile._definition(current)) != snapshot["definition_digest"] or current.get("@odata.etag") != snapshot["etag"]: + raise _failure("definition-drift", "Current Search state differs from the original producer snapshot.") + if target["type"] == "knowledge-source": + if dependencies.generated(current) != record["acknowledgement"]["generated"]: + raise _failure("generated-ownership-unproven", "Current generated identities differ from the native create acknowledgement.") + plan["dependency_guard"] = dependencies.search_snapshot( + plan, current, token, transport=transport, bounds=dependencies.limits(request.get("inventory_limits"))) + if plan["dependency_guard"]["generated"] != snapshot["generated"]: + raise _failure("generated-incarnation-drift", "Generated resources differ from original producer snapshots.") + else: + reject_secrets(current) + plan["desired"] = current + search_reconcile._validate_plan(plan) + planned = _planned(target, plan, "helpers/search_reconcile.py", []) + if target["type"] == "knowledge-source": + planned["approval_summary"]["delete"].extend( + {"type": child["type"], "name": child["name"], "endpoint": target["endpoint"], "service_managed": True} + for child in snapshot["generated"]) + else: + planned["approval_summary"]["retained_targets"] = [ + {**target, "type": "knowledge-source", "name": item["name"]} for item in current["knowledgeSources"]] + return planned + if prior.get("operation") == "create-initial-prompt-agent": + try: + from . import _initial_prompt + except ImportError: + import _initial_prompt + _initial_prompt.validate(prior) + if (target["type"] != "prompt-agent-version" or record["acknowledgement"]["operation"] != "agents.create" + or snapshot["definition_digest"] != digest(prior["agent"]["definition"])): + raise _failure("ownership-unproven", "Initial receipts authorize only the acknowledged version, never a connection/container.") + else: + prompt_connect._validate_plan(prior) + if target["type"] == "prompt-agent-version" and record["acknowledgement"]["operation"] != "agents.create_version": + raise _failure("ownership-unproven", "Connect receipts must come from the native new-version operation.") + project, endpoint = prompt_connect._project_identity(prior) + if project.casefold() != target["project_resource_id"] or endpoint != target["project_endpoint"]: + raise _failure("ownership-unproven", "Receipt project differs from original creation approval.") + plan.update(sdk_major=2, project_resource_id=target["project_resource_id"], project_endpoint=target["project_endpoint"]) + if target["type"] == "prompt-agent-version": + if prior["agent"]["name"] != target["name"] or prior["agent"].get("version") == target["version"]: + raise _failure("ownership-unproven", "Only the acknowledged new version is eligible, never the baseline.") + Client, _, _, _, extras = sdk_loader() + Credential, AzureError = extras + client = Client(endpoint=endpoint, credential=Credential()) + try: + current = client.agents.get_version(agent_name=target["name"], agent_version=target["version"]) + if (current.name != target["name"] or str(current.version) != target["version"] + or digest(current.definition.as_dict()) != snapshot["definition_digest"] + or receipts.version_identity(current) != snapshot["version_identity"]): + raise _failure("definition-drift", "Selected version differs from the original SDK return.") + except AzureError as exc: + if sdk_error_status(exc) == 404: + return _absent(target) + raise HelperFailure(message="Exact version readback failed.", blocked_at="cleanup-planning", + **sdk_error_metadata(exc, "agent-readback-failed")) from exc + finally: + client.close() + plan["agent"] = {"name": target["name"], "version": target["version"], "run_owned": True, + "owned_definition_digest": snapshot["definition_digest"], "owned_version_identity": snapshot["version_identity"]} + prompt_cleanup._validate_plan(plan) + return _planned(target, plan, "helpers/prompt_cleanup.py", []) + if prior["connection"]["name"] != target["name"] or prior["connection"]["action"] != "create": + raise _failure("ownership-unproven", "Connection receipt must bind original create intent.") + plan["connection"] = {"name": target["name"], "run_owned": True, "expected_etag": snapshot["etag"], + "owned_definition_digest": snapshot["definition_digest"]} + token = token_provider(MANAGEMENT_AUDIENCE) + url = prompt_connect._connection_url(plan) + current, request_id = prompt_cleanup._get_connection(url, token, transport=transport) + if current is not None and (digest(current) != snapshot["definition_digest"] or not prompt_connect._connection_readback(prior, current)[0]): + raise _failure("definition-drift", "Connection differs from its original acknowledged producer state.") + selected = None + if "agent_version" in request: + selected = plan_cleanup({ + "schema_version": "1.0", "owner": request["owner"], + "target": {**target, "type": "prompt-agent-version", "name": prior["agent"]["name"], "version": request["agent_version"]}, + "creation_input_file": request["creation_input_file"], + "creation_receipt_file": request["agent_creation_receipt_file"], + }, base_dir=base_dir, token_provider=token_provider, transport=transport, sdk_loader=sdk_loader) + if selected["status"] == "planned": + plan["agent"] = selected["execution_input"]["plan"]["agent"] + if current is None: + if selected and selected["status"] == "planned": + selected["approval_summary"]["already_absent"] = [target] + return selected + return _absent(target, request_id) + plan["dependency_guard"] = dependencies.prompt_snapshot( + plan, current, sdk_loader=sdk_loader, bounds=dependencies.limits(request.get("inventory_limits"))) + if prompt_cleanup._get_connection(url, token, transport=transport)[0] != current: + raise _failure("definition-drift", "Connection changed during consumer discovery.") + prompt_cleanup._validate_plan(plan) + planned = _planned(target, plan, "helpers/prompt_cleanup.py", []) + if selected and selected["status"] == "planned": + planned["approval_summary"]["delete"].insert(0, selected["approval_summary"]["delete"][0]) + return planned + + +def plan_cleanup( + request: dict[str, Any], *, base_dir: Path = Path("."), + token_provider: TokenProvider = azure_cli_token, transport: Transport = http_request, + sdk_loader: Callable[[], tuple[Any, Any, Any, Any, Any]] = prompt_cleanup.load_cleanup_sdk, +) -> dict[str, Any]: + _object(request, "Cleanup planning request") + reject_secrets(request) + require_allowed_fields(request, REQUEST_FIELDS, label="Cleanup planning request") + if request.get("schema_version") != "1.0" or not _text(request.get("owner")): + raise _failure("input-schema-invalid", "schema_version 1.0 and accountable owner are required.") + target = _target(request.get("target")) + dependencies.limits(request.get("inventory_limits")) + selection_fields = {"agent_version", "agent_creation_response_file", "agent_creation_receipt_file"} & set(request) + evidence_field = "agent_creation_receipt_file" if "agent_creation_receipt_file" in request else "agent_creation_response_file" + if selection_fields and (selection_fields != {"agent_version", evidence_field} or target["type"] != "project-connection"): + raise _failure("input-schema-invalid", "Only connection cleanup can explicitly select both agent_version and its original create response.") + if selection_fields and ( + not isinstance(request["agent_version"], str) or re.fullmatch(r"[1-9][0-9]*", request["agent_version"]) is None + or not _text(request[evidence_field]) + ): + raise _failure("target-selection-required", "Select an exact numeric created agent version and original SDK response file.") + if "creation_receipt_file" in request: + try: + from .blob_recheck import read_private + except ImportError: + from blob_recheck import read_private + path = _path(request, "creation_receipt_file", base_dir) + if read_private(path).get("kind") == "cleanup-creation-receipt": + if ("creation_response_file" in request or "creation_result_file" in request + or (selection_fields and evidence_field != "agent_creation_receipt_file")): + raise _failure("input-schema-invalid", "Protected receipts cannot be mixed with manual response adapters.") + prior, record = receipts.load(_path(request, "creation_input_file", base_dir), path, target) + return _plan_protected(request, target, prior, record, base_dir=base_dir, + token_provider=token_provider, transport=transport, sdk_loader=sdk_loader) + if evidence_field == "agent_creation_receipt_file": + raise _failure("input-schema-invalid", "Selected producer receipts require a protected connection receipt.") + prior, result, response = _records(request, target, base_dir) + if target["type"] in {"knowledge-base", "knowledge-source"}: + return _plan_search( + request, target, prior, result, response, token_provider=token_provider, transport=transport, + ) + if target["type"] == "project-connection": + return _plan_connection( + request, target, prior, result, response, base_dir=base_dir, + token_provider=token_provider, transport=transport, sdk_loader=sdk_loader, + ) + return _plan_prompt(request, target, prior, result, response, sdk_loader=sdk_loader) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--plan", type=Path, required=True) + args = parser.parse_args(argv) + request: dict[str, Any] = {} + try: + request = _read_json(args.plan) + result = plan_cleanup(request, base_dir=args.plan.resolve().parent) + emit_result(result, preserve_unapproved_input=result["status"] == "planned") + except HelperFailure as failure: + result = blocked_result(failure, outcome="plan-cleanup", fingerprint=None, owner=request.get("owner")) + result["approval_summary"] = { + "delete": [], "retain": RETAIN + ["selected target and all its dependencies"], + "blocked": [failure.code], "order": [], "hosted_cleanup": "unsupported", + } + try: + result["approval_summary"]["retained_targets"] = [_target(request.get("target"))] + except HelperFailure: + pass + emit_result(result) + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/contracts.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/contracts.md new file mode 100644 index 00000000..48d63fa2 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/contracts.md @@ -0,0 +1,154 @@ +# Deterministic helper contracts + +Read before planning/approval; scope discovery. +Owners assess content/ambiguity/roles/consent. [Private output](private-artifacts.md). +Inspect source only on `blocked`/`partial`. + +## Common envelope and process contract + +UTF-8 JSON: + +```json +{"schema_version":"1.0","plan":{},"approval":{"confirmed":true,"fingerprint":"sha256:"}} +``` + +Canonical JSON: UTF-8, key-sorted, ASCII-escaped, compact. Approve the unchanged +plan; helpers verify fingerprints. Exclude credentials, tokens, keys, SAS, +passwords, content and secret-bearing connection strings. +Use CLI tokens. File CU: MI or approved ARM/ENV; never retain secrets. +Envelope, approval, plan, nested controls and file records are +closed schemas: undeclared fields block before authentication. +`desired` remains a complete service body. Results recursively redact secret fields. +Azure Policy evaluation belongs to the creation owner, not these helpers. +[Bootstrap](bootstrap-contracts.md): no preflight policy reads. +Bind known requirements and child fingerprint in the parent plan; no policy precheck. +Never add undeclared policy/tag fields; if required settings cannot reach the supported request body, +return `azure-policy-setting-unsupported` before creation. + +Mutation commands emit compact JSON to stdout: exit `0` is `completed`, `2` no-write `blocked`, +`3` written/ambiguous `partial`. Preserve JSON/exit status. +Read same identity after ambiguous writes. Unacknowledged Search creates stay +`partial`: `resources_remaining.reused`, not owned. Otherwise prove approved state/absence. + +429: [read recovery](throttle-recovery.md), plus File queue retry below. + +File/Blob/ADLS and Search apply: [_progress.py](_progress.py): +stderr JSONL; `--no-progress` disables. Planning/discovery stays quiet. +Libraries: `progress=Progress("")`; children share one terminal event. +Bounded activity/remaining checks, elapsed seconds and counts. +Same stage: once/second; transitions/terminal: immediate. +File POST: five-second heartbeat, no polling/ETA; metadata proves File ingestion; +cycle updates aren't unique documents; ARM readiness isn't KB/retrieval/data-plane proof. +I/O failure: `progress-output-failed` warning; primary failures/execution unchanged. +Failed native stderr drains to null at shutdown; custom sinks untouched. Final JSON is authoritative. + +## File source application + +`python helpers/file_source.py --plan request.json`: read-only; +unapproved `execution_input`/`approval_summary`; exit `0` is `planned`, not readiness. +See [request/example](../knowledge-sources/create-file.md#proposed-plan). +Require `extraction_mode`; [optional embeddings](vector-contracts.md) are independent. +Planning: no bootstrap/roles/uploads/cleanup/approval; owner-owned requirements. +Fresh exact reuse sets `execution_required`/`mutation_approval_required` false; +no approval/execution. Create: approve the unchanged envelope: + +```text +python helpers/file_source.py --input approved.json --cleanup-receipt-dir --upload-receipt-dir +``` + +Require `operation: reconcile-and-ingest`, `cleanup_approved: false`, +`owner`, `source` and `ingestion`. Children target the same endpoint/name/API/owner. +Validate both before mutation; reconcile before upload/readback. +ACKs/journal: [resume](file-upload-recovery.md). +Reuse requires exact markers, never new files or per-file cleanup. +Parent/approved upload-only routes. `source` uses the +[reconciliation shape](#search-resource-reconciliation) with `kind: "file"`; +`ingestion`: + +```json +{"operation":"ingest","endpoint":"https://svc.search.windows.net","name":"src","api_version":"2026-08-01-preview","local_root":"","service_tier":"basic","extraction_mode":"minimal","owner":"o@x","cleanup_approved":false,"files":[{"path":"guide.txt","size":123,"mtime_ns":1700000000000000000,"sha256":"sha256:<64-hex>","media_type":"text/plain"}],"inventory_digest":"sha256:<64-hex>","expected_server_inventory_digest":"sha256:<64-hex>"} +``` + +## Search resource reconciliation + +```text +python helpers/search_reconcile.py --input +``` + +Require `operation: reconcile`, `outcome`, `resource_type`, `endpoint`, `name`, +`api_version`, `action`, complete approved `desired`, `owner`, and +`cleanup_approved: false`. Use `action: create`, or approved `action: update` +with `expected_etag`. Exact existing state is zero-write. + +Blob/ADLS plans also require `source_evidence` with `verified: true` and an +`inventory_digest`. ADLS requires `path_verified: true`, `acl_verified: true`. +These attest reconciliation, not evidence collection/readiness: apply through +the Blob parent. File `standard`: `ai_services_managed_identity` or +`ai_services_key_acquisition` via File parent, or existing `ai_services_api_key_environment`. + +KB `--plan`: [intent/wire/approval](kb-contracts.md); save unapproved `execution_input`, +never hand-build bodies/hashes. `verified_source` binds `verified: true`, `name`, +normalized `definition_digest`; fresh same-API GET must match before create/update/reuse. +GA omits preview fields/models. Low/medium needs a separate chat model; +model-free agent retrieval requires preview minimal/extractive. + +For cleanup, [plan](../lifecycle/cleanup.md), then use the same executor. +Bind `operation: delete`, `plan_kind: cleanup`, `cleanup_approved: true`, +`expected_etag`, and `owned_definition_digest`. Recheck ownership; verify absence. + +## Blob/ADLS source application + +Blob/ADLS alone loads [folder scope/bounds/drift/cleanup](blob-contracts.md). +`blob_source.py --discover` inventories; `--plan` builds unapproved artifacts +or verifies approval-free reuse; +`--input` applies the approved parent and monitors ingestion. Search +reconciliation alone never proves Blob readiness. + +## File ingestion internals + +`file_source.py` invokes this child after validating both plans. +Standalone CLI never authorizes new uploads. + +Reject path/inventory/extraction/server-state/marker drift. Sequential uploads; +one final inventory. Only per-file 415 permits continuation; systemic failures stop. +[File 429](file-upload-recovery.md): one bounded retry. POST: 180s; reads: 60s. +Timeout/409/5xx: read proof, never POST replay. Metadata proves ingestion, not +retrieval; status-only ACKs remain pending. +Never return bytes/local root. + +## Existing Prompt Agent fallback + +Only for unavailable MCP/missing connection/version operations; +never authorization denial/conflicting state: + +```text +python helpers/prompt_connect.py --input +``` + +The plan requires `operation: connect`, `sdk_major: 2`, exact project resource +ID and matching endpoint identity, `connection`, `agent`, `rbac_verified`, `allowed_tools: +["knowledge_base_retrieve"]`, `require_approval: "never"`, +`permission_forwarding`, `owner`, and `cleanup_approved: false`. +The [fallback](../agents/connect-prompt-sdk-fallback.md) defines `--plan`: +intent yields unapproved `execution_input` + summary; hashes stay internal. +Legacy inputs omit dependency/version snapshots. +Approve grounding changes. Forwarding +is either `{"mode":"not-applicable"}` or +`{"mode":"structured-input","name":"search_auth_token"}`. +Reuse one complete match; multiple matches block. +ARM uses `Microsoft.CognitiveServices/accounts/projects/connections`. +Optional boolean `connection.is_shared_to_all` defaults to `true`. +Fallback defines normalization/bounded warnings for creation, +ambiguous-write recovery/reuse. `agent_invocation: not-run` isn't E2E proof. + +Prompt cleanup (planner above blocks unproven connection consumers): + +```text +python helpers/prompt_cleanup.py --input +``` + +Bind `operation: delete`, `plan_kind: cleanup`, `cleanup_approved: true`, +`sdk_major: 2`, exact project identity and `agent`/`connection` with +`run_owned: true`, exact identity and `owned_definition_digest`. +Connections also bind `expected_etag`. Delete version before connection; +verify absence; preserve prior versions/project/model/KB/roles. diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/cu_ingestion_auth.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/cu_ingestion_auth.py new file mode 100644 index 00000000..06c6d1bd --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/cu_ingestion_auth.py @@ -0,0 +1,245 @@ +"""Shared CU auth disclosure; private acquisition is restricted to File source PUTs.""" +from __future__ import annotations + +import base64 +import copy +import json +import re + +try: + from . import _bootstrap_io + from ._common import HelperFailure, HttpResult, MANAGEMENT_AUDIENCE, azure_cli_token, digest, require_allowed_fields +except ImportError: + import _bootstrap_io + from _common import HelperFailure, HttpResult, MANAGEMENT_AUDIENCE, azure_cli_token, digest, require_allowed_fields + + +API_VERSION = "2024-10-01" +PURPOSE = "file-standard-cu-source-put" +RESOURCE = re.compile( + r"/subscriptions/([0-9a-fA-F-]{36})/resourceGroups/[A-Za-z0-9_.()-]{1,90}" + r"/providers/Microsoft\.CognitiveServices/accounts/[A-Za-z0-9][A-Za-z0-9_.-]{1,63}", + re.IGNORECASE, +) +ENDPOINT = re.compile(r"https://[a-z0-9][a-z0-9-]{0,62}\.services\.ai\.azure\.com/?") +UUID = re.compile(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}") + + +def approval_summary(adapter, auth, *, creating): + modes = { + "file": {"system-assigned", "api-key-arm", "api-key-environment"}, + "azureBlob": {"system-assigned"}, "adlsGen2": {"system-assigned"}, + } + if adapter not in modes or auth not in modes[adapter]: + raise failure("cu-adapter-auth-unsupported", "Select the supported adapter's verified CU auth contract; no cross-adapter auth fallback.") + return { + "adapter": adapter, "mode": auth, + "contract": ( + "File system MI is implemented in inspected service code; live compatibility is unverified. Explicit legacy key modes remain supported, without fallback." + if adapter == "file" else + "Documented Search system-assigned CU identity; Cognitive Services User on the selected CU account. No key fallback." + ), + "credential_read": ( + "none: exact source reuse" if not creating else + "private ARM listKeys/key1 for the exact source PUT only" if auth == "api-key-arm" else + "explicit existing ENV only" if auth == "api-key-environment" else + "none: managed identity" + ), + "setup": ( + "Reuse verified dependencies. Resolve only missing changes through packaged native bootstrap with exact identity/role/scope and explicit permission/local-auth approval; source execution does not alter them and unsupported updates block." + if creating else "No CU setup or reingestion for exact source reuse." + ), + "consent": "Disclose auth and cost/data/access in the concrete source approval; no separate manual credential/MI/ENV confirmation. Material changes require refreshed approval.", + } + + +def failure(code, message, *, status=None, partial=False): + return HelperFailure(code, message, blocked_at="cu-authentication", status=status, partial=partial) + + +def token_principal(token): + # This binds CLI token identity, not token validity; ARM validates the credential. + try: + payload = token.split(".")[1] + claims = json.loads(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4))) + if any(not isinstance(claims.get(k), str) or UUID.fullmatch(claims[k]) is None for k in ("tid", "oid")): + raise ValueError() + return claims["tid"], digest({"object_id": claims["oid"]}) + except Exception: + raise failure("cu-context-unavailable", "The signed-in ARM token lacks a usable tenant/object identity binding; token details withheld.") from None + + +def account_context(): + """Read non-secret CLI identity metadata; never run a key-returning subprocess.""" + try: + code, stdout, _ = _bootstrap_io.run_cli(["account", "show"], 30) + value = json.loads(stdout) if code == 0 else None + user = value.get("user") if isinstance(value, dict) else None + if ( + not isinstance(user, dict) or user.get("type") not in ("user", "servicePrincipal") + or not isinstance(user.get("name"), str) or not user["name"] + or value.get("environmentName") != "AzureCloud" or value.get("state") != "Enabled" + ): + raise ValueError() + tenant, principal = token_principal(azure_cli_token(MANAGEMENT_AUDIENCE)) + if tenant != value["tenantId"]: + raise ValueError() + context = { + "subscription_id": value["id"], "tenant_id": value["tenantId"], + "principal_digest": principal, + } + validate_context(context) + return context + except Exception: + raise failure("cu-context-unavailable", "Use an existing signed-in AzureCloud CLI context; identity details withheld.") from None + + +def validate_context(context): + if not isinstance(context, dict): + raise failure("cu-context-invalid", "Retain the planner's exact signed-in context.") + require_allowed_fields(context, {"subscription_id", "tenant_id", "principal_digest"}, label="File CU context") + if ( + any(not isinstance(context.get(k), str) or UUID.fullmatch(context[k]) is None + for k in ("subscription_id", "tenant_id")) + or not isinstance(context.get("principal_digest"), str) + or re.fullmatch(r"sha256:[0-9a-f]{64}", context["principal_digest"]) is None + ): + raise failure("cu-context-invalid", "Retain complete tenant, subscription and principal binding.") + + +def acquisition(choice, context): + result = { + "resource_id": choice["resource_id"], "endpoint": choice["endpoint"].rstrip("/"), + "context": copy.deepcopy(context), "api_version": API_VERSION, "key_name": "key1", "purpose": PURPOSE, + } + validate_acquisition(result, choice["endpoint"]) + return result + + +def validate_acquisition(value, endpoint): + if not isinstance(value, dict): + raise failure("cu-acquisition-invalid", "Retain the exact approved File CU acquisition contract.") + require_allowed_fields(value, {"resource_id", "endpoint", "context", "api_version", "key_name", "purpose"}, + label="File CU acquisition") + resource = RESOURCE.fullmatch(value.get("resource_id", "")) if isinstance(value.get("resource_id"), str) else None + if ( + resource is None or not isinstance(value.get("endpoint"), str) + or ENDPOINT.fullmatch(value["endpoint"]) is None + or value["endpoint"] != endpoint.rstrip("/") + or value.get("api_version") != API_VERSION or value.get("key_name") != "key1" + or value.get("purpose") != PURPOSE + ): + raise failure("cu-acquisition-invalid", "Only the exact selected AIServices account, endpoint and key1 source-PUT purpose are supported.") + validate_context(value.get("context")) + if resource.group(1).casefold() != value["context"]["subscription_id"].casefold(): + raise failure("cu-subscription-mismatch", "Select the approved CU account's existing CLI subscription, then refresh the plan; no context switching was performed.") + + +def check_context(expected, provider): + current = provider() + validate_context(current) + if current != expected: + raise failure("cu-context-drift", "Signed-in tenant, subscription or principal changed; restore the approved context or refresh the plan and approval.") + + +class PrivateKey: + def __init__(self, contract, *, token_provider, transport, context_provider, recheck): + self.contract = contract + self.token_provider = token_provider + self.raw_transport = transport + self.context_provider = context_provider + self.recheck = recheck + self._secrets = () + self._attempted = False + + def acquire(self): + if self._attempted: + raise failure("cu-acquisition-repeated", "Credential acquisition is single-attempt; no alternate key, account or auth channel was tried.") + self._attempted = True + expected = self.contract["context"] + check_context(expected, self.context_provider) + self.recheck() + try: + token = self.token_provider(MANAGEMENT_AUDIENCE) + if token_principal(token) != (expected["tenant_id"], expected["principal_digest"]): + raise failure("cu-context-drift", "ARM token identity differs from the approved caller.") + check_context(expected, self.context_provider) + result = self.raw_transport( + "POST", f"{MANAGEMENT_AUDIENCE}{self.contract['resource_id']}/listKeys?api-version={API_VERSION}", + token, follow_redirects=False, max_response_bytes=16384, + ) + except HelperFailure as error: + if error.code in ("cu-context-drift", "cu-context-unavailable"): + raise failure(error.code, "Signed-in ARM identity no longer matches the usable approved context; refresh approval.") from None + raise self._acquisition_failure(error.http_status) from None + except Exception: + raise self._acquisition_failure(None) from None + if result.status != 200: + raise self._acquisition_failure(result.status) + body = result.body + if ( + not isinstance(body, dict) + or not isinstance(body.get("key1"), str) + or not 1 <= len(body["key1"]) <= 4096 + or not body["key1"].isascii() or any(c.isspace() or ord(c) < 33 for c in body["key1"]) + or (body.get("key2") is not None and ( + not isinstance(body["key2"], str) or len(body["key2"]) > 4096 + )) + ): + raise failure("cu-key-response-invalid", "ARM listKeys did not return a usable key1; no alternate key was selected.") + self._secrets = tuple(v for k in ("key1", "key2") if isinstance(v := body.get(k), str) and v) + check_context(expected, self.context_provider) + return body["key1"] + + @staticmethod + def _acquisition_failure(status): + status = status if type(status) is int and 100 <= status <= 599 else None + if status in (401, 403): + return failure( + "cu-key-access-denied", + "The signed-in caller needs Microsoft.CognitiveServices/accounts/listKeys/action on the exact approved CU account. Have its owner resolve access under separate concrete approval; no roles or policies were changed.", + status=status, + ) + return failure("cu-key-acquisition-failed", "ARM listKeys failed for the approved CU account; details withheld and no retry or auth fallback performed.", status=status) + + def _redact(self, value): + if isinstance(value, str): + for secret in self._secrets: + value = value.replace(secret, "[REDACTED]") + return value + if isinstance(value, dict): + return {self._redact(k): self._redact(v) for k, v in value.items()} + if isinstance(value, list): + return [self._redact(v) for v in value] + if isinstance(value, tuple): + return tuple(self._redact(v) for v in value) + return value + + def transport(self, method, url, token, **kwargs): + if not self._secrets: + return self.raw_transport(method, url, token, **kwargs) + kwargs["follow_redirects"] = False + try: + result = self.raw_transport(method, url, token, **kwargs) + except HelperFailure as error: + raise HelperFailure( + self._redact(error.code), "Azure request failed after private CU credential acquisition; response details withheld.", + blocked_at=self._redact(error.blocked_at), + status=error.http_status if type(error.http_status) is int and 100 <= error.http_status <= 599 else None, + partial=error.partial, request_id=self._redact(error.request_id), + writes=self._redact(error.writes), resources_remaining=self._redact(error.resources_remaining), + resources_reused=self._redact(error.resources_reused), warnings=self._redact(error.warnings), + ) from None + except Exception: + raise failure("cu-private-request-failed", "Azure request failed; private request/response details withheld.", + partial=method not in ("GET", "HEAD")) from None + etags = self._redact(result.etag_values) + versions = [ + *(result.etag_values or ()), + *(value for name, value in result.headers.items() if name.lower() == "etag"), + result.body.get("@odata.etag") if isinstance(result.body, dict) else None, + ] + if any(self._redact(value) != value for value in versions): + # Redaction must not turn conflicting secret-bearing versions into equal evidence. + etags = ("",) + return HttpResult(result.status, self._redact(result.body), self._redact(result.headers), etags) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/document-assessment.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/document-assessment.md new file mode 100644 index 00000000..9e288e63 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/document-assessment.md @@ -0,0 +1,124 @@ +# Bounded document assessment + +Read **only when document sample inspection is selected** from content-fit intake. +Structural facts only; no conversion or admission proof. +Refusal remains valid. No upload, OCR, rasterization, model call, +macro/script/formula execution, external URI/object resolution or file extraction +to disk. Never inspect binary documents using text viewers. + +## Exact approval and read contract + +Before any read, approve exact local file/SHA-256/scope/ceilings. Separately approve +whole-file hashing if identity is unknown. No inspection/hash/worker on decline. +One file/invocation; no automatic batches, retries, wider scope or larger limits. + +PDF: explicit 1-based pages, maximum eight selected pages/200 total pages. +Others require `--whole-document`, not guessed page subsets; narrower non-PDF +scope is unsupported. Pagination/rendering is not assessed. +Whole-file hashing and PDF trailer/xref/object-stream/page-tree parsing remain +necessary; compressed containers may include unselected objects. Only selected +PDF streams are assessed, never other pages to guess modality. Decline if this +whole-file structural metadata parsing is unwanted. + +Use a customer-controlled private local scope. No URLs, device/UNC/mapped-network +paths, symlinks/reparse points or untrusted writable directories. Linux mount +locality and concurrent file replacement remain operator responsibilities. +Blob/ADLS: reuse customer-provided, separately authorized local sample copies. +This tool implements **no remote read/download adapter**. Remote requests return +`remote-sample-access-not-supported`, not a container/subscription crawl. +Separate sample-access approval names exact account/container/blob/version/ETag, +byte range/full-file read, destination, retention and limits. Local path/hash +approval remains required; no remote provenance verification or wider prefix. + +```text +python "\helpers\document_assess.py" --approve-inspection --file "C:\approved\sample.pdf" --sha256 <64-lowercase-hex> --pages 1,3 +python "\helpers\document_assess.py" --approve-inspection --file "C:\approved\sample.docx" --sha256 <64-lowercase-hex> --whole-document +``` + +POSIX: native paths/quoting. Optional `--format`: +`auto`, `pdf`, `text`, `markdown`, `html`, `json`, `docx`, `pptx`, `xlsx`, `png`, +`jpeg`. Byte signatures/OOXML content types determine binary adapters, never +extensions/MIME. Text auto mode reports UTF-8 text; HTML/JSON need explicit hints for structure, +Markdown for lexical counts. Hints cannot override binary detection, ingestion +mode or Search server admission. Auto-detected PDF still needs exact pages. + +## Declared installation and limits + +Python 3.11+, Windows/Linux. Separately approve installation outside inspection; +no automatic install or library guessing: + +```text +python -m pip install -r "\helpers\requirements-assessment.txt" +``` + +`pypdf==6.8.0` (BSD-3-Clause): PDF; `defusedxml==0.7.1` (PSF license): Office XML; +others use stdlib. Missing/wrong selected-adapter versions block, without +affecting other adapters. Nothing vendored; MIT-compatible distribution requires +retaining dependency notices/non-endorsement terms when bundling. +Repository development: `uv sync --locked` (`pyproject.toml`/`uv.lock`). + +| Bound | Fixed ceiling | +|---|---| +| File / worker / output | 16 MiB input; 256 MiB worker; 10 CPU seconds; 15 seconds wall including startup/read; 8 KiB JSON | +| Text / HTML / JSON | 2 MiB UTF-8 input; JSON depth 64/100,000 nodes; HTML 100,000 start tags | +| Office ZIP/XML | 256 entries; 2 MiB per inflated entry; 16 MiB aggregate declared inflation; compression ratio 100; 100,000 inspected XML nodes/depth 64 | +| Images | PNG IHDR/JPEG SOF0/1/2 headers only; 100 million declared pixels; 4,096 JPEG markers; no pixel decompression | + +ZIP: stored/deflated only; encrypted/macro packages, duplicate/unsafe paths and +inflation excess block. Capped in-memory reads, no disk extraction. XML forbids +DTDs/entities/external references. OS limits cover forged sizes, decompression, +cycles and allocations before post-parse node checks. + +Windows Job Object: process commit-memory/CPU/active-process limits. Linux: +`RLIMIT_AS`/`RLIMIT_CPU`, zero core/file-output limits; not RSS quotas. +Install before source reads/parser imports or block. Parent timeout/output guards +kill only their owned PID. Audit denies writes/network/processes; after trusted +parser/codec imports, file/directory reads too. Unsupported lazy features block. +This is **not an OS security sandbox or proof against every hostile document**. +OS paging/crash dumps remain outside the helper's privacy boundary. + +## Fact semantics and coverage + +| Adapter | Reported structural facts; exclusions | +|---|---| +| PDF | Page-tree count; selected-page Unicode length/nonempty flag; direct image/path paint and Form calls. Not unique/visible images, lines or tables. Forms untraversed/their text unassessed; total images/drawings unknown. | +| Text/Markdown | UTF-8 character count including whitespace/markup, not an AST or visual classification. Other encodings unassessed. | +| HTML | Data-event characters excluding script/style; recognized img/table/tr/td/th tag counts. Tolerant syntax, not rendering, table relationships or fetched images. | +| JSON | String-value characters, object/array/key counts, not keys' contents, schema semantics or numeric text interpretation. | +| DOCX | Main-document text/paragraph/table/blip elements; headers/footers/notes/embedded objects unassessed. | +| PPTX | Standard numbered slide parts' text/paragraph/table/blip elements and part count; not order, notes, diagrams or linked objects. | +| XLSX | Standard worksheet/shared-string/table parts: stored text, formula/cell/table counts, declared sheets. Shared strings counted once, not cell occurrences; no evaluation/numeric/chart interpretation. | +| PNG/JPEG | Dimensions/numeric headers; PNG IHDR CRC checked. Pixels, orientation, further frames and completeness unassessed. | + +Office: conventional transitional namespaces/parts only. Unreferenced standard +slide/sheet parts may be counted; relationships unresolved. Image references/ +table elements prove neither visibility, preserved relationships nor semantics. +Every report keeps input profile `unknown`, OCR need/answerability/layout +relationships `not-assessed`. `not-applicable` distinguishes non-page formats or +irrelevant typed counters; unknown facts never become false zero counts. +Text output/Markdown representation cannot prove text-only input or choose CU. + +## Return and fallback + +Exit `0`/`completed`, `assessment: assessed`: adapter facts only. +Exit `2`/`blocked`, `assessment: not-assessed`, `facts: null`: approval/scope/identity/ +limits, malformed/encrypted/unsupported content, missing adapter, warning or +worker failure. Do not invent worker-death causes. Legacy Office, other formats +and unsupported profiles are unassessed—not rejected for service ingestion. +All reports say `service_admission: not-assessed`; preserve the blocker and use +description/answer questions, never assume text-only or silently downgrade. + +After identity verification, bind source SHA-256 and canonical path SHA-256 +(UTF-8 absolute path, Windows case-normalized). Changed bytes block before parsing; +original File bytes remain unchanged. +No raw text/keys/snippets/images/drawing operands/archive names in stdout/stderr/ +artifacts. Only aggregate JSON leaves worker memory; no temporary files/deletion. +Return coverage/uncertainty to content-fit, then source-owner plan/ingestion +approvals—not fabricated evidence. + +Authorities: failure/conflict/uncertainty only. +- [pypdf contract/license](https://pypi.org/project/pypdf/6.8.0/) +- [XML parser/license](https://github.com/tiran/defusedxml) +- [OOXML](https://ecma-international.org/publications-and-standards/standards/ecma-376/) +- [PNG specification](https://www.w3.org/TR/png-3/) +- [JPEG T.81](https://www.itu.int/rec/T-REC-T.81) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/document_assess.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/document_assess.py new file mode 100644 index 00000000..6f8833f6 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/document_assess.py @@ -0,0 +1,353 @@ +"""Consent-gated input-document structural facts; never emit document content.""" +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import os +from pathlib import Path +import re +import stat +import subprocess +import sys +import threading + +PARSER_VERSION = "6.8.0" +MAX_INPUT = 16 * 1024 * 1024 +MAX_PAGES = 200 +MAX_SELECTED = 8 +MAX_OUTPUT = 8192 +MAX_REQUEST = 16384 +WALL_SECONDS = 15 +FORMATS = {"auto", "pdf", "text", "markdown", "html", "json", "docx", "pptx", + "xlsx", "png", "jpeg"} + + +class Blocked(ValueError): + """A public, content-free blocker code.""" + + +def blocked(code): + return {"schema_version": "1.0", "status": "blocked", "assessment": "not-assessed", "code": code, + "service_admission": "not-assessed", "facts": None} + + +def validate_request(request): + if not isinstance(request, dict) or request.get("approved") is not True: + raise Blocked("inspection-not-approved") + if not {"approved", "path", "sha256"} <= set(request) or set(request) - { + "approved", "path", "sha256", "pages", "whole_document", "format" + }: + raise Blocked("invalid-request") + path, digest, pages = request["path"], request["sha256"], request.get("pages") + if isinstance(path, str) and path.lower().startswith(("http:", "https:", "abfs:", "abfss:")): + raise Blocked("remote-sample-access-not-supported") + if (not isinstance(path, str) or len(path) > 4096 or "\0" in path + or not Path(path).is_absolute()): + raise Blocked("invalid-source-path") + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise Blocked("invalid-source-identity") + if (pages is not None and (not isinstance(pages, list) or not 1 <= len(pages) <= MAX_SELECTED + or any(type(page) is not int or not 1 <= page <= MAX_PAGES for page in pages) + or len(set(pages)) != len(pages))): + raise Blocked("invalid-page-selection") + if type(request.get("whole_document", False)) is not bool or ( + (pages is not None) == request.get("whole_document", False) + ): + raise Blocked("explicit-document-scope-required") + if not isinstance(request.get("format", "auto"), str) or request.get("format", "auto") not in FORMATS: + raise Blocked("unsupported-format-hint") + + +def read_source(request): + path = Path(request["path"]) + if sys.platform == "win32": + import ctypes + + # Reject device namespaces, UNC, ADS and mapped network drives. + if (str(path).startswith("\\\\") or ":" in str(path)[2:] + or ctypes.windll.kernel32.GetDriveTypeW(str(path.anchor)) not in (3, 6)): + raise Blocked("nonlocal-source") + if any(item.is_symlink() or ( + getattr(item.lstat(), "st_file_attributes", 0) & 0x400 + ) for item in (path, *path.parents)): + raise Blocked("source-link-unsupported") + canonical = path.resolve(strict=True) + with canonical.open("rb") as source: + before = os.fstat(source.fileno()) + if not stat.S_ISREG(before.st_mode): + raise Blocked("nonregular-source") + if before.st_size > MAX_INPUT: + raise Blocked("input-limit") + data = source.read(MAX_INPUT + 1) + after = os.fstat(source.fileno()) + if len(data) > MAX_INPUT: + raise Blocked("input-limit") + if (before.st_size, before.st_mtime_ns, before.st_ino) != ( + after.st_size, after.st_mtime_ns, after.st_ino + ) or len(data) != before.st_size: + raise Blocked("source-changed") + if hashlib.sha256(data).hexdigest() != request["sha256"]: + raise Blocked("source-identity-mismatch") + binding = hashlib.sha256(os.path.normcase(str(canonical)).encode("utf-8")).hexdigest() + return data, binding + + +def deny_document_reads(event, args): + if event in {"open", "os.listdir", "os.scandir"}: + raise PermissionError("Document external read denied") + + +def assess_bytes(data, selected): + import codecs + import logging + import warnings + + try: + import pypdf + except ImportError: + raise Blocked("parser-dependency-missing") from None + if pypdf.__version__ != PARSER_VERSION: + raise Blocked("parser-version-unsupported") + from pypdf.errors import PdfReadError, PdfStreamError + from pypdf.generic import ContentStream + + class RejectWarning(logging.Handler): + def emit(self, record): + raise Blocked("pdf-parser-warning") + + logger = logging.getLogger("pypdf") + logger.handlers = [RejectWarning()] + logger.propagate = False + logger.setLevel(logging.WARNING) + warnings.simplefilter("error") + for encoding in ("charmap", "utf-16-be", "utf-16-le", "utf-8", "latin-1", "ascii"): + codecs.lookup(encoding) + # Parser imports are trusted installation reads; once bytes are supplied, + # no PDF-directed file read (even local) is permitted. + sys.addaudithook(deny_document_reads) + if not data.startswith(b"%PDF-") or not data.rstrip().endswith(b"%%EOF"): + raise Blocked("invalid-pdf") + try: + reader = pypdf.PdfReader(io.BytesIO(data), strict=True) + if reader.is_encrypted: + raise Blocked("encrypted-pdf") + page_count = len(reader.pages) + if page_count > MAX_PAGES: + raise Blocked("page-limit") + if max(selected) > page_count: + raise Blocked("page-out-of-range") + pages = [] + for number in selected: + page = reader.pages[number - 1] + # Direct stream operators, not object inventory or rendered entities. + content = page.get_contents() + operations = ContentStream(content, reader).operations if content is not None else [] + images, drawings, forms = 0, 0, 0 + for operands, operator in operations: + if operator == b"INLINE IMAGE": + images += 1 + elif operator in {b"S", b"s", b"f", b"F", b"f*", b"B", b"B*", b"b", b"b*"}: + drawings += 1 + elif operator == b"Do": + obj = page["/Resources"]["/XObject"][operands[0]].get_object() + subtype = obj["/Subtype"] + if subtype == "/Image": + images += 1 + elif subtype == "/Form": + forms += 1 + else: + raise Blocked("unsupported-paint-object") + # Forms can reference other pages/resources. Do not traverse them to + # estimate modality; report text unknown instead of widening scope. + characters = None if forms else len(page.extract_text()) + pages.append({ + "page": number, + "text_characters": characters, + "text_available": None if characters is None else characters > 0, + "text_status": "not-assessed-form-content" if forms else "assessed", + "direct_image_paints": images, + "direct_path_paints": drawings, + "form_invocations": forms, + "total_images": None, + "total_drawings": None, + "table_structure": "not-assessed", + "layout_relationships": "not-assessed", + }) + return {"page_count": page_count, "pages": pages, "input_profile": "unknown", + "answerability": "not-assessed", "ocr_need": "not-assessed"} + except (PdfReadError, PdfStreamError, KeyError, IndexError, TypeError, + UnicodeError, NotImplementedError): + raise Blocked("pdf-parse-unsupported") from None + except Warning: + raise Blocked("pdf-parser-warning") from None + except PermissionError: + raise Blocked("pdf-external-read-blocked") from None + + +def dispatch(data, request): + hint = request.get("format", "auto") + if data.startswith(b"%PDF-"): + if hint not in {"auto", "pdf"}: + raise Blocked("format-hint-mismatch") + if request.get("pages") is None: + raise Blocked("pdf-page-selection-required") + return "pdf", f"pypdf=={PARSER_VERSION}", assess_bytes(data, request["pages"]) + if request.get("pages") is not None: + raise Blocked("non-pdf-requires-whole-document-scope") + from _document_adapters import AdapterBlocked, assess_document + + try: + return assess_document(data, hint, deny_document_reads) + except AdapterBlocked as error: + raise Blocked(str(error)) from None + + +def worker(): + # Isolated mode excludes ambient PYTHONPATH; add only this packaged helper. + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from _document_limits import deny_side_effects, install_limits + + try: + job_handle = install_limits() + except (OSError, ValueError): + return blocked("worker-limits-unavailable") + sys.addaudithook(deny_side_effects) + binding = {} + try: + raw = sys.stdin.buffer.read(MAX_REQUEST + 1) + if len(raw) > MAX_REQUEST: + raise Blocked("request-limit") + request = json.loads(raw) + validate_request(request) + data, path_digest = read_source(request) + binding = {"source_sha256": request["sha256"], "path_sha256": path_digest} + kind, parser, facts = dispatch(data, request) + return {"schema_version": "1.0", "status": "completed", "assessment": "assessed", + "service_admission": "not-assessed", "format": kind, "parser": parser, + **binding, "facts": facts} + except Blocked as error: + return dict(blocked(str(error)), **binding) + except MemoryError: + return dict(blocked("worker-memory-limit"), **binding) + except RecursionError: + return dict(blocked("document-recursion-limit"), **binding) + except (OSError, ValueError, OverflowError): + return dict(blocked("document-input-or-parse-failed"), **binding) + finally: + # This local deliberately remains live through parsing on Windows. + _ = job_handle + + +def run_worker(command, request, timeout=WALL_SECONDS): + """Pipe-only IPC; bounded output reader, no sample or stderr artifacts.""" + encoded = json.dumps(request, separators=(",", ":")).encode() + if len(encoded) > MAX_REQUEST: + return blocked("request-limit") + output = bytearray() + exceeded = threading.Event() + io_failed = threading.Event() + with subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL) as process: + def drain(): + try: + chunk = process.stdout.read(MAX_OUTPUT + 1) + if len(chunk) > MAX_OUTPUT: + exceeded.set() + process.kill() + else: + output.extend(chunk) + except OSError: + io_failed.set() + + def send(): + try: + process.stdin.write(encoded) + process.stdin.close() + except OSError: + io_failed.set() + + thread = threading.Thread(target=drain, daemon=True) + writer = threading.Thread(target=send, daemon=True) + thread.start() + writer.start() + try: + process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + return blocked("worker-timeout") + finally: + if process.poll() is None: + process.kill() + process.wait() + thread.join() + writer.join() + if exceeded.is_set(): + return blocked("worker-output-limit") + if process.returncode not in (0, 2): + return blocked("worker-failed-or-resource-limit") + if io_failed.is_set(): + return blocked("worker-io-failed") + try: + result = json.loads(output) + except (ValueError, UnicodeError): + return blocked("worker-invalid-output") + if not isinstance(result, dict) or result.get("status") not in {"completed", "blocked"}: + return blocked("worker-invalid-output") + return result + + +def assess(request): + try: + validate_request(request) + return run_worker([sys.executable, "-I", "-B", str(Path(__file__).resolve()), + "--worker"], request) + except Blocked as error: + return blocked(str(error)) + except OSError: + return blocked("worker-start-failed") + + +class SafeArgumentParser(argparse.ArgumentParser): + def error(self, message): + raise Blocked("invalid-cli-arguments") + + +def main(): + def private_failure(exc_type, exc_value, traceback): + print(json.dumps(blocked("worker-failed-or-resource-limit"))) + + # Unexpected parser/runtime failures must not print attacker-controlled + # exception strings or stack traces, including when --worker is invoked. + sys.excepthook = private_failure + if sys.argv[1:] == ["--worker"]: + result = worker() + else: + parser = SafeArgumentParser(description=__doc__) + parser.add_argument("--approve-inspection", action="store_true") + parser.add_argument("--file") + parser.add_argument("--sha256") + parser.add_argument("--pages", help="Explicit 1-based pages, e.g. 1,3; no ranges") + parser.add_argument("--whole-document", action="store_true", + help="Approve whole bounded non-PDF document, not rendered pages") + parser.add_argument("--format", choices=sorted(FORMATS), default="auto") + try: + args = parser.parse_args() + if not args.approve_inspection: + raise Blocked("inspection-not-approved") + pages = None if args.pages is None else [int(value) for value in args.pages.split(",")] + result = assess({"approved": True, "path": args.file, + "sha256": args.sha256, "pages": pages, + "whole_document": args.whole_document, "format": args.format}) + except Blocked as error: + result = blocked(str(error)) + except ValueError: + result = blocked("invalid-page-selection") + print(json.dumps(result, separators=(",", ":"), ensure_ascii=True)) + return 0 if result["status"] == "completed" else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file-upload-recovery.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file-upload-recovery.md new file mode 100644 index 00000000..67e82a91 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file-upload-recovery.md @@ -0,0 +1,135 @@ +# File upload continuation and partial batches + +After partial File creation, retain the source. Never rerun creation, infer MI +from redacted GET, or recover by recreation/deletion/reset/new roles. +Generic File MI reuse/provenance guards remain unchanged. + +## Retain original evidence before creation + +```text +python helpers/file_source.py --input --cleanup-receipt-dir --upload-receipt-dir +``` + +Use the [private directory helper](private-artifacts.md); no automatic directory/ +ACL changes. The required journal retains original approval/context, actual +conditional source PUT ACK/request ID/ETag, exclusive **pre-POST** attempts and +separate upload responses. Never edit/remove/reconstruct entries. Integrity isn't +a service signature: retain exclusive control, no concurrent writers. + +File CLI creation requires protected `--cleanup-receipt-dir` and +`--upload-receipt-dir` pre-write; separate directories, no guessed paths. Plan with +`--execution-output `; show reference/counts, not +hash-bearing envelopes. Plan JSON/library and non-File policies are unchanged. + +Persistence failure stops requests. An attempt without result stays uncertain, +even if POST wasn't sent. Missing ACK/ETag cannot come from GET. Journals neither +replace [cleanup provenance](../lifecycle/cleanup.md) nor authorize deletion. +Without originals: retain source, `file-upload-provenance-missing`, not +"recreate required". + +## Supported upload-only plan + +Closed request: + +```json +{"schema_version":"1.0","receipt_directory":""} +``` + +```text +python helpers/file_upload.py --plan resume-request.json --execution-output +python helpers/file_upload.py --input +``` + +Planning is read-only: an **unapproved**, closed version `1.0` `resume-file-uploads` +envelope binds original plan/creation ACK/journal digests, owner, exact +never-attempted ordinals and receipt directory; +`cleanup_approved` stays false. Review and newly approve these uploads. An empty +eligible list needs no mutation or execution; use the returned observation. +Never edit ordinals/hashes to authorize attempted files. + +Execution validates the original unchanged approved corpus (paths, full hashes, +size, timestamps, mode, metadata), tenant/subscription/principal, source URL/API/ +definition/ACK ETag, and applicable CU account/Search MI/role/network snapshots. +It neither acquires CU keys nor changes authentication. It invokes only existing +metadata GETs and direct File upload POSTs using the original Search identity; +no source PUT, source update, new role, indexer operation, deletion, KB or model call. +POST targets the original validated +`/knowledgesources('')/files?api-version=2026-08-01-preview&pageSize=200` +multipart contract; approval/privacy gates apply. + +Continuation excludes previously attempted files, including exhausted 429 retries, +timeout/409/5xx and interrupted attempts. Only the original operation may retry its +received File-upload 429 once, as below. An immediate or delayed **empty** inventory +never proves in-flight termination for unknown outcomes. Positive exact file-ID/ +marker/hash/size readback can prove completed ingestion, but never invents an +upload ACK or source ownership. +Drift blocks; never repair by write. + +## Bounded batching and truthful progress + +File upload is synchronous: extraction, chunking, embedding, indexing and metadata +persistence finish before success. No File indexer, schedule or asynchronous status +polling. Each sequential POST has a 180-second processing/response allowance, +separate from the 60-second read budget; stricter approved outer deadlines win. +Timeout is not remote cancellation. Retain 200/201 ACKs without relisting source/ +full inventory after every healthy file. One final bounded inventory checks the +batch; follow validated `@odata.nextLink` exactly, without rebuilding parameters. +A per-file 415 permits independent continuation. Auth/access, deadlines, +persistence, drift and unknown outcomes stop new uploads. Repeated 429, or later +429 after the operation's one retry opportunity, stops without more reads/sleeps. +Preserve prior confirmed files; never hammer the remaining corpus. + +[429 recovery](throttle-recovery.md) supplies one server-respecting delay/read +opportunity, at most 30 seconds waiting/60 seconds per complete read sequence, +including all pages and response reads. Overlong delays stop, not shorten. +Journal-retained UTC backoff blocks resume planning/execution before any network; +fresh approval never waives it. `file_batch.backoff` reports waiting/unresolved/ +elapsed. Legacy 429 without timing stays unresolved; no invented deadline. +`--no-progress` disables observations. Otherwise stderr emits counts, a `waiting` +event with the bounded HTTP-429 delay, and terminal state. During POST, an immediate +and then five-second content-free heartbeat reports ordinal/total/attempt, elapsed +time and already ingested count: "1/25 ingested; processing file 2/25". This observer +never makes requests. No within-file percentage, ETA, filenames or private content. +Increment completion only after ACK/metadata proof and required receipt persistence. + +`file_batch.files` records bounded relative names/full hashes, upload state, +verification state and sanitized request IDs. Counts distinguish: +`accepted` (original 200/201 ACK), `confirmed` (exact file-ID/markers without +reported error), `failed`, `pending` (ACK but not confirmed), +`unverified` (uncertain attempt), and `not_attempted`. +Accepted overlaps verification counts; do not sum it with them. +`ingested` counts synchronous completion proved by valid ACK metadata or exact +completed-file readback. A status-only ACK is insufficient. Original ACK file IDs +remain binding even if its remaining metadata is incomplete. Failed readback stays +partial while preserving known prior ingestion; `retrieval` stays `unverified`. +File completion does not prove OCR quality or KB retrieval. + +Report "24 ingested, one failed/unverified" only with this proof; otherwise +distinguish accepted/pending. Preserve outcomes, not a blanket blocker. A subset +handoff must disclose pending files may appear and subset isolation is not proven; +any KB use still needs its existing separate approval. No automatic KB writes. + +## One documented queue-rejection retry + +The official File guide identifies upload HTTP 429 as a full processing queue and +recommends bounded parallelism/exponential backoff; Retry-After is not guaranteed. +Only this File upload route permits one same-operation retry across the batch, +not per file. No private throttle marker/error text is required. Honor valid +Retry-After; missing/invalid metadata uses the diagnosed first backoff step of one +second. Shared 30-second wait/60-second complete preflight limits apply. +After waiting, revalidate unchanged source/corpus and one fresh bounded inventory. +Positive proof avoids another POST; absence is not the retry authority: the +received queue-rejection 429 is. Persist exclusive original/retry attempt and +result records when journaling. Retry the original URL, token, multipart bytes/ +boundary and approved payload; source creation happens once. + +The MI canary's stricter one-upload approval forbids this retry. No PUT/DELETE/ +update retry, new roles or auth substitution. Each upload creates a new file ID, +even for the same filename; no filename-idempotency key exists. Timeout/504/409/ +5xx remain unknown, never replayable from empty inventory or elapsed time. +`file-upload-retry-safety-unproven` retains that blocker. Exhausted files stay +failed; continuation uploads only never-attempted files under fresh approval, +not another attempt at an exhausted file. + +Authorities: failure/conflict/uncertainty only. +[File upload/troubleshooting](https://learn.microsoft.com/azure/search/agentic-knowledge-source-how-to-file). diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_cu_canary.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_cu_canary.py new file mode 100644 index 00000000..92d65254 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_cu_canary.py @@ -0,0 +1,540 @@ +"""Approval-gated one-file OCR canary; never invoked automatically by ingestion.""" +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import re +import secrets +import sys +import time +import zlib +from email import policy +from email.parser import BytesParser +from pathlib import Path +from urllib.parse import urlencode + +try: + from . import _bootstrap_io, file_source, file_ingest, file_cu_mi, search_reconcile + from ._common import (HelperFailure, MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, azure_cli_token, + blocked_result, digest, emit_result, http_request, load_approved_input, + require_allowed_fields, reject_secrets) +except ImportError: + import _bootstrap_io, file_source, file_ingest, file_cu_mi, search_reconcile + from _common import (HelperFailure, MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, azure_cli_token, + blocked_result, digest, emit_result, http_request, load_approved_input, + require_allowed_fields, reject_secrets) + +PDF_NAME = "cu-mi-probe.pdf" +MARKER = re.compile(r"CUOCR[2-9]{8}") +FIELD = re.compile(r"[A-Za-z][A-Za-z0-9_]{0,127}") +# Pinned implementation, not deployed proof; see references/file-cu-canary.md. +# Never populate this from caller attestations or an arbitrary readback field. +OCR_CONTENT_MAPPINGS = { + "2026-08-01-preview": { + "field": "snippet", "parent_field": "snippet_parent_id", "path_field": "metadata_storage_path", + "authority": { + "repository": "AzureSearch", "commit": "2e241af8939a0891836b78278df113dab31a5964", + "contract": "file-standard-cu-index-v1", + }, + }, +} +DISCLOSURE = ( + "One synthetic image-only PDF is uploaded to Search and processed by billable CU OCR; " + "CU may create an analyzer, content may cross regions, and Search retains generated data. " + "No embeddings, KB/chat or direct CU probes. Existing resources/roles/network/local-auth stay unchanged. " + "Client time/attempt caps are not a monetary cap or cancellation of remote processing. " + "Cleanup needs separate approval; retain the private source/file/index inventory." +) +GLYPHS = { + "C": ("01111", "10000", "10000", "10000", "10000", "10000", "01111"), + "U": ("10001", "10001", "10001", "10001", "10001", "10001", "01110"), + "O": ("01110", "10001", "10001", "10001", "10001", "10001", "01110"), + "R": ("11110", "10001", "10001", "11110", "10100", "10010", "10001"), + "2": ("01110", "10001", "00001", "00010", "00100", "01000", "11111"), + "3": ("11110", "00001", "00001", "01110", "00001", "00001", "11110"), + "4": ("00010", "00110", "01010", "10010", "11111", "00010", "00010"), + "5": ("11111", "10000", "10000", "11110", "00001", "00001", "11110"), + "6": ("01110", "10000", "10000", "11110", "10001", "10001", "01110"), + "7": ("11111", "00001", "00010", "00100", "01000", "01000", "01000"), + "8": ("01110", "10001", "10001", "01110", "10001", "10001", "01110"), + "9": ("01110", "10001", "10001", "01111", "00001", "00001", "01110"), +} + + +def fail(code, message, *, partial=False, request_id=None, status=None): + return HelperFailure(code, message, blocked_at="file-cu-canary", partial=partial, request_id=request_id, status=status) + + +def pdf_bytes(marker): + if not isinstance(marker, str) or MARKER.fullmatch(marker) is None: + raise fail("canary-marker-invalid", "Use CUOCR followed by eight digits 2–9.") + scale, margin = 12, 24 + width, height = len(marker) * 6 * scale + 2 * margin, 7 * scale + 2 * margin + image = bytearray(b"\xff" * width * height) + for n, letter in enumerate(marker): + for y, row in enumerate(GLYPHS[letter]): + for x, pixel in enumerate(row): + if pixel == "1": + for yy in range(scale): + start = (margin + y * scale + yy) * width + margin + (n * 6 + x) * scale + image[start:start + scale] = b"\x00" * scale + compressed = zlib.compress(bytes(image), 9) + draw = b"q 600 0 0 84 20 20 cm /Image0 Do Q\n" + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 640 124] /Resources << /XObject << /Image0 4 0 R >> >> /Contents 5 0 R >>", + f"<< /Type /XObject /Subtype /Image /Width {width} /Height {height} /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /FlateDecode /Length {len(compressed)} >>\nstream\n".encode() + + compressed + b"\nendstream", + f"<< /Length {len(draw)} >>\nstream\n".encode() + draw + b"endstream", + ] + out = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n") + offsets = [0] + for n, obj in enumerate(objects, 1): + offsets.append(len(out)) + out.extend(f"{n} 0 obj\n".encode() + obj + b"\nendobj\n") + xref = len(out) + out.extend(b"xref\n0 6\n0000000000 65535 f \n") + for offset in offsets[1:]: + out.extend(f"{offset:010d} 00000 n \n".encode()) + out.extend(f"trailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode()) + if len(out) > 65536 or marker.encode() in out: + raise fail("canary-pdf-invalid", "Probe must be bounded image-only bytes without a text marker.") + return bytes(out) + + +def prepare(directory, marker=None): + directory = _bootstrap_io.private_directory(directory) + marker = marker or "CUOCR" + "".join(secrets.choice("23456789") for _ in range(8)) + data = pdf_bytes(marker) + _bootstrap_io.private_bytes(directory, PDF_NAME, data) + manifest = {"file": PDF_NAME, "marker": marker, "sha256": hashlib.sha256(data).hexdigest(), + "bytes": len(data), "pages": 1, "text_layer": False} + path = _bootstrap_io.private_file(directory, "cu-mi-probe-manifest.json", manifest) + return {"status": "prepared", "manifest": str(path), "writes_performed": [], + "azure_operations": 0, "note": "Local synthetic artifacts only; no live approval or resource selection."} + + +def _bounds(value): + if not isinstance(value, dict): + raise fail("canary-bounds-missing", "Select explicit timeout, HTTP request, polling attempt and interval caps.") + ranges = {"timeout_seconds": (30, 600), "max_requests": (16, 100), + "max_poll_attempts": (1, 10), "poll_interval_seconds": (1, 30)} + require_allowed_fields(value, set(ranges), label="canary bounds") + if any(type(value.get(k)) is not int or not low <= value[k] <= high for k, (low, high) in ranges.items()): + raise fail("canary-bounds-invalid", "Use timeout 30–600s, HTTP requests 16–100, polls 1–10 and interval 1–30s.") + + +def _ocr_mapping(api_version, field): + mapping = OCR_CONTENT_MAPPINGS.get(api_version) + if mapping is None: + raise fail("canary-ocr-mapping-unverified", + "No pinned File Standard OCR-to-index field mapping is verified for this API. " + "Do not create/upload; verify the first-party mapping and update the supported contract before planning.") + if field != mapping["field"]: + raise fail("canary-field-unverified", "The selected field does not match the pinned File Standard OCR-content mapping.") + return copy.deepcopy(mapping) + + +def _verified_file(source_result, ingestion): + files = source_result["verification"]["readback"].get("files") + expected = ingestion["files"][0] + if ( + not isinstance(files, list) or len(files) != 1 or not isinstance(files[0], dict) + or not isinstance(files[0].get("fileId"), str) or not files[0]["fileId"].strip() + or files[0].get("fileName") != expected["path"] + or files[0].get("sha256") != expected["sha256"] or files[0].get("size") != expected["size"] + ): + raise fail("canary-file-identity-unverified", "Require the uploaded synthetic file's exact ID, path, hash and size readback.") + return files[0] + + +def _verify_index_fields(index, mapping): + fields = index.get("fields") + if not isinstance(fields, list): + raise fail("canary-field-unverified", "Generated index fields must match the pinned File OCR mapping.") + for name in (mapping["field"], mapping["parent_field"], mapping["path_field"]): + selected = [field for field in fields if isinstance(field, dict) and field.get("name") == name] + if len(selected) != 1 or selected[0].get("type") != "Edm.String" or selected[0].get("retrievable") is not True: + raise fail("canary-field-unverified", "Require unique retrievable string fields for pinned OCR content and file identity.") + if name == mapping["field"] and ( + selected[0].get("searchable") is not True + or any(selected[0].get(flag) is not False for flag in ("filterable", "sortable", "facetable")) + ): + raise fail("canary-field-unverified", "The canonical snippet field does not match the pinned searchable OCR-content schema.") + + +def _contains_marker(value, marker): + if isinstance(value, str): + return marker in re.sub(r"[^A-Z0-9]", "", value.upper()) + if isinstance(value, dict): + return any(_contains_marker(k, marker) or _contains_marker(v, marker) for k, v in value.items()) + if isinstance(value, (list, tuple)): + return any(_contains_marker(item, marker) for item in value) + return False + + +def _check_non_image_channels(plan, marker): + ingestion = plan["ingestion"] + # Produce the actual multipart envelope with only the image bytes omitted. + try: + body, boundary = file_ingest._multipart(ingestion, ingestion["files"][0], b"", digest(plan)) + message = BytesParser(policy=policy.default).parsebytes( + f"Content-Type: multipart/form-data; boundary={boundary}\r\n\r\n".encode("ascii") + body + ) + except (ValueError, UnicodeError, TypeError): + raise fail("canary-upload-envelope-unverified", "The non-image upload envelope could not be produced safely.") from None + parts = list(message.iter_parts()) + if ( + message.defects or len(parts) != 2 or any(part.defects for part in parts) + or [part.get_param("name", header="content-disposition") for part in parts] != ["metadata", "content"] + or parts[0].get_content_type() != "application/json" or parts[1].get_payload(decode=True) != b"" + ): + raise fail("canary-upload-envelope-unverified", "The produced non-image upload envelope is not the supported multipart contract.") + try: + metadata = json.loads(parts[0].get_payload(decode=True)) + channels = [plan["source"]["desired"], file_ingest._list_url(ingestion), + body.decode("utf-8"), metadata, [list(part.items()) for part in parts]] + except (ValueError, UnicodeError, TypeError): + raise fail("canary-upload-envelope-unverified", "The produced non-image upload envelope could not be inspected.") from None + if _contains_marker(channels, marker): + raise fail("canary-marker-in-metadata", "The OCR marker must not occur in source/upload names, owner, metadata or other non-image upload channels.") + + +def _file_contract(plan, marker): + expected = pdf_bytes(marker) + if not isinstance(plan, dict) or not isinstance(plan.get("ingestion"), dict): + raise fail("canary-file-contract-invalid", "Retain the complete File plan.") + root = file_ingest.resolve_local_root(plan["ingestion"].get("local_root")) + actual = file_ingest._resolve_inventory_path(root, PDF_NAME) + try: + if actual.stat().st_size != len(expected): + raise fail("canary-content-drift", "Probe size differs from the bounded synthetic image-only PDF.") + except OSError: + raise fail("canary-content-drift", "Selected synthetic PDF is unreadable.") from None + file_source._validate_plan(plan) + source, ingestion = plan["source"], plan["ingestion"] + settings = source["desired"]["fileParameters"]["ingestionParameters"] + if ( + plan.get("file_cu_plan_version") != "1.2" or source.get("action") != "create" + or source["api_version"] != "2026-08-01-preview" + or settings.get("contentExtractionMode") != "standard" or "embedding" in plan + or settings.get("embeddingModel") is not None or settings.get("chatCompletionModel") is not None + or settings.get("disableImageVerbalization") is not True + or len(ingestion["files"]) != 1 or ingestion["files"][0]["path"] != PDF_NAME + ): + raise fail("canary-file-contract-invalid", "Canary requires fresh conditional File Standard MI creation, one synthetic PDF, no embedding/chat and unchanged August API.") + _check_non_image_channels(plan, marker) + records = file_ingest.snapshot_inventory(root, [PDF_NAME], service_tier=ingestion["service_tier"]) + if records != ingestion["files"] or records[0]["sha256"] != "sha256:" + hashlib.sha256(expected).hexdigest(): + raise fail("canary-content-drift", "Synthetic inventory changed; regenerate the concrete plan before approval.") + + +def plan_canary(request, *, token_provider=azure_cli_token, transport=http_request, now=time.time): + if not isinstance(request, dict): + raise fail("canary-input-invalid", "Canary intent must be an object.") + require_allowed_fields(request, {"schema_version", "file_request", "marker", "content_field", "bounds", "receipt_directory"}, + label="File CU canary intent") + reject_secrets(request) + _bounds(request.get("bounds")) + if request.get("schema_version") != "1.0" or not isinstance(request.get("content_field"), str) or not FIELD.fullmatch(request["content_field"]): + raise fail("canary-input-invalid", "Use schema 1.0 and an exact simple generated content field name.") + pdf_bytes(request.get("marker")) + directory = _bootstrap_io.private_directory(request.get("receipt_directory")) + fr = request.get("file_request") + if not isinstance(fr, dict): + raise fail("canary-input-invalid", "Supply an explicit existing Search/CU File request; no resource defaults or provisioning.") + # Reject a key mode before even read-only resource discovery. + cu = fr.get("content_understanding") + if not isinstance(cu, dict) or cu.get("auth", "system-assigned") != "system-assigned": + raise fail("canary-auth-invalid", "This canary only tests File managed identity; no keys or fallback.") + if fr.get("paths") != [PDF_NAME] or fr.get("extraction_mode") != "standard" or fr.get("vectorization") != "none": + raise fail("canary-file-contract-invalid", "Select exactly the generated probe PDF, Standard extraction and no vectors.") + version = fr.get("api_version", file_ingest.API_VERSION) + file_ingest.validate_api_version(version) + mapping = _ocr_mapping(version, request["content_field"]) + root = file_ingest.resolve_local_root(fr.get("local_root")) + selected = file_ingest._resolve_inventory_path(root, PDF_NAME) + try: + if selected.stat().st_size != len(pdf_bytes(request["marker"])): + raise fail("canary-content-drift", "Selected PDF is not the bounded synthetic probe.") + except OSError: + raise fail("canary-content-drift", "Selected probe is unreadable.") from None + result = file_source.plan_source(fr, token_provider=token_provider, transport=transport) + fp = result["execution_input"]["plan"] + _file_contract(fp, request["marker"]) + created = int(now()) + plan = { + "operation": "validate-file-cu-mi", "version": "1.1", "file_plan": fp, "ocr_mapping": mapping, + "marker": request["marker"], "content_field": request["content_field"], + "bounds": copy.deepcopy(request["bounds"]), "receipt_directory": str(directory), + "created_at": created, "expires_at": created + 900, "disclosure": DISCLOSURE, + } + fingerprint = digest(plan) + return { + "status": "planned", "plan_fingerprint": fingerprint, + "execution_input": {"schema_version": "1.0", "plan": plan, + "approval": {"confirmed": False, "fingerprint": fingerprint}}, + "approval_summary": {"source": result["approval_summary"], "bounds": plan["bounds"], + "expires_at": plan["expires_at"], "disclosure": DISCLOSURE, + "expected_outcome": "Indexed OCR marker without client keys; principal attribution remains separate."}, + "writes_performed": [], + } + + +class BoundedTransport: + def __init__(self, plan, raw, clock, record): + self.plan, self.raw, self.clock, self.record = plan, raw, clock, record + self.deadline = clock() + plan["bounds"]["timeout_seconds"] + self.calls, self.puts, self.uploads = 0, 0, 0 + self.upload_ack_failure = None + self.index_url = None + fp = plan["file_plan"] + self.source_url = search_reconcile.resource_url(fp["source"]) + self.files_prefix = self.source_url.split("?")[0] + "/files" + cu = fp["content_understanding"] + mi = cu["managed_identity"] + self.arm_urls = { + f"{MANAGEMENT_AUDIENCE}{cu['resource_id']}?api-version=2024-10-01", + f"{MANAGEMENT_AUDIENCE}{mi['search_resource_id']}?api-version={file_cu_mi.SEARCH_API}", + f"{MANAGEMENT_AUDIENCE}{mi['role_assignment_id']}?api-version={file_cu_mi.ROLE_API}", + } + + def __call__(self, method, url, token, **kwargs): + if self.upload_ack_failure is not None: + raise self.upload_ack_failure + remaining = self.deadline - self.clock() + if remaining <= 0 or self.calls >= self.plan["bounds"]["max_requests"]: + raise fail("canary-bound-reached", "Client request/deadline cap reached; remote processing may continue.", partial=bool(self.puts)) + file_url = url.startswith(self.files_prefix + "?") + allowed = method == "GET" and ( + url in self.arm_urls or url == self.source_url or file_url + or self.index_url is not None and (url == self.index_url or url.startswith(self.index_url.split("?")[0] + "/docs?")) + ) + if method == "PUT" and url == self.source_url and self.puts == 0 and kwargs.get("headers", {}).get("If-None-Match") == "*": + allowed = True + self.puts += 1 + elif method == "POST" and file_url and self.uploads == 0: + allowed = True + self.uploads += 1 + if not allowed: + raise fail("canary-operation-forbidden", "Only exact resource GETs, one conditional source PUT and one selected upload are approved.", partial=bool(self.puts)) + self.calls += 1 + kwargs.update(follow_redirects=False, max_response_bytes=min(524288, kwargs.get("max_response_bytes", 524288)), + response_deadline=min(self.deadline, kwargs.get("response_deadline", self.deadline)), + timeout=max(0.01, min(float(kwargs.get("timeout", 60)), remaining))) + try: + response = self.raw(method, url, token, **kwargs) + except HelperFailure as failure: + failure.recovery_deadline = min(self.deadline, failure.recovery_deadline or self.deadline) + raise + if method == "POST" and response.status in (200, 201, 202): + try: + self.record("upload-http-ack", {"status": response.status, "request_id": response.request_id, + "ingestion": "not yet verified"}) + except HelperFailure as error: + # Return the real ACK: persistence is not an ambiguous upload. + self.upload_ack_failure = error + return search_reconcile.HttpResult( + response.status, response.body, response.headers, response.etag_values, + min(self.deadline, response.recovery_deadline or self.deadline), + self.upload_ack_failure, + ) + + +def execute(document, *, token_provider=azure_cli_token, transport=http_request, now=time.time, + clock=time.monotonic, sleep=time.sleep): + if not isinstance(document, dict): + raise fail("canary-input-invalid", "Canary execution requires an object envelope.") + require_allowed_fields(document, {"schema_version", "plan", "approval", "_computed_fingerprint"}, label="canary envelope") + if document.get("schema_version") != "1.0": + raise fail("canary-input-invalid", "Use canary envelope schema_version 1.0.") + plan = document.get("plan") + if not isinstance(plan, dict): + raise fail("canary-input-invalid", "Use the unchanged canary execution envelope.") + require_allowed_fields(plan, {"operation", "version", "file_plan", "marker", "content_field", "ocr_mapping", "bounds", + "receipt_directory", "created_at", "expires_at", "disclosure"}, label="canary plan") + fingerprint = digest(plan) + if document.get("approval") != {"confirmed": True, "fingerprint": fingerprint} or document.get("_computed_fingerprint") != fingerprint: + raise fail("approval-missing", "Exact canary scope, synthetic content/cost and bounded verification need unchanged fingerprinted approval.") + if ( + plan.get("operation") != "validate-file-cu-mi" or plan.get("version") != "1.1" + or plan.get("disclosure") != DISCLOSURE or not isinstance(plan.get("content_field"), str) + or not FIELD.fullmatch(plan["content_field"]) + or type(plan.get("created_at")) is not int or type(plan.get("expires_at")) is not int + or plan["expires_at"] - plan["created_at"] != 900 or not plan["created_at"] <= now() < plan["expires_at"] + ): + raise fail("canary-plan-stale", "Canary plan is invalid or outside its 15-minute approval window; refresh discovery.") + _bounds(plan.get("bounds")) + _file_contract(plan["file_plan"], plan["marker"]) + if plan.get("ocr_mapping") != _ocr_mapping(plan["file_plan"]["source"]["api_version"], plan["content_field"]): + raise fail("canary-ocr-mapping-stale", "The approved OCR mapping differs from the pinned helper contract; a fresh plan and approval are required.") + directory = _bootstrap_io.private_directory(plan["receipt_directory"]) + prefix = "cu-mi-" + fingerprint.split(":")[1][:16] + refs = [] + + def record(stage, value): + path = _bootstrap_io.private_file(directory, prefix + "-" + stage + ".json", { + "fingerprint": fingerprint, "stage": stage, "value": value, + }) + refs.append(str(path)) + + record("started", {"approved_input": {k: v for k, v in document.items() if k != "_computed_fingerprint"}, + "state": "No Azure mutation yet; later completion is not atomic."}) + bounded = BoundedTransport(plan, transport, clock, record) + fp = plan["file_plan"] + child_digest = digest(fp) + source_result = None + def source_ack(**evidence): + response = evidence["response"] + record("source-http-ack", {"status": response.status, "request_id": response.request_id, + "etag_evidence": search_reconcile.response_etags(response)}) + + try: + source_result = file_source.execute( + {"schema_version": "1.0", "plan": fp, "_computed_fingerprint": child_digest, + "approval": {"confirmed": True, "fingerprint": child_digest}}, + token_provider=token_provider, transport=bounded, mi_on_created=source_ack, + allow_upload_retry=False, + ) + if bounded.upload_ack_failure is not None: + raise bounded.upload_ack_failure + record("file-completed", source_result) + original_file = _verified_file(source_result, fp["ingestion"]) + mapping = plan["ocr_mapping"] + token = token_provider(SEARCH_AUDIENCE) + current, _ = search_reconcile.read_resource(bounded.source_url, token, transport=bounded) + file_source.verify_content_understanding_readback(fp["content_understanding"], current) + created = current.get("fileParameters", {}).get("createdResources") + if not isinstance(created, dict) or set(created) != {"index"} or not isinstance(created["index"], str) or not re.fullmatch(r"[a-z0-9][a-z0-9_-]{1,127}", created["index"]): + raise fail("canary-index-unverified", "Fresh File readback must identify exactly one generated index.") + retained = source_result["verification"]["readback"]["source"] + if current.get("@odata.etag") != retained["etag"] or not search_reconcile.definitions_match(fp["source"]["desired"], current): + raise fail("canary-source-drift", "Source version changed before indexed OCR verification.") + version = fp["source"]["api_version"] + bounded.index_url = f"{fp['source']['endpoint'].rstrip('/')}/indexes('{created['index']}')?api-version={version}" + index, _ = search_reconcile.read_resource(bounded.index_url, token, transport=bounded) + if not isinstance(index, dict) or index.get("name") != created["index"] or not isinstance(index.get("@odata.etag"), str) or not index["@odata.etag"]: + raise fail("canary-index-unverified", "Require the generated index's matching name and fresh ETag.") + _verify_index_fields(index, mapping) + query = bounded.index_url.split("?")[0] + "/docs?" + urlencode({ + "api-version": version, "search": "*", + "$select": ",".join((mapping["field"], mapping["parent_field"], mapping["path_field"])), "$top": 3, + }) + matched = False + for attempt in range(plan["bounds"]["max_poll_attempts"]): + response = bounded("GET", query, token) + if response.status != 200 or not isinstance(response.body, dict) or not isinstance(response.body.get("value"), list): + raise fail("canary-index-query-failed", "Generated-index query did not return bounded document rows.", + request_id=response.request_id, status=response.status) + rows = response.body["value"] + if len(rows) > 3: + raise fail("canary-index-query-failed", "Generated-index response exceeded the approved row cap.") + if any( + not isinstance(row, dict) or not isinstance(row.get(mapping["field"]), str) + or row.get(mapping["parent_field"]) != original_file["fileId"] + or row.get(mapping["path_field"]) != original_file["fileName"] + for row in rows + ): + raise fail("canary-document-binding-unverified", "Indexed rows must contain canonical OCR text and the uploaded file's exact parent ID/path; chunk IDs or metadata alone are not proof.") + matched = any(_contains_marker(row[mapping["field"]], plan["marker"]) for row in rows) + if matched: + break + if attempt + 1 < plan["bounds"]["max_poll_attempts"]: + sleep(min(plan["bounds"]["poll_interval_seconds"], max(0, bounded.deadline - clock()))) + if not matched: + raise fail("canary-ocr-unverified", "Upload/source creation is not extraction proof: indexed OCR marker was not observed within the caps.") + after, _ = search_reconcile.read_resource(bounded.source_url, token, transport=bounded) + if not isinstance(after, dict) or after.get("@odata.etag") != current.get("@odata.etag") or after.get("fileParameters", {}).get("createdResources") != created: + raise fail("canary-source-drift", "Generated source/index binding changed during OCR verification.") + final_index, _ = search_reconcile.read_resource(bounded.index_url, token, transport=bounded) + if final_index != index: + raise fail("canary-index-drift", "Generated index changed during OCR verification.") + binding, _ = file_cu_mi.read_binding(fp["content_understanding"], fp["source"]["endpoint"], + token_provider=token_provider, transport=bounded) + if binding != fp["cu_identity_state"]: + raise fail("canary-identity-drift", "Search/CU identity-role binding changed during validation.") + result = {**copy.deepcopy(source_result), "outcome": "validate-file-cu-mi", + "approved_plan": {"fingerprint": fingerprint, "confirmed": True}, + "status": "completed", "verdict": "keyless-functional-pass", "indexed_ocr_marker": True, + "principal_attribution": "unverified", "backend_rollout": "unverified", + "identity_evidence": fp["cu_identity_state"], "source_result": source_result, + "ocr_evidence": {"mapping": mapping, "file": original_file}, + "index": created["index"], "request_count": bounded.calls, + "cleanup": {"status": "separate-plan-and-approval-required", + "instructions": "Retain inventory; separately approved guarded source cleanup only."}, + "warnings": ["Functional evidence is for this run only, not an atomic deployment or release claim.", + "No backend CU principal/telemetry was observed."]} + except HelperFailure as error: + if bounded.upload_ack_failure is not None and error is not bounded.upload_ack_failure: + checkpoint_error = bounded.upload_ack_failure + checkpoint_error.writes = error.writes + checkpoint_error.resources_remaining = error.resources_remaining + checkpoint_error.resources_reused = error.resources_reused + checkpoint_error.resources_unverified = error.resources_unverified + checkpoint_error.warnings.extend(error.warnings) + checkpoint_error.partial = error.partial + error = checkpoint_error + if source_result is not None: + error.partial = True + error.writes = [ + {**resource, "action": "created", "type": resource.get("type", "knowledge-source-file")} + for resource in source_result["resources"]["created"] + ] + error.writes + error.resources_remaining = source_result["ownership"]["run_owned"] + error.resources_remaining + result = blocked_result(error, outcome="validate-file-cu-mi", fingerprint=fingerprint, owner=fp["owner"]) + result.update(verdict="unverified", indexed_ocr_marker=False, principal_attribution="unverified", + backend_rollout="unverified", request_count=bounded.calls) + result.setdefault("warnings", []).append("Remote processing/costs may continue; do not replay mutations or infer ownership from an ambiguous create.") + if bounded.upload_ack_failure is not None: + result["warnings"].append("Upload HTTP ACK could not be retained; validation stopped. Only listed receipt_refs are confirmed; do not replay the upload.") + try: + record("result", result) + except HelperFailure: + if result["status"] == "completed": + error = fail("canary-receipt-failed", "Validation finished but its final private receipt could not be persisted.", partial=True) + error.writes = [ + {**resource, "action": "created", "type": resource.get("type", "knowledge-source-file")} + for resource in source_result["resources"]["created"] + ] + error.resources_remaining = source_result["ownership"]["run_owned"] + result = {**blocked_result(error, outcome="validate-file-cu-mi", fingerprint=fingerprint, owner=fp["owner"]), + "indexed_ocr_marker": True, "principal_attribution": "unverified", "backend_rollout": "unverified", + "source_result": source_result, "request_count": bounded.calls} + result["verdict"] = "unverified" + result.setdefault("warnings", []).append("Final private receipt persistence failed; retain this ownership handoff and existing checkpoints. Do not replay.") + return {**result, "receipt_refs": refs} + + +def main(argv=None): + parser = argparse.ArgumentParser() + modes = parser.add_mutually_exclusive_group(required=True) + modes.add_argument("--prepare", metavar="EXISTING_PRIVATE_DIRECTORY") + modes.add_argument("--plan", type=Path) + modes.add_argument("--input", type=Path) + args = parser.parse_args(argv) + try: + if args.prepare: + result = prepare(args.prepare) + elif args.plan: + result = plan_canary(_bootstrap_io.read_json(args.plan)) + directory = result["execution_input"]["plan"]["receipt_directory"] + path = _bootstrap_io.private_file(directory, "cu-mi-plan-" + result["plan_fingerprint"].split(":")[1][:16] + ".json", + result["execution_input"]) + result = {k: v for k, v in result.items() if k != "execution_input"} + result["execution_input_ref"] = str(path) + else: + document, _, fingerprint = load_approved_input(args.input) + document["_computed_fingerprint"] = fingerprint + result = execute(document) + emit_result(result) + return 0 if result["status"] in ("prepared", "planned", "completed") else 3 if result["status"] == "partial" else 2 + except HelperFailure as error: + emit_result(blocked_result(error, outcome="validate-file-cu-mi", fingerprint=None, owner=None)) + return 3 if error.partial else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_cu_mi.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_cu_mi.py new file mode 100644 index 00000000..768e0662 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_cu_mi.py @@ -0,0 +1,206 @@ +"""File Standard system-MI binding; implementation evidence is not rollout proof.""" +from __future__ import annotations + +import copy +import re + +try: + from . import _bootstrap_io, cu_ingestion_auth, search_reconcile + from ._common import MANAGEMENT_AUDIENCE, HelperFailure, digest, require_allowed_fields +except ImportError: + import _bootstrap_io, cu_ingestion_auth, search_reconcile + from _common import MANAGEMENT_AUDIENCE, HelperFailure, digest, require_allowed_fields + +SEARCH_API = "2025-05-01" +ROLE_API = "2022-04-01" +CU_ROLE = "a97b65f3-24c7-4388-baec-2e87135dc908" +SEARCH_ID = re.compile( + r"/subscriptions/[0-9a-fA-F-]{36}/resourceGroups/[A-Za-z0-9_.()-]{1,90}" + r"/providers/Microsoft\.Search/searchServices/(?=.{2,60}$)((?a:[a-z0-9][a-z0-9]+(?:-[a-z0-9]+)*))", re.I, +) +REDACTED = (None, "", "", "***") + + +def fail(code, message, *, request_id=None, status=None): + return HelperFailure(code, message, blocked_at="cu-managed-identity", request_id=request_id, status=status) + + +def validate_choice(value, resource_id): + if not isinstance(value, dict): + raise fail("cu-mi-prerequisite-missing", "Resolve existing Search resource ID and exact CU-scoped Cognitive Services User assignment. Missing identity/role changes need packaged bootstrap plans and explicit approval.") + require_allowed_fields(value, {"search_resource_id", "role_assignment_id"}, label="File CU managed identity") + search = value.get("search_resource_id") + role = value.get("role_assignment_id") + prefix = resource_id + "/providers/Microsoft.Authorization/roleAssignments/" + if ( + not isinstance(search, str) or SEARCH_ID.fullmatch(search) is None + or not isinstance(role, str) or not role.casefold().startswith(prefix.casefold()) + or cu_ingestion_auth.UUID.fullmatch(role[len(prefix):]) is None + ): + raise fail("cu-mi-binding-invalid", "Select exact Search and CU-account-scoped role assignment IDs; no inferred account, inherited/custom role or user-assigned identity.") + return copy.deepcopy(value) + + +def read_binding(choice, endpoint, *, token_provider, transport): + selected = validate_choice(choice.get("managed_identity"), choice["resource_id"]) + match = SEARCH_ID.fullmatch(selected["search_resource_id"]) + if endpoint.rstrip("/").casefold() != f"https://{match.group(1)}.search.windows.net".casefold(): + raise fail("cu-mi-search-mismatch", "Search resource ID must match the exact approved Search endpoint.") + ids = [] + + def get(resource, version): + response = transport( + "GET", f"{MANAGEMENT_AUDIENCE}{resource}?api-version={version}", + token_provider(MANAGEMENT_AUDIENCE), follow_redirects=False, max_response_bytes=65536, + ) + if response.status != 200 or not isinstance(response.body, dict): + raise fail("cu-mi-prerequisite-unavailable", "Cannot GET the exact Search identity or scoped CU role. Have its owner resolve read access or plan missing setup; no key fallback or permission changes.", + request_id=response.request_id, status=response.status) + if str(response.body.get("id", "")).casefold() != resource.casefold(): + raise fail("cu-mi-resource-mismatch", "ARM returned another resource or scope; refresh the selected bindings.", request_id=response.request_id) + if response.request_id: + ids.append(response.request_id) + return response.body + + search = get(selected["search_resource_id"], SEARCH_API) + identity, props = search.get("identity"), search.get("properties") + if ( + not isinstance(identity, dict) or not isinstance(props, dict) + or "SystemAssigned" not in str(identity.get("type", "")).replace(" ", "").split(",") + or any(not isinstance(identity.get(k), str) or cu_ingestion_auth.UUID.fullmatch(identity[k]) is None + for k in ("principalId", "tenantId")) + or str(props.get("provisioningState", "")).casefold() != "succeeded" + or str(props.get("status", "")).casefold() != "running" + or str(props.get("endpoint", "")).rstrip("/").casefold() not in ( + endpoint.rstrip("/").casefold(), endpoint.rstrip("/").casefold().removeprefix("https://"), + ) + ): + raise fail("cu-mi-identity-unverified", "Require ready Search system-assigned principal/tenant and endpoint readback; resolve missing identity with a separately approved bootstrap plan.") + role = get(selected["role_assignment_id"], ROLE_API) + rp = role.get("properties") + expected_role = re.compile(r"/subscriptions/[0-9a-fA-F-]{36}/providers/Microsoft.Authorization/roleDefinitions/" + CU_ROLE, re.I) + if ( + not isinstance(rp, dict) + or str(rp.get("principalId", "")).casefold() != identity["principalId"].casefold() + or rp.get("principalType") != "ServicePrincipal" + or str(rp.get("scope", "")).casefold() != choice["resource_id"].casefold() + or expected_role.fullmatch(str(rp.get("roleDefinitionId", ""))) is None + or rp.get("condition") not in (None, "") + ): + raise fail("cu-mi-role-unverified", "Require the selected Search principal's unconditional Cognitive Services User assignment on this CU account. Plan a missing scoped role explicitly; no self-grant or key fallback.") + return { + **selected, "search_endpoint": endpoint.rstrip("/"), + "principal_id": identity["principalId"], "tenant_id": identity["tenantId"], + "identity_type": identity["type"], "search_api_version": SEARCH_API, "role_api_version": ROLE_API, + "role_definition_id": rp["roleDefinitionId"], "cu_resource_id": choice["resource_id"], + "search_network": {k: copy.deepcopy(props.get(k)) for k in ("publicNetworkAccess", "networkRuleSet")}, + }, ids + + +def validate_state(state, choice, endpoint): + if not isinstance(state, dict): + raise fail("cu-mi-prerequisite-missing", "Retain the planner's verified Search identity/role binding.") + fields = {"search_resource_id", "role_assignment_id", "search_endpoint", "principal_id", "tenant_id", + "identity_type", "search_api_version", "role_api_version", "role_definition_id", + "cu_resource_id", "search_network"} + require_allowed_fields(state, fields, label="File CU identity state") + selected = validate_choice(choice["managed_identity"], choice["resource_id"]) + if ( + set(state) != fields or any(state.get(k) != v for k, v in selected.items()) + or state["search_endpoint"] != endpoint.rstrip("/") + or state["cu_resource_id"] != choice["resource_id"] + or state["search_api_version"] != SEARCH_API or state["role_api_version"] != ROLE_API + or not isinstance(state["search_network"], dict) + or any(not isinstance(state[k], str) or cu_ingestion_auth.UUID.fullmatch(state[k]) is None + for k in ("principal_id", "tenant_id")) + or "SystemAssigned" not in str(state["identity_type"]).replace(" ", "").split(",") + or not str(state["role_definition_id"]).casefold().endswith("/roledefinitions/" + CU_ROLE) + ): + raise fail("cu-mi-binding-invalid", "Retain complete, unchanged identity, role, API and account bindings.") + + +def verify_reuse(request, plan, current): + paths = (request.get("reuse_input_file"), request.get("reuse_result_file")) + if not all(isinstance(p, str) and p for p in paths): + raise fail("cu-mi-provenance-required", "A redacted source GET cannot establish key versus MI auth. Retain the original approved version 1.2 File input and completed creation result for exact reuse.") + prior, result = (_bootstrap_io.read_json(path) for path in paths) + if ( + not isinstance(prior, dict) or not isinstance(result, dict) + or not isinstance(prior.get("plan"), dict) or not isinstance(prior.get("approval"), dict) + or not isinstance(result.get("resources"), dict) + or not isinstance(result["resources"].get("created"), list) + ): + raise fail("cu-mi-provenance-mismatch", "Retain complete approved input and completed File result objects.") + require_allowed_fields(prior, {"schema_version", "plan", "approval"}, label="MI creation input") + try: + from . import file_source + except ImportError: + import file_source + approved = prior.get("approval", {}) + pp = prior.get("plan", {}) + file_source._validate_plan(pp) + fingerprint = digest(pp) + created = result.get("resources", {}).get("created", []) + source = pp.get("source", {}) + if ( + prior.get("schema_version") != "1.0" or pp.get("file_cu_plan_version") != "1.2" + or approved != {"confirmed": True, "fingerprint": fingerprint} + or source.get("action") != "create" or source.get("endpoint") != plan["source"]["endpoint"] + or source.get("name") != plan["source"]["name"] or pp.get("owner") != plan["owner"] + or pp.get("cu_identity_state") != plan["cu_identity_state"] + or pp.get("cu_resource_state") != plan["cu_resource_state"] + or result.get("status") != "completed" + or result.get("approved_plan") != {"confirmed": True, "fingerprint": fingerprint} + or not search_reconcile.definitions_match(source.get("desired"), current) + or not any(isinstance(item, dict) and item.get("type") == "knowledge-source" + and item.get("name") == source["name"] and item.get("etag") == current.get("@odata.etag") + and item.get("definition_digest") == digest(search_reconcile._definition(current)) + for item in created) + ): + raise fail("cu-mi-provenance-mismatch", "Retained MI creation evidence does not bind the fresh source/ETag/account/identity. No auth inference, reingestion or ownership claim.") + + +def guard_create(plan, transport, recheck, checkpoint=None): + """An expected-absent MI plan must not adopt another actor's redacted source.""" + url = search_reconcile.resource_url(plan["source"]) + acknowledged = False + created_etag = None + + def on_created(**evidence): + nonlocal created_etag + # The reconciler calls this after ACK, outside ambiguous transport recovery. + if checkpoint is not None: + try: + checkpoint(**evidence) + except HelperFailure: + raise + except Exception: + raise fail("cu-mi-checkpoint-failed", "Acknowledged MI creation checkpoint failed; private details withheld.") from None + try: + response = evidence["response"] + created_etag = search_reconcile.resolve_etag(search_reconcile.response_etags(response), response.request_id) + except HelperFailure: + raise fail("cu-mi-ack-version-unverified", "MI creation was acknowledged, but its version evidence is invalid. Retain ownership; do not replay.") from None + if created_etag is None: + raise fail("cu-mi-ack-version-unverified", "MI creation was acknowledged without an ETag; redacted auth cannot be bound to a created version. Retain ownership; do not replay.") + + def guarded(method, target, token, **kwargs): + nonlocal acknowledged, created_etag + if method == "PUT" and target == url: + recheck() + result = transport(method, target, token, **kwargs) + if target == url: + if method == "PUT" and result.status in (200, 201): + acknowledged = True + if method == "GET" and result.status == 200 and plan["source"]["action"] == "create" and not acknowledged: + raise fail("cu-mi-source-drift", "Expected source absence changed before creation; do not infer MI from redacted readback. Refresh discovery and provenance.") + if method == "GET" and result.status == 200 and acknowledged: + try: + observed_etag = search_reconcile.resolve_etag(search_reconcile.response_etags(result), result.request_id) + except HelperFailure: + raise fail("cu-mi-readback-version-unverified", "Created MI source readback has invalid version evidence.") from None + if observed_etag != created_etag: + raise fail("cu-mi-source-drift", "Source version changed after acknowledged MI creation; redacted auth is unproven. Inspect ownership before cleanup.") + return result + + return guarded, on_created diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_ingest.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_ingest.py new file mode 100644 index 00000000..810d663b --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_ingest.py @@ -0,0 +1,1040 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import stat +import sys +import time +from pathlib import Path, PurePosixPath +from typing import Any +from urllib.parse import parse_qs, urlencode, urlsplit + +try: + from ._progress import Progress, add_progress_argument, reporting +except ImportError: + from _progress import Progress, add_progress_argument, reporting + +try: + from ._common import ( + SEARCH_AUDIENCE, + HelperFailure, + ReadRecovery, + TokenProvider, + Transport, + azure_cli_token, + blocked_result, + digest, + emit_result, + file_digest, + http_request, + load_approved_input, + odata_name, + reject_secrets, + require_allowed_fields, + validate_search_endpoint, + ) +except ImportError: + from _common import ( # type: ignore[no-redef] + SEARCH_AUDIENCE, + HelperFailure, + ReadRecovery, + TokenProvider, + Transport, + azure_cli_token, + blocked_result, + digest, + emit_result, + file_digest, + http_request, + load_approved_input, + odata_name, + reject_secrets, + require_allowed_fields, + validate_search_endpoint, + ) + + +API_VERSION = "2026-08-01-preview" +MAX_INVENTORY_PAGES = 200 +MAX_SERVER_FILES = 200 +INVENTORY_READ_TIMEOUT_SECONDS = 60 +MAX_INVENTORY_RESPONSE_BYTES = 1024 * 1024 +MAX_FILE_BYTES = { + "free": 50 * 1024 * 1024, + "basic": 50 * 1024 * 1024, + "dedicated": 100 * 1024 * 1024, + "serverless": 100 * 1024 * 1024, +} +MEDIA_TYPE = re.compile( + r"^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]*/[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]*$" +) +FILE_MEDIA_HINTS = { + ".txt": "text/plain", ".md": "text/markdown", + ".pdf": "application/pdf", ".html": "text/html", ".htm": "text/html", + ".csv": "text/csv", ".json": "application/json", ".sh": "application/x-sh", + ".doc": "application/msword", + ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".ppt": "application/vnd.ms-powerpoint", + ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ".xls": "application/vnd.ms-excel", + ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".jpeg": "image/jpeg", ".jpg": "image/jpeg", ".png": "image/png", + ".bmp": "image/bmp", ".heif": "image/heif", ".heic": "image/heic", + ".tiff": "image/tiff", ".tif": "image/tiff", ".gif": "image/gif", + ".webp": "image/webp", ".svg": "image/svg+xml", ".avif": "image/avif", + ".ico": "image/vnd.microsoft.icon", +} + + +def validate_api_version(value: Any) -> None: + if value != API_VERSION: + raise HelperFailure( + "api-version-invalid", + f"This File helper requires {API_VERSION} for its metadata upload contract. " + "The service also supports 2026-05-01-preview minimal extraction; use a compatible client " + "or explicitly approve August, never silently change the requested API.", + blocked_at="input-resolution", + ) + + +def media_type_hint(path: str) -> str: + """Return a portable filename hint, never a claim of server content detection.""" + return FILE_MEDIA_HINTS.get(PurePosixPath(path).suffix.lower(), "application/octet-stream") + + +def _reject_credential_path(path: Path) -> None: + lowered = [part.lower() for part in path.parts] + if ( + any(part in {".ssh", ".aws", ".azure", ".git", "credentials"} or part == ".env" or part.startswith(".env.") + for part in lowered) + or lowered[-1] in {"id_rsa", "id_dsa", "id_ecdsa", "id_ed25519"} + or path.suffix.lower() in {".pem", ".key", ".pfx", ".p12", ".kdbx"} + ): + raise HelperFailure( + "credential-file-forbidden", "Credential files are outside document ingestion.", + blocked_at="input-resolution", + ) + + +def _odata_name(name: Any) -> str: + return odata_name(name) + + +def _list_url(plan: dict[str, Any]) -> str: + endpoint = validate_search_endpoint(plan.get("endpoint")) + validate_api_version(plan.get("api_version")) + name = _odata_name(plan.get("name")) + return ( + f"{endpoint}/knowledgesources('{name}')/files?" + + urlencode({"api-version": API_VERSION, "pageSize": 200}) + ) + + +def _normalize_inventory(files: list[dict[str, Any]]) -> list[dict[str, Any]]: + fields = ( + "fileId", + "fileName", + "prefix", + "metadata", + "parsingMode", + "extractionMode", + "fileSizeBytes", + "errorMessage", + ) + normalized = [ + {field: item.get(field) for field in fields if field in item} + for item in files + ] + return sorted( + normalized, + key=lambda item: (str(item.get("fileName")), str(item.get("fileId"))), + ) + + +def _list_files( + url: str, + token: str, + *, + transport: Transport, + recovery: ReadRecovery | None = None, +) -> tuple[list[dict[str, Any]], list[str]]: + try: + origin = urlsplit(url) + origin_port = origin.port + except ValueError as exc: + raise HelperFailure( + "continuation-url-invalid", "File inventory URL is malformed.", + blocked_at="verification", + ) from exc + current_url: str | None = url + seen: set[str] = set() + files: list[dict[str, Any]] = [] + request_ids: list[str] = [] + requests = 0 + deadline = time.monotonic() + INVENTORY_READ_TIMEOUT_SECONDS + if recovery is not None: + deadline = min(deadline, recovery.deadline) + + def inventory_transport(method, target, credential, **options): + nonlocal requests + if requests >= MAX_INVENTORY_PAGES: + raise HelperFailure( + "file-list-limit-exceeded", "Complete file inventory exceeds 200 pages/requests.", + blocked_at="verification", + ) + requests += 1 + return transport(method, target, credential, **options) + + while current_url: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise HelperFailure( + "file-list-timeout", "Complete file inventory read exceeded its 60-second deadline.", + blocked_at="verification", + request_id=request_ids[-1] if request_ids else None, + ) + if len(seen) >= MAX_INVENTORY_PAGES: + raise HelperFailure( + "file-list-limit-exceeded", "Complete file inventory exceeds 200 pages/requests.", + blocked_at="verification", + request_id=request_ids[-1] if request_ids else None, + ) + try: + current_url.encode("ascii") + current = urlsplit(current_url) + current_port = current.port + except (ValueError, UnicodeEncodeError) as exc: + raise HelperFailure( + "continuation-url-invalid", "Search returned a malformed continuation URL.", + blocked_at="verification", + ) from exc + query = parse_qs(current.query) + if ( + current_url in seen + or current.scheme != "https" + or current.hostname != origin.hostname + or current_port != origin_port + or current.path != origin.path + or query.get("api-version") != [API_VERSION] + or current.username + or current.password + ): + raise HelperFailure( + "continuation-url-invalid", + "Search returned a continuation URL outside the approved service.", + blocked_at="verification", + ) + seen.add(current_url) + try: + options = dict(timeout=remaining, response_deadline=deadline, + max_response_bytes=MAX_INVENTORY_RESPONSE_BYTES, follow_redirects=False) + result = (recovery.get(current_url, token, transport=inventory_transport, + max_requests=MAX_INVENTORY_PAGES - requests, **options) + if recovery is not None else inventory_transport("GET", current_url, token, **options)) + except HelperFailure as failure: + if failure.code not in {"response-deadline-exceeded", "read-recovery-budget-exhausted"}: + raise + raise HelperFailure( + "file-list-timeout", "Complete file inventory read exceeded its effective deadline (at most 60 seconds).", + blocked_at="verification", request_id=failure.request_id, + status=failure.http_status, warnings=failure.warnings, + ) from failure + if time.monotonic() >= deadline: + raise HelperFailure( + "file-list-timeout", "Complete file inventory read exceeded its 60-second deadline.", + blocked_at="verification", + request_id=result.request_id, + ) + if result.status != 200 or not isinstance(result.body, dict): + raise HelperFailure( + "file-list-invalid", + "File-list readback did not return a JSON object.", + blocked_at="reconciliation", + request_id=result.request_id, + status=result.status, + ) + values = result.body.get("value") + if isinstance(values, list) and len(files) + len(values) > MAX_SERVER_FILES: + raise HelperFailure( + "file-list-limit-exceeded", "Complete file inventory exceeds 200 records.", + blocked_at="verification", + request_id=result.request_id, + ) + if not isinstance(values, list) or not all( + isinstance(item, dict) for item in values + ): + raise HelperFailure( + "file-list-invalid", + "File-list readback did not contain an object array.", + blocked_at="reconciliation", + request_id=result.request_id, + ) + files.extend(values) + if result.request_id: + request_ids.append(result.request_id) + next_link = result.body.get("@odata.nextLink") + if next_link is not None and not isinstance(next_link, str): + raise HelperFailure( + "continuation-url-invalid", + "Search returned an invalid continuation URL.", + blocked_at="verification", + ) + current_url = next_link + return files, recovery.request_ids if recovery is not None else request_ids + + +def _validate_relative_path(value: Any) -> str: + if not isinstance(value, str) or not value or "\\" in value: + raise HelperFailure( + "inventory-path-invalid", + "Every inventory path must be a non-empty normalized POSIX path.", + blocked_at="input-resolution", + ) + path = PurePosixPath(value) + if ( + path.is_absolute() + or path.as_posix() != value + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise HelperFailure( + "inventory-path-invalid", + f"Inventory path is not safely relative: {value!r}.", + blocked_at="input-resolution", + ) + if any("\r" in part or "\n" in part or ":" in part for part in path.parts): + raise HelperFailure( + "inventory-path-invalid", + f"Inventory path contains a forbidden segment: {value!r}.", + blocked_at="input-resolution", + ) + return path.as_posix() + + +def _reject_links(path: Path) -> None: + for probe in reversed((path, *path.parents)): + attributes = getattr(probe.lstat(), "st_file_attributes", 0) + if probe.is_symlink() or attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT: + raise HelperFailure( + "inventory-path-invalid", + "Local inventory cannot traverse links or reparse points.", + blocked_at="input-resolution", + ) + + +def resolve_local_root(value: Any) -> Path: + """Resolve the explicit data boundary without following links or reparse points.""" + if not isinstance(value, str) or not os.path.isabs(value): + raise HelperFailure( + "local-root-invalid", + "local_root must be an explicit absolute path.", + blocked_at="input-resolution", + ) + try: + path = Path(value) + _reject_links(path) + root = path.resolve(strict=True) + if not root.is_dir(): + raise OSError("not a directory") + except OSError as exc: + raise HelperFailure( + "local-root-invalid", + "local_root must be an existing readable real directory.", + blocked_at="input-resolution", + ) from exc + return root + + +def _resolve_inventory_path(root: Path, value: Any) -> Path: + relative = _validate_relative_path(value) + path = root.joinpath(*PurePosixPath(relative).parts) + try: + _reject_links(path) + resolved = path.resolve(strict=True) + resolved.relative_to(root) + except (OSError, ValueError) as exc: + raise HelperFailure( + "inventory-path-invalid", + f"Inventory path escapes or is unreadable: {relative}.", + blocked_at="input-resolution", + ) from exc + if not resolved.is_file() or resolved.is_symlink(): + raise HelperFailure( + "inventory-path-invalid", + f"Inventory entry is not a regular file: {relative}.", + blocked_at="input-resolution", + ) + _reject_credential_path(resolved) + return resolved + + +def _resolve_file(root: Path, record: dict[str, Any]) -> Path: + relative = _validate_relative_path(record.get("path")) + resolved = _resolve_inventory_path(root, relative) + try: + observed = resolved.stat() + matches = ( + record.get("size") == observed.st_size + and record.get("mtime_ns") == observed.st_mtime_ns + and record.get("sha256") == file_digest(resolved) + ) + except OSError as exc: + raise HelperFailure( + "inventory-unreadable", f"Selected file cannot be read: {relative}.", + blocked_at="input-resolution", + ) from exc + if not matches: + raise HelperFailure( + "inventory-drift", + f"Size, modification time, or SHA-256 changed for {relative}.", + blocked_at="confirmation", + ) + return resolved + + +def snapshot_inventory( + root: Path, paths: list[str], *, service_tier: str +) -> list[dict[str, Any]]: + """Freeze selected files with MIME hints; Search determines actual support.""" + if not isinstance(service_tier, str) or service_tier not in MAX_FILE_BYTES: + raise HelperFailure( + "service-tier-invalid", "Select a supported service_tier.", + blocked_at="input-resolution", + ) + if not isinstance(paths, list) or not 1 <= len(paths) <= 200: + raise HelperFailure( + "inventory-invalid", "Select between 1 and 200 explicit file paths.", + blocked_at="input-resolution", + ) + relative_paths = [_validate_relative_path(path) for path in paths] + if len(set(relative_paths)) != len(relative_paths): + raise HelperFailure( + "inventory-invalid", "Selected file paths must be unique.", + blocked_at="input-resolution", + ) + records = [] + for relative in sorted(relative_paths): + path = _resolve_inventory_path(root, relative) + media_type = media_type_hint(relative) + try: + observed = path.stat() + if not 0 < observed.st_size <= MAX_FILE_BYTES[service_tier]: + raise HelperFailure( + "inventory-invalid", "File size is empty or exceeds the selected tier limit.", + blocked_at="input-resolution", + ) + record = { + "path": relative, "size": observed.st_size, + "mtime_ns": observed.st_mtime_ns, "sha256": file_digest(path), + "media_type": media_type, + } + _resolve_file(root, record) + except OSError as exc: + raise HelperFailure( + "inventory-unreadable", "Selected file cannot be read.", + blocked_at="input-resolution", + ) from exc + records.append(record) + return records + + +def read_inventory( + plan: dict[str, Any], token: str, *, transport: Transport +) -> tuple[list[dict[str, Any]], list[str]]: + """Read all pages for the exact File source, using guarded continuations.""" + return _list_files(_list_url(plan), token, transport=transport) + + +def inventory_digest(files: list[dict[str, Any]]) -> str: + return digest(_normalize_inventory(files)) + + +def reconcile_inventory( + plan: dict[str, Any], + before: list[dict[str, Any]], + *, + allow_new_uploads: bool = False, +) -> dict[str, dict[str, Any]]: + """Validate all existing markers before planning or performing any upload.""" + expected_names = {record["path"] for record in plan["files"]} + if any(item.get("fileName") not in expected_names for item in before): + raise HelperFailure( + "server-inventory-conflict", + "The server inventory contains files outside the approved inventory.", + blocked_at="reconciliation", + ) + matched = {} + for record in plan["files"]: + matches = [item for item in before if item.get("fileName") == record["path"]] + if len(matches) > 1: + raise HelperFailure( + "duplicate-file-record", f"Multiple server records exist for {record['path']}.", + blocked_at="reconciliation", + ) + if matches: + if not _matches(matches[0], plan, record): + raise HelperFailure( + "file-record-conflict", f"Server record conflicts with approved file {record['path']}.", + blocked_at="reconciliation", + ) + matched[record["path"]] = matches[0] + elif not allow_new_uploads: + raise HelperFailure( + "reused-source-upload-forbidden", + "New files cannot be uploaded into a reused source because individual-file cleanup is unsupported.", + blocked_at="reconciliation", + ) + return matched + + +def _metadata(plan: dict[str, Any], record: dict[str, Any]) -> dict[str, str]: + supplied = record.get("metadata") or {} + if not isinstance(supplied, dict) or not all( + isinstance(key, str) and isinstance(value, str) + for key, value in supplied.items() + ): + raise HelperFailure( + "file-metadata-invalid", + "File metadata must contain only string keys and values.", + blocked_at="input-resolution", + ) + metadata = dict(supplied) + metadata.update( + { + "foundryIqSha256": record["sha256"], + "foundryIqSizeBytes": str(record["size"]), + "foundryIqInventory": plan["inventory_digest"], + "foundryIqOwner": str(plan["owner"]), + "foundryIqSource": str(plan["name"]), + } + ) + return metadata + + +def _matches( + server: dict[str, Any], + plan: dict[str, Any], + record: dict[str, Any], +) -> bool: + metadata = server.get("metadata") or {} + expected = _metadata(plan, record) + return ( + server.get("fileName") == record["path"] + and server.get("fileSizeBytes") == record["size"] + and server.get("errorMessage") is None + and isinstance(metadata, dict) + and all(metadata.get(key) == value for key, value in expected.items()) + ) + + +def _multipart( + plan: dict[str, Any], + record: dict[str, Any], + content: bytes, + fingerprint: str, +) -> tuple[bytes, str]: + boundary = "foundry-iq-" + fingerprint.removeprefix("sha256:")[:24] + metadata = { + "fileName": record["path"], + "metadata": _metadata(plan, record), + } + media_type = record.get("media_type") + if not isinstance(media_type, str) or not media_type: + media_type = media_type_hint(record["path"]) + pieces = [ + f"--{boundary}\r\n".encode("ascii"), + b'Content-Disposition: form-data; name="metadata"\r\n', + b"Content-Type: application/json\r\n\r\n", + json.dumps(metadata, sort_keys=True, separators=(",", ":")).encode("utf-8"), + b"\r\n", + f"--{boundary}\r\n".encode("ascii"), + b'Content-Disposition: form-data; name="content"; filename="upload"\r\n', + f"Content-Type: {media_type}\r\n\r\n".encode("ascii"), + content, + b"\r\n", + f"--{boundary}--\r\n".encode("ascii"), + ] + return b"".join(pieces), boundary + + +def _validate_plan(plan: dict[str, Any]) -> tuple[Path, list[dict[str, Any]]]: + reject_secrets(plan) + require_allowed_fields( + plan, + { + "operation", + "outcome", + "endpoint", + "name", + "api_version", + "local_root", + "files", + "inventory_digest", + "expected_server_inventory_digest", + "service_tier", + "extraction_mode", + "rbac", + "network", + "owner", + "cleanup_approved", + }, + label="File ingestion plan", + ) + for field, allowed in ( + ("rbac", {"assignments"}), + ("network", {"posture", "evidence"}), + ): + section = plan.get(field) + if section is not None: + if not isinstance(section, dict): + raise HelperFailure( + "input-schema-invalid", + f"{field} must be an object.", + blocked_at="input-resolution", + ) + require_allowed_fields(section, allowed, label=field) + if plan.get("operation") != "ingest" or plan.get("cleanup_approved") is not False: + raise HelperFailure( + "operation-invalid", + "File ingestion requires operation ingest and cleanup_approved false.", + blocked_at="input-resolution", + ) + if not isinstance(plan.get("name"), str) or not plan["name"]: + raise HelperFailure( + "name-invalid", + "Knowledge source name is required.", + blocked_at="input-resolution", + ) + if not isinstance(plan.get("owner"), str) or not plan["owner"]: + raise HelperFailure( + "owner-invalid", + "File ingestion owner is required.", + blocked_at="input-resolution", + ) + root = resolve_local_root(plan.get("local_root")) + records = plan.get("files") + if not isinstance(records, list) or not records or len(records) > 200: + raise HelperFailure( + "inventory-invalid", + "files must contain between 1 and 200 entries.", + blocked_at="input-resolution", + ) + if not all(isinstance(record, dict) for record in records): + raise HelperFailure( + "inventory-invalid", + "Every file inventory entry must be an object.", + blocked_at="input-resolution", + ) + for record in records: + require_allowed_fields( + record, + {"path", "size", "mtime_ns", "sha256", "media_type", "metadata"}, + label="File inventory record", + ) + service_tier = plan.get("service_tier") + if service_tier not in MAX_FILE_BYTES: + raise HelperFailure( + "service-tier-invalid", + "service_tier must be free, basic, dedicated, or serverless.", + blocked_at="input-resolution", + ) + extraction_mode = plan.get("extraction_mode") + if extraction_mode not in {"minimal", "standard"}: + raise HelperFailure( + "extraction-mode-invalid", + "extraction_mode must be minimal or standard.", + blocked_at="input-resolution", + ) + paths = [_validate_relative_path(record.get("path")) for record in records] + if paths != sorted(paths) or len(paths) != len(set(paths)): + raise HelperFailure( + "inventory-invalid", + "File inventory paths must be unique and byte-sorted.", + blocked_at="input-resolution", + ) + if digest(records) != plan.get("inventory_digest"): + raise HelperFailure( + "inventory-drift", + "inventory_digest does not match the approved file records.", + blocked_at="confirmation", + ) + for record in records: + size = record.get("size") + mtime_ns = record.get("mtime_ns") + media_type = record.get("media_type") + if ( + not isinstance(size, int) + or size <= 0 + or size > MAX_FILE_BYTES[service_tier] + or not isinstance(mtime_ns, int) + or mtime_ns <= 0 + or not isinstance(media_type, str) + or MEDIA_TYPE.fullmatch(media_type) is None + ): + raise HelperFailure( + "inventory-invalid", + "Every file requires positive size/mtime, media type, and a tier-valid size.", + blocked_at="input-resolution", + ) + _resolve_file(root, record) + _metadata(plan, record) + return root, records + + +def _confirm_ambiguous_upload( + list_url: str, + token: str, + transport: Transport, + record: dict[str, Any], + plan: dict[str, Any], + request_ids: list[str], + failure: HelperFailure, + warnings: list[str], +) -> dict[str, Any] | None: + """Resolve an ambiguous upload with bounded readback, never upload replay. + + Only ambiguous outcomes (transport failure, timeout, 409/429/5xx) reach + this helper. A definitive 200/201 response never calls it. Returns the + matching server record when the readback proves the approved file + exists, otherwise ``None`` so the caller reports ``partial``. + """ + recovery = ReadRecovery() + try: + recovery.delay(failure) + observed, _ = _list_files( + list_url, token, transport=transport, recovery=recovery, + ) + except HelperFailure as read_failure: + warnings.append(f"Upload readback failed ({read_failure.code}); original upload failure retained.") + return None + finally: + request_ids.extend(recovery.request_ids) + warnings.extend(recovery.diagnostics()) + matches = [item for item in observed if item.get("fileName") == record["path"]] + if len(matches) != 1 or not _matches(matches[0], plan, record): + return None + return matches[0] + + +def _remaining_files(resources: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [ + { + "type": "knowledge-source-file", + "fileName": resource["fileName"], + "sha256": resource["sha256"], + } + for resource in resources + ] + + +@reporting("file-upload") +def execute( + document: dict[str, Any], + *, + token_provider: TokenProvider = azure_cli_token, + transport: Transport = http_request, + allow_new_uploads: bool = False, + progress: Progress | None = None, + upload_session=None, + source_check=None, + allow_upload_retry=True, +) -> dict[str, Any]: + if allow_new_uploads or upload_session is not None or len(document.get("plan", {}).get("files", [])) > 1: + try: + from .file_upload import run_batch + except ImportError: + from file_upload import run_batch + return run_batch(document, token_provider=token_provider, transport=transport, progress=progress, + allow_new_uploads=allow_new_uploads, session=upload_session, + source_check=source_check, allow_upload_retry=allow_upload_retry) + progress.update("file-inventory") + plan = document["plan"] + fingerprint = document["_computed_fingerprint"] + root, records = _validate_plan(plan) + list_url = _list_url(plan) + token = token_provider(SEARCH_AUDIENCE) + readonly = ReadRecovery() if not allow_new_uploads else None + before, request_ids = _list_files(list_url, token, transport=transport, recovery=readonly) + warnings: list[str] = list(readonly.warnings) if readonly is not None else [] + if inventory_digest(before) != plan.get("expected_server_inventory_digest"): + raise HelperFailure( + "server-inventory-drift", + "Server file inventory changed after approval.", + blocked_at="reconciliation", + warnings=readonly.diagnostics() if readonly is not None else [], + ) + matched = reconcile_inventory(plan, before, allow_new_uploads=allow_new_uploads) + + created: list[dict[str, Any]] = [] + reused: list[dict[str, Any]] = [] + acknowledged_ids: list[str] = [] + progress.update("file-upload", uploads_acknowledged=0, files_reused=0) + for record in records: + progress.update("file-upload", uploads_acknowledged=len(created), files_reused=len(reused)) + if record["path"] in matched: + reused.append( + { + "fileId": matched[record["path"]].get("fileId"), + "fileName": record["path"], + "sha256": record["sha256"], + } + ) + continue + + path = _resolve_file(root, record) + try: + content = path.read_bytes() + except OSError as exc: + raise HelperFailure( + "inventory-unreadable", + f"Approved file became unreadable: {record['path']}.", + blocked_at="execution", + writes=created, + resources_remaining=_remaining_files(created), + partial=bool(created), + ) from exc + try: + post_read_stat = path.stat() + except OSError as exc: + raise HelperFailure( + "inventory-unreadable", + f"Approved file became unreadable: {record['path']}.", + blocked_at="execution", + writes=created, + resources_remaining=_remaining_files(created), + partial=bool(created), + ) from exc + content_digest = "sha256:" + hashlib.sha256(content).hexdigest() + if ( + content_digest != record["sha256"] + or len(content) != record["size"] + or post_read_stat.st_mtime_ns != record["mtime_ns"] + ): + raise HelperFailure( + "inventory-drift", + f"Approved file changed before upload: {record['path']}.", + blocked_at="confirmation", + writes=created, + resources_remaining=_remaining_files(created), + partial=bool(created), + ) + body, boundary = _multipart(plan, record, content, fingerprint) + try: + result = transport( + "POST", + list_url, + token, + body=body, + headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, + ) + except HelperFailure as failure: + if failure.http_status is not None and 400 <= failure.http_status < 500 and failure.http_status not in {408, 409, 429}: + raise HelperFailure( + failure.code, failure.message, blocked_at=failure.blocked_at, + writes=created, resources_remaining=_remaining_files(created), + request_id=failure.request_id, status=failure.http_status, partial=bool(created), + ) from failure + # The transport outcome is ambiguous (we do not know whether the + # server received the write): attempt one readback before + # concluding partial, per the ambiguous-write contract. + confirmed = _confirm_ambiguous_upload( + list_url, token, transport, record, plan, request_ids, failure, warnings + ) + if confirmed is not None: + created.append({"fileName": record["path"], "sha256": record["sha256"]}) + if failure.request_id: + request_ids.append(failure.request_id) + continue + uncertain = { + "action": "upload-unverified", + "type": "knowledge-source-file", + "fileName": record["path"], + "sha256": record["sha256"], + } + raise HelperFailure( + failure.code, + failure.message, + blocked_at=failure.blocked_at, + writes=created + [uncertain], + resources_remaining=_remaining_files(created) + [uncertain], + request_id=failure.request_id, + status=failure.http_status, + partial=True, + warnings=warnings, + ) from failure + if result.status not in {200, 201}: + ambiguous = result.status in {408, 409, 429} or result.status >= 500 + if ambiguous: + confirmed = _confirm_ambiguous_upload( + list_url, token, transport, record, plan, request_ids, + HelperFailure("upload-failed", "Upload response is ambiguous.", + blocked_at="execution", status=result.status, + request_id=result.request_id, retry_after=result.retry_after, + recovery_deadline=result.recovery_deadline), + warnings, + ) + if confirmed is not None: + created.append( + {"fileName": record["path"], "sha256": record["sha256"]} + ) + if result.request_id: + request_ids.append(result.request_id) + continue + uncertain = { + "action": "upload-unverified", + "type": "knowledge-source-file", + "fileName": record["path"], + "sha256": record["sha256"], + } + raise HelperFailure( + "upload-failed", + f"Upload returned unexpected HTTP {result.status}.", + blocked_at="execution", + writes=created + ([uncertain] if ambiguous else []), + resources_remaining=( + _remaining_files(created) + ([uncertain] if ambiguous else []) + ), + request_id=result.request_id, + status=result.status, + partial=bool(created) or ambiguous, + warnings=warnings, + ) + # A definitive 200/201 response is not ambiguous: trust it rather than + # re-listing the whole source after every single file. The complete + # inventory is verified once, in bulk, after the loop. + created.append({"fileName": record["path"], "sha256": record["sha256"]}) + if result.request_id: + request_ids.append(result.request_id) + acknowledged_ids.append(ReadRecovery.safe_id(result.request_id)) + + progress.update("file-readback", uploads_acknowledged=len(created), files_reused=len(reused)) + ack_warnings = (["Acknowledged upload request IDs: " + ", ".join(acknowledged_ids)] + if acknowledged_ids else []) + recovery = ReadRecovery() + try: + after, after_request_ids = _list_files( + list_url, token, transport=transport, recovery=recovery, + ) + except HelperFailure as failure: + raise HelperFailure( + failure.code, + failure.message, + blocked_at=failure.blocked_at, + writes=created + failure.writes, + resources_remaining=( + _remaining_files(created) + failure.resources_remaining + ), + resources_reused=failure.resources_reused, + resources_unverified=failure.resources_unverified, + warnings=[*warnings, *ack_warnings, *failure.warnings, *recovery.diagnostics()], + request_id=failure.request_id, + status=failure.http_status, + partial=bool(created or failure.partial), + ) from failure + request_ids.extend(after_request_ids) + warnings.extend(recovery.warnings) + if len(after) != len(records): + raise HelperFailure( + "readback-mismatch", + "Final server inventory count differs from the approved inventory.", + blocked_at="verification", + writes=created, + resources_remaining=_remaining_files(created), + request_id=request_ids[-1] if request_ids else None, + partial=bool(created), + warnings=[*warnings, *ack_warnings, *recovery.diagnostics()], + ) + verified: list[dict[str, Any]] = [] + for record in records: + matches = [item for item in after if item.get("fileName") == record["path"]] + if len(matches) != 1 or not _matches(matches[0], plan, record): + raise HelperFailure( + "readback-mismatch", + f"File readback failed for {record['path']}.", + blocked_at="verification", + writes=created, + resources_remaining=_remaining_files(created), + request_id=request_ids[-1] if request_ids else None, + partial=bool(created), + warnings=[*warnings, *ack_warnings, *recovery.diagnostics()], + ) + verified.append( + { + "fileId": matches[0].get("fileId"), + "fileName": record["path"], + "sha256": record["sha256"], + "size": record["size"], + } + ) + + progress.update("file-readback", files_verified=len(verified)) + return { + "status": "completed", + "outcome": str(plan.get("outcome") or "file-knowledge-source-ingestion"), + "approved_plan": {"fingerprint": fingerprint, "confirmed": True}, + "resources": { + "created": created, + "reused": reused, + "updated": [], + "skipped": [], + }, + "api_contracts": [ + {"operation": "upload-file", "version": API_VERSION, "preview": True} + ], + "data_movement": { + "boundary": {"local_root_digest": digest(str(root))}, + "result": "exact approved files uploaded directly to Search", + }, + "auth": {"mode": "entra-user", "principals": []}, + "rbac": plan.get("rbac", {"assignments": []}), + "network": plan.get("network", {"posture": "preserved", "evidence": None}), + "verification": { + "readback": verified, + "server_inventory_digest": digest(_normalize_inventory(after)), + "request_ids": request_ids, + "idempotency": "matching marker metadata is zero-write", + }, + "warnings": warnings, + "ownership": { + "run_owned": created, + "reused_not_owned": reused, + "owner": plan.get("owner"), + }, + "cleanup": { + "status": "not-requested", + "separate_confirmation_required": True, + }, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--input", type=Path, required=True) + add_progress_argument(parser) + args = parser.parse_args(argv) + fingerprint: str | None = None + owner: Any = None + outcome = "file-knowledge-source-ingestion" + try: + document, plan, fingerprint = load_approved_input(args.input) + document["_computed_fingerprint"] = fingerprint + owner = plan.get("owner") + outcome = str(plan.get("outcome") or outcome) + result = execute(document, progress=Progress("file-upload", enabled=args.progress)) + except HelperFailure as failure: + result = blocked_result( + failure, + outcome=outcome, + fingerprint=fingerprint, + owner=owner, + ) + emit_result(result) + return 3 if result["status"] == "partial" else 2 + emit_result(result) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_source.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_source.py new file mode 100644 index 00000000..2b3f73b3 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_source.py @@ -0,0 +1,858 @@ +from __future__ import annotations + +import argparse +import copy +import json +import re +import sys +from pathlib import Path +from typing import Any + +try: + from ._progress import Progress, add_progress_argument, reporting + from . import file_ingest, search_reconcile, source_vector, file_cu_mi + from . import cu_ingestion_auth as file_cu_auth + from ._common import ( + SEARCH_AUDIENCE, + MANAGEMENT_AUDIENCE, + HelperFailure, + TokenProvider, + Transport, + azure_cli_token, + blocked_result, + digest, + emit_result, + http_request, + load_approved_input, + normalize_azure_location, + reject_secrets, + require_allowed_fields, + ) +except ImportError: + from _progress import Progress, add_progress_argument, reporting + import file_ingest # type: ignore[no-redef] + import search_reconcile # type: ignore[no-redef] + import source_vector + import file_cu_mi + import cu_ingestion_auth as file_cu_auth + from _common import ( # type: ignore[no-redef] + SEARCH_AUDIENCE, + MANAGEMENT_AUDIENCE, + HelperFailure, + TokenProvider, + Transport, + azure_cli_token, + blocked_result, + digest, + emit_result, + http_request, + load_approved_input, + normalize_azure_location, + reject_secrets, + require_allowed_fields, + ) + + +def _cu_failure(code: str, message: str, *, request_id: str | None = None) -> HelperFailure: + return HelperFailure(code, message, blocked_at="cu-prerequisites", request_id=request_id) + + +def validate_content_understanding(value: Any, *, enabled: bool) -> dict[str, Any] | None: + if not enabled: + if value is not None: + raise _cu_failure("cu-choice-conflict", "Minimal extraction must omit CU choices.") + return None + if not isinstance(value, dict): + raise _cu_failure("cu-prerequisite-missing", "Standard planning requires a resolved CU account, disclosed auth channel and owner-verified prerequisites.") + value = copy.deepcopy(value) + value.setdefault("auth", "system-assigned") + reject_secrets(value) + require_allowed_fields(value, { + "endpoint", "resource_id", "auth", "api_key_environment", "prerequisites", "managed_identity", + }, label="File CU choice") + if ( + not isinstance(value.get("endpoint"), str) + or re.fullmatch(r"https://[a-z0-9][a-z0-9-]{0,62}\.services\.ai\.azure\.com/?", value["endpoint"]) is None + or not isinstance(value.get("resource_id"), str) + or re.fullmatch( + r"/subscriptions/[0-9a-fA-F-]{36}/resourceGroups/[A-Za-z0-9_.()-]{1,90}" + r"/providers/Microsoft\.CognitiveServices/accounts/[A-Za-z0-9][A-Za-z0-9_.-]{1,63}", + value["resource_id"], re.IGNORECASE, + ) is None + or value.get("auth") not in ("api-key-environment", "api-key-arm", "system-assigned") + or (value.get("auth") == "api-key-environment" and ( + not isinstance(value.get("api_key_environment"), str) + or search_reconcile.ENVIRONMENT_NAME.fullmatch(value["api_key_environment"]) is None + )) + or (value.get("auth") != "api-key-environment" and "api_key_environment" in value) + or (value.get("auth") != "system-assigned" and "managed_identity" in value) + ): + raise _cu_failure("cu-choice-invalid", "Select exact AIServices and system-assigned MI, or explicitly retain approved ARM/ENV key auth. No automatic auth fallback or setup changes.") + if value["auth"] == "system-assigned": + file_cu_mi.validate_choice(value.get("managed_identity"), value["resource_id"]) + prerequisites = value.get("prerequisites") + if not isinstance(prerequisites, dict): + raise _cu_failure("cu-prerequisite-missing", "Supply CU region/capability, selected processing/required deployments, identity/local-auth and network evidence references.") + fields = {"resource", "configuration", "identity", "network"} + require_allowed_fields(prerequisites, fields, label="File CU prerequisites") + if any(not source_vector._text(prerequisites.get(key)) for key in fields): + raise _cu_failure("cu-prerequisite-missing", "Owner-verified CU capability/region, selected processing/required deployments, auth/access and reachability evidence is required.") + source_vector._json_valid(value) + return copy.deepcopy(value) + + +def _cu_account_state(choice: dict[str, Any], account: Any) -> dict[str, Any]: + properties = account.get("properties") if isinstance(account, dict) else None + if not isinstance(properties, dict): + raise _cu_failure("cu-prerequisite-invalid", "CU account readback is incomplete.") + endpoints = properties.get("endpoints", {}) + candidates = [properties.get("endpoint")] + if isinstance(endpoints, dict): + candidates.extend(endpoints.values()) + if ( + str(account.get("id", "")).casefold() != choice["resource_id"].casefold() + or account.get("kind") != "AIServices" + or normalize_azure_location(account.get("location")) is None + or properties.get("provisioningState") != "Succeeded" + or (choice["auth"] != "system-assigned" and properties.get("disableLocalAuth") is not False) + or properties.get("publicNetworkAccess") not in ("Enabled", "Disabled") + or choice["endpoint"].rstrip("/") not in [v.rstrip("/") for v in candidates if isinstance(v, str)] + ): + raise _cu_failure("cu-prerequisite-invalid", "Readback must bind the selected ready AIServices account/endpoint/location/network. Key modes also need enabled local auth; MI does not. Any required setup change needs separate approval.") + state = { + "id": choice["resource_id"], "kind": "AIServices", "location": account["location"], + "identity": copy.deepcopy(account.get("identity")), + "properties": { + "endpoint": choice["endpoint"].rstrip("/"), "provisioningState": "Succeeded", + "disableLocalAuth": properties.get("disableLocalAuth"), "publicNetworkAccess": properties["publicNetworkAccess"], + "networkAcls": copy.deepcopy(properties.get("networkAcls")), + }, + } + if choice["auth"] == "system-assigned": + acl = properties.get("networkAcls") + if properties["publicNetworkAccess"] != "Enabled" or ( + acl is not None and (not isinstance(acl, dict) or acl.get("defaultAction") != "Allow") + ): + raise _cu_failure("cu-mi-network-unverified", "This MI path requires existing public CU reachability without default-deny ACLs. Restricted/private network compatibility needs separate verified setup and approval; no network changes or key fallback.") + reject_secrets(state) + source_vector._json_valid(state) + return state + + +def _cu_states_match(current: dict[str, Any], retained: dict[str, Any]) -> bool: + location = normalize_azure_location(retained.get("location")) + return location is not None and ( + {**current, "location": normalize_azure_location(current.get("location"))} + == {**retained, "location": location} + ) + + +def read_content_understanding( + choice: dict[str, Any], *, token_provider: TokenProvider, transport: Transport, +) -> tuple[dict[str, Any], list[str]]: + url = f"{MANAGEMENT_AUDIENCE}{choice['resource_id']}?api-version=2024-10-01" + response = transport("GET", url, token_provider(MANAGEMENT_AUDIENCE)) + try: + if response.status != 200: + raise _cu_failure("cu-prerequisite-unavailable", "Selected CU account metadata could not be read; no provisioning or auth changes are allowed.") + state = _cu_account_state(choice, response.body) + except HelperFailure as failure: + failure.request_id = response.request_id + if response.status != 200: + failure.http_status = response.status + raise + return state, [response.request_id] if response.request_id else [] + + +def verify_content_understanding_readback(choice: dict[str, Any], current: Any) -> None: + parameters = current.get("fileParameters") if isinstance(current, dict) else None + ingestion = parameters.get("ingestionParameters") if isinstance(parameters, dict) else None + ai = ingestion.get("aiServices") if isinstance(ingestion, dict) else None + if ( + not isinstance(ai, dict) or ingestion.get("contentExtractionMode") != "standard" + or not isinstance(ai.get("uri"), str) + or ai["uri"].rstrip("/") != choice["endpoint"].rstrip("/") + or ingestion.get("identity") is not None + or (choice["auth"] == "system-assigned" and ai.get("apiKey") not in file_cu_mi.REDACTED) + ): + raise _cu_failure("cu-readback-mismatch", "Observed File CU endpoint/extraction/auth conflicts with the selected configuration; credential details withheld.") + # File's approved key may be redacted in GET. It is not embedding auth, + # and source readback cannot prove its value or CU processing readiness. + + +def plan_source( + request: dict[str, Any], + *, + token_provider: TokenProvider = azure_cli_token, + transport: Transport = http_request, + context_provider=file_cu_auth.account_context, +) -> dict[str, Any]: + """Build an unapproved File workflow using only local reads and GETs.""" + if not isinstance(request, dict): + raise HelperFailure( + "input-schema-invalid", "Planning input must be an object.", + blocked_at="input-resolution", + ) + reject_secrets(request) + require_allowed_fields( + request, + {"schema_version", "api_version", "endpoint", "name", "owner", "local_root", "paths", + "service_tier", "extraction_mode", "vectorization", "embedding", "rbac", "network", + "content_understanding", "reuse_input_file", "reuse_result_file"}, + label="File planning input", + ) + if request.get("schema_version") != "1.0": + raise HelperFailure( + "input-schema-invalid", "Planning requires schema_version 1.0.", + blocked_at="input-resolution", + ) + file_ingest.validate_api_version(request.get("api_version", file_ingest.API_VERSION)) + for field in ("name", "owner"): + if not isinstance(request.get(field), str) or not request[field].strip(): + raise HelperFailure( + "input-schema-invalid", f"An explicit non-empty {field} is required.", + blocked_at="input-resolution", + ) + if request.get("extraction_mode") not in ("minimal", "standard") or request.get("vectorization") not in ("none", "azureOpenAI"): + raise HelperFailure( + "planning-processing-unsupported", + "Select minimal or standard extraction and independent vectorization none or azureOpenAI.", + blocked_at="input-resolution", + ) + embedding = source_vector.validate_choice( + request.get("embedding"), enabled=request["vectorization"] == "azureOpenAI", + api_version=file_ingest.API_VERSION, + ) + cu = validate_content_understanding( + request.get("content_understanding"), enabled=request["extraction_mode"] == "standard", + ) + if (request.get("reuse_input_file") is not None or request.get("reuse_result_file") is not None) and ( + cu is None or cu["auth"] != "system-assigned" + ): + raise _cu_failure("cu-choice-conflict", "File MI provenance inputs are only for managed-identity exact reuse.") + rbac, network = request.get("rbac"), request.get("network") + if ( + not isinstance(rbac, dict) + or not isinstance(rbac.get("assignments"), list) + or not rbac["assignments"] + or not all(isinstance(item, dict) and item for item in rbac["assignments"]) + or not isinstance(network, dict) + or not isinstance(network.get("posture"), str) + or not network["posture"].strip() + or not isinstance(network.get("evidence"), str) + or not network["evidence"].strip() + ): + raise HelperFailure( + "planning-evidence-missing", + "Supply observed RBAC assignments and network posture/evidence; the policy owner must refresh and verify them before approval.", + blocked_at="input-resolution", + ) + root = file_ingest.resolve_local_root(request.get("local_root")) + records = file_ingest.snapshot_inventory( + root, request.get("paths"), service_tier=request.get("service_tier") + ) + common = { + "endpoint": request.get("endpoint"), "name": request.get("name"), + "api_version": file_ingest.API_VERSION, "owner": request.get("owner"), + "cleanup_approved": False, "rbac": copy.deepcopy(rbac), + "network": copy.deepcopy(network), + } + source = { + **common, "operation": "reconcile", "resource_type": "knowledge-source", + "outcome": "create-file-knowledge-source", "action": "create", + "desired": { + "name": common["name"], "kind": "file", + "fileParameters": {"ingestionParameters": {"contentExtractionMode": request["extraction_mode"]}}, + }, + } + ingestion = { + **copy.deepcopy(common), "operation": "ingest", "local_root": str(root), + "files": records, "inventory_digest": digest(records), + "expected_server_inventory_digest": file_ingest.inventory_digest([]), + "service_tier": request["service_tier"], "extraction_mode": request["extraction_mode"], + } + plan = { + "operation": "reconcile-and-ingest", "outcome": "create-file-knowledge-source", + "owner": common["owner"], "cleanup_approved": False, + "source": source, "ingestion": ingestion, + } + if embedding is not None: + plan["embedding"] = embedding + source["desired"]["fileParameters"]["ingestionParameters"]["embeddingModel"] = source_vector.model_definition(embedding) + if cu is not None: + automatic = cu["auth"] == "api-key-arm" + mi = cu["auth"] == "system-assigned" + plan.update(file_cu_plan_version="1.2" if mi else "1.1" if automatic else "1.0", content_understanding=cu) + if automatic: + source["ai_services_key_acquisition"] = file_cu_auth.acquisition(cu, context_provider()) + elif mi: + source["ai_services_managed_identity"] = True + else: + source["ai_services_api_key_environment"] = cu["api_key_environment"] + source["desired"]["fileParameters"]["ingestionParameters"].update( + aiServices={"uri": cu["endpoint"].rstrip("/")}, disableImageVerbalization=True, + ) + # Validate the local inventory and all choices before any authentication. + _validate_plan(plan, require_cu_readback=False) + cu_request_ids = [] + if cu is not None: + state, cu_request_ids = read_content_understanding(cu, token_provider=token_provider, transport=transport) + plan["cu_resource_state"] = state + if cu["auth"] == "system-assigned": + plan["cu_identity_state"], ids = file_cu_mi.read_binding( + cu, common["endpoint"], token_provider=token_provider, transport=transport, + ) + cu_request_ids.extend(ids) + _validate_plan(plan) + transport = source_vector.guard_readback_transport(plan, transport) + url = search_reconcile.resource_url(source) + token = token_provider(SEARCH_AUDIENCE) + current, request_id = search_reconcile.read_resource(url, token, transport=transport) + request_ids = cu_request_ids + ([request_id] if request_id else []) + matched = {} + if current is not None: + if cu is not None and cu["auth"] == "system-assigned": + file_cu_mi.verify_reuse(request, plan, current) + if not search_reconcile.definitions_match(source["desired"], current): + raise HelperFailure( + "definition-conflict", + "The exact source has a different definition; planning never overwrites or chooses another name.", + blocked_at="reconciliation", + ) + etag = current.get("@odata.etag") + if not isinstance(etag, str) or not etag: + raise HelperFailure( + "definition-evidence-missing", "Exact reuse requires the current source ETag.", + blocked_at="reconciliation", + ) + before, ids = file_ingest.read_inventory(ingestion, token, transport=transport) + request_ids.extend(ids) + matched = file_ingest.reconcile_inventory(ingestion, before) + file_ids = [item.get("fileId") for item in matched.values()] + if ( + not all(isinstance(value, str) and value for value in file_ids) + or len(set(file_ids)) != len(file_ids) + ): + raise HelperFailure( + "file-identity-ambiguous", "Exact reuse requires unique non-empty server file IDs.", + blocked_at="reconciliation", + ) + source.update(action="reuse", expected_etag=etag) + ingestion["expected_server_inventory_digest"] = file_ingest.inventory_digest(before) + refreshed, refresh_id = search_reconcile.read_resource(url, token, transport=transport) + if refresh_id: + request_ids.append(refresh_id) + if ( + refreshed is None + or refreshed.get("@odata.etag") != etag + or not search_reconcile.definitions_match(source["desired"], refreshed) + ): + raise HelperFailure( + "definition-drift", + "Source definition or ETag changed during file inventory readback.", + blocked_at="reconciliation", + ) + # Do not return a snapshot that changed while Search discovery was running. + _validate_plan(plan) + if cu is not None: + state, ids = read_content_understanding(cu, token_provider=token_provider, transport=transport) + request_ids.extend(ids) + if not _cu_states_match(state, plan["cu_resource_state"]): + raise _cu_failure( + "cu-prerequisite-drift", "CU account access or configuration changed during planning; refresh the plan.", + request_id=ids[-1] if ids else None, + ) + if cu["auth"] == "api-key-arm": + file_cu_auth.check_context(source["ai_services_key_acquisition"]["context"], context_provider) + elif cu["auth"] == "system-assigned": + binding, ids = file_cu_mi.read_binding(cu, common["endpoint"], token_provider=token_provider, transport=transport) + request_ids.extend(ids) + if binding != plan["cu_identity_state"]: + raise _cu_failure("cu-mi-identity-drift", "Search identity, CU scoped role or network changed during planning.") + fingerprint = digest(plan) + mutation_required = source["action"] == "create" + return { + "status": "planned", "outcome": plan["outcome"], + "plan_fingerprint": fingerprint, + "execution_input": { + "schema_version": "1.0", "plan": plan, + "approval": {"confirmed": False, "fingerprint": fingerprint}, + }, + "approval_summary": { + "target": {"endpoint": common["endpoint"], "name": common["name"], + "api_version": common["api_version"], "preview": True}, + "source_action": source["action"], "owner": common["owner"], + "execution_required": mutation_required, + "mutation_approval_required": mutation_required, + "processing": ( + "standard CU extraction" + (" with source embeddings" if embedding else "; source vectors off") + if cu else "minimal extraction with embeddings" if embedding else "minimal lexical; no models" + ), + **({"content_understanding": { + "purpose": "Standard document extraction only; not source vectorization or KB answer synthesis.", + "endpoint": cu["endpoint"], + "authentication": file_cu_auth.approval_summary("file", cu["auth"], creating=mutation_required), + "auth": ( + "Search system-assigned MI for CU; no API key or credential reads, no local-auth requirement. Service implementation inspected; live compatibility unverified." + if cu["auth"] == "system-assigned" else + "Existing File CU key-auth configuration reused; no credential acquisition. Search remains keyless." + if not mutation_required else + "Approved private ARM listKeys acquisition of key1 for this source PUT only; Search remains keyless. Local authentication means key auth, not manual local setup." + if cu["auth"] == "api-key-arm" else "Explicit existing CU API-key ENV channel; Search remains keyless." + ), + **({"credential_acquisition": copy.deepcopy(source["ai_services_key_acquisition"])} if cu["auth"] == "api-key-arm" and mutation_required else {}), + **({"managed_identity": copy.deepcopy(plan["cu_identity_state"])} if cu["auth"] == "system-assigned" else {}), + "cost_and_data": "Billable CU processing, no daily free document allowance; uploaded content moves from Search to CU, possibly across regions. Search retains outputs.", + "verification": ( + "ARM verifies Search identity, exact CU-scoped role and account/network metadata, not backend MI rollout, effective access or extraction. A separately approved bounded OCR canary can validate functionality." + if cu["auth"] == "system-assigned" else + "ARM verifies account binding/local-auth/network metadata, not effective access, key validity or processing success. Verify selected processing/required deployment evidence; no blanket account-defaults confirmation." + ), + "kb_reasoning": "Unchanged; source CU does not enable KB chat or source vectors.", + }} if cu else {}), + **({"embedding": source_vector.summary(embedding)} if embedding else {}), + "data_boundary": {"paths": [r["path"] for r in records], + "file_count": len(records), "total_bytes": sum(r["size"] for r in records)}, + "uploads": 0 if matched else len(records), + "reused_files": [{"path": path, "fileId": item["fileId"]} for path, item in matched.items()], + "service_tier": request["service_tier"], + "rbac": rbac, "network": network, + "cost_and_retention": ( + "Existing Search charges remain; review File ingestion/storage charges and retention before approval." + if mutation_required else "Existing charges and retention are unchanged; no new uploads or resources." + ), + "ownership": "New source and uploaded files only; reused Search/source/files are not run-owned.", + "format_verification": "Filename/MIME hints are not detected types; Search checks actual content support during ingestion.", + "verification": ( + "Execution rechecks definition/ETag and local/server inventories, then verifies uploaded file markers." + if mutation_required else + "Fresh source definition/ETag and complete file markers/IDs match the local inventory; this does not verify ingestion readiness or retrieval." + ), + "cleanup": "Excluded; separate run-owned source cleanup plan and approval required.", + "next_step": ( + "Creation owner refreshes identity, RBAC, network, source state and cost/data consent, then obtains explicit approval of these changes before applying the unchanged execution input." + if mutation_required else + "Reuse the verified identity and file markers without mutation approval or invoking the mutation helper. Refresh discovery before later use; this observation is not future consent." + ), + }, + "read_only_evidence": {"request_ids": request_ids, "source_state": source["action"]}, + "writes_performed": [], + "warnings": [ + "Planning is not approval or completed ingestion. Uploader RBAC/network remain supplied evidence; MI identity/role metadata readbacks do not prove backend attribution or effective access." + if cu is not None and cu["auth"] == "system-assigned" else + "Planning is not approval, policy evaluation, or proof of completed ingestion. RBAC/network are caller-supplied evidence, not verified by this helper." + ], + } + + +def _validate_plan( + plan: dict[str, Any], + *, + require_cu_readback: bool = True, +) -> tuple[dict[str, Any], dict[str, Any]]: + reject_secrets(plan) + require_allowed_fields( + plan, + { + "operation", + "outcome", + "cleanup_approved", + "owner", + "source", + "ingestion", + "embedding", + "content_understanding", + "file_cu_plan_version", + "cu_resource_state", + "cu_identity_state", + }, + label="File source plan", + ) + if ( + plan.get("operation") != "reconcile-and-ingest" + or plan.get("cleanup_approved") is not False + ): + raise HelperFailure( + "operation-invalid", + "File source application requires reconcile-and-ingest with cleanup excluded.", + blocked_at="input-resolution", + ) + source = plan.get("source") + ingestion = plan.get("ingestion") + if not isinstance(source, dict) or not isinstance(ingestion, dict): + raise HelperFailure( + "input-schema-invalid", + "File source application requires source and ingestion objects.", + blocked_at="input-resolution", + ) + desired = source.get("desired") + if ( + source.get("operation") != "reconcile" + or source.get("resource_type") != "knowledge-source" + or source.get("action") not in {"create", "reuse"} + or not isinstance(desired, dict) + or desired.get("kind") != "file" + or ingestion.get("operation") != "ingest" + or plan.get("owner") != source.get("owner") + or any( + source.get(field) != ingestion.get(field) + for field in ("endpoint", "name", "api_version", "owner") + ) + ): + raise HelperFailure( + "step-contract-mismatch", + "Source reconciliation and ingestion must target the same approved File source.", + blocked_at="input-resolution", + ) + source_mode = desired.get("fileParameters", {}).get( + "ingestionParameters", {} + ).get("contentExtractionMode") + if source_mode != ingestion.get("extraction_mode"): + raise HelperFailure( + "step-contract-mismatch", + "Source and ingestion extraction modes must match exactly.", + blocked_at="input-resolution", + ) + search_reconcile._validate_plan(source) + file_ingest._validate_plan(ingestion) + source_vector.validate_plan_choice(plan) + if any(field in plan for field in ("file_cu_plan_version", "content_understanding", "cu_resource_state")): + if plan.get("file_cu_plan_version") not in ("1.0", "1.1", "1.2") or source_mode != "standard": + raise _cu_failure("cu-plan-mismatch", "New File CU plans require their supported CU-specific version and standard extraction.") + cu = validate_content_understanding(plan.get("content_understanding"), enabled=True) + automatic = cu["auth"] == "api-key-arm" + mi = cu["auth"] == "system-assigned" + acquisition = source.get("ai_services_key_acquisition") + if automatic: + file_cu_auth.validate_acquisition(acquisition, cu["endpoint"]) + if ( + plan["file_cu_plan_version"] != ("1.2" if mi else "1.1" if automatic else "1.0") + or (automatic and acquisition["resource_id"] != cu["resource_id"]) + or (not automatic and acquisition is not None) + or source.get("ai_services_managed_identity") is not (True if mi else None) + or (not mi and "cu_identity_state" in plan) + ): + raise _cu_failure("cu-plan-mismatch", "CU auth mode, version and exact acquisition scope must match the approved plan.") + settings = desired["fileParameters"]["ingestionParameters"] + if ( + settings.get("aiServices") != {"uri": cu["endpoint"].rstrip("/")} + or settings.get("identity") is not None + or settings.get("disableImageVerbalization") is not True + or settings.get("chatCompletionModel") is not None + or source.get("ai_services_api_key_environment") != cu.get("api_key_environment") + or ("embedding" in plan) != (settings.get("embeddingModel") is not None) + ): + raise _cu_failure("cu-plan-mismatch", "CU/embedding choices, credential channel and source processing must match the approved definition.") + if require_cu_readback: + state = plan.get("cu_resource_state") + if not isinstance(state, dict) or state != _cu_account_state(cu, state): + raise _cu_failure("cu-prerequisite-missing", "Retain the planner's selected CU account readback.") + if mi: + file_cu_mi.validate_state(plan.get("cu_identity_state"), cu, source["endpoint"]) + elif source.get("ai_services_key_acquisition") is not None or source.get("ai_services_managed_identity") is not None or "cu_identity_state" in plan: + raise _cu_failure("cu-plan-mismatch", "Automatic acquisition requires the complete versioned File CU workflow.") + return source, ingestion + + +def _writes(result: dict[str, Any]) -> list[dict[str, Any]]: + writes: list[dict[str, Any]] = [] + for action in ("created", "updated"): + for resource in result["resources"].get(action, []): + writes.append( + { + "action": action, + "type": resource["type"], + "name": resource["name"], + } + ) + return writes + + +@reporting("file-source") +def execute( + document: dict[str, Any], + *, + token_provider: TokenProvider = azure_cli_token, + transport: Transport = http_request, + progress: Progress | None = None, + context_provider=file_cu_auth.account_context, + mi_on_created=None, + cleanup_capture=None, + upload_receipt_dir=None, + allow_upload_retry=True, +) -> dict[str, Any]: + progress.update("validation") + plan = document["plan"] + fingerprint = document["_computed_fingerprint"] + source, ingestion = _validate_plan(plan) + upload_session = None + if upload_receipt_dir is not None: + try: + from .file_upload import Session + except ImportError: + from file_upload import Session + upload_session = Session(upload_receipt_dir, document, context_provider=context_provider) + private_key = None + acquisition = source.get("ai_services_key_acquisition") + mi = source.get("ai_services_managed_identity") is True + mi_callback = None + if mi_on_created is not None and (not mi or not callable(mi_on_created)): + raise _cu_failure("creation-callback-unsupported", "Private MI checkpoints cannot receive File key-auth wire.") + if acquisition is not None or mi: + approval = document.get("approval") + if ( + not isinstance(approval, dict) or approval.get("confirmed") is not True + or approval.get("fingerprint") != digest(plan) or fingerprint != digest(plan) + ): + raise _cu_failure("approval-missing", "Private credential acquisition requires the unchanged fingerprinted source approval.") + if acquisition is not None: + file_cu_auth.check_context(acquisition["context"], context_provider) + child = {"_computed_fingerprint": fingerprint} + cu_ids = [] + if "content_understanding" in plan: + state, cu_ids = read_content_understanding( + plan["content_understanding"], token_provider=token_provider, transport=transport, + ) + if not _cu_states_match(state, plan["cu_resource_state"]): + raise _cu_failure( + "cu-prerequisite-drift", "CU account access or configuration changed since approval; refresh the plan.", + request_id=cu_ids[-1] if cu_ids else None, + ) + if mi: + binding, ids = file_cu_mi.read_binding( + plan["content_understanding"], source["endpoint"], token_provider=token_provider, transport=transport, + ) + cu_ids.extend(ids) + if binding != plan["cu_identity_state"]: + raise _cu_failure("cu-mi-identity-drift", "Search identity, CU role assignment or network changed since approval; refresh the concrete plan.") + raw_transport = transport + + def recheck_mi(): + state, _ = read_content_understanding( + plan["content_understanding"], token_provider=token_provider, transport=raw_transport, + ) + binding, _ = file_cu_mi.read_binding( + plan["content_understanding"], source["endpoint"], token_provider=token_provider, transport=raw_transport, + ) + if not _cu_states_match(state, plan["cu_resource_state"]) or binding != plan["cu_identity_state"]: + raise _cu_failure("cu-mi-identity-drift", "CU account or Search identity/role/network changed immediately before source PUT.") + + transport, mi_callback = file_cu_mi.guard_create(plan, transport, recheck_mi, mi_on_created) + if acquisition is not None: + def recheck(): + current, _ = read_content_understanding( + plan["content_understanding"], token_provider=token_provider, transport=transport, + ) + if not _cu_states_match(current, plan["cu_resource_state"]): + raise _cu_failure("cu-prerequisite-drift", "CU account changed before credential acquisition; refresh the plan and approval.") + + private_key = file_cu_auth.PrivateKey( + acquisition, token_provider=token_provider, transport=transport, + context_provider=context_provider, recheck=recheck, + ) + transport = private_key.transport + + if upload_session is not None: + transport = upload_session.transport(transport) + progress.update("source-reconciliation") + source_result = search_reconcile.execute( + {**child, "plan": source}, + token_provider=token_provider, + transport=source_vector.guard_readback_transport(plan, transport), + credential_provider=private_key.acquire if private_key else None, + **({"managed_identity_verified": True, "on_created": mi_callback} if mi else {}), + **({"cleanup_capture": cleanup_capture} if cleanup_capture is not None else {}), + **({"on_file_acknowledged": upload_session.acknowledge} if upload_session is not None else {}), + ) + source_writes = _writes(source_result) + + def check_retry_source(recovery, token): + _validate_plan(plan) + current, _ = search_reconcile._get( + search_reconcile.resource_url(source), token, + transport=source_vector.guard_readback_transport(plan, transport), recovery=recovery, + ) + etag = source_result["verification"]["readback"]["etag"] + if (not etag or current is None or current.get("@odata.etag") != etag + or not search_reconcile.definitions_match(source["desired"], current)): + raise HelperFailure("file-upload-source-drift", "Retry requires the unchanged acknowledged source version/definition.", + blocked_at="verification") + + try: + if upload_session is not None: + etag = upload_session.require_ack() + if source_result["verification"]["readback"]["etag"] != etag: + raise HelperFailure("file-upload-source-drift", "Source readback differs from the retained creation ACK; retain the source.", + blocked_at="verification") + ingestion_result = file_ingest.execute( + {**child, "plan": ingestion}, + token_provider=token_provider, + transport=transport, + allow_new_uploads=bool(source_result["resources"]["created"]), + progress=progress, + source_check=check_retry_source, + allow_upload_retry=allow_upload_retry, + **({"upload_session": upload_session} if upload_session is not None else {}), + ) + except HelperFailure as failure: + combined = HelperFailure( + failure.code, + failure.message, + blocked_at=failure.blocked_at, + writes=source_writes + failure.writes, + resources_remaining=( + [ + { + "type": str(write["type"]), + "name": str(write["name"]), + } + for write in source_writes + ] + + failure.resources_remaining + ), + resources_reused=failure.resources_reused, + resources_unverified=failure.resources_unverified, + warnings=failure.warnings, + request_id=failure.request_id, + status=failure.http_status, + partial=bool(source_writes or failure.writes or failure.partial), + ) + combined.file_batch = failure.file_batch + raise combined from failure + + resources = {"created": [], "reused": [], "updated": [], "skipped": []} + for action in resources: + resources[action].extend(source_result["resources"].get(action, [])) + resources[action].extend(ingestion_result["resources"].get(action, [])) + return { + "status": "completed", + "outcome": str(plan.get("outcome") or "create-file-knowledge-source"), + "approved_plan": {"fingerprint": fingerprint, "confirmed": True}, + "resources": resources, + "api_contracts": source_result["api_contracts"] + + ingestion_result["api_contracts"], + "data_movement": ingestion_result["data_movement"], + "auth": ingestion_result["auth"], + "rbac": ingestion_result["rbac"], + "network": ingestion_result["network"], + "verification": { + **({"cu_managed_identity": { + "mode": "system-assigned", "binding_digest": digest(plan["cu_identity_state"]), + "processing": "unverified until indexed OCR marker validation", + "principal_attribution": "unverified; no backend identity telemetry collected", + }} if mi else {}), + "readback": { + "source": source_result["verification"]["readback"], + "files": ingestion_result["verification"]["readback"], + }, + "request_ids": cu_ids + source_result["verification"]["request_ids"] + + ingestion_result["verification"]["request_ids"], + "idempotency": ( + "exact source and file marker readback is zero-write" + ), + }, + "warnings": source_result["warnings"] + ingestion_result["warnings"], + **({"file_batch": ingestion_result["file_batch"]} if "file_batch" in ingestion_result else {}), + "ownership": { + "run_owned": source_result["ownership"]["run_owned"] + + ingestion_result["ownership"]["run_owned"], + "reused_not_owned": source_result["ownership"]["reused_not_owned"] + + ingestion_result["ownership"]["reused_not_owned"], + "owner": plan.get("owner"), + }, + "cleanup": { + "status": "not-requested", + "separate_confirmation_required": True, + }, + } + + +def main(argv: list[str] | None = None) -> int: + try: + from . import _cleanup_receipts as cleanup_receipts + from .private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode + except ImportError: + import _cleanup_receipts as cleanup_receipts + from private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode + parser = argparse.ArgumentParser() + modes = parser.add_mutually_exclusive_group(required=True) + modes.add_argument("--input", type=Path) + modes.add_argument("--plan", type=Path) + add_execution_output_argument(parser) + add_progress_argument(parser) + cleanup_receipts.add_argument(parser) + parser.add_argument("--upload-receipt-dir", type=Path, + help="Required for creation: existing empty private directory for original File ACK and pre-upload attempts.") + args = parser.parse_args(argv) + fingerprint: str | None = None + owner: Any = None + outcome = "create-file-knowledge-source" + capture = None + try: + validate_execution_output_mode(args) + if args.plan and (args.cleanup_receipt_dir or args.upload_receipt_dir): + raise HelperFailure("input-schema-invalid", "Creation capture requires approved --input.", blocked_at="confirmation") + if args.plan: + try: + request = json.loads(args.plan.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise HelperFailure( + "input-unreadable", "Planning input must be readable UTF-8 JSON.", + blocked_at="input-resolution", + ) from exc + result = plan_source(request) + emit_plan_result(result, args.execution_output) + return 0 + document, plan, fingerprint = load_approved_input(args.input) + document["_computed_fingerprint"] = fingerprint + owner = plan.get("owner") + outcome = str(plan.get("outcome") or outcome) + source, _ = _validate_plan(plan) + if source["action"] == "create" and args.cleanup_receipt_dir is None: + raise HelperFailure( + "file-creation-receipt-required", + "File creation requires explicit --cleanup-receipt-dir pointing to an existing protected private directory; no default is inferred.", + blocked_at="confirmation", + ) + if args.cleanup_receipt_dir is not None: + directory = cleanup_receipts.private_io.validate_private_artifact_directory(str(args.cleanup_receipt_dir)) + if args.upload_receipt_dir is not None: + upload_directory = cleanup_receipts.private_io.validate_private_artifact_directory(str(args.upload_receipt_dir)) + if directory == upload_directory: + raise HelperFailure("file-receipt-directory-conflict", + "Use separate private cleanup and upload journal directories.", + blocked_at="confirmation") + elif source["action"] == "create": + raise HelperFailure( + "file-upload-receipt-required", + "File creation requires explicit --upload-receipt-dir for durable original ACK and pre-upload attempt records; use a separate existing empty private directory.", + blocked_at="confirmation", + ) + capture = cleanup_receipts.Capture(directory, document) + result = execute(document, progress=Progress("file-source", enabled=args.progress), + **({"cleanup_capture": capture} if capture else {}), + **({"upload_receipt_dir": args.upload_receipt_dir} if args.upload_receipt_dir else {})) + except HelperFailure as failure: + result = blocked_result( + failure, + outcome=outcome, + fingerprint=fingerprint, + owner=owner, + ) + if capture is not None: + result["cleanup_receipts"] = capture.summaries + result["safe_next_decision"] = ( + "Retain any acknowledged source and confirmed uploads. Do not rerun the creation envelope, " + "replay uncertain uploads, reset or delete resources. Use file_upload.py --plan with the " + "original upload journal for newly approved never-attempted files; missing evidence blocks continuation, not retention." + ) + emit_result(result) + return 3 if result["status"] == "partial" else 2 + if capture is not None: + result["cleanup_receipts"] = capture.summaries + emit_result(result) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_upload.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_upload.py new file mode 100644 index 00000000..9a95a62f --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/file_upload.py @@ -0,0 +1,853 @@ +"""Synchronous File batches; one in-operation queue-429 retry, never unknown-outcome replay.""" +from __future__ import annotations + +import argparse +import copy +import hashlib +import re +import sys +import time +import uuid +from dataclasses import replace +from pathlib import Path + +try: + from . import _bootstrap_io as private_io, cu_ingestion_auth, file_ingest, search_reconcile + from ._common import (HelperFailure, ReadRecovery, RetryAfter, RetryAfterTiming, SEARCH_AUDIENCE, + retry_after_timing, retry_after_not_before, valid_utc_timestamp, azure_cli_token, blocked_result, + digest, emit_result, http_request, load_approved_input, reject_secrets, require_allowed_fields) + from ._progress import Progress, add_progress_argument, reporting +except ImportError: + import _bootstrap_io as private_io, cu_ingestion_auth, file_ingest, search_reconcile + from _common import (HelperFailure, ReadRecovery, RetryAfter, RetryAfterTiming, SEARCH_AUDIENCE, + retry_after_timing, retry_after_not_before, valid_utc_timestamp, azure_cli_token, blocked_result, + digest, emit_result, http_request, load_approved_input, reject_secrets, require_allowed_fields) + from _progress import Progress, add_progress_argument, reporting + + +RETAIN = " Retain the source and evidence; no creation replay, reset, deletion or new roles." +UPLOAD_TIMEOUT = 180 +MAX_BACKOFF_RECORDS = 200 + + +def _rejected(status): + return type(status) is int and 400 <= status < 500 and status not in (408, 409) + + +def fail(code, message): + return HelperFailure(code, message + RETAIN, blocked_at="file-upload-resume") + + +class Session: + def __init__(self, directory, document=None, *, context_provider=cu_ingestion_auth.account_context): + self.directory = private_io.validate_private_artifact_directory(str(directory)) + self.context_provider = context_provider + self.records = {} + self.network_failure = None + self.active_attempt = None + if document is not None: + self._validate_document(document) + if any(self.directory.iterdir()): + raise fail("file-upload-journal-exists", "Use a new empty private receipt directory for original creation.") + context = context_provider() + cu_ingestion_auth.validate_context(context) + self.seed = {"document": {key: copy.deepcopy(document[key]) for key in ("schema_version", "plan", "approval")}, + "context": context, "backoff_version": "1.0"} + self._write("run.json", self.seed) + else: + self._load() + self.seed = self.records["run.json"] + self._validate_document(self.seed["document"]) + cu_ingestion_auth.validate_context(self.seed["context"]) + self.document = self.seed["document"] + self.plan = self.document["plan"] + self.ingestion = self.plan["ingestion"] + + @staticmethod + def _validate_document(document): + try: + from . import file_source + except ImportError: + import file_source + if (not isinstance(document, dict) or document.get("schema_version") != "1.0" + or not isinstance(document.get("plan"), dict) + or "_computed_fingerprint" in document and document["_computed_fingerprint"] != digest(document.get("plan")) + or not isinstance(document.get("approval"), dict) or document["approval"].get("confirmed") is not True + or document.get("approval") != {"confirmed": True, "fingerprint": digest(document.get("plan"))}): + raise fail("file-upload-approval-missing", "Original unchanged File creation approval is required.") + require_allowed_fields(document, {"schema_version", "plan", "approval", "_computed_fingerprint"}, label="original File envelope") + file_source._validate_plan(document["plan"]) + if document["plan"]["source"]["action"] != "create": + raise fail("file-upload-provenance-missing", "This continuation is only for original acknowledged creation, not generic reuse.") + + def _write(self, name, payload): + value = {"schema_version": "1.0", "kind": "file-upload-journal", "payload": payload, "integrity": digest(payload)} + reject_secrets(value) + try: + private_io.atomic_private_file(self.directory, name, value, max_bytes=private_io.MAX_BYTES) + except HelperFailure as error: + raise HelperFailure("file-upload-receipt-failed", "Private upload evidence persistence failed." + RETAIN, + blocked_at="local-persistence", warnings=error.warnings) from error + self.records[name] = copy.deepcopy(payload) + + def _load(self): + # Reuse the existing bounded, private, no-link evidence reader. + try: + from .blob_recheck import read_private + except ImportError: + from blob_recheck import read_private + try: + paths = list(self.directory.iterdir()) + except OSError as error: + raise fail("file-upload-evidence-unreadable", "Original upload evidence is inaccessible.") from error + if len(paths) > 803 + MAX_BACKOFF_RECORDS or not {"run.json", "source-ack.json"}.issubset({p.name for p in paths}): + raise fail("file-upload-provenance-missing", "Necessary original ACK and journal were not retained.") + for path in paths: + if (path.name not in {"run.json", "source-ack.json", "pending-request.json"} + and not re.fullmatch(r"(?:[0-9]{4}-(retry-)?(attempt|result)|backoff-[0-9]{4})\.json", path.name)): + raise fail("file-upload-evidence-invalid", "Unexpected or incomplete journal entry; do not discard it to resume.") + value = read_private(path) + if (set(value) != {"schema_version", "kind", "payload", "integrity"} + or value["schema_version"] != "1.0" or value["kind"] != "file-upload-journal" + or value["integrity"] != digest(value["payload"])): + raise fail("file-upload-evidence-invalid", "Original private upload evidence changed.") + self.records[path.name] = value["payload"] + seed = self.records["run.json"] + if (not isinstance(seed, dict) or set(seed) not in ({"document", "context"}, {"document", "context", "backoff_version"}) + or "backoff_version" in seed and seed["backoff_version"] != "1.0"): + raise fail("file-upload-evidence-invalid", "Original approval/context evidence is incomplete.") + + def acknowledge(self, metadata): + self._write("source-ack.json", {"original_plan_digest": digest(self.plan), + "source_url": search_reconcile.resource_url(self.plan["source"]), + "acknowledgement": metadata}) + + def require_ack(self): + value = self.records.get("source-ack.json") + if (not isinstance(value, dict) or set(value) != {"original_plan_digest", "source_url", "acknowledgement"} + or value["original_plan_digest"] != digest(self.plan) + or value["source_url"] != search_reconcile.resource_url(self.plan["source"])): + raise fail("file-upload-provenance-missing", "An original acknowledged conditional source creation must be retained.") + ack = value["acknowledgement"] + if (not isinstance(ack, dict) or set(ack) != {"status", "request_id", "etag_evidence"} + or ack["status"] not in (200, 201) or not ack["request_id"] + or not isinstance(ack["request_id"], str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}", ack["request_id"]) + or not isinstance(ack["etag_evidence"], dict) or set(ack["etag_evidence"]) != {"body", "headers"} + or not isinstance(ack["etag_evidence"]["headers"], list)): + raise fail("file-upload-provenance-missing", "Original successful creation ACK/request ID is required, not GET ownership.") + etag = search_reconcile.resolve_etag(ack["etag_evidence"], ack["request_id"]) + if not etag: + raise fail("file-upload-provenance-missing", "The original creation ACK lacks version proof; GET cannot supply it.") + return etag + + def states(self): + states = [] + events = self.backoff_events() + allowed = {"run.json", "source-ack.json", "pending-request.json"} | set(events) + pending = self.records.get("pending-request.json") + if "pending-request.json" in self.records and ( + not isinstance(pending, dict) or set(pending) != {"version", "plan_digest", "nonce", "method", "url_digest"} + or pending["version"] != "1.0" or pending["plan_digest"] != digest(self.plan) + or not isinstance(pending["nonce"], str) or not re.fullmatch(r"[0-9a-f]{32}", pending["nonce"]) + or pending["method"] not in ("GET", "POST", "PUT", "HEAD") + or not isinstance(pending["url_digest"], str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", pending["url_digest"]) + ): + raise fail("file-upload-evidence-invalid", "Pending request evidence is invalid.") + for index, record in enumerate(self.ingestion["files"]): + state = {"upload": "not_attempted", "request_ids": [], "ingested": False} + for retry in (False, True): + prefix = f"{index:04}-" + ("retry-" if retry else "") + attempt_name, result_name = prefix + "attempt.json", prefix + "result.json" + allowed.update((attempt_name, result_name)) + attempt, result = self.records.get(attempt_name), self.records.get(result_name) + expected = self._attempt_record(index, retry=retry) if attempt is not None else None + if attempt is not None and attempt != expected or result is not None and attempt is None: + raise fail("file-upload-evidence-invalid", "Upload attempt evidence does not bind the original source/corpus.") + if attempt is not None: + state.update(upload="unverified", ingested=False) + state.pop("file_proof", None) + state.pop("ack_file_id", None) + if result is not None: + if (not isinstance(result, dict) or not {"upload", "status", "request_id"} <= set(result) + or set(result) - {"upload", "status", "request_id", "file_proof", "file_id", "throttle_event"} + or not isinstance(result["upload"], str) + or result["upload"] not in {"accepted", "rejected", "unverified"} + or result["upload"] == "accepted" and result["status"] not in (200, 201) + or result["upload"] == "rejected" and not _rejected(result["status"]) + or result["status"] is not None and (type(result["status"]) is not int or not 100 <= result["status"] <= 599) + or result["request_id"] is not None and ReadRecovery.safe_id(result["request_id"]) != result["request_id"]): + raise fail("file-upload-evidence-invalid", "Upload result is not an original supported ACK or failure.") + linked = [event for event in events.values() if event["attempt"] == attempt_name] + if "throttle_event" in result: + if (result["status"] != 429 or len(linked) != 1 + or result["throttle_event"] != digest(linked[0]) + or result["request_id"] != linked[0]["request_id"]): + raise fail("file-upload-evidence-invalid", "Upload 429 timing does not bind its original attempt/result.") + elif result["status"] == 429 and self.seed.get("backoff_version") is not None: + raise fail("file-upload-evidence-invalid", "Required original upload 429 timing evidence is missing.") + if "file_id" in result: + if result["upload"] != "accepted" or not _file_id(result["file_id"]): + raise fail("file-upload-evidence-invalid", "Original upload ACK file identity is invalid.") + state["ack_file_id"] = result["file_id"] + proof = result.get("file_proof") + if proof is not None: + if (result["upload"] != "accepted" or not isinstance(proof, dict) + or set(proof) != {"file_id", "record_digest"} or not _file_id(proof["file_id"]) + or proof["record_digest"] != digest(record) + or result.get("file_id", proof["file_id"]) != proof["file_id"]): + raise fail("file-upload-evidence-invalid", "Persisted File completion proof does not match the approved record.") + state.update(file_proof=proof, ingested=True) + state.update(upload=result["upload"], status=result["status"]) + if result["request_id"]: + state["request_ids"].append(result["request_id"]) + states.append(state) + if set(self.records) - allowed: + raise fail("file-upload-evidence-invalid", "Upload journal contains records outside the approved corpus.") + return states + + def _attempt_record(self, index, *, retry=False): + record = { + "original_plan_digest": digest(self.plan), "record_digest": digest(self.ingestion["files"][index]), + "creation_ack_digest": digest(self.records["source-ack.json"]), + } + if retry: + rejection = self.records.get(f"{index:04}-result.json") + if not isinstance(rejection, dict) or rejection.get("status") != 429 or rejection.get("upload") != "rejected": + raise fail("file-upload-replay-forbidden", "A retry attempt requires the same operation's retained File upload 429.") + record["rejected_result_digest"] = digest(rejection) + return record + + def attempt(self, index, *, retry=False): + prefix = f"{index:04}-" + ("retry-" if retry else "") + self._write(prefix + "attempt.json", self._attempt_record(index, retry=retry)) + self.active_attempt = prefix + "attempt.json" + + def result(self, index, upload, status, request_id, *, retry=False, proof=None, file_id=None): + prefix = f"{index:04}-" + ("retry-" if retry else "") + event = {} + if status == 429: + linked = [value for value in self.backoff_events().values() if value["attempt"] == prefix + "attempt.json"] + if len(linked) != 1: + raise fail("file-upload-evidence-invalid", "Original upload 429 timing was not retained.") + event["throttle_event"] = digest(linked[0]) + self._write(prefix + "result.json", {"upload": upload, "status": status, + "request_id": ReadRecovery.safe_id(request_id) if request_id else None, + **({"file_id": file_id} if file_id is not None else {}), + **({"file_proof": proof} if proof is not None else {}), **event}) + self.active_attempt = None + + def backoff_events(self): + events = {key: self.records[key] for key in sorted(self.records) if key.startswith("backoff-")} + if len(events) > MAX_BACKOFF_RECORDS: + raise fail("file-upload-evidence-invalid", "Too many retained backoff records.") + plan_digest = digest(self.plan) + previous = None + for index, (name, event) in enumerate(events.items()): + if (name != f"backoff-{index:04}.json" or not isinstance(event, dict) + or set(event) != {"version", "plan_digest", "previous", "method", "url_digest", + "attempt", "attempt_digest", "request_id", "metadata", "timing", "origin"} + or event["version"] != "1.0" or event["plan_digest"] != plan_digest + or event["previous"] != previous or event["method"] not in ("GET", "POST", "PUT", "HEAD") + or not isinstance(event["url_digest"], str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", event["url_digest"]) + or event["request_id"] is not None and ReadRecovery.safe_id(event["request_id"]) != event["request_id"] + or event["origin"] not in ("response-headers", "native-http-error", "typed-metadata", "unavailable")): + raise fail("file-upload-evidence-invalid", "Original backoff provenance is invalid or incomplete.") + if event["attempt"] is not None: + attempt = self.records.get(event["attempt"]) if isinstance(event["attempt"], str) else None + if (attempt is None or not re.fullmatch(r"[0-9]{4}-(retry-)?attempt\.json", event["attempt"]) + or event["attempt_digest"] != digest(attempt) or event["method"] != "POST" + or event["url_digest"] != digest(file_ingest._list_url(self.ingestion))): + raise fail("file-upload-evidence-invalid", "Backoff does not bind the original upload attempt.") + elif event["attempt_digest"] is not None: + raise fail("file-upload-evidence-invalid", "Backoff attempt provenance is inconsistent.") + metadata, timing = event["metadata"], event["timing"] + if (not isinstance(metadata, dict) or set(metadata) != {"kind", "value"} + or metadata["kind"] not in ("seconds", "date", "date-rfc850", "missing", "invalid", "overlong") + or not valid_utc_timestamp(metadata["value"]) + or metadata["kind"] == "seconds" and (type(metadata["value"]) is not int or not 0 <= metadata["value"] <= 30) + or metadata["kind"] in ("missing", "invalid", "overlong") and metadata["value"] != 0 + or not isinstance(timing, dict) or set(timing) != {"received_at_utc", "not_before_utc", "server_delay_seconds"}): + raise fail("file-upload-evidence-invalid", "Retained Retry-After metadata is invalid.") + received, deadline = timing["received_at_utc"], timing["not_before_utc"] + seconds = timing["server_delay_seconds"] + if (received is not None and not valid_utc_timestamp(received) + or deadline is not None and (received is None or not valid_utc_timestamp(deadline)) + or seconds is not None and (type(seconds) is not int or not 0 <= seconds < 315537897600 + or metadata["kind"] not in ("seconds", "overlong")) + or metadata["kind"] == "seconds" and seconds not in (None, metadata["value"])): + raise fail("file-upload-evidence-invalid", "Retained backoff UTC timing is invalid.") + if deadline is not None: + if (event["origin"] == "unavailable" + or event["origin"] == "typed-metadata" and metadata["kind"] not in ("seconds", "date", "date-rfc850") + or metadata["kind"] == "overlong" and (seconds is None or seconds <= 30 or deadline != received + seconds) + or metadata["kind"] != "overlong" + and deadline != retry_after_not_before(RetryAfter(**metadata), received)): + raise fail("file-upload-evidence-invalid", "Retained backoff deadline contradicts original metadata.") + previous = digest(event) + return events + + def backoff_status(self): + events = self.backoff_events() + attempts = {name: value for name, value in self.records.items() + if re.fullmatch(r"[0-9]{4}-(retry-)?(attempt|result)\.json", name)} + if any(not isinstance(value, dict) for value in attempts.values()): + raise fail("file-upload-evidence-invalid", "Attempt/result evidence is malformed.") + # Legacy 429 or an interrupted receipt cannot prove what restriction was received. + unknown = "pending-request.json" in self.records or any( + name.endswith("result.json") and value.get("status") == 429 and "throttle_event" not in value + or name.endswith("attempt.json") and name.replace("attempt.json", "result.json") not in self.records + and name != self.active_attempt + for name, value in attempts.items() + ) + now = time.time() + if unknown or any(event["timing"]["not_before_utc"] is None for event in events.values()): + return {"status": "unresolved", "reason": "Original response timing is unavailable; no deadline is inferred."} + if not events: + return {"status": "clear"} + if (not valid_utc_timestamp(now) + or any(now < event["timing"]["received_at_utc"] for event in events.values())): + return {"status": "unresolved", "reason": "UTC clock is invalid or precedes retained response receipt."} + deadline = max(event["timing"]["not_before_utc"] for event in events.values()) + return {"status": "waiting" if now < deadline else "elapsed", "not_before_utc": deadline, + "seconds_remaining": max(0, deadline - now)} + + def require_backoff(self): + status = self.backoff_status() + if status["status"] in ("waiting", "unresolved"): + error = fail("file-upload-backoff-" + status["status"], + "Server backoff still applies; fresh approval does not waive it. " + + ("Wait until the retained UTC not-before time." if status["status"] == "waiting" + else "Original timing evidence or scoped operator recovery is required.")) + error.partial = "source-ack.json" in self.records + error.file_batch = _summary(self.ingestion, self.states(), self) + raise error + + def transport(self, raw): + if getattr(raw, "_file_journal_owner", None) is self: + return raw + + def invoke(method, url, token, **kwargs): + if self.network_failure is not None: + raise self.network_failure + self.require_backoff() + if len(self.backoff_events()) >= MAX_BACKOFF_RECORDS: + raise fail("file-upload-backoff-limit", "Backoff journal limit reached; no more requests.") + pending = {"version": "1.0", "plan_digest": digest(self.plan), "nonce": uuid.uuid4().hex, + "method": method, "url_digest": digest(url)} + self._write("pending-request.json", pending) + try: + response = raw(method, url, token, **kwargs) + except HelperFailure as error: + if error.http_status == 429: + self.record_backoff(method, url, error, None) + self.finish_request(error) + raise + if response.status == 429: + error = HelperFailure("azure-http-error", "Azure request failed with HTTP 429.", + blocked_at="execution", status=429, request_id=response.request_id, + retry_after=response.retry_after, recovery_deadline=response.recovery_deadline) + self.record_backoff(method, url, error, response.headers) + try: + self.finish_request() + except HelperFailure as persistence: + if method in ("PUT", "POST") and response.status in (200, 201): + return replace(response, ack_failure=persistence) + raise + return response + + invoke._file_journal_owner = self + return invoke + + def finish_request(self, original=None): + try: + path = self.directory / "pending-request.json" + expected = self.records["pending-request.json"] + retained = private_io.read_json(path) + if not isinstance(retained, dict) or retained.get("payload") != expected or retained.get("integrity") != digest(expected): + raise fail("file-upload-evidence-invalid", "Pending request receipt changed.") + path.unlink() + self.records.pop("pending-request.json") + except (OSError, HelperFailure) as cleanup: + error = original or HelperFailure("file-upload-receipt-failed", "Pending request evidence could not be finalized.", + blocked_at="local-persistence") + error.blocked_at = "local-persistence" + error.warnings.append("Pending request evidence remains unresolved; no further requests.") + self.network_failure = error + raise error from cleanup + + def record_backoff(self, method, url, error, headers): + metadata = error.retry_after + origin = "response-headers" if headers is not None else "native-http-error" + timing = retry_after_timing(headers) if headers is not None else error.retry_after_timing + if timing is None: + received = time.time() + received = received if valid_utc_timestamp(received) else None + known = isinstance(metadata, RetryAfter) and metadata.kind in ("seconds", "date", "date-rfc850") + timing = RetryAfterTiming(received, retry_after_not_before(metadata, received) if known else None, + metadata.value if known and metadata.kind == "seconds" else None) + origin = "typed-metadata" if known else "unavailable" + if not isinstance(metadata, RetryAfter): + metadata, origin = RetryAfter("missing"), "unavailable" + timing = RetryAfterTiming(timing.received_at_utc, None) + events = self.backoff_events() + attempt = self.active_attempt if method == "POST" and url == file_ingest._list_url(self.ingestion) else None + event = {"version": "1.0", "plan_digest": digest(self.plan), + "previous": digest(next(reversed(events.values()))) if events else None, + "method": method, "url_digest": digest(url), "attempt": attempt, + "attempt_digest": digest(self.records[attempt]) if attempt else None, + "request_id": ReadRecovery.safe_id(error.request_id) if error.request_id else None, + "metadata": metadata._asdict(), "timing": timing._asdict(), "origin": origin} + try: + self._write(f"backoff-{len(events):04}.json", event) + except HelperFailure as persistence: + error.blocked_at = "local-persistence" + error.warnings.extend(persistence.warnings) + error.warnings.append("Original HTTP 429 retained; backoff receipt persistence failed. No further requests.") + self.network_failure = error + raise error from persistence + + +def _file_id(value): + return isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}", value) is not None + + +def _proof(plan, record, item): + if isinstance(item, dict) and _file_id(item.get("fileId")) and file_ingest._matches(item, plan, record): + return {"file_id": item["fileId"], "record_digest": digest(record)} + return None + + +def _observe(plan, states, files): + names = {record["path"] for record in plan["files"]} + if any(item.get("fileName") not in names for item in files): + raise fail("server-inventory-conflict", "File inventory contains an unapproved identity.") + observed = {} + for index, record in enumerate(plan["files"]): + matches = [item for item in files if item.get("fileName") == record["path"]] + if len(matches) > 1 or matches and not file_ingest._matches({**matches[0], "errorMessage": None}, plan, record): + raise fail("file-record-conflict", "File inventory has duplicate or conflicting approved markers.") + state = states[index] + state["verification"] = {"accepted": "pending", "rejected": "failed"}.get(state["upload"], state["upload"]) + if matches: + item = matches[0] + proof = _proof(plan, record, item) + if (proof and state.get("file_proof") and proof != state["file_proof"] + or state.get("ack_file_id") and item.get("fileId") is not None + and item["fileId"] != state["ack_file_id"]): + raise fail("file-record-conflict", "Current File identity differs from the persisted upload ACK.") + state["verification"] = ("failed" if state["upload"] == "rejected" and state.get("status") != 429 + or item.get("errorMessage") is not None else + "confirmed" if proof else "unverified") + state["ingested"] = state["verification"] == "confirmed" + observed[index] = item + return observed + + +def _summary(plan, states, session): + counts = {name: 0 for name in ("accepted", "confirmed", "failed", "pending", "unverified", "not_attempted")} + files = [] + for record, state in zip(plan["files"], states): + counts["accepted"] += state["upload"] == "accepted" + verification = state.get("verification", {"accepted": "pending", "rejected": "failed"}.get(state["upload"], state["upload"])) + counts[verification] += 1 + files.append({"fileName": record["path"], "sha256": record["sha256"], "upload": state["upload"], + "verification": verification, "http_status": state.get("status"), + "request_ids": state["request_ids"][:3]}) + ingested = sum(bool(state.get("ingested")) for state in states) + return {"counts": counts, "files": files, "ingested": ingested, + "readiness": "ingested" if ingested == len(states) and counts["confirmed"] == len(states) else "partial", + "retrieval": "unverified", + **({"backoff": session.backoff_status()} if session else {}), + **({"upload_retry": { + "status": "blocked", "code": "file-upload-retry-safety-unproven", + "reason": "Unknown upload outcomes cannot be replayed; only a received File upload 429 permits one bounded in-operation retry.", + }} if counts["unverified"] else {}), + "resume": ("Plan newly approved upload-only continuation for never-attempted files; uncertain attempts are never replayed." + if session else "Original durable ACK/attempt evidence was not retained; uploads cannot safely resume from GET alone."), + "subset_handoff": "Report the proved ingested subset and remaining outcomes; subset isolation and KB retrieval are separate. KB changes need separate approval."} + + +def _progress(progress, plan, states): + summary = _summary(plan, states, None) + counts = summary["counts"] + progress.update("file-upload", uploads_acknowledged=counts["accepted"], files_verified=counts["confirmed"], + files_ingested=summary["ingested"], + files_reused=sum(state["upload"] == "not_attempted" and state.get("verification") == "confirmed" for state in states), + files_failed=counts["failed"], files_unverified=counts["unverified"], + files_not_attempted=counts["not_attempted"], files_pending=counts["pending"]) + + +def run_batch(document, *, token_provider, transport, progress, allow_new_uploads=False, session=None, + eligible=None, source_check=None, allow_upload_retry=True): + plan, fingerprint = document["plan"], document["_computed_fingerprint"] + root, records = file_ingest._validate_plan(plan) + states = session.states() if session else [{"upload": "not_attempted", "request_ids": []} for _ in records] + if session: + session.require_backoff() + transport = session.transport(transport) + created, reused, request_ids, warnings = [], [], [], [] + primary = None + stopped = False + retry_used = False + readback_failed = False + token = token_provider(SEARCH_AUDIENCE) + url = file_ingest._list_url(plan) + recovery = ReadRecovery(on_wait=progress.waiting) + progress.update("file-inventory") + try: + before, ids = file_ingest._list_files(url, token, transport=transport, recovery=recovery) + request_ids.extend(ids) + if session is None and file_ingest.inventory_digest(before) != plan["expected_server_inventory_digest"]: + raise fail("server-inventory-drift", "Approved initial file inventory changed.") + observed = _observe(plan, states, before) + if not allow_new_uploads and len(observed) != len(records): + raise fail("reused-source-upload-forbidden", "Generic reuse cannot authorize new uploads.") + except HelperFailure as failure: + failure.file_batch = _summary(plan, states, session) + failure.file_batch["request_ids"] = recovery.request_ids + raise + warnings.extend(recovery.warnings) + eligible = set(range(len(records))) if eligible is None else set(eligible) + after = None + for index, record in enumerate(records): + _progress(progress, plan, states) + state = states[index] + if index in observed: + reused.append({"fileId": observed[index].get("fileId"), "fileName": record["path"], "sha256": record["sha256"]}) + continue + if index not in eligible or state["upload"] != "not_attempted": + continue + try: + path = file_ingest._resolve_file(root, record) + content = path.read_bytes() + if (len(content) != record["size"] or "sha256:" + hashlib.sha256(content).hexdigest() != record["sha256"] + or path.stat().st_mtime_ns != record["mtime_ns"]): + raise fail("inventory-drift", "Approved corpus changed before upload.") + except OSError: + primary = primary or fail("inventory-unreadable", "Approved corpus became inaccessible.") + warnings.append("Batch stopped: approved corpus became inaccessible.") + stopped = True + break + except HelperFailure as failure: + primary = primary or failure + warnings.append(f"Batch stopped ({failure.code}); no further uploads were attempted.") + stopped = True + break + body, boundary = file_ingest._multipart(plan, record, content, fingerprint) + for attempt in range(2): + try: + if attempt: + file_ingest._resolve_file(root, record) + if session: + session.attempt(index, retry=bool(attempt)) + except HelperFailure as failure: + primary = primary or failure + warnings.append(f"Upload attempt evidence/drift check failed ({failure.code}); no further requests.") + stopped = True + break + state.update(upload="unverified", verification="unverified", status=None) + _progress(progress, plan, states) + response = None + transport_failed = False + try: + with progress.processing_file(index + 1, len(records), attempt + 1): + response = transport("POST", url, token, body=body, + headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, + follow_redirects=False, timeout=UPLOAD_TIMEOUT, + response_deadline=time.monotonic() + UPLOAD_TIMEOUT, + max_response_bytes=1024 * 1024) + except HelperFailure as caught: + transport_failed = True + failure = caught + failure_status, failure_id = failure.http_status, failure.request_id + else: + failure_status, failure_id = response.status, response.request_id + failure = HelperFailure("upload-failed", f"Upload returned HTTP {response.status}.", + blocked_at="execution", status=response.status, request_id=response.request_id, + retry_after=response.retry_after, recovery_deadline=response.recovery_deadline) + if failure_id: + safe_id = ReadRecovery.safe_id(failure_id) + failure.request_id = safe_id + state["request_ids"].append(safe_id) + request_ids.append(safe_id) + state["status"] = failure_status + proof = None + if failure_status in (200, 201): + state.update(upload="accepted", verification="pending") + created.append({"fileName": record["path"], "sha256": record["sha256"]}) + if response is not None: + proof = _proof(plan, record, response.body) + if isinstance(response.body, dict) and _file_id(response.body.get("fileId")): + state["ack_file_id"] = response.body["fileId"] + elif _rejected(failure_status): + state.update(upload="rejected", verification="failed") + try: + if response is not None and response.ack_failure is not None: + raise response.ack_failure + if session: + session.result(index, state["upload"], failure_status, failure_id, retry=bool(attempt), + proof=proof, file_id=state.get("ack_file_id")) + except HelperFailure as persistence: + primary = primary or (failure if failure_status not in (200, 201) else persistence) + primary.warnings.extend(persistence.warnings) + warnings.append("Upload journal persistence failed; no further requests were issued.") + stopped = True + break + if failure_status in (200, 201) and not transport_failed: + if proof: + state.update(file_proof=proof, ingested=True, verification="confirmed") + elif isinstance(response.body, dict) and ( + response.body.get("errorMessage") is not None + or "fileName" in response.body and not file_ingest._matches(response.body, plan, record) + ): + primary = primary or fail("upload-metadata-unverified", "Upload ACK metadata conflicts with the approved file.") + stopped = True + _progress(progress, plan, states) + break + primary = primary or failure + if failure_status == 415: + break + _progress(progress, plan, states) + if failure_status == 429 and not retry_used and allow_upload_retry and failure.blocked_at != "local-persistence": + retry_used = True + recovery = ReadRecovery(on_wait=progress.waiting) + try: + recovery.delay(failure) + if source_check is not None: + source_check(recovery, token) + inventory, _ = file_ingest._list_files(url, token, transport=transport, recovery=recovery) + seen = _observe(plan, states, inventory) + if index in seen: + if state["verification"] != "confirmed": + raise fail("file-upload-retry-conflict", "Existing file does not prove the approved completed upload.") + created.append({"fileName": record["path"], "sha256": record["sha256"]}) + break + file_ingest._resolve_file(root, record) + if time.monotonic() >= recovery.deadline: + raise fail("read-recovery-budget-exhausted", "Retry preflight exceeded its complete read budget.") + warnings.append("File upload HTTP 429: one same-operation retry after bounded backoff; no source recreation.") + except HelperFailure as read_failure: + warnings.append(f"File retry stopped ({read_failure.code}; HTTP {read_failure.http_status}).") + warnings.extend(read_failure.warnings) + stopped = True + finally: + request_ids.extend(recovery.request_ids) + warnings.extend(recovery.diagnostics()) + if not stopped: + continue + else: + stopped = True + warnings.append(f"Batch stopped ({failure.code}; HTTP {failure_status}); no further uploads.") + if failure.blocked_at != "local-persistence" and ( + failure_status in (408, 409) or failure_status is None or failure_status >= 500 + ): + recovery = ReadRecovery(on_wait=progress.waiting) + try: + after, _ = file_ingest._list_files(url, token, transport=transport, recovery=recovery) + _observe(plan, states, after) + if state["verification"] == "confirmed": + created.append({"fileName": record["path"], "sha256": record["sha256"]}) + except HelperFailure as read_failure: + warnings.append(f"File readback stopped ({read_failure.code}; HTTP {read_failure.http_status}).") + warnings.extend(read_failure.warnings) + request_ids.extend(recovery.request_ids) + warnings.extend(recovery.diagnostics()) + break + if stopped: + break + if session: + session.active_attempt = None + _progress(progress, plan, states) + progress.update("file-readback") + if all(state["upload"] == "rejected" and state["verification"] != "confirmed" for state in states): + stopped = True + if after is None and not stopped: + recovery = ReadRecovery(on_wait=progress.waiting) + try: + after, ids = file_ingest._list_files(url, token, transport=transport, recovery=recovery) + _observe(plan, states, after) + except HelperFailure as failure: + primary = primary or failure + readback_failed = True + warnings.append(f"File readback stopped ({failure.code}; HTTP {failure.http_status}).") + request_ids.extend(recovery.request_ids) + warnings.extend(recovery.diagnostics()) + batch = _summary(plan, states, session) + batch["request_ids"] = request_ids[:804] + batch["upload_retry_used"] = retry_used + progress.update("file-readback", files_verified=batch["counts"]["confirmed"], files_failed=batch["counts"]["failed"], + files_pending=batch["counts"]["pending"], files_unverified=batch["counts"]["unverified"], + files_not_attempted=batch["counts"]["not_attempted"], files_ingested=batch["ingested"]) + if batch["counts"]["confirmed"] != len(records) or readback_failed: + failure = primary or fail("readback-mismatch", "Some approved files remain pending, failed or unverified.") + uncertain = [{"action": "upload-unverified", "type": "knowledge-source-file", + "fileName": record["path"], "sha256": record["sha256"]} + for record, state in zip(records, states) + if state["upload"] == "unverified" and state["verification"] != "confirmed"] + failure.writes = created + uncertain + failure.writes + failure.resources_remaining = file_ingest._remaining_files(created) + uncertain + failure.resources_remaining + failure.partial = bool(created) or any(s["upload"] == "unverified" for s in states) or ( + failure.partial and not _rejected(failure.http_status)) + failure.warnings.extend(warnings) + failure.file_batch = batch + raise failure + verified = [{"fileId": item.get("fileId"), "fileName": record["path"], "sha256": record["sha256"], "size": record["size"]} + for record in records for item in after or before if item.get("fileName") == record["path"]] + return {"status": "completed", "outcome": plan.get("outcome", "file-knowledge-source-ingestion"), + "approved_plan": {"fingerprint": fingerprint, "confirmed": True}, + "resources": {"created": created, "reused": reused, "updated": [], "skipped": []}, + "api_contracts": [{"operation": "upload-file", "version": file_ingest.API_VERSION, "preview": True}], + "data_movement": {"boundary": {"local_root_digest": digest(str(root))}, "result": "Approved direct File uploads"}, + "auth": {"mode": "entra-user", "principals": []}, "rbac": plan.get("rbac", {"assignments": []}), + "network": plan.get("network", {"posture": "preserved", "evidence": None}), + "verification": {"readback": verified, "request_ids": request_ids, "server_inventory_digest": file_ingest.inventory_digest(after or before), + "idempotency": "Only a received File upload 429 permits one bounded same-operation retry; unknown outcomes are never replayed."}, + "warnings": warnings, "file_batch": batch, + "ownership": {"run_owned": created, "reused_not_owned": reused, "owner": plan["owner"]}, + "cleanup": {"status": "not-requested", "separate_confirmation_required": True}} + + +def _source_check(session, token, transport, recovery): + try: + from . import source_vector + except ImportError: + import source_vector + plan = session.plan + file_ingest._validate_plan(session.ingestion) + etag = session.require_ack() + current, _ = search_reconcile._get(search_reconcile.resource_url(plan["source"]), token, + transport=source_vector.guard_readback_transport(plan, transport), recovery=recovery) + if (current is None or current.get("@odata.etag") != etag + or not search_reconcile.definitions_match(plan["source"]["desired"], current)): + raise fail("file-upload-source-drift", "Current source does not match original acknowledged identity/version/definition.") + return current + + +def _verify(session, token_provider, transport, context_provider): + try: + from . import file_source, file_cu_mi + except ImportError: + import file_source, file_cu_mi + session.require_ack() + session.states() + session.require_backoff() + transport = session.transport(transport) + context = context_provider() + cu_ingestion_auth.validate_context(context) + if context != session.seed["context"]: + raise fail("file-upload-context-drift", "Original tenant/subscription/principal changed.") + plan = session.plan + if "content_understanding" in plan: + state, _ = file_source.read_content_understanding(plan["content_understanding"], token_provider=token_provider, transport=transport) + if not file_source._cu_states_match(state, plan["cu_resource_state"]): + raise fail("file-upload-auth-drift", "Original CU account/auth/network state changed.") + if plan["source"].get("ai_services_managed_identity"): + binding, _ = file_cu_mi.read_binding(plan["content_understanding"], plan["source"]["endpoint"], + token_provider=token_provider, transport=transport) + if binding != plan["cu_identity_state"]: + raise fail("file-upload-auth-drift", "Original Search MI/CU role/network binding changed.") + return _source_check(session, token_provider(SEARCH_AUDIENCE), transport, ReadRecovery()) + + +def plan_resume(request, *, token_provider=azure_cli_token, transport=http_request, + context_provider=cu_ingestion_auth.account_context): + if not isinstance(request, dict) or not isinstance(request.get("receipt_directory"), str): + raise fail("file-upload-input-invalid", "Use an object with the original private receipt directory.") + require_allowed_fields(request, {"schema_version", "receipt_directory"}, label="upload resume request") + if request.get("schema_version") != "1.0": + raise fail("file-upload-input-invalid", "Use the supported resume request schema.") + session = Session(request.get("receipt_directory")) + transport = session.transport(transport) + _verify(session, token_provider, transport, context_provider) + states = session.states() + inventory, _ = file_ingest._list_files(file_ingest._list_url(session.ingestion), token_provider(SEARCH_AUDIENCE), + transport=transport, recovery=ReadRecovery()) + observed = _observe(session.ingestion, states, inventory) + eligible = [i for i, state in enumerate(states) if state["upload"] == "not_attempted" and i not in observed] + plan = {"operation": "resume-file-uploads", "version": "1.0", "receipt_directory": str(session.directory), + "original_plan_digest": digest(session.plan), "creation_ack_digest": digest(session.records["source-ack.json"]), + "journal_digest": digest(session.records), "eligible": eligible, "owner": session.plan["owner"], "cleanup_approved": False} + return {"status": "planned", "execution_input": {"schema_version": "1.0", "plan": plan, + "approval": {"confirmed": False, "fingerprint": digest(plan)}}, + "approval_summary": {"execution_required": bool(eligible), "mutation_approval_required": bool(eligible), "uploads": len(eligible)}, + "file_batch": _summary(session.ingestion, states, session), "writes_performed": [], + "next_step": "Approve only never-attempted uploads. No eligible files: retain this observation; do not replay uncertain files."} + + +@reporting("file-source") +def execute(document, *, token_provider=azure_cli_token, transport=http_request, + context_provider=cu_ingestion_auth.account_context, progress=None): + if not isinstance(document, dict) or not isinstance(document.get("plan"), dict): + raise fail("file-upload-input-invalid", "Use the newly approved upload-only envelope.") + require_allowed_fields(document, {"schema_version", "plan", "approval", "_computed_fingerprint"}, label="upload resume envelope") + plan = document.get("plan") + reject_secrets(document) + require_allowed_fields(plan, {"operation", "version", "receipt_directory", "original_plan_digest", "creation_ack_digest", + "journal_digest", "eligible", "owner", "cleanup_approved"}, label="upload resume plan") + if (document.get("schema_version") != "1.0" or plan.get("operation") != "resume-file-uploads" or plan.get("version") != "1.0" + or set(plan) != {"operation", "version", "receipt_directory", "original_plan_digest", "creation_ack_digest", + "journal_digest", "eligible", "owner", "cleanup_approved"} + or not isinstance(document.get("approval"), dict) or document["approval"].get("confirmed") is not True + or plan.get("cleanup_approved") is not False or document.get("approval") != {"confirmed": True, "fingerprint": digest(plan)} + or document.get("_computed_fingerprint") != digest(plan)): + raise fail("file-upload-approval-missing", "A newly approved unchanged upload-only plan is required.") + session = Session(plan.get("receipt_directory")) + transport = session.transport(transport) + if (plan["original_plan_digest"] != digest(session.plan) or plan["journal_digest"] != digest(session.records) + or plan["creation_ack_digest"] != digest(session.records["source-ack.json"]) or plan["owner"] != session.plan["owner"] + or not isinstance(plan["eligible"], list) or any(type(i) is not int or not 0 <= i < len(session.ingestion["files"]) for i in plan["eligible"]) + or len(set(plan["eligible"])) != len(plan["eligible"])): + raise fail("file-upload-plan-drift", "Original approval/ACK/journal changed; refresh the upload-only plan.") + states = session.states() + if any(states[i]["upload"] != "not_attempted" for i in plan["eligible"]): + raise fail("file-upload-replay-forbidden", "Attempted files cannot be submitted again, even after an empty inventory.") + progress.update("source-reconciliation") + try: + _verify(session, token_provider, transport, context_provider) + result = run_batch({"plan": session.ingestion, "_computed_fingerprint": digest(session.plan)}, token_provider=token_provider, + transport=transport, progress=progress, allow_new_uploads=True, session=session, eligible=plan["eligible"], + source_check=lambda recovery, token: _source_check(session, token, transport, recovery)) + except HelperFailure as failure: + if failure.file_batch is None: + failure.file_batch = _summary(session.ingestion, states, session) + failure.partial = True + failure.resources_remaining.insert(0, {"type": "knowledge-source", "name": session.plan["source"]["name"]}) + raise + result["approved_plan"] = document["approval"] + result["original_run"] = {"plan_digest": digest(session.plan), "creation_ack_digest": plan["creation_ack_digest"], + "source_retained": session.plan["source"]["name"]} + return result + + +def main(argv=None): + try: + from .private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode + except ImportError: + from private_artifacts import add_execution_output_argument, emit_plan_result, validate_execution_output_mode + parser = argparse.ArgumentParser() + modes = parser.add_mutually_exclusive_group(required=True) + modes.add_argument("--plan", type=Path) + modes.add_argument("--input", type=Path) + add_execution_output_argument(parser) + add_progress_argument(parser) + args = parser.parse_args(argv) + fingerprint = None + try: + validate_execution_output_mode(args) + if args.plan: + emit_plan_result(plan_resume(private_io.read_json(args.plan)), args.execution_output) + return 0 + document, _, fingerprint = load_approved_input(args.input) + document["_computed_fingerprint"] = fingerprint + result = execute(document, progress=Progress("file-source", enabled=args.progress)) + except HelperFailure as failure: + result = blocked_result(failure, outcome="resume-file-uploads", fingerprint=fingerprint) + result["safe_next_decision"] = "Retain the source. Missing/changed original evidence blocks upload continuation, not retention; no creation replay or cleanup workaround." + emit_result(result) + return 3 if result["status"] == "partial" else 2 + emit_result(result) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/hosted_connect.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/hosted_connect.py new file mode 100644 index 00000000..524e3306 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/hosted_connect.py @@ -0,0 +1,488 @@ +"""GET-only assessment of an existing Hosted agent's toolbox connection. + +Promotion is deliberately unavailable until a conditional shared-default update +contract is verified. No intermediate resources are created while that gate blocks. +""" +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path +from urllib.parse import parse_qs, quote, unquote, urlsplit + +try: + from . import _prompt_read as read + from ._bootstrap_io import read_json, run_cli + from ._common import ( + MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, HelperFailure, azure_cli_token, + blocked_result, digest, emit_result, http_request, reject_secrets, + ) + from .prompt_connect import _project_identity, _project_endpoint, PROJECT_ID +except ImportError: + import _prompt_read as read + from _bootstrap_io import read_json, run_cli + from _common import ( + MANAGEMENT_AUDIENCE, SEARCH_AUDIENCE, HelperFailure, azure_cli_token, + blocked_result, digest, emit_result, http_request, reject_secrets, + ) + from prompt_connect import _project_identity, _project_endpoint, PROJECT_ID + +AI_AUDIENCE = "https://ai.azure.com" +FOUNDRY_USER = "53ca6127-db72-4b80-b1b0-d745d6d5456d" +OUTCOME = "assess-existing-hosted-toolbox" +FIELDS = { + "schema_version", "scope", "project", "search_service", "knowledge_base", + "agent_name", "agent_version", "toolbox_name", "tool_label", "connection_name", + "reader_assignment_id", "project_assignment_id", "retention_owner", +} +OPTIONAL = {"known_agents", "supported_question", "unrelated_question"} + + +class ProjectSelectionFailure(HelperFailure): + def __init__(self, candidates): + super().__init__( + "project-ambiguous" if candidates else "project-absent", + "Scoped project name did not resolve uniquely; select an exact account/project without widening scope.", + blocked_at="input-resolution", + ) + self.candidates = candidates + + +def fail(code, message, **metadata): + return read.fail(code, message, **metadata) + + +def name(value): + return isinstance(value, str) and read.NAME.fullmatch(value) is not None + + +def url(value): + try: + parsed = urlsplit(value) + if (parsed.scheme != "https" or not parsed.hostname or parsed.username + or parsed.password or parsed.port not in (None, 443) or parsed.fragment): + raise ValueError() + return parsed + except ValueError as error: + raise fail("selection-invalid", "Use an exact public-cloud HTTPS endpoint without credentials or fragments.") from error + + +def validate(request): + if not isinstance(request, dict): + raise fail("input-schema-invalid", "Hosted assessment requires a resolved intent object.") + reject_secrets(request) + if set(request) - FIELDS - OPTIONAL or not FIELDS <= set(request) or request["schema_version"] != "1.0": + raise fail("input-schema-invalid", "Use only the documented Hosted assessment decisions.") + scope = request["scope"] + if (not isinstance(scope, dict) or not {"subscription_id", "resource_group"} <= set(scope) + or set(scope) - {"subscription_id", "resource_group", "account_name"} + or any(not name(value) for value in scope.values())): + raise fail("scope-invalid", "Select one subscription/resource group and optionally one account; no automatic widening.") + for key in ("agent_name", "agent_version", "toolbox_name", "tool_label", "connection_name"): + if not name(request[key]): + raise fail("input-schema-invalid", "Select bounded exact agent/version, toolbox and connection names.") + if request["agent_version"].casefold() in {"latest", "default"}: + raise fail("agent-version-unresolved", "Pin the observed agent version, never latest.") + for key in ("project", "search_service", "knowledge_base"): + value = request[key] + if not isinstance(value, str) or not 1 <= len(value) <= 2048: + raise fail("selection-invalid", "Supply a name, exact resource ID or documented endpoint.") + if value.startswith("https:"): + parsed = url(value) + if key == "project": + _project_endpoint(value) + if (not name(parsed.hostname.removesuffix(".services.ai.azure.com")) + or not name(unquote(parsed.path.rstrip("/").rsplit("/", 1)[-1]))): + raise fail("selection-invalid", "The project endpoint must identify a valid account/project name.") + elif key == "search_service": + if (not parsed.hostname.endswith(".search.windows.net") or parsed.path not in ("", "/") + or parsed.query): + raise fail("selection-invalid", "Select one Search service endpoint, not an index.") + elif (not parsed.hostname.endswith(".search.windows.net") + or re.fullmatch(r"/knowledgebases/[A-Za-z0-9_.-]+/mcp", parsed.path) is None + or parse_qs(parsed.query) != {"api-version": ["2026-08-01-preview"]}): + raise fail("selection-invalid", "Select an exact preview KB MCP endpoint.") + elif value.startswith("/"): + pattern = PROJECT_ID if key == "project" else read.SEARCH_ID if key == "search_service" else None + if pattern is None or pattern.fullmatch(value) is None: + raise fail("selection-invalid", "The supplied resource ID does not identify the selected resource kind.") + elif not name(value): + raise fail("selection-invalid", "The selected name is malformed.") + if key == "search_service" and not value.startswith("/"): + service = url(value).hostname.removesuffix(".search.windows.net") if value.startswith("https:") else value + resource_id = (f"/subscriptions/{scope['subscription_id']}/resourceGroups/{scope['resource_group']}" + "/providers/Microsoft.Search/searchServices/" + service) + if read.SEARCH_ID.fullmatch(resource_id) is None: + raise fail("selection-invalid", "Use a valid Search service name, not a display label or index.") + for key in ("reader_assignment_id", "project_assignment_id"): + value = request[key] + if (not isinstance(value, str) or len(value) > 2048 + or re.fullmatch(r"/subscriptions/[^/?#\s]+/resourceGroups/[^?#\s]+/providers/" + r"Microsoft.Authorization/roleAssignments/[0-9a-fA-F-]+", value, re.I) is None + or not read.GUID.fullmatch(value.rsplit("/", 1)[-1])): + raise fail("role-selection-invalid", "Select exact resource-scoped role-assignment IDs, not principals to invent.") + if (not isinstance(request["retention_owner"], str) or not request["retention_owner"].strip() + or len(request["retention_owner"]) > 256): + raise fail("owner-unresolved", "Name the retained connection/toolbox-version owner; cleanup is unsupported.") + agents = request.get("known_agents", []) + if (not isinstance(agents, list) or len(agents) > 20 + or any(not isinstance(item, dict) or set(item) != {"name", "version"} + or not name(item["name"]) or not name(item["version"]) + or item["version"].casefold() in {"latest", "default"} for item in agents)): + raise fail("known-bindings-invalid", "Supply at most 20 known exact agent/version bindings; no exclusive-consumer inference.") + for key in ("supported_question", "unrelated_question"): + if key in request and (not isinstance(request[key], str) or not request[key].strip() or len(request[key]) > 4096): + raise fail("acceptance-invalid", "Questions are optional resolved candidates, never implicit invocation approval.") + + +class Reads: + def __init__(self, token_provider, transport): + self.token_provider, self.transport = token_provider, transport + self.tokens, self.request_ids = {}, [] + self.last_request_id = None + + def get(self, endpoint, audience, *, absent=False, label="resource"): + if audience not in self.tokens: + self.tokens[audience] = self.token_provider(audience) + value, ids = read.get_object(endpoint, self.tokens[audience], self.transport, absent=absent, label=label) + self.request_ids.extend(ids) + self.last_request_id = ids[-1] if ids else None + return value + + def collection(self, endpoint): + initial = urlsplit(endpoint) + result, seen = [], set() + while endpoint: + current = url(endpoint) + if (endpoint in seen or len(seen) >= 20 or current.netloc != initial.netloc + or current.path != initial.path + or parse_qs(current.query).get("api-version") != parse_qs(initial.query).get("api-version")): + raise fail("inventory-unverified", "Scoped inventory continuation is unsafe or exceeds 20 pages.") + seen.add(endpoint) + page = self.get(endpoint, MANAGEMENT_AUDIENCE, label="scoped-inventory") + items = page.get("value") + if not isinstance(items, list) or any(not isinstance(item, dict) for item in items): + raise fail("inventory-unverified", "Scoped inventory is malformed; no inferred absence.") + result.extend(items) + if len(result) > 100: + raise fail("inventory-limit", "Scoped inventory exceeds 100 resources; select an exact parent instead.") + endpoint = page.get("nextLink") + if endpoint is not None and not isinstance(endpoint, str): + raise fail("inventory-unverified", "Scoped continuation must be a URL.") + return result + + +def resolve(request, reads): + scope = request["scope"] + prefix = f"/subscriptions/{scope['subscription_id']}/resourceGroups/{scope['resource_group']}" + account_prefix = prefix + "/providers/Microsoft.CognitiveServices/accounts/" + selection = request["project"] + if selection.startswith("/"): + project_id = selection + elif selection.startswith("https:"): + parsed = url(selection) + project_id = account_prefix + parsed.hostname.removesuffix(".services.ai.azure.com") + "/projects/" + unquote(parsed.path.rstrip("/").rsplit("/", 1)[-1]) + elif scope.get("account_name"): + project_id = account_prefix + scope["account_name"] + "/projects/" + selection + else: + accounts = reads.collection(MANAGEMENT_AUDIENCE + account_prefix.rstrip("/") + "?api-version=2025-06-01") + if len(accounts) > 20: + raise fail("account-selection-required", "Select one account; do not enumerate more than 20 accounts for a project name.") + matches, seen = [], set() + for account in accounts: + account_id = account.get("id", "") + if (not isinstance(account_id, str) or not account_id.casefold().startswith(account_prefix.casefold()) + or not name(account_id[len(account_prefix):]) or account_id.casefold() in seen): + raise fail("inventory-unverified", "Account inventory contains a duplicate or out-of-scope resource.") + seen.add(account_id.casefold()) + projects_seen = set() + for project in reads.collection(MANAGEMENT_AUDIENCE + account_id + "/projects?api-version=" + read.PROJECT_API): + candidate = project.get("id", "") + if (not isinstance(candidate, str) or PROJECT_ID.fullmatch(candidate) is None + or not candidate.casefold().startswith((account_id + "/projects/").casefold()) + or candidate.casefold() in projects_seen): + raise fail("inventory-unverified", "Project inventory contains an unverified identity.") + projects_seen.add(candidate.casefold()) + if candidate.rsplit("/", 1)[-1].casefold() == selection.casefold(): + matches.append(candidate) + if len(matches) != 1: + raise ProjectSelectionFailure(matches) + project_id = matches[0] + match = PROJECT_ID.fullmatch(project_id) + if match is None: + raise fail("selection-invalid", "The resolved project resource identity is malformed.") + endpoint = f"https://{match['account']}.services.ai.azure.com/api/projects/{quote(match['project'], safe='')}" + _project_identity({"project_resource_id": project_id, "project_endpoint": endpoint}) + selection = request["search_service"] + if selection.startswith("/"): + search_id = selection + else: + service = url(selection).hostname.removesuffix(".search.windows.net") if selection.startswith("https:") else selection + search_id = prefix + "/providers/Microsoft.Search/searchServices/" + service + match = read.SEARCH_ID.fullmatch(search_id) + if match is None: + raise fail("selection-invalid", "The selected Search service identity is invalid.") + search_endpoint = "https://" + match["name"].lower() + ".search.windows.net" + selection = request["knowledge_base"] + if selection.startswith("https:"): + parsed = url(selection) + if parsed.hostname != urlsplit(search_endpoint).hostname: + raise fail("kb-binding-invalid", "The KB endpoint belongs to another selected Search service.") + kb_name = parsed.path.split("/")[2] + else: + kb_name = selection + return project_id, endpoint, search_id, search_endpoint, kb_name + + +def role(reads, assignment_id, scope, principal, role_id): + prefix = scope + "/providers/Microsoft.Authorization/roleAssignments/" + if (not assignment_id.casefold().startswith(prefix.casefold()) + or not read.GUID.fullmatch(assignment_id[len(prefix):])): + raise fail("role-scope-invalid", "Role assignment must use the exact selected resource scope.") + value = reads.get(MANAGEMENT_AUDIENCE + assignment_id + "?api-version=" + read.ROLE_API, + MANAGEMENT_AUDIENCE, label="role-assignment") + properties = value.get("properties") + definitions = { + "/providers/microsoft.authorization/roledefinitions/" + role_id, + "/subscriptions/" + scope.split("/")[2].lower() + "/providers/microsoft.authorization/roledefinitions/" + role_id, + } + if (str(value.get("id", "")).casefold() != assignment_id.casefold() or not isinstance(properties, dict) + or str(properties.get("principalId", "")).lower() != principal + or str(properties.get("scope", "")).casefold() != scope.casefold() + or str(properties.get("roleDefinitionId", "")).casefold() not in definitions + or properties.get("principalType", "ServicePrincipal") != "ServicePrincipal" + or properties.get("condition") not in (None, "")): + raise fail("hosted-runtime-role-unverified", "Require exact-scope grants to the published Hosted principal, not the project or blueprint.") + return value + + +def agent(reads, endpoint, agent_name, version): + value = reads.get(endpoint + f"/agents/{quote(agent_name, safe='')}/versions/{quote(version, safe='')}?api-version=v1", + AI_AUDIENCE, label="hosted-agent") + if value.get("name") != agent_name or str(value.get("version")) != version or not isinstance(value.get("definition"), dict): + raise fail("agent-identity-unverified", "Read the exact observed agent/version; never select latest.") + return value + + +def toolbox_binding(environment, endpoint, toolbox_name): + """Match the first-party FoundryToolbox environment resolver, not arbitrary code.""" + if not isinstance(environment, dict): + return None + consumer = endpoint + "/toolboxes/" + toolbox_name + "/mcp?api-version=v1" + if "TOOLBOX_ENDPOINT" in environment: + return "endpoint" if environment["TOOLBOX_ENDPOINT"] == consumer else None + project = environment.get("FOUNDRY_PROJECT_ENDPOINT") + if (isinstance(project, str) and project.rstrip("/") == endpoint + and environment.get("TOOLBOX_NAME") == toolbox_name): + return "name" + return None + + +def assess(request, *, token_provider=azure_cli_token, transport=http_request, cli=run_cli): + validate(request) + scope = request["scope"] + selected_project = request["project"] + context_id = selected_project if selected_project.startswith("/") else "/subscriptions/" + scope["subscription_id"] + context = read.cli_context(context_id, cli=cli) + reads = Reads(token_provider, transport) + project_id, endpoint, search_id, search_endpoint, kb_name = resolve(request, reads) + project = reads.get(MANAGEMENT_AUDIENCE + project_id + "?api-version=" + read.PROJECT_API, MANAGEMENT_AUDIENCE, label="project") + properties = project.get("properties", {}) + endpoints = properties.get("endpoints") if isinstance(properties, dict) else None + if (str(project.get("id", "")).casefold() != project_id.casefold() or not isinstance(properties, dict) + or str(properties.get("provisioningState", "")).lower() != "succeeded" + or not isinstance(endpoints, dict) or endpoint not in endpoints.values()): + raise fail("project-unverified", "The exact ready project endpoint must be confirmed remotely.") + selected = agent(reads, endpoint, request["agent_name"], request["agent_version"]) + definition, identity = selected["definition"], selected.get("instance_identity") + other_principals = [ + value.get("principalId") or value.get("principal_id") + for value in (project.get("identity"), selected.get("blueprint")) + if isinstance(value, dict) + ] + if (definition.get("kind") != "hosted" or selected.get("status") != "active" + or not isinstance(identity, dict) or identity.get("status") != "active" + or not isinstance(identity.get("principal_id"), str) or not read.GUID.fullmatch(identity["principal_id"]) + or not isinstance(identity.get("client_id"), str) or not read.GUID.fullmatch(identity["client_id"]) + or identity["principal_id"].casefold() in { + value.casefold() for value in other_principals if isinstance(value, str) + }): + raise fail("hosted-principal-unverified", "Require an active published Hosted agent and observed instance principal.") + principal = identity["principal_id"].lower() + consumer = endpoint + "/toolboxes/" + request["toolbox_name"] + "/mcp?api-version=v1" + environment = definition.get("environment_variables") + binding_mode = toolbox_binding(environment, endpoint, request["toolbox_name"]) + if binding_mode is None: + raise fail("hosted-runtime-change-required", "Observed settings do not establish this unversioned FoundryToolbox binding. The runtime owner must verify custom/overridden behavior or the actual configuration change before requesting source; a version-pinned developer endpoint is not equivalent.") + search = reads.get(MANAGEMENT_AUDIENCE + search_id + "?api-version=" + read.SEARCH_API, MANAGEMENT_AUDIENCE, label="search") + properties = search.get("properties") + if (str(search.get("id", "")).casefold() != search_id.casefold() or not isinstance(properties, dict) + or str(properties.get("status", "")).lower() not in {"running", "provisioning", "degraded"} + or str(properties.get("provisioningState", "")).lower() not in {"succeeded", "provisioning"}): + raise fail("search-operation-blocked", "Search failed/deleting/disabled/unresolved state blocks this assessment.") + kb = reads.get(search_endpoint + "/knowledgebases('" + quote(kb_name, safe="") + "')?api-version=2026-08-01-preview", + SEARCH_AUDIENCE, absent=True, label="knowledge-base") + if kb is None: + raise fail("knowledge-base-absent", "The exact KB is absent; Search indexes are not KB evidence.", + status=404, request_id=reads.last_request_id) + _, profile = read.kb_state(kb, kb_name) + reader = role(reads, request["reader_assignment_id"], search_id, principal, read.READER_ROLE) + project_role = role(reads, request["project_assignment_id"], project_id, principal, FOUNDRY_USER) + toolbox_url = endpoint + "/toolboxes/" + request["toolbox_name"] + toolbox = reads.get(toolbox_url + "?api-version=v1", AI_AUDIENCE, label="toolbox") + default = toolbox.get("default_version") + if (toolbox.get("name") != request["toolbox_name"] or not name(default) + or default.casefold() in {"latest", "default"}): + raise fail("toolbox-default-unverified", "Read the exact toolbox and its current default version.") + version = reads.get(toolbox_url + "/versions/" + default + "?api-version=v1", AI_AUDIENCE, label="toolbox-version") + tools = version.get("tools") + if (version.get("name") != request["toolbox_name"] or str(version.get("version")) != default + or not isinstance(tools, list) or len(tools) > 200 or any(not isinstance(tool, dict) for tool in tools)): + raise fail("toolbox-version-unverified", "The default immutable version and its complete tool list must be verified.") + matching = [tool for tool in tools if tool.get("server_label") == request["tool_label"]] + if len(matching) > 1: + raise fail("toolbox-binding-ambiguous", "Multiple selected KB tool labels require explicit reconciliation.") + if matching: + tool = matching[0] + allowed = tool.get("allowed_tools") or [] + allowed = allowed.get("tool_names", []) if isinstance(allowed, dict) else allowed + if (tool.get("type") != "mcp" or not isinstance(allowed, list) + or any(not isinstance(item, str) for item in allowed) + or not isinstance(tool.get("headers") or {}, dict) + or not isinstance(tool.get("server_url"), str) or len(tool["server_url"]) > 2048 + or not isinstance(tool.get("project_connection_id"), str)): + raise fail("toolbox-binding-unverified", "The selected MCP tool definition is malformed or has another type.") + if tool.get("authorization") is not None or tool.get("connector_id") is not None or tool.get("headers"): + raise fail("toolbox-auth-unverified", "Inline authorization, connectors or custom headers are outside this agentic-identity recipe; preserve them rather than silently replacing their auth.") + if "allowed_tools" in tool and tool["allowed_tools"] is not None and "knowledge_base_retrieve" not in allowed: + raise fail("toolbox-policy-unverified", "The explicit tool filter does not establish access to knowledge_base_retrieve; review the policy delta separately, never silently widen it.") + old_endpoint = url(tool["server_url"]) + api = parse_qs(old_endpoint.query) + reference = tool["project_connection_id"] + prefix = project_id + "/connections/" + if (not old_endpoint.hostname.endswith(".search.windows.net") + or re.fullmatch(r"/knowledgebases/[A-Za-z0-9_.-]+/mcp", old_endpoint.path) is None + or set(api) != {"api-version"} or len(api["api-version"]) != 1 + or re.fullmatch(r"\d{4}-\d{2}-\d{2}(?:-preview)?", api["api-version"][0]) is None + or not (name(reference) or (reference.casefold().startswith(prefix.casefold()) + and name(reference[len(prefix):])))): + raise fail("toolbox-binding-unverified", "The selected label is not a verified Search KB MCP binding.") + target = search_endpoint + "/knowledgebases/" + kb_name + "/mcp?api-version=2026-08-01-preview" + connection_url = MANAGEMENT_AUDIENCE + project_id + "/connections/" + request["connection_name"] + "?api-version=" + read.PROJECT_API + connection = reads.get(connection_url, MANAGEMENT_AUDIENCE, absent=True, label="connection") + if connection is not None: + props = connection.get("properties") + expected_id = project_id + "/connections/" + request["connection_name"] + if (connection.get("name") != request["connection_name"] or not isinstance(props, dict) + or str(connection.get("id", expected_id)).casefold() != expected_id.casefold() + or any(props.get(key) != value for key, value in { + "category": "RemoteTool", "authType": "AgenticIdentityToken", + "target": target, "audience": "https://search.azure.com/", + }.items())): + raise fail("connection-conflict", "This exact connection NAME has incompatible Hosted recipe auth/binding. Preserve it; explicitly choose a new name, even for the same KB endpoint.") + old_connection, old_connection_url = None, None + if matching: + old_name = matching[0]["project_connection_id"].rsplit("/", 1)[-1] + old_connection_url = MANAGEMENT_AUDIENCE + project_id + "/connections/" + old_name + "?api-version=" + read.PROJECT_API + old_connection = connection if old_name == request["connection_name"] else reads.get( + old_connection_url, MANAGEMENT_AUDIENCE, label="previous-connection", + ) + old_id = project_id + "/connections/" + old_name + if (old_connection is None or old_connection.get("name") != old_name + or str(old_connection.get("id", old_id)).casefold() != old_id.casefold() + or not isinstance(old_connection.get("properties"), dict) + or old_connection["properties"].get("target") != matching[0]["server_url"]): + raise fail("toolbox-existing-binding-unverified", "The previous connection and selected toolbox tool do not establish one consistent KB binding.") + exact = bool(connection and matching + and matching[0].get("project_connection_id") in { + request["connection_name"], project_id + "/connections/" + request["connection_name"], + } + and matching[0]["server_url"] == target) + known = [{"name": request["agent_name"], "version": request["agent_version"]}] + known_states = [] + for item in request.get("known_agents", []): + observed = agent(reads, endpoint, item["name"], item["version"]) + known_states.append(observed) + env = observed["definition"].get("environment_variables", {}) + configured_tools = observed["definition"].get("tools") or [] + tool_binding = isinstance(configured_tools, list) and any( + isinstance(tool, dict) and tool.get("type") == "mcp" and tool.get("server_url") == consumer + for tool in configured_tools + ) + if (toolbox_binding(env, endpoint, request["toolbox_name"]) or tool_binding) and item not in known: + known.append(item) + refreshed = reads.get(toolbox_url + "?api-version=v1", AI_AUDIENCE, label="toolbox") + refreshed_agent = agent(reads, endpoint, request["agent_name"], request["agent_version"]) + refreshed_connection = reads.get(connection_url, MANAGEMENT_AUDIENCE, absent=True, label="connection") + refreshed_old = old_connection + if old_connection_url and old_connection_url != connection_url: + refreshed_old = reads.get(old_connection_url, MANAGEMENT_AUDIENCE, label="previous-connection") + refreshed_version = reads.get(toolbox_url + "/versions/" + default + "?api-version=v1", AI_AUDIENCE, label="toolbox-version") + if (refreshed != toolbox or refreshed_agent != selected or refreshed_connection != connection + or refreshed_old != old_connection or refreshed_version != version + or read.cli_context(project_id, cli=cli) != context): + raise fail("hosted-protected-state-drift", "Agent, toolbox default/metadata or CLI context changed during assessment.") + summary = { + "branch": "toolbox-only", "agent": request["agent_name"], "agent_version": request["agent_version"], + "project_resource_id": project_id, "project_endpoint": endpoint, "search_resource_id": search_id, + "toolbox_name": request["toolbox_name"], "consumer_endpoint": consumer, + "runtime_binding": {"resolver": "FoundryToolbox environment", "mode": binding_mode, + "runtime_usage_verified": False}, + "agent_change": "none", "runtime_principal_id": principal, "kb_profile": profile, + "before": {"kb_endpoint": matching[0].get("server_url") if matching else None, "default_version": default, + "connection": matching[0].get("project_connection_id") if matching else None}, + "after": {"kb_endpoint": target, "default_version": default if exact else "new immutable version (not created)", + "connection": request["connection_name"]}, + "connection_action": "reuse" if connection else "create (blocked)", + "known_consumers": known, "unknown_consumers": True, + "shared_default_approval_required": not exact, "mutation_approval_required": not exact, + "retention_owner": request["retention_owner"], "cleanup": "unsupported; retain previous versions and legacy connections", + "rollback": "separate explicit plan; never automatic", + "tool_policy": "Preserve existing tool approval/filter/configuration; runtime enforcement is not proven by this assessment.", + "acceptance": { + "supported_candidate_needed": "supported_question" not in request, + "unrelated_question_needed": "unrelated_question" not in request, + "invocation_approval": "not granted by assessment", "agent_tool_retrieval": "not-run", + }, + } + result = { + "status": "planned" if exact else "blocked", "outcome": OUTCOME, "approval_summary": summary, + "writes_performed": [], "execution_input": None, "execution_available": False, + "private_evidence": {"fingerprint": digest({ + "project": project, "agent": selected, "search": search, "kb": kb, + "reader": reader, "project_role": project_role, "toolbox": toolbox, + "version": version, "connection": connection, "previous_connection": old_connection, + "known_agents": known_states, "cli": context, + })}, + "request_ids": reads.request_ids, + "warnings": ["Configured runtime binding is not actual agent tool-use proof; acceptance invocations remain separate.", + "Known bindings are not a complete consumer inventory; external consumers may follow this default."], + } + if (str(search["properties"].get("status")).lower() != "running" + or str(search["properties"].get("provisioningState")).lower() != "succeeded"): + result["warnings"].append("Search is provisioning/degraded; healthy KB GET is configuration evidence, not retrieval readiness.") + if not exact: + result["first_blocker"] = { + "code": "toolbox-promotion-concurrency-unverified", + "message": "SDK/REST version creation and default promotion exist, but their conditional ETag/default update contract is unverified. Obtain service-owner confirmation before any connection/version creation; arbitrary If-Match headers are not proof.", + } + return result + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument("--plan", type=Path, required=True) + args = parser.parse_args(argv) + try: + result = assess(read_json(args.plan)) + except HelperFailure as error: + result = blocked_result(error, outcome=OUTCOME, fingerprint=None, owner=None) + if isinstance(error, ProjectSelectionFailure): + result["selection_candidates"] = error.candidates + emit_result(result) + return 3 if result["status"] == "partial" else 2 if result["status"] == "blocked" else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/kb-contracts.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/kb-contracts.md new file mode 100644 index 00000000..eaf6fb0c --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/kb-contracts.md @@ -0,0 +1,132 @@ +# Knowledge-base planning contract + +Use for KB creation/reuse or the explicit model-free agent transition. +The [KB owner](../knowledge-bases/create.md) resolves goals, source readiness, +access and costs; [common contracts](contracts.md) govern approval and execution. +No helper source inspection, hand-built `desired`/models or hash scripts on the +normal path. Reuse supplied choices instead of asking another questionnaire. + +## Read-only planning + +```text +python helpers/search_reconcile.py --plan +``` + +Closed UTF-8 input, at most one MiB: + +```json +{"schema_version":"1.0","endpoint":"https://svc.search.windows.net","name":"docs-kb","owner":"operator@example.com","source_name":"docs-file","api_version":"2026-08-01-preview","reasoning_effort":"minimal","output_mode":"extractiveData"} +``` + +All shown fields are required. `name`/`source_name` are exact names, not patterns; +owner is workflow metadata, not wire data. Names are URL-encoded, never code. +`action` defaults to `create-or-reuse`. Renaming after collision needs approval. + +| Choice | Supported shape | +|---|---| +| API | `2026-04-01` for Blob minimal/extractive; `2026-08-01-preview` for File/ADLS, reasoning/synthesis or agents | +| Reasoning/output | Preview `minimal`/`low`/`medium`: `extractiveData` or `answerSynthesis`; GA minimal/extractive only | +| Model | Omit/null for minimal/extractive; one explicit `model` below for low/medium or synthesis | +| Optional text | `description`, `retrieval_instructions`, `answer_instructions`: text/null, each at most 4096 characters; instructions preview-only | +| Optional controls | Existing closed `data_movement`, `rbac`, `network` objects from common contracts | + +No implicit API/mode/model choice, escalation or CU/vector inference. Source +processing and embeddings remain unchanged. Unknown fields and invalid choices +block before authentication. No keys, secret environments, model provisioning, +network/RBAC changes or model calls. + +### Reasoning or synthesis with a chat model + +Use resolved deployment/model choices, not guesses or a hardcoded default: + +```json +{"schema_version":"1.0","endpoint":"https://svc.search.windows.net","name":"docs-kb-synthesis","owner":"operator@example.com","source_name":"docs-file","api_version":"2026-08-01-preview","reasoning_effort":"low","output_mode":"answerSynthesis","model":{"endpoint":"https://models.services.ai.azure.com","deployment":"chat-deployment","model":"gpt-4.1-mini","auth":"system-assigned","prerequisites":{"deployment":"","identity":"","network":""}}} +``` + +`model` accepts only those five fields. Prerequisites are nonempty text, at most +4096 characters each: owner attestations, not helper proof of capacity/readiness/access. +Use a supported Azure OpenAI chat model, exact public-cloud account endpoint and deployment. +The Search system identity needs model access. CU/embeddings are not KB chat. +For medium or minimal synthesis, change `reasoning_effort` accordingly; retain +the model. Minimal synthesis adds model cost without query planning; use `intents`, +not `messages`, for retrieval. Never auto-escalate or change extractive defaults. + +The helper generates the complete wire definition, including: + +```json +{"name":"docs-kb-synthesis","knowledgeSources":[{"name":"docs-file"}],"retrievalReasoningEffort":{"kind":"low"},"outputMode":"answerSynthesis","models":[{"kind":"azureOpenAI","azureOpenAIParameters":{"resourceUri":"https://models.services.ai.azure.com","deploymentId":"chat-deployment","modelName":"gpt-4.1-mini","authIdentity":null}}]} +``` + +No `apiKey`. Only minimal/extractive omits models. GA omits preview fields/models. +New artifacts bind +`kb_plan_version: "1.0"` and `kb_model` (null or the selected choice) to that wire +definition. Existing approved artifacts retain their fingerprints; preview +minimal/synthesis is now admitted with one model. +Masked/nonempty keys or a different model identity in KB readback block; they +cannot be hidden by generic definition normalization. + +## Observation, result and approval + +Only selected Search `GET knowledgesources('{source}')` and +`GET knowledgebases('{name}')`, using the requested API and signed-in CLI token. +Source GETs bracket KB observation; existing KBs are reread for ETag/definition +stability. Three GETs for an absent KB, four for existing; no resource enumeration, +status polling, uploads, retrieval, PUT/PATCH/DELETE or writes to local artifacts. +These observations are not locks or ingestion/retrieval proof. + +The source must exist and return the exact selected File/Blob/ADLS definition. +The helper derives `verified_source` name/normalized definition digest from +those reads; never supply or fabricate it in the intent. Finish source-owner +readiness and prerequisite verification before mutation approval. + +Exit `0`: `status: planned`, `execution_input`, `plan_fingerprint`, +`approval_summary`, read request IDs and `writes_performed: []`. +Save only `execution_input` in a private UTF-8 JSON file. It already contains +the complete plan and computed fingerprint, with `approval.confirmed: false`. +Do not wrap it again, edit its generated body or calculate a fingerprint. +Summaries omit prerequisite references; retain private artifact data privately. + +An exact existing KB is observed reuse, not a prediction: stable name/definition/ +ETag, `action: reuse`, `execution_required: false`, +`mutation_approval_required: false`. Do not execute an unapproved envelope just +to repeat this read-only verification. Continue authorized retrieval; ingestion +and retrieval remain explicitly unverified by planning. + +Otherwise present concrete actions/source/mode/model, access, cost/data movement, +acceptance and cleanup. An already-confirmed concrete plan is not a reason to +repeat choice questions; confirm any material difference before proceeding. +Only actual approval of the unchanged generated plan permits setting its existing +`approval.confirmed` to true; preserve the supplied fingerprint. Tool permission, +bootstrap consent or a proposed future source does not establish that approval. +The planner never infers consent from the input. + +```text +python helpers/search_reconcile.py --input +``` + +Apply freshly verifies the source binding, conditional create or current ETag +update, exact readback and keyless model auth. Source/KB drift blocks; regenerate +the proposal and review changes, never hand-repair nested JSON/hashes. +Exit `2` is structured blocked/no-write; post-write failures are `3`/partial +with retained ownership. No automatic retries of writes or cleanup. +An updated KB remains pre-existing/reused, including failed or ambiguous +post-write verification; an update never establishes creation ownership. + +## Explicit agent-minimal transition + +This is optional KB-side model-free normalization, not an MCP prerequisite. +Never downgrade a valid reasoning/synthesis KB to connect an agent. +For the owner's existing model-free GA/extractive KB transition, set +`action: agent-minimal-transition`, preview API, minimal effort and extractive +output. Omit model and optional text changes. The source/KB must exist and match. +Existing absent/minimal/low effort is allowed only without models and synthesis; +the summary discloses prior mode fields, including an observed default low. +Only the two preview mode fields change, guarded by the current ETag; unrelated +fields/models remain. The update needs separate concrete approval. +An equivalent KB is reuse. Verify actual agent MCP invocation separately from GA REST retrieval. + +## Authorities + +Authorities: failure/conflict/uncertainty only. +[preview KB wire/model fields](https://learn.microsoft.com/rest/api/searchservice/knowledge-bases/create-or-update?view=rest-searchservice-2026-08-01-preview), +[KB creation and model access](https://learn.microsoft.com/azure/search/agentic-retrieval-how-to-create-knowledge-base). diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/knowledge_base_retrieve.py b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/knowledge_base_retrieve.py new file mode 100644 index 00000000..080db5aa --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/knowledge_base_retrieve.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any +from urllib.parse import quote + +try: + from ._common import ( + SEARCH_AUDIENCE, HelperFailure, HttpResult, TokenProvider, Transport, + azure_cli_token, blocked_result, emit_result, http_request, + reject_secrets, require_allowed_fields, validate_search_endpoint, + ) +except ImportError: + from _common import ( # type: ignore[no-redef] + SEARCH_AUDIENCE, HelperFailure, HttpResult, TokenProvider, Transport, + azure_cli_token, blocked_result, emit_result, http_request, + reject_secrets, require_allowed_fields, validate_search_endpoint, + ) + + +API_VERSIONS = {"2026-04-01", "2026-08-01-preview"} + + +def _invalid(message: str) -> HelperFailure: + return HelperFailure("input-schema-invalid", message, blocked_at="input-resolution") + + +def _target(endpoint: Any, name: Any, api_version: Any) -> str: + if not isinstance(api_version, str) or api_version not in API_VERSIONS: + raise _invalid("Unsupported API version.") + if not isinstance(name, str) or not name.strip() or any(ord(c) < 32 for c in name): + raise _invalid("An exact nonempty resource name is required.") + try: + return validate_search_endpoint(endpoint) + except ValueError as exc: + raise _invalid("Invalid Search endpoint.") from exc + + +def retrieval_request( + endpoint: str, name: str, api_version: str, effort: str, query: str, +) -> tuple[str, str, dict[str, Any]]: + base = _target(endpoint, name, api_version) + "/knowledgebases" + if not isinstance(effort, str) or effort not in {"minimal", "low", "medium"}: + raise _invalid("Unsupported reasoning effort.") + if api_version == "2026-04-01" and effort != "minimal": + raise _invalid("GA requires minimal retrieval.") + if not isinstance(query, str) or not query.strip(): + raise _invalid("A nonempty query is required.") + definition_url = base + "('" + quote(name.replace("'", "''"), safe="") + "')" + retrieve_url = base + "/" + quote(name, safe="") + "/retrieve" + suffix = "?api-version=" + api_version + body = ( + {"intents": [{"type": "semantic", "search": query}]} + if effort == "minimal" else + {"messages": [{"role": "user", "content": [{"type": "text", "text": query}]}]} + ) + return definition_url + suffix, retrieve_url + suffix, body + + +def read_json( + method: str, url: str, body: dict[str, Any] | None = None, + forward_permissions: bool = False, *, + token_provider: TokenProvider = azure_cli_token, + transport: Transport = http_request, +) -> HttpResult: + token = token_provider(SEARCH_AUDIENCE) + headers = {"Content-Type": "application/json"} + if forward_permissions: + headers["x-ms-query-source-authorization"] = token + result = transport( + method, url, token, + body=json.dumps(body).encode("utf-8") if body is not None else None, + headers=headers, follow_redirects=False, max_response_bytes=5 * 1024 * 1024, + ) + if result.status != 200 or not isinstance(result.body, dict): + raise HelperFailure( + "retrieval-incomplete", "Expected complete JSON response.", + blocked_at="verification", status=result.status, request_id=result.request_id, + ) + return result + + +def _verify_signed_in_user() -> None: + executable = shutil.which("az") + if executable is None: + raise HelperFailure( + "azure-cli-unavailable", "Azure CLI is required for keyless authentication.", + blocked_at="execution", + ) + try: + result = subprocess.run( + [executable, "account", "show", "--query", "user.type", "--output", "tsv"], + check=True, capture_output=True, text=True, timeout=60, + ) + except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc: + raise HelperFailure( + "permission-forwarding-unavailable", "Could not verify the signed-in user.", + blocked_at="input-resolution", + ) from exc + if result.stdout.strip().casefold() != "user": + raise HelperFailure( + "permission-forwarding-unavailable", "Permission forwarding requires a signed-in user.", + blocked_at="input-resolution", + ) + + +def execute( + document: Any, *, token_provider: TokenProvider = azure_cli_token, + transport: Transport = http_request, +) -> dict[str, Any]: + if not isinstance(document, dict): + raise _invalid("Input must be a JSON object.") + reject_secrets(document) + operation = document.get("operation") + if not isinstance(operation, str) or operation not in { + "get-knowledge-base", "get-knowledge-source", "retrieve", + }: + raise _invalid("Choose get-knowledge-base, get-knowledge-source, or retrieve.") + fields = {"operation", "endpoint", "name", "api_version"} + if operation == "retrieve": + fields |= {"effort", "query", "forward_permissions"} + require_allowed_fields(document, fields, label="read-only request") + endpoint = _target(document.get("endpoint"), document.get("name"), document.get("api_version")) + name, api_version = document["name"], document["api_version"] + body = None + forwarding = False + if operation == "retrieve": + forwarding = document.get("forward_permissions") + if type(forwarding) is not bool: + raise _invalid("Explicit forward_permissions true or false is required.") + _, url, body = retrieval_request( + endpoint, name, api_version, document.get("effort"), document.get("query"), + ) + if forwarding: + _verify_signed_in_user() + method = "POST" + else: + collection = "knowledgebases" if operation == "get-knowledge-base" else "knowledgesources" + url = endpoint + "/" + collection + "('" + quote(name.replace("'", "''"), safe="") + "')" + url += "?api-version=" + api_version + method = "GET" + result = read_json( + method, url, body, forwarding, token_provider=token_provider, transport=transport, + ) + return { + "status": "response-received", "operation": operation, "mutation": "none", + "writes_performed": [], "http_status": result.status, "request_id": result.request_id, + "response_body": result.body, "verification": "not-performed", "cleanup": "not-applicable", + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Read Search KB/source definitions or retrieve from one KB.") + parser.add_argument("--input", required=True, type=Path) + args = parser.parse_args(argv) + try: + try: + document = json.loads(args.input.read_text(encoding="utf-8")) + except OSError as exc: + raise HelperFailure("input-unreadable", "Input file cannot be read.", blocked_at="input-resolution") from exc + except (UnicodeError, json.JSONDecodeError) as exc: + raise HelperFailure("input-invalid-json", "Input must be UTF-8 JSON.", blocked_at="input-resolution") from exc + result = execute(document) + except HelperFailure as failure: + # POST retrieve is read-only even when the generic transport marks it ambiguous. + readonly_failure = HelperFailure( + failure.code, failure.message, blocked_at=failure.blocked_at, + status=failure.http_status, request_id=failure.request_id, + ) + result = blocked_result(readonly_failure, outcome="knowledge-base-retrieval", fingerprint=None) + result.update( + mutation="none", cleanup="not-applicable", + safe_next_decision="Resolve the first blocker before repeating this read-only request.", + ) + emit_result(result) + return 2 + emit_result(result) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/model-discovery-contracts.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/model-discovery-contracts.md new file mode 100644 index 00000000..7f0f8759 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/model-discovery-contracts.md @@ -0,0 +1,150 @@ +# Read-only model and CU selection + +`purpose: none` means no dependencies, +not minimal retrieval effort. Minimal+extractive can be model-free; +minimal+answerSynthesis needs `purpose: chat`. +Only FIND inventories; supplied IDs exact-read. KB/retrieve verifies API; +no readiness/access/approval proof. + +## Invocation and closed input + +From skill directory: + +```text +python helpers/model_discovery.py --input choices.json +``` + +Required JSON (UTF-8): + +```json +{"schema_version":"1.0","purpose":"embedding","subscription_id":null,"resource_group":null,"account_id":null,"account_name":null,"deployment":null} +``` + +`purpose`: `none`, `embedding`, `chat`, `cu`. Null is unresolved. +GUID subscription; ARM names. +RGs: 1-90 Unicode letters/decimal digits or `_-.()`; no final period. +`account_id`: exact Cognitive Services account ARM ID. `deployment`: name or +deployment ARM ID. IDs supply parent/scope; conflicting fields block before CLI. +CU/none reject deployment selectors. +Optional `endpoint`: observed HTTPS origin, only with an exact account selector. +Null/omitted requires a unique origin; explicit selection must match ARM. +Reuse IDs/choices; no setup attestations during discovery. + +Three dependency types: CU service capability (Content Understanding, Standard extraction); +embedding deployment (vectorization); chat deployment (enabled image verbalization/KB synthesis). +No project/agent gate. +Keep separate CU, embedding and chat selections. CU never selects models. +Unresolved models keep scope; set +`account_id`, `account_name`, `deployment` and `endpoint` to null; never repurpose +CU's `selection_input`. Explicit model selectors win. +Prefer one compatible existing Foundry/AIServices account: +CU capability + two model deployments. +Separate embedding/KB-chat: per consumer contract. CU image-description models +must be in the Foundry resource attached to the skillset. +Never provision/migrate/redeploy to consolidate. +CU proves no model deployments. Empty model lists are not scope-wide absence: +inspect another selected candidate before proposing creation. +A 403/timeout is unresolved, not absence; no denied-scope broadening. + +- Known account ID or account name plus group: exact account GET. +- Name without group: selected-subscription typed listing; unique match exact GET, + otherwise choices. +- Unknown account: group-scoped, otherwise subscription-scoped Cognitive Services + accounts: AIServices/OpenAI for models; AIServices for CU. + No deployment listing until an account is selected, even if only one exists. +- Selected model account: exact deployment GET when named, else its deployment list. + Validate each role's deployment/capabilities/auth; never infer support + from names. Missing capabilities need owner validation. +- CU: exact account metadata only; no deployment reads. + +Bootstrap's shell-free runner disables installs/auto-upgrade/telemetry and raw logs. +No login/context changes, secrets, writes, role/Policy/catalog/quota scans, +content reads/uploads or model calls. + +## Exact CLI surfaces + +Unresolved subscription only: `az account show --query "{id:id}" --output json --only-show-errors`. +Otherwise skip context lookup; explicit scope wins. +ARM calls: + +```text +az rest --method get --url --query --subscription --output json --only-show-errors +``` + +`URL`: `https://management.azure.com?api-version=2024-10-01`. +Encode paths, not IDs. `A` is the exact account path: +`/subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts/`. + +| Read | Path | +| --- | --- | +| Accounts in subscription | `/subscriptions//providers/Microsoft.CognitiveServices/accounts` | +| Accounts in group | `/subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts` | +| Exact account | `A` | +| Selected account deployments | `A/deployments` | +| Exact deployment | `A/deployments/` | + +Complete pagination: 10 pages/200 raw rows aggregate; 20 seconds/command, +120 seconds total. Partial results block; narrow `resource_group` or use an exact ID. +At most ten supported endpoint origins per account. +Continuation retains ARM host/collection/API; only skip-token parameters. +Native CLI JMESPath projects before capture: inventory IDs/names/kinds/locations, +exact account endpoints/state, deployment identity/model/state/public capabilities. +No ARM `$select`; all rows and `nextLink` survive. One MiB per stream, aggregate +captured bytes and final JSON; post-capture checks are not memory bounds. +No retries, fanout, denied-scope broadening or absence inference. + +## Closed output + +Fields: `schema_version`, `status`, `purpose`, `scope`, +`accounts`, `deployments`, `selected`, `limits`, `warnings`, `first_failure`, +`writes_performed` (always `[]`). Exit 0 returns `skipped`, `no-candidates`, +`account-choice-required`, `deployment-choice-required` or `selected`; +exit 2: `blocked`. No-candidates covers only the completed query. +Scope: null or `{subscription_id, resource_group}`; `limits`: bounds above. +`first_failure` is null or `{code, status, message, request_id, message_digest}`. +Original service code/status/request ID and message digest survive CLI failure; +No raw text/invented IDs; unreadable is not absent. + +Account rows: `{account_id, name, resource_group, location, kind, +provisioning_state, endpoint, endpoint_sources, endpoint_state, endpoint_candidates, selection_input}`. +Deployment rows: `{deployment_id, name, model_name, model_version, model_format, +capabilities, provisioning_state, selection_input}`. Metadata fields may be null; +capabilities: safe string map, not support/capacity proof: +`embeddings`, `chatCompletion` and validated numeric `maxContextToken`, `maxOutputToken` +only. Missing/invalid values remain unknown; internal/unknown fields never emit. +`selected` is null or `{account, deployment}` (deployment null for CU). +`endpoint_candidates` is the sorted observed-origin list. Endpoint resolution +failure retains the exact account row, never deployments or a selected result. +Other failures discard choices. +Inventory `endpoint_state` is `not-assessed`; endpoints/state are null/empty until GET. + +Present observed compatible deployments together in existing flow. +Keep complete scoped rows internally for ambiguity checks; never paste full +inventories/JSON to the customer. Show at most five concise account/deployment +choices, total count and a more/other choice from retained rows, not new discovery. +Show account/group/region and deployment/model names distinctly. Known Storage +location or selected region can rank suggestions, not prove CU/model availability. +Explicit locations/reuse win; no hidden regional filter or absence claim. +For choices, save the chosen row's **`selection_input`** unchanged. +For ambiguous origins, copy it and set `endpoint` to a listed origin; rerun. +No caller filtering, ID stitching or CLI glue. +Exact readback revalidates choices. Selection is metadata only: owner verifies +model suitability, version/capacity/cost/residency, +selected CU dependencies, actual access and network before concrete approval. + +## Endpoint metadata, not a CU readiness assertion + +ARM `2024-10-01` does **not** document `endpoints["ContentUnderstanding"]` as a +stable CU discriminator. Never infer hosts from map keys/customSubDomainName. + +Inspect the exact account's primary endpoint and every map **value** locally. +CU: `services.ai.azure.com`; embedding/chat also accept `openai.azure.com` and +`cognitiveservices.azure.com`. HTTPS origins only; strip only a trailing slash; +without a selector require exactly one matching origin. +Missing/multiple matching origins block; no rewriting, arbitrary fetch, +paths/query/user-info/ports or readiness inference. For diagnostics: +[scope/endpoint rules](model-discovery-scopes.md). + +File CU MI/explicit key and Blob Search identity stay distinct; neither is accessed. +Existing source planners own exact-name collision checks; no extra preflight, +suffixes, writes or auth changes. diff --git a/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/model-discovery-scopes.md b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/model-discovery-scopes.md new file mode 100644 index 00000000..7598e428 --- /dev/null +++ b/.github/plugins/foundry-iq-skills/skills/foundry-iq/helpers/model-discovery-scopes.md @@ -0,0 +1,131 @@ +# Discovery scope and endpoint diagnostics + +Read for ambiguous/missing endpoints, CU/model handoff confusion, unusual RG names +or selector/readback errors. +Metadata only; no setup or billable probes. + +## Separate dependency selections + +CU extraction uses `content_understanding.endpoint` / `aiServices.uri`; source +vectors use `embedding.endpoint` / `embeddingModel.azureOpenAIParameters.resourceUri`. +Retain each role's account ID, observed origin and selected deployment separately. +The CU key/identity is not evidence of embedding access. + +For example, CU account A can have no embedding deployment while model account B +already hosts a suitable one. Leave A selected for CU; start unresolved +`purpose: embedding` discovery in the agreed scope without A's account/name/endpoint. +The returned model candidates include `OpenAI` and `AIServices` accounts. A Foundry +resource is an `AIServices` model-hosting account, not another mandatory resource. +Choose a model account before listing its deployments. An account-list result is +not a deployment search; one empty deployment list does not exhaust the scope. +Keep the shortlist and choose another candidate, not repeated reads of A. + +An explicit embedding deployment fixes its own parent account; never overwrite +that binding with CU's. Same-account reuse remains valid when both roles are +independently verified. Carry the selected embedding origin/deployment unchanged +into the source plan; never rewrite hosts or substitute the CU origin. + +Prefer compatible existing deployments. If the completed agreed search finds +none suitable, propose a deployment in a compatible existing account before a +new account, with separate approval and model/SKU/quota/access/cost checks. +An explicitly selected target failure, denial, timeout or incomplete inventory +remains unresolved: no automatic scope expansion, creation or silent fallback. + +## Resource groups are not account names + +`Microsoft.Resources/resourcegroups` permits 1-90 characters: underscores, +hyphens, periods, parentheses, and Unicode categories Lu/Ll/Lt/Lm/Lo/Nd +(letters or decimal digits). No final period. No alphanumeric-first requirement: +`_shared-ai`, `.shared` and `(ops)` are valid. Combining marks, nondecimal numeric +characters, whitespace, slashes, percent escapes and controls are not allowed. +Use literal names in selectors/ARM IDs, not URL-encoded strings. + +The helper uses one RG predicate for supplied groups, account/deployment IDs and +observed IDs, before filtering account kinds. Thus an unrelated Speech account +in a valid RG does not poison a model listing; malformed/foreign scope still +blocks the entire result, never silently skips a row. Account/deployment name +rules remain separate. Unicode case handling must not merge distinct groups +through multi-character folds. + +Only HTTP paths are UTF-8 percent-encoded. Returned IDs and selection inputs stay +literal. Continuations must decode to the same collection and retain the exact +ARM host/API and allowed skip-token parameters; they are sent on the validated +encoded path. No context change, broader listing or Unicode transliteration. + +## Purpose-specific observed origins + +| Purpose | Accepted suffixes | +|---|---| +| `embedding` | `openai.azure.com`, `services.ai.azure.com`, `cognitiveservices.azure.com` | +| `chat` | Same model origins, independently checked against the KB consumer | +| `cu` | Only `services.ai.azure.com`; no model-host substitution | + +Require `https://