[CI] Generate ApiHash during extended build

eng/pipelines/templates/jobs/apireview-hub-job-python.yml

@@ -58,7 +58,7 @@ jobs:
 
           mkdir -p "$tooling_dir/eng/common/scripts" "$tooling_dir/eng/scripts"
           cp "$source_repo/eng/common/scripts/Export-APIViewMarkdown.ps1" "$tooling_dir/eng/common/scripts/Export-APIViewMarkdown.ps1"
-          cp "$source_repo/eng/scripts/Extract-APIViewMetadata-Python.ps1" "$tooling_dir/eng/scripts/Extract-APIViewMetadata-Python.ps1"
+          cp "$source_repo/eng/scripts/extract_apiview_metadata.py" "$tooling_dir/eng/scripts/extract_apiview_metadata.py"
 
           python -m pip install --upgrade pip
           python -m pip install virtualenv

eng/pipelines/templates/steps/build-extended-artifacts.yml

@@ -82,6 +82,12 @@ steps:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
         AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"'
 
+    - script: >-
+        python eng/scripts/save_package_api_hash.py
+        --artifact-staging-directory "$(Build.ArtifactStagingDirectory)"
+        --repo-root "$(Build.SourcesDirectory)"
+      displayName: 'Update package properties with API hashes'
+
   - ${{ parameters.BeforePublishSteps }}
 
   - ${{ if eq(parameters.RunApiStubGen, 'true') }}:

eng/pipelines/templates/steps/create-apireview-hub-artifacts-python.yml

@@ -70,7 +70,7 @@ steps:
               exit 1
             fi
             install -D "$tooling_dir/eng/common/scripts/Export-APIViewMarkdown.ps1" "$source_repo/eng/common/scripts/Export-APIViewMarkdown.ps1"
-            install -D "$tooling_dir/eng/scripts/Extract-APIViewMetadata-Python.ps1" "$source_repo/eng/scripts/Extract-APIViewMetadata-Python.ps1"
+            install -D "$tooling_dir/eng/scripts/extract_apiview_metadata.py" "$source_repo/eng/scripts/extract_apiview_metadata.py"
             (cd "$source_repo" && PYTHONPATH="$common_tasks_dir${PYTHONPATH:+:$PYTHONPATH}" python -m azpysdk.main apistub --dest-dir "$package_dir" "${{ parameters.packageName }}")
 
             if [ ! -f "$package_dir/api.md" ]; then
@@ -88,4 +88,4 @@ steps:
             version=$(python -c 'import pathlib, re, sys; package_dir = pathlib.Path(sys.argv[1]); pattern = re.compile(r"^\s*VERSION\s*[:=]\s*[\"'"'"']([^\"'"'"']+)[\"'"'"']", re.MULTILINE); candidates = sorted(package_dir.rglob("*_version.py")) + sorted(package_dir.rglob("version.py")); match = next((pattern.search(candidate.read_text(encoding="utf-8")) for candidate in candidates if "_generated" not in candidate.parts and pattern.search(candidate.read_text(encoding="utf-8"))), None); print(match.group(1) if match else "")' "$package_dir")
             printf '%s' "$package_relative_path" > "$state_dir/package-relative-path.txt"
             printf '%s' "$version" > "$state_dir/version.txt"
-          displayName: 'Generate ${{ parameters.kind }} Python API review bundle'
+          displayName: 'Generate ${{ parameters.kind }} Python API review bundle'

eng/scripts/extract_apiview_metadata.py

@@ -0,0 +1,63 @@
+import argparse
+import hashlib
+import pathlib
+import re
+from typing import Dict, List
+
+
+_METADATA_PATTERN = re.compile(
+    r"^# Package is parsed using apiview-stub-generator\(version:([^\)]+)\), Python version:\s*([^\s]+)\s*$"
+)
+
+
+def extract_metadata(api_markdown_path: pathlib.Path) -> Dict[str, str]:
+    with api_markdown_path.open(encoding="utf-8-sig", newline="") as api_markdown_file:
+        file_text = api_markdown_file.read()
+    line_ending = "\r\n" if "\r\n" in file_text else "\n"
+    lines = re.split(r"\r?\n", file_text)
+
+    metadata: Dict[str, str] = {}
+    filtered: List[str] = []
+    for line in lines:
+        match = _METADATA_PATTERN.match(line)
+        if match:
+            metadata["parserVersion"] = match.group(1)
+            metadata["pythonVersion"] = match.group(2)
+        else:
+            filtered.append(line)
+
+    if filtered and filtered[0].startswith("```"):
+        body = filtered[1:]
+        while body and not body[0].strip():
+            body.pop(0)
+        filtered = [filtered[0], *body]
+    else:
+        while filtered and not filtered[0].strip():
+            filtered.pop(0)
+
+    normalized_text = "\n".join(line.rstrip() for line in filtered)
+    metadata["apiMdSha256"] = hashlib.sha256(normalized_text.encode("utf-8")).hexdigest()
+
+    api_markdown_path.write_text(line_ending.join(filtered), encoding="utf-8", newline="")
+    metadata_path = api_markdown_path.parent / "api.metadata.yml"
+    metadata_text = line_ending.join(f"{key}: {metadata[key]}" for key in sorted(metadata)) + line_ending
+    metadata_path.write_text(metadata_text, encoding="utf-8", newline="")
+    print(f"Updated markdown: {api_markdown_path}")
+    print(f"Generated metadata: {metadata_path}")
+    return metadata
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Extract Python APIView metadata from API markdown")
+    parser.add_argument("--api-markdown-path")
+    parser.add_argument("--output-path", default=".")
+    args = parser.parse_args()
+
+    api_markdown_path = pathlib.Path(args.api_markdown_path) if args.api_markdown_path else pathlib.Path(args.output_path) / "api.md"
+    if not api_markdown_path.is_file():
+        parser.error(f"API markdown file not found: {api_markdown_path}")
+    extract_metadata(api_markdown_path)
+
+
+if __name__ == "__main__":
+    main()

eng/scripts/save_package_api_hash.py

@@ -0,0 +1,55 @@
+import argparse
+import json
+import pathlib
+import subprocess
+
+import yaml
+
+from extract_apiview_metadata import extract_metadata
+
+
+def update_package_info(metadata_path: pathlib.Path, package_info_path: pathlib.Path) -> None:
+    metadata = yaml.safe_load(metadata_path.read_text(encoding="utf-8-sig"))
+    api_hash = metadata.get("apiMdSha256") if isinstance(metadata, dict) else None
+    if not isinstance(api_hash, str) or not api_hash:
+        raise ValueError(f"apiMdSha256 was not found in {metadata_path}")
+
+    package_info = json.loads(package_info_path.read_text(encoding="utf-8-sig"))
+    package_info["ApiHash"] = api_hash
+    package_info_path.write_text(json.dumps(package_info, indent=2) + "\n", encoding="utf-8")
+    print(f"Stored ApiHash in {package_info_path}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Add API markdown hashes to PackageInfo files")
+    parser.add_argument("--artifact-staging-directory", required=True)
+    parser.add_argument("--repo-root", required=True)
+    args = parser.parse_args()
+
+    artifact_dir = pathlib.Path(args.artifact_staging_directory)
+    repo_root = pathlib.Path(args.repo_root)
+    package_info_dir = artifact_dir / "PackageInfo"
+    export_script = repo_root / "eng" / "common" / "scripts" / "Export-APIViewMarkdown.ps1"
+
+    for package_info_path in package_info_dir.glob("*.json"):
+        package_info = json.loads(package_info_path.read_text(encoding="utf-8-sig"))
+        package_name = package_info["Name"]
+        package_artifact_dir = artifact_dir / package_name
+        token_file = package_artifact_dir / f"{package_name}_python.json"
+        # API stub generation intentionally omits management-plane packages, so they have no token file.
+        # Revisit this behavior if management-plane packages are included in API stub generation.
+        if not token_file.is_file():
+            print(f"API token file was not found for {package_name}; skipping ApiHash update")
+            continue
+
+        subprocess.run(
+            ["pwsh", str(export_script), "-TokenJsonPath", str(token_file), "-OutputPath", str(package_artifact_dir)],
+            check=True,
+        )
+        extract_metadata(package_artifact_dir / "api.md")
+
+        update_package_info(package_artifact_dir / "api.metadata.yml", package_info_path)
+
+
+if __name__ == "__main__":
+    main()

eng/tools/azure-sdk-tools/azpysdk/apistub.py

@@ -233,7 +233,7 @@ def run(self, args: argparse.Namespace) -> int:
                         results.append(1)
                 else:
                     md_script = os.path.join(REPO_ROOT, "eng", "common", "scripts", "Export-APIViewMarkdown.ps1")
-                    metadata_script = os.path.join(REPO_ROOT, "eng", "scripts", "Extract-APIViewMetadata-Python.ps1")
+                    metadata_script = os.path.join(REPO_ROOT, "eng", "scripts", "extract_apiview_metadata.py")
                     logger.info(f"Generating api.md for {package_name}")
                     try:
                         result = run(
@@ -248,7 +248,7 @@ def run(self, args: argparse.Namespace) -> int:
 
                         logger.info(f"Extracting API metadata for {package_name}")
                         metadata_result = run(
-                            ["pwsh", metadata_script, "-OutputPath", out_token_path],
+                            [executable, metadata_script, "--output-path", out_token_path],
                             check=True,
                             capture_output=True,
                             text=True,

eng/tools/azure-sdk-tools/tests/test_apistub.py

@@ -266,9 +266,10 @@ def fake_apistub_run(exe, cmds, **kwargs):
             open(os.path.join(out_dir, "azure-core_python.json"), "w").close()
 
         def fake_pwsh(cmd, **kwargs):
-            out_idx = cmd.index("-OutputPath")
+            output_arg = "--output-path" if "extract_apiview_metadata.py" in cmd[1] else "-OutputPath"
+            out_idx = cmd.index(output_arg)
             out_dir = cmd[out_idx + 1]
-            if "Extract-APIViewMetadata-Python.ps1" in cmd[1]:
+            if "extract_apiview_metadata.py" in cmd[1]:
                 open(os.path.join(out_dir, "api.metadata.yml"), "w").close()
             else:
                 open(os.path.join(out_dir, "api.md"), "w").close()
@@ -325,9 +326,11 @@ def fake_apistub_run(exe, cmds, **kwargs):
             open(os.path.join(out_dir, "azure-core_python.json"), "w").close()
 
         def fake_pwsh(cmd, **kwargs):
-            out_idx = cmd.index("-OutputPath")
+            output_arg = "--output-path" if "extract_apiview_metadata.py" in cmd[1] else "-OutputPath"
+            out_idx = cmd.index(output_arg)
             out_dir = cmd[out_idx + 1]
-            open(os.path.join(out_dir, "api.md"), "w").close()
+            output_file = "api.metadata.yml" if "extract_apiview_metadata.py" in cmd[1] else "api.md"
+            open(os.path.join(out_dir, output_file), "w").close()
             return MagicMock(returncode=0)
 
         with patch.object(stub, "get_targeted_directories", return_value=[fake_parsed]), patch.object(

sdk/template/azure-template/pyproject.toml

@@ -49,7 +49,7 @@ pyright = true
 pylint = true
 black = true
 generate = false
-apistub = false
+apistub = true
 
 [tool.azure-sdk-conda]
 in_bundle = false