Add repository-wide TypeSpec emitter regeneration pipeline

doc/dev/dataplane_generation.md

@@ -107,6 +107,46 @@ tsp-client update
 
 tsp-client will look for a `tsp-location.yaml` file in your local directory. `tsp-location.yaml` contains the configuration information that will be used to sync your TypeSpec project and generate your SDK. Please make sure that the commit is targeting the correct TypeSpec project updates you wish to generate your SDK from.
 
+## Validate emitter updates
+
+The [TypeSpec regeneration pipeline](../../eng/pipelines/typespec-regeneration.yml)
+checks emitter updates by regenerating selected SDKs at their recorded spec
+revisions. It runs when `eng/emitter-package.json` or its lockfile changes on
+`main` or in a pull request, and can also be queued manually.
+
+The default run covers three SDKs:
+
+- Advisor (management plane).
+- App Configuration (data plane).
+- Agent Server Responses (nested client).
+
+To choose other SDKs, set **PackageDirectories** when queuing a run. Use
+comma-separated repository paths, such as `sdk/advisor/azure-mgmt-advisor`.
+Package paths include their nested clients. Set it to `*` to run all SDKs across
+eight jobs instead of the default single job.
+
+Download the `typespec-regeneration-<shard>` artifacts for `results.json`,
+per-client logs, and `changes.txt` / `changes.patch`. Generation failures fail
+the job without skipping later clients. Code differences do not fail the job:
+review the patches for unexpected changes.
+
+Maintainers must register the pipeline YAML in Azure Pipelines and enable its
+YAML CI and PR triggers before automatic runs can start.
+
+For a local run, use a **clean, disposable checkout**, Node.js 24, Python 3.10+,
+and PyYAML. Configure feed access using [CONTRIBUTING.md](../../CONTRIBUTING.md),
+then run:
+
+```shell
+npm ci --prefix eng/common/tsp-client
+npm exec --prefix eng/common/tsp-client --no -- tsp-client install-dependencies
+python eng/scripts/regenerate_typespec.py --output ../regeneration-results --packages sdk/advisor/azure-mgmt-advisor
+```
+
+Omitting `--packages` runs all clients. Keep the output directory outside the
+checkout. The runner leaves generated changes in place and marks new SDK files
+as intent-to-add for patch collection; it does not commit them.
+
 ## Post-Generation Steps
 
 The generated code is not enough to release directly. Follow these steps regardless of whether you generated from TypeSpec or Swagger.

eng/pipelines/typespec-regeneration.yml

@@ -0,0 +1,111 @@
+# Validate the pinned emitter against existing clients without advancing specs.
+parameters:
+  - name: PackageDirectories
+    displayName: 'SDK directories (comma-separated repository paths; * for all)'
+    type: string
+    default: sdk/advisor/azure-mgmt-advisor,sdk/appconfiguration/azure-appconfiguration,sdk/agentserver/azure-ai-agentserver-responses/type_spec
+
+trigger:
+  branches:
+    include:
+      - main
+  paths:
+    include:
+      - eng/emitter-package.json
+      - eng/emitter-package-lock.json
+
+pr:
+  branches:
+    include:
+      - main
+  paths:
+    include:
+      - eng/emitter-package.json
+      - eng/emitter-package-lock.json
+      - eng/common/tsp-client/**
+      - eng/pipelines/typespec-regeneration.yml
+      - eng/scripts/regenerate_typespec.py
+
+extends:
+  template: /eng/pipelines/templates/stages/1es-redirect.yml
+  parameters:
+    stages:
+      - stage: Regenerate
+        displayName: Regenerate TypeSpec SDKs
+        jobs:
+          - job: Regenerate
+            timeoutInMinutes: 360
+            variables:
+              ${{ if eq(parameters.PackageDirectories, '*') }}:
+                ShardCount: 8
+              ${{ else }}:
+                ShardCount: 1
+            pool:
+              name: azsdk-pool
+              image: ubuntu-24.04
+              os: linux
+            strategy:
+              maxParallel: 8
+              matrix:
+                Batch0:
+                  ShardIndex: 0
+                ${{ if eq(parameters.PackageDirectories, '*') }}:
+                  Batch1:
+                    ShardIndex: 1
+                  Batch2:
+                    ShardIndex: 2
+                  Batch3:
+                    ShardIndex: 3
+                  Batch4:
+                    ShardIndex: 4
+                  Batch5:
+                    ShardIndex: 5
+                  Batch6:
+                    ShardIndex: 6
+                  Batch7:
+                    ShardIndex: 7
+            templateContext:
+              outputs:
+                - output: pipelineArtifact
+                  targetPath: $(Build.ArtifactStagingDirectory)/typespec-regeneration
+                  artifactName: typespec-regeneration-$(ShardIndex)
+                  condition: succeededOrFailed()
+                  sbomEnabled: false
+            steps:
+              - checkout: self
+
+              - task: NodeTool@0
+                displayName: Install Node.js
+                inputs:
+                  versionSpec: 24.x
+
+              - task: UsePythonVersion@0
+                displayName: Install Python
+                inputs:
+                  versionSpec: '3.10'
+
+              - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml
+
+              - template: /eng/pipelines/templates/steps/auth-dev-feed.yml
+                parameters:
+                  EnableTwineAuth: false
+                  EnableUvAuth: false
+
+              - script: python -m pip install PyYAML==6.0.2
+                displayName: Install YAML parser
+
+              - script: npm ci --prefix eng/common/tsp-client
+                displayName: Install pinned tsp-client
+
+              - script: npm exec --prefix eng/common/tsp-client --no -- tsp-client install-dependencies
+                displayName: Install pinned emitter dependencies
+
+              - script: >-
+                  python eng/scripts/regenerate_typespec.py
+                  --output "$(Build.ArtifactStagingDirectory)/typespec-regeneration"
+                  --shard-index $(ShardIndex)
+                  --shard-count $(ShardCount)
+                  --packages "$TYPESPEC_REGENERATION_PACKAGES"
+                displayName: Regenerate clients and collect changes
+                env:
+                  TYPESPEC_REGENERATION_PACKAGES: ${{ parameters.PackageDirectories }}

eng/scripts/regenerate_typespec.py

@@ -0,0 +1,132 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+"""Regenerate existing TypeSpec clients and retain reviewable changes and logs."""
+
+import argparse
+import json
+from pathlib import Path
+import shutil
+import subprocess
+
+import yaml
+
+
+def discover_clients(repo: Path) -> list[Path]:
+    result = subprocess.run(
+        ["git", "ls-files", "-z", "--", "sdk/**/tsp-location.yaml"],
+        cwd=repo,
+        check=True,
+        capture_output=True,
+    )
+    locations = sorted(repo / name.decode("utf-8") for name in result.stdout.split(b"\0") if name)
+    clients = []
+    for location in locations:
+        try:
+            config = yaml.safe_load(location.read_text(encoding="utf-8"))
+        except yaml.YAMLError:
+            # Let tsp-client report invalid configurations as individual failures
+            # rather than preventing regeneration of every other client.
+            config = None
+        if (
+            isinstance(config, dict)
+            and set(config) == {"batch"}
+            and isinstance(config["batch"], list)
+            and config["batch"]
+        ):
+            children = config["batch"]
+            if all(
+                isinstance(child, str) and (location.parent / child / "tsp-location.yaml").resolve() in locations
+                for child in children
+            ):
+                # Run each tracked child independently so a failed batch member
+                # cannot prevent the remaining members from being attempted.
+                continue
+        clients.append(location.parent)
+    return clients
+
+
+def select_clients(repo: Path, clients: list[Path], packages: str) -> list[Path]:
+    if packages.strip() == "*":
+        return clients
+    selected = set()
+    for directory in packages.split(","):
+        relative = Path(directory.strip().replace("\\", "/"))
+        if relative.is_absolute() or len(relative.parts) < 3 or relative.parts[0] != "sdk" or ".." in relative.parts:
+            raise ValueError(f"Expected a repository-relative SDK package or client directory, got: {directory!r}")
+        target = repo / relative
+        matches = [client for client in clients if client == target or target in client.parents]
+        if not matches:
+            raise ValueError(f"No tracked TypeSpec clients match directory: {directory!r}")
+        selected.update(matches)
+    return [client for client in clients if client in selected]
+
+
+def regenerate(repo: Path, output: Path, shard_index: int = 0, shard_count: int = 1, packages: str = "*") -> int:
+    if shard_count < 1 or not 0 <= shard_index < shard_count:
+        raise ValueError("Require shard-count > 0 and 0 <= shard-index < shard-count")
+    repo = repo.resolve()
+    output = output.resolve()
+    if output == repo or repo in output.parents:
+        raise ValueError("The output directory must be outside the checkout")
+    status = subprocess.run(["git", "status", "--porcelain"], cwd=repo, check=True, capture_output=True, text=True)
+    if status.stdout.strip():
+        raise ValueError("Regeneration requires a clean checkout, including untracked files")
+
+    clients = discover_clients(repo)
+    if not clients:
+        raise ValueError("No TypeSpec clients were found under sdk")
+    clients = select_clients(repo, clients, packages)
+    clients = clients[shard_index::shard_count]
+    node = shutil.which("node")
+    if node is None:
+        raise FileNotFoundError("Node.js is required for TypeSpec regeneration")
+    cli = repo / "eng" / "common" / "tsp-client" / "node_modules" / "@azure-tools"
+    cli = cli / "typespec-client-generator-cli" / "cmd" / "tsp-client.js"
+    if not cli.is_file():
+        raise FileNotFoundError("Install the pinned CLI with: npm ci --prefix eng/common/tsp-client")
+    command = [node, str(cli), "update", "--skip-install", "--debug"]
+    output.mkdir(parents=True, exist_ok=True)
+    results = []
+    for index, client in enumerate(clients):
+        relative = client.relative_to(repo).as_posix()
+        log_name = f"{index:04d}.log"
+        print(f"Regenerating {relative}", flush=True)
+        with (output / log_name).open("wb") as log:
+            result = subprocess.run(command, cwd=client, stdout=log, stderr=subprocess.STDOUT, check=False)
+        results.append({"directory": relative, "exit_code": result.returncode, "log": log_name})
+        if result.returncode:
+            print(f"Generation failed for {relative}: exit {result.returncode}; see {log_name}", flush=True)
+        (output / "results.json").write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8")
+
+    # Include new generated files without creating commits. The clean checkout
+    # requirement ensures this cannot stage a developer's existing work.
+    subprocess.run(["git", "add", "--intent-to-add", "--", "sdk"], cwd=repo, check=True)
+    with (output / "changes.patch").open("wb") as patch:
+        subprocess.run(["git", "diff", "--binary", "HEAD", "--", "sdk"], cwd=repo, stdout=patch, check=True)
+    with (output / "changes.txt").open("wb") as summary:
+        subprocess.run(["git", "diff", "--stat", "HEAD", "--", "sdk"], cwd=repo, stdout=summary, check=True)
+    (output / "results.json").write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8")
+    return int(any(result["exit_code"] for result in results))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--repo", type=Path, default=Path(__file__).resolve().parents[2])
+    parser.add_argument("--output", type=Path, required=True, help="Artifact directory outside the checkout")
+    parser.add_argument("--shard-index", type=int, default=0)
+    parser.add_argument("--shard-count", type=int, default=1)
+    parser.add_argument(
+        "--packages",
+        default="*",
+        help="Comma-separated repository-relative SDK package or client directories; '*' selects all clients",
+    )
+    args = parser.parse_args()
+    return regenerate(args.repo, args.output, args.shard_index, args.shard_count, args.packages)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())