Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
changeKind: fix
packages:
- "@azure-tools/typespec-java"
---

Fix XML serialization to only apply for `azure-v1` data-plane clients, and to skip the XML `ObjectSerializer` for raw `byte[]`/`BinaryData` payloads that are not structured XML models. This avoids emitting a reference to a non-generated `XmlSerializerProviders` helper (which caused a build break) for operations that return raw XML bytes.
5 changes: 5 additions & 0 deletions packages/typespec-java/Build-Generator.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@

$ErrorActionPreference = "Stop"

# See Copy-Sources.ps1: silence progress so no cmdlet touches the controlling
# terminal from turbo's background process group (which would SIGTTIN/SIGTTOU-stop
# this process and hang the parallel build).
$ProgressPreference = "SilentlyContinue"

$packageRoot = $PSScriptRoot
$generatorRoot = Join-Path $packageRoot "generator"
$generatorPom = Join-Path $generatorRoot "pom.xml"
Expand Down
40 changes: 31 additions & 9 deletions packages/typespec-java/Copy-Sources.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,33 @@
# Azure-flavored `src/options.ts` (excluded from the copy).
#
# The desired core commit is pinned in core-commit.json; CoreCommit.ps1's
# Enter-CoreCommit temporarily checks it out (when newer than the submodule's
# current checkout) to copy the sources from, and Restore-CoreCommit always puts
# the submodule back to its original SHA (repo-wide `pnpm build` runs this and CI
# checks git status).
# Get-CoreSourceRoot returns a directory to read the emitter/generator sources from
# (extracted from the pinned commit via `git archive` when it is newer than the
# submodule's current checkout), without ever mutating the core/ submodule working
# tree. Remove-CoreSourceRoot cleans up any temporary extraction afterwards.
#
# Mirrors the "Copy TypeScript code" step of autorest.java's Build-TypeSpec.ps1.

$ErrorActionPreference = "Stop"

# Disable progress rendering. Under a full parallel `pnpm build`, turbo runs each
# task's process in its own background process group under the build's controlling
# terminal. When a cmdlet here (notably `Copy-Item -Recurse`, which emits a
# Write-Progress) renders progress, PowerShell touches that terminal from a
# background process group and the OS stops the process with SIGTTIN/SIGTTOU -- it
# never resumes, so the emitter build appears to hang forever (intermittently, and
# only for this package since it is the only one that shells out to pwsh). Silencing
# progress removes that terminal interaction entirely.
$ProgressPreference = "SilentlyContinue"

$packageRoot = $PSScriptRoot
$repoRoot = Resolve-Path (Join-Path $packageRoot ".." "..")
$coreRoot = Join-Path $repoRoot "core"

. (Join-Path $packageRoot "CoreCommit.ps1")
$originSha = Enter-CoreCommit -CoreRoot $coreRoot -PackageRoot $packageRoot
$core = Get-CoreSourceRoot -CoreRoot $coreRoot -PackageRoot $packageRoot
try {
$emitterRoot = Resolve-Path (Join-Path $coreRoot "packages" "http-client-java" "emitter")
$emitterRoot = Join-Path $core.Root "emitter"

Copy-Item -Path (Join-Path $emitterRoot "src") -Destination $packageRoot -Exclude "options.ts" -Recurse -Force
Copy-Item -Path (Join-Path $emitterRoot "test") -Destination $packageRoot -Recurse -Force
Expand All @@ -29,15 +39,27 @@ try {
# customization patch (core.patch) to the copy, so building emitter.jar
# (Build-Generator.ps1) never mutates the core/ submodule. The patch paths are
# relative to this generator folder.
$srcGenerator = Join-Path $coreRoot "packages" "http-client-java" "generator"
$srcGenerator = Join-Path $core.Root "generator"
$destGenerator = Join-Path $packageRoot "generator"
$patchFile = Join-Path $packageRoot "core.patch"

# The emitter.jar reactor (generator/pom.xml) only builds http-client-generator,
# http-client-generator-mgmt and http-client-generator-core. The two test modules
# below live in a separate (never-activated) Maven `test` profile and hold ~38MB
# of generated Java test sources that are not part of the emitter build, so skip
# copying them -- it is the bulk of the copy time. (The Azure emitter-tests
# project syncs http-client-generator-test straight from core via SyncTests.ps1,
# not from this copy.)
$excludedGeneratorModules = @("http-client-generator-test", "http-client-generator-clientcore-test")

Write-Host "Copy generator sources from core"
if (Test-Path $destGenerator) {
Remove-Item $destGenerator -Recurse -Force
}
Copy-Item -Path $srcGenerator -Destination $destGenerator -Recurse -Force
New-Item -ItemType Directory -Path $destGenerator | Out-Null
Get-ChildItem -Path $srcGenerator -Force |
Where-Object { $excludedGeneratorModules -notcontains $_.Name } |
ForEach-Object { Copy-Item -Path $_.FullName -Destination $destGenerator -Recurse -Force }

Write-Host "Apply Azure customization patch to copied generator"
# Run from the repo root with --directory so git apply resolves the patch's
Expand All @@ -50,5 +72,5 @@ try {
}
}
finally {
Restore-CoreCommit -CoreRoot $coreRoot -OriginSha $originSha
Remove-CoreSourceRoot $core
}
150 changes: 104 additions & 46 deletions packages/typespec-java/CoreCommit.ps1
Original file line number Diff line number Diff line change
@@ -1,26 +1,36 @@
# Shared helpers for running work against the desired `core/` submodule commit.
# Shared helper for reading sources from the desired `core/` submodule commit
# WITHOUT mutating the submodule's working tree.
#
# Dot-source this file to import Enter-CoreCommit / Restore-CoreCommit, then wrap
# the work in a try/finally:
# Dot-source this file to import Get-CoreSourceRoot / Remove-CoreSourceRoot, then
# wrap the work in a try/finally so the temporary extraction (if any) is cleaned up:
#
# . (Join-Path $packageRoot "CoreCommit.ps1")
# $originSha = Enter-CoreCommit -CoreRoot $coreRoot -PackageRoot $packageRoot
# $core = Get-CoreSourceRoot -CoreRoot $coreRoot -PackageRoot $packageRoot
# try {
# # copy/sync from the (possibly pinned) core checkout
# # read sources from under $core.Root (e.g. emitter/, generator/)
# }
# finally {
# Restore-CoreCommit -CoreRoot $coreRoot -OriginSha $originSha
# Remove-CoreSourceRoot $core
# }
#
# The desired core commit is pinned in <PackageRoot>/core-commit.json. When that
# file is present and its commit is newer than the submodule's current checkout,
# Enter-CoreCommit temporarily checks it out; Restore-CoreCommit puts the submodule
# back to its original SHA (repo-wide `pnpm build` runs these scripts and CI checks
# git status).
# Get-CoreSourceRoot extracts that commit's `packages/http-client-java` subtree into
# a temporary directory via `git archive`. It never checks out a different SHA or
# otherwise touches the submodule's working tree, so it is safe to run while a
# parallel monorepo build (`pnpm build`, `regen-all-packages-docs`, ...) reads the
# core/ submodule concurrently, and `git status` on core always stays clean.
#
# When core-commit.json is absent, or its commit is not newer than the current
# checkout, the sources are read straight from the submodule's current working tree.

# Path (relative to the core repo root) of the subtree the Java package reads from.
$script:CoreJavaSubtree = "packages/http-client-java"

# Check out the pinned core commit (when newer) and return the submodule's original
# SHA so the caller can restore it in a finally block.
function Enter-CoreCommit {
# Resolve where to read the core Java sources from and return a descriptor:
# @{ Root = <dir containing emitter/, generator/, ...>; TempDir = <temp dir or $null> }
# Pass the descriptor to Remove-CoreSourceRoot in a finally block to clean up.
function Get-CoreSourceRoot {
param(
# Path to the `core/` submodule.
[Parameter(Mandatory = $true)]
Expand All @@ -31,59 +41,107 @@ function Enter-CoreCommit {
[string] $PackageRoot
)

# Capture the submodule's current SHA so the caller can restore it.
# Never let any git invocation below block on an interactive credential or
# host-key prompt. During a parallel `pnpm build` the emitter build runs
# headless (turbo captures its stdio / attaches a pseudo-terminal), so a git
# prompt has nowhere to read from and would hang the whole build forever --
# observed as an intermittent, Java-only build hang. Make git fail fast instead.
$env:GIT_TERMINAL_PROMPT = "0"

# Silence progress: a cmdlet rendering Write-Progress (e.g. the archive
# extraction below) touches the controlling terminal, and under turbo's
# background process group that stops this process with SIGTTIN/SIGTTOU and
# hangs the build. Scoped to this function so callers are unaffected.
$ProgressPreference = "SilentlyContinue"

$originSha = (git -C $CoreRoot rev-parse HEAD).Trim()
if ($LASTEXITCODE -ne 0) {
Write-Error "Failed to read the current SHA of the 'core' submodule at: $CoreRoot`nMake sure it is checked out (git submodule update --init --recursive)."
}

$liveRoot = Join-Path $CoreRoot $script:CoreJavaSubtree

# core-commit.json is optional: when present it pins the core commit to use;
# when absent we just use the submodule's current checkout.
$configPath = Join-Path $PackageRoot "core-commit.json"
$targetSha = $null
if (Test-Path $configPath) {
$targetSha = ((Get-Content -Raw $configPath | ConvertFrom-Json).sha).Trim()
if ([string]::IsNullOrWhiteSpace($targetSha)) {
Write-Error "No 'sha' found in $configPath."
}
}
else {
if (-not (Test-Path $configPath)) {
Write-Host "core-commit.json not found; using current core checkout $originSha."
return @{ Root = $liveRoot; TempDir = $null }
}

$targetSha = ((Get-Content -Raw $configPath | ConvertFrom-Json).sha).Trim()
if ([string]::IsNullOrWhiteSpace($targetSha)) {
Write-Error "No 'sha' found in $configPath."
}

if ($targetSha -and $targetSha -ne $originSha) {
# Make sure the target commit is available locally (it may be newer than
# what has been fetched); ignore failure if it is already present.
if ($targetSha -eq $originSha) {
return @{ Root = $liveRoot; TempDir = $null }
}

# Make sure the target commit is available locally (the pin may be newer than
# what has been fetched). Only hit the network when the commit is genuinely
# missing: this code runs on EVERY emitter build and doc regen, and a `git
# fetch` on core/ is the only per-build network operation in the whole repo
# build, so doing it unconditionally turns any transient network/credential
# stall into a Java-only build hang. `git cat-file -e <sha>^{commit}` is a
# cheap, offline existence check.
git -C $CoreRoot cat-file -e "${targetSha}^{commit}" 2>$null
if ($LASTEXITCODE -ne 0) {
git -C $CoreRoot fetch --quiet origin $targetSha 2>$null
}

# Only move forward: check out the target when it is a descendant (newer)
# of the current checkout. `merge-base --is-ancestor A B` exits 0 when A is
# an ancestor of B, i.e. B is newer than A.
git -C $CoreRoot merge-base --is-ancestor $originSha $targetSha
if ($LASTEXITCODE -eq 0) {
Write-Host "Checking out newer core commit $targetSha (was $originSha)"
git -C $CoreRoot checkout --quiet $targetSha
if ($LASTEXITCODE -ne 0) {
Write-Error "Failed to checkout core commit $targetSha."
}
}
else {
Write-Host "core-commit.json SHA $targetSha is not newer than current $originSha; using current checkout."
}
# Only move forward: use the target when it is a descendant (newer) of the
# current checkout. `merge-base --is-ancestor A B` exits 0 when A is an ancestor
# of B, i.e. B is newer than A.
git -C $CoreRoot merge-base --is-ancestor $originSha $targetSha
if ($LASTEXITCODE -ne 0) {
Write-Host "core-commit.json SHA $targetSha is not newer than current $originSha; using current checkout."
return @{ Root = $liveRoot; TempDir = $null }
}

return $originSha
# The target is newer: extract only its packages/http-client-java subtree into a
# temp directory with `git archive`. This never modifies the submodule's working
# tree, so it stays safe under a parallel monorepo build that reads core/.
Write-Host "Reading core sources from pinned commit $targetSha (was $originSha) without checking it out"
$tempDir = Join-Path ([System.IO.Path]::GetTempPath()) ("typespec-java-core-" + [System.Guid]::NewGuid().ToString("N"))
New-Item -ItemType Directory -Path $tempDir | Out-Null
$zipPath = Join-Path $tempDir "core.zip"
$extractDir = Join-Path $tempDir "src"
try {
# Produce a zip (not tar) and expand it with .NET's ZipFile.ExtractToDirectory:
# this keeps the whole flow inside PowerShell/.NET and avoids depending on an
# external `tar`, whose behavior varies by platform (notably Git for Windows'
# GNU tar treats a drive-letter path like C:\...\core.tar as a remote host).
#
# ExtractToDirectory is used instead of Expand-Archive on purpose: the archived
# subtree holds several thousand files, and Expand-Archive emits a per-entry
# Write-Progress that makes it pathologically slow on that many files (worse when
# its output is captured by a parallel `pnpm build`), to the point the emitter
# build looks hung. ZipFile.ExtractToDirectory has no such overhead.
git -C $CoreRoot archive --format=zip -o $zipPath "${targetSha}:$script:CoreJavaSubtree"
if ($LASTEXITCODE -ne 0) {
Write-Error "Failed to archive core commit ${targetSha}:$script:CoreJavaSubtree."
}
Add-Type -AssemblyName System.IO.Compression.FileSystem
[System.IO.Compression.ZipFile]::ExtractToDirectory($zipPath, $extractDir)
}
catch {
Remove-Item $tempDir -Recurse -Force -ErrorAction SilentlyContinue
throw
}
Remove-Item $zipPath -Force
return @{ Root = $extractDir; TempDir = $tempDir }
}

# Restore the submodule to the SHA captured by Enter-CoreCommit.
function Restore-CoreCommit {
# Clean up the temporary extraction created by Get-CoreSourceRoot (no-op when the
# submodule's current checkout was read directly).
function Remove-CoreSourceRoot {
param(
[Parameter(Mandatory = $true)]
[string] $CoreRoot,

[Parameter(Mandatory = $true)]
[string] $OriginSha
$CoreSource
)

git -C $CoreRoot checkout --quiet $OriginSha
if ($CoreSource.TempDir) {
Remove-Item $CoreSource.TempDir -Recurse -Force -ErrorAction SilentlyContinue
}
}
2 changes: 1 addition & 1 deletion packages/typespec-java/core-commit.json
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{ "sha": "d012e83129830c7b71a44999c4f3681dcdd9041c" }
{ "sha": "43ff00415eb571c0c9e2d1274f621a61886e4a9b" }
2 changes: 1 addition & 1 deletion packages/typespec-java/emitter-tests/Setup.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ if (Test-Path package-lock.json) {
# typespec-tests references typespec-java via a local file path.
# npm ci will fail when the hash of the package is different from the one in package-lock.json.
# To avoid this, we remove package-lock.json and run npm install instead.
npm install --registry=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js/npm/registry/
npm install

# delete output
if (Test-Path tsp-output) {
Expand Down
12 changes: 7 additions & 5 deletions packages/typespec-java/emitter-tests/SyncTests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@
# hand-written tests and specs synced here are committed.
#
# The desired core commit is pinned in ../core-commit.json; CoreCommit.ps1's
# Enter-CoreCommit transiently checks it out (when newer) and Restore-CoreCommit
# restores the submodule afterwards.
# Get-CoreSourceRoot returns a directory to read the test sources from (extracted
# from the pinned commit via `git archive` when it is newer than the submodule's
# current checkout), without mutating the core/ submodule working tree.
# Remove-CoreSourceRoot cleans up any temporary extraction afterwards.

$ErrorActionPreference = "Stop"

Expand All @@ -20,9 +22,9 @@ $packageRoot = Resolve-Path (Join-Path $scriptRoot "..")
$coreRoot = Resolve-Path (Join-Path $scriptRoot ".." ".." ".." "core")

. (Join-Path $packageRoot "CoreCommit.ps1")
$originSha = Enter-CoreCommit -CoreRoot $coreRoot -PackageRoot $packageRoot
$core = Get-CoreSourceRoot -CoreRoot $coreRoot -PackageRoot $packageRoot
try {
$testRoot = Resolve-Path (Join-Path $coreRoot "packages" "http-client-java" "generator" "http-client-generator-test")
$testRoot = Join-Path $core.Root "generator" "http-client-generator-test"

$localSrc = Join-Path $scriptRoot "src"
$localTsp = Join-Path $scriptRoot "tsp"
Expand Down Expand Up @@ -67,7 +69,7 @@ try {
Write-Host "Synced dependency and override versions from $corePackageJsonPath"
}
finally {
Restore-CoreCommit -CoreRoot $coreRoot -OriginSha $originSha
Remove-CoreSourceRoot $core
}

# Point the local @azure-tools/typespec-java dependency at the .tgz produced by
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

package tsptest.xmlbytesverify;

import java.lang.reflect.Field;
import java.util.Arrays;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

/**
* Regression test for XML serializer gating.
*
* An operation whose response content-type is {@code application/xml} but whose body is raw {@code bytes} (not a
* structured model) must NOT generate the static XML {@code ObjectSerializer XML_SERIALIZER} field, because the
* {@code XmlSerializerProviders} helper it references is only generated for models actually used in XML.
*/
public class XmlBytesVerifyTests {

private static boolean hasSerializerField(Class<?> clazz) {
return Arrays.stream(clazz.getDeclaredFields()).anyMatch(f -> "XML_SERIALIZER".equals(f.getName()));
}

@Test
public void noXmlSerializerFieldForBytesPayload() {
Assertions.assertFalse(hasSerializerField(XmlBytesVerifyClient.class),
"XmlBytesVerifyClient should not declare a static XML_SERIALIZER field for a raw bytes XML payload.");
Assertions.assertFalse(hasSerializerField(XmlBytesVerifyAsyncClient.class),
"XmlBytesVerifyAsyncClient should not declare a static XML_SERIALIZER field for a raw bytes XML payload.");
}

@Test
public void xmlSerializerFieldPresentForModelPayload() throws ClassNotFoundException {
// Positive control: a real XML model client does declare the SERIALIZER field, ensuring the reflection
// check above is meaningful and would catch a regression that emits XML_SERIALIZER unconditionally.
Class<?> modelClient = Class.forName("payload.xml.SimpleModelValueClient");
Field serializer = Arrays.stream(modelClient.getDeclaredFields())
.filter(f -> "XML_SERIALIZER".equals(f.getName()))
.findFirst()
.orElse(null);
Assertions.assertNotNull(serializer,
"payload.xml.SimpleModelValueClient should declare a XML_SERIALIZER field.");
}
}
20 changes: 20 additions & 0 deletions packages/typespec-java/emitter-tests/tsp/xml-bytes-verify.tsp
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import "@typespec/rest";
import "@typespec/http";
import "@azure-tools/typespec-client-generator-core";

using TypeSpec.Http;
using TypeSpec.Rest;
using Azure.ClientGenerator.Core;

@service(#{ title: "XmlBytesVerify" })
namespace TspTest.XmlBytesVerify;

// Operation that returns a raw XML document as bytes (like planetarycomputer getWmtsCapabilities).
// The response content-type is application/xml but the body is unstructured bytes, so NO XML
// ObjectSerializer (SERIALIZER) should be generated/used.
@route("/xml-bytes")
@get
op getWmtsCapabilities(): OkResponse & {
@header("content-type") contentType: "application/xml";
@body body: bytes;
};
Loading