-
Notifications
You must be signed in to change notification settings - Fork 482
Add WorkerProxy assignment, readiness, and state polling APIs #12018
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
Shyju Krishnankutty (kshyju)
wants to merge
8
commits into
dev
Choose a base branch
from
shkr/workerproxy_readyness
base: dev
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from 3 commits
Commits
Show all changes
8 commits
Select commit
Hold shift + click to select a range
6ded017
Add WorkerProxy one-worker assignment and readiness APIs
kshyju 74b3e37
Fix WorkerProxy development startup and review feedback
kshyju 08a46ea
Synchronize delayed worker read processing in readiness test
kshyju 04f70cd
Document explicit management API JSON parsing rationale
kshyju 49e6d5c
Align WorkerProxy management APIs with resource-oriented contract
kshyju 1141b17
Fix assignment charset validation and synchronize timer disposal asse…
kshyju 67a85fc
Make relay shutdown test logging gates deterministic
kshyju fa48ddb
Supply pod identity to WorkerProxy CI smoke container
kshyju File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Some comments aren't visible on the classic Files Changed page.
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
12 changes: 12 additions & 0 deletions
12
src/Functions.WorkerProxy/Management/InstanceStatePollRequest.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the MIT License. See License.txt in the project root for license information. | ||
|
|
||
| namespace Azure.Functions.WorkerProxy.Management; | ||
|
|
||
| /// <summary> | ||
| /// Requests a current snapshot when the revision is omitted or null, or waits for a newer revision when supplied. | ||
| /// </summary> | ||
| internal sealed class InstanceStatePollRequest | ||
| { | ||
| public long? LastKnownRevision { get; init; } | ||
| } |
66 changes: 66 additions & 0 deletions
66
src/Functions.WorkerProxy/Management/ManagementApiEndpoints.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the MIT License. See License.txt in the project root for license information. | ||
|
|
||
| using System.Text.Json; | ||
| using System.Threading.Tasks; | ||
| using Azure.Functions.WorkerProxy.State; | ||
| using Microsoft.AspNetCore.Builder; | ||
| using Microsoft.AspNetCore.Http; | ||
| using Microsoft.AspNetCore.Routing; | ||
|
|
||
| namespace Azure.Functions.WorkerProxy.Management; | ||
|
|
||
| /// <summary> | ||
| /// Registers worker lifecycle APIs on the management listener. | ||
| /// </summary> | ||
| internal static class ManagementApiEndpoints | ||
| { | ||
| public static void Map(IEndpointRouteBuilder endpoints) | ||
| { | ||
| endpoints.MapGet("/admin/worker/ready", ManagementApiHandlers.GetWorkerReady).AllowAnonymous(); | ||
| endpoints.MapPost("/admin/worker/assign", AssignWorkerAsync).AllowAnonymous(); | ||
| endpoints.MapPost("/admin/infra/instanceState", GetInstanceStateAsync).AllowAnonymous(); | ||
| } | ||
|
|
||
| private static async Task<IResult> AssignWorkerAsync(HttpRequest request, WorkerPodStateManager manager) | ||
| { | ||
| if (!request.HasJsonContentType()) | ||
| { | ||
| return ManagementApiHandlers.InvalidBody(); | ||
| } | ||
|
|
||
| WorkerAssignRequest? assignment; | ||
| try | ||
| { | ||
| assignment = await request.ReadFromJsonAsync( | ||
| WorkerProxyJsonContext.Default.WorkerAssignRequest, request.HttpContext.RequestAborted); | ||
| } | ||
| catch (JsonException) | ||
| { | ||
| return ManagementApiHandlers.InvalidBody(); | ||
| } | ||
|
|
||
| return ManagementApiHandlers.AssignWorker(assignment, manager); | ||
| } | ||
|
|
||
| private static async Task<IResult> GetInstanceStateAsync(HttpRequest request, WorkerPodStateManager manager) | ||
| { | ||
| if (!request.HasJsonContentType()) | ||
| { | ||
| return ManagementApiHandlers.InvalidBody(); | ||
| } | ||
|
|
||
| InstanceStatePollRequest? poll; | ||
| try | ||
| { | ||
| poll = await request.ReadFromJsonAsync( | ||
| WorkerProxyJsonContext.Default.InstanceStatePollRequest, request.HttpContext.RequestAborted); | ||
| } | ||
| catch (JsonException) | ||
| { | ||
| return ManagementApiHandlers.InvalidBody(); | ||
| } | ||
|
|
||
| return await ManagementApiHandlers.GetInstanceStateAsync(poll, manager, request.HttpContext.RequestAborted); | ||
| } | ||
| } | ||
136 changes: 136 additions & 0 deletions
136
src/Functions.WorkerProxy/Management/ManagementApiHandlers.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,136 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the MIT License. See License.txt in the project root for license information. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Azure.Functions.WorkerProxy.State; | ||
| using Microsoft.AspNetCore.Http; | ||
|
|
||
| namespace Azure.Functions.WorkerProxy.Management; | ||
|
|
||
| /// <summary> | ||
| /// Maps validated management requests and worker lifecycle outcomes to HTTP results. | ||
| /// </summary> | ||
| internal static class ManagementApiHandlers | ||
| { | ||
| public static IResult GetWorkerReady(WorkerPodStateManager manager) => | ||
| manager.State.IsWorkerReady ? TypedResults.Ok() : TypedResults.StatusCode(StatusCodes.Status503ServiceUnavailable); | ||
|
|
||
| public static IResult AssignWorker(WorkerAssignRequest? request, WorkerPodStateManager manager) | ||
| { | ||
| if (request is null) | ||
| { | ||
| return InvalidBody(); | ||
| } | ||
|
|
||
| List<RequestValidationError> errors = []; | ||
| if (string.IsNullOrWhiteSpace(request.FunctionAppName)) | ||
| { | ||
| errors.Add(new(WorkerApiErrorCodes.Required, "functionAppName")); | ||
| } | ||
|
|
||
| if (string.IsNullOrWhiteSpace(request.FunctionGroupName)) | ||
| { | ||
| errors.Add(new(WorkerApiErrorCodes.Required, "functionGroupName")); | ||
| } | ||
|
|
||
| if (request.IsAlwaysReady is null) | ||
| { | ||
| errors.Add(new(WorkerApiErrorCodes.Required, "isAlwaysReady")); | ||
| } | ||
|
|
||
| if (string.IsNullOrWhiteSpace(request.FunctionAppDirectory)) | ||
| { | ||
| errors.Add(new(WorkerApiErrorCodes.Required, "functionAppDirectory")); | ||
| } | ||
|
|
||
| Dictionary<string, string> environment = new(StringComparer.Ordinal); | ||
| if (request.Environment is null) | ||
| { | ||
| errors.Add(new(WorkerApiErrorCodes.Required, "environment")); | ||
| } | ||
| else | ||
| { | ||
| foreach ((string key, string? value) in request.Environment) | ||
| { | ||
| if (string.IsNullOrEmpty(key) || value is null) | ||
| { | ||
| // Report the invalid field once without exposing environment keys or values. | ||
| errors.Add(new(WorkerApiErrorCodes.InvalidValue, "environment")); | ||
| break; | ||
| } | ||
|
|
||
| environment.Add(key, value); | ||
| } | ||
| } | ||
|
|
||
| if (errors.Count > 0 | ||
| || request.FunctionAppName is not { } functionAppName | ||
| || request.FunctionGroupName is not { } functionGroupName | ||
| || request.FunctionAppDirectory is not { } functionAppDirectory | ||
| || request.IsAlwaysReady is not { } isAlwaysReady) | ||
| { | ||
| return ValidationError(errors); | ||
| } | ||
|
|
||
| WorkerAssignment assignment = new( | ||
| functionAppName, | ||
| functionGroupName, | ||
| isAlwaysReady, | ||
| environment, | ||
| functionAppDirectory); | ||
| return manager.Assign(assignment) switch | ||
| { | ||
| WorkerAssignmentResult.Success => TypedResults.Ok(), | ||
| WorkerAssignmentResult.WorkerNotReady => Error( | ||
| StatusCodes.Status503ServiceUnavailable, WorkerApiErrorCodes.WorkerNotReady, "The worker has not established a valid StartStream."), | ||
| WorkerAssignmentResult.AssignmentConflict => Error( | ||
| StatusCodes.Status409Conflict, WorkerApiErrorCodes.AssignmentConflict, "The pod is already assigned to a different assignment."), | ||
| WorkerAssignmentResult.WorkerTerminated => Error( | ||
| StatusCodes.Status503ServiceUnavailable, WorkerApiErrorCodes.WorkerTerminated, "The assigned worker stream has terminated."), | ||
| _ => throw new InvalidOperationException("Unexpected worker assignment result.") | ||
| }; | ||
| } | ||
|
|
||
| public static async Task<IResult> GetInstanceStateAsync( | ||
| InstanceStatePollRequest? request, | ||
| WorkerPodStateManager manager, | ||
| CancellationToken cancellationToken = default) | ||
| { | ||
| cancellationToken.ThrowIfCancellationRequested(); | ||
| if (request is null) | ||
| { | ||
| return InvalidBody(); | ||
| } | ||
|
|
||
| if (request.LastKnownRevision is not { } lastKnownRevision) | ||
| { | ||
| return StateResponse(manager.State); | ||
| } | ||
|
|
||
| // Revisions never decrease, so a revision valid here remains valid when the manager registers the poll. | ||
| if (lastKnownRevision < 0 || lastKnownRevision > manager.State.Revision) | ||
| { | ||
| return ValidationError([new(WorkerApiErrorCodes.InvalidRevision, "lastKnownRevision")]); | ||
| } | ||
|
|
||
| WorkerStatePollResult result = await manager.WaitForChangeAsync(lastKnownRevision, cancellationToken); | ||
| return result.State is { } state ? StateResponse(state) : TypedResults.NoContent(); | ||
| } | ||
|
|
||
| internal static IResult InvalidBody() => | ||
| ValidationError([new(WorkerApiErrorCodes.InvalidBody, "request")]); | ||
|
|
||
| private static IResult ValidationError(IReadOnlyList<RequestValidationError> errors) => | ||
| TypedResults.Json(new RequestValidationResponse(errors), | ||
| WorkerProxyJsonContext.Default.RequestValidationResponse, statusCode: StatusCodes.Status400BadRequest); | ||
|
|
||
| private static IResult Error(int statusCode, string code, string detail) => | ||
| TypedResults.Json(new WorkerApiErrorResponse(new(code, detail)), | ||
| WorkerProxyJsonContext.Default.WorkerApiErrorResponse, statusCode: statusCode); | ||
|
|
||
| private static IResult StateResponse(WorkerPodState state) => | ||
| TypedResults.Json(WorkerInstanceState.FromState(state), WorkerProxyJsonContext.Default.WorkerInstanceState); | ||
| } |
9 changes: 9 additions & 0 deletions
9
src/Functions.WorkerProxy/Management/RequestValidationError.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the MIT License. See License.txt in the project root for license information. | ||
|
|
||
| namespace Azure.Functions.WorkerProxy.Management; | ||
|
|
||
| /// <summary> | ||
| /// Identifies an invalid request field, or request for a body-level error, without echoing its value. | ||
| /// </summary> | ||
| internal sealed record RequestValidationError(string Code, string Target); |
11 changes: 11 additions & 0 deletions
11
src/Functions.WorkerProxy/Management/RequestValidationResponse.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the MIT License. See License.txt in the project root for license information. | ||
|
|
||
| using System.Collections.Generic; | ||
|
|
||
| namespace Azure.Functions.WorkerProxy.Management; | ||
|
|
||
| /// <summary> | ||
| /// Contains detected field errors, or one request-body error, returned with HTTP 400. | ||
| /// </summary> | ||
| internal sealed record RequestValidationResponse(IReadOnlyList<RequestValidationError> Errors); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the MIT License. See License.txt in the project root for license information. | ||
|
|
||
| namespace Azure.Functions.WorkerProxy.Management; | ||
|
|
||
| /// <summary> | ||
| /// Provides a stable error code and optional diagnostic detail without echoing request values. | ||
| /// </summary> | ||
| /// <param name="Code">A case-sensitive contract identifier; existing codes must not be renamed or repurposed.</param> | ||
| /// <param name="Detail">Diagnostic text that may change and must not be used for client decisions.</param> | ||
| /// <remarks> | ||
| /// WorkerNotReady (503) permits retry after readiness. WorkerTerminated (503) is terminal for the | ||
| /// assigned session; retrying the same assignment on this pod cannot recover it. | ||
| /// AssignmentConflict (409) rejects a different assignment; do not retry that request unchanged. | ||
| /// Clients must inspect Code to distinguish the two 503 outcomes and handle unknown codes gracefully. | ||
| /// </remarks> | ||
| internal sealed record WorkerApiError(string Code, string? Detail = null); |
20 changes: 20 additions & 0 deletions
20
src/Functions.WorkerProxy/Management/WorkerApiErrorCodes.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the MIT License. See License.txt in the project root for license information. | ||
|
|
||
| namespace Azure.Functions.WorkerProxy.Management; | ||
|
|
||
| /// <summary> | ||
| /// Defines the stable error codes returned by the management APIs. | ||
| /// </summary> | ||
| internal static class WorkerApiErrorCodes | ||
| { | ||
| // Clients branch on these exact, case-sensitive wire values. Do not change or repurpose them. | ||
| // Keep explicit literals rather than nameof so symbol renames cannot change the contract. | ||
| public const string Required = "Required"; | ||
| public const string InvalidBody = "InvalidBody"; | ||
| public const string InvalidValue = "InvalidValue"; | ||
| public const string InvalidRevision = "InvalidRevision"; | ||
| public const string WorkerNotReady = "WorkerNotReady"; | ||
| public const string WorkerTerminated = "WorkerTerminated"; | ||
| public const string AssignmentConflict = "AssignmentConflict"; | ||
| } |
9 changes: 9 additions & 0 deletions
9
src/Functions.WorkerProxy/Management/WorkerApiErrorResponse.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the MIT License. See License.txt in the project root for license information. | ||
|
|
||
| namespace Azure.Functions.WorkerProxy.Management; | ||
|
|
||
| /// <summary> | ||
| /// Wraps validation and lifecycle failures in the same management API error envelope. | ||
|
|
||
| /// </summary> | ||
| internal sealed record WorkerApiErrorResponse(WorkerApiError Error); | ||
22 changes: 22 additions & 0 deletions
22
src/Functions.WorkerProxy/Management/WorkerAssignRequest.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the MIT License. See License.txt in the project root for license information. | ||
|
|
||
| using System.Collections.Generic; | ||
|
|
||
| namespace Azure.Functions.WorkerProxy.Management; | ||
|
|
||
| /// <summary> | ||
| /// Describes assignment identity for an already-specialized worker. | ||
| /// </summary> | ||
| internal sealed class WorkerAssignRequest | ||
| { | ||
| public string? FunctionAppName { get; init; } | ||
|
|
||
| public string? FunctionGroupName { get; init; } | ||
|
|
||
| public bool? IsAlwaysReady { get; init; } | ||
|
|
||
| public Dictionary<string, string?>? Environment { get; init; } | ||
|
|
||
| public string? FunctionAppDirectory { get; init; } | ||
| } |
20 changes: 20 additions & 0 deletions
20
src/Functions.WorkerProxy/Management/WorkerInstanceState.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the MIT License. See License.txt in the project root for license information. | ||
|
|
||
| using Azure.Functions.WorkerProxy.State; | ||
|
|
||
| namespace Azure.Functions.WorkerProxy.Management; | ||
|
|
||
| /// <summary> | ||
| /// Exposes platform-facing pod state without internal session details or assignment environment values. | ||
| /// </summary> | ||
| internal sealed record WorkerInstanceState( | ||
| string PodName, | ||
| long RevisionId, | ||
| WorkerPodStateResponse WorkerPodState) | ||
| { | ||
| public string FunctionsContainerType => "FunctionsWorkerPod"; | ||
|
|
||
| public static WorkerInstanceState FromState(WorkerPodState state) => | ||
| new(state.PodName, state.Revision, new(state.PodStatus, state.FunctionGroupName, state.IsAlwaysReady)); | ||
| } |
15 changes: 15 additions & 0 deletions
15
src/Functions.WorkerProxy/Management/WorkerPodStateResponse.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the MIT License. See License.txt in the project root for license information. | ||
|
|
||
| using System.Text.Json.Serialization; | ||
| using Azure.Functions.WorkerProxy.State; | ||
|
|
||
| namespace Azure.Functions.WorkerProxy.Management; | ||
|
|
||
| /// <summary> | ||
| /// Reports worker-pod eligibility and known assignment identity, not runtime serving readiness. | ||
| /// </summary> | ||
| internal sealed record WorkerPodStateResponse( | ||
| [property: JsonConverter(typeof(JsonStringEnumConverter<WorkerPodStatus>))] WorkerPodStatus PodStatus, | ||
| string? FunctionGroupName, | ||
| bool? IsAlwaysReady); |
17 changes: 17 additions & 0 deletions
17
src/Functions.WorkerProxy/Management/WorkerProxyJsonContext.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the MIT License. See License.txt in the project root for license information. | ||
|
|
||
| using System.Text.Json.Serialization; | ||
|
|
||
| namespace Azure.Functions.WorkerProxy.Management; | ||
|
|
||
| [JsonSourceGenerationOptions( | ||
| PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, | ||
| PropertyNameCaseInsensitive = true, | ||
| DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] | ||
| [JsonSerializable(typeof(WorkerAssignRequest))] | ||
| [JsonSerializable(typeof(InstanceStatePollRequest))] | ||
| [JsonSerializable(typeof(WorkerInstanceState))] | ||
| [JsonSerializable(typeof(WorkerApiErrorResponse))] | ||
| [JsonSerializable(typeof(RequestValidationResponse))] | ||
| internal sealed partial class WorkerProxyJsonContext : JsonSerializerContext; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.