diff --git a/src/WebJobs.Script.WebHost/Metrics/FlexConsumptionMetricsPublisher.cs b/src/WebJobs.Script.WebHost/Metrics/FlexConsumptionMetricsPublisher.cs index d375e72b5..dcd4776d3 100644 --- a/src/WebJobs.Script.WebHost/Metrics/FlexConsumptionMetricsPublisher.cs +++ b/src/WebJobs.Script.WebHost/Metrics/FlexConsumptionMetricsPublisher.cs @@ -8,6 +8,7 @@ using Microsoft.Azure.WebJobs.Script.Diagnostics; using Microsoft.Azure.WebJobs.Script.Metrics; using Microsoft.Azure.WebJobs.Script.WebHost.Configuration; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -18,6 +19,7 @@ public sealed partial class FlexConsumptionMetricsPublisher : IMetricsPublisher, private readonly IOptionsMonitor _standbyOptions; private readonly FlexConsumptionMetricsPublisherOptions _options; private readonly IEnvironment _environment; + private readonly IConfiguration _configuration; private readonly ILogger _logger; private readonly IHostMetricsProvider _metricsProvider; private readonly object _lock = new(); @@ -28,13 +30,15 @@ public sealed partial class FlexConsumptionMetricsPublisher : IMetricsPublisher, private DateTime _activityIntervalHighWatermark = DateTime.MinValue; private IDisposable _standbyOptionsOnChangeSubscription; private DateTime _lastPublishTime = DateTime.UtcNow; + private bool _firstExecutionObserved; private Lifecycle _lifecycle; - public FlexConsumptionMetricsPublisher(IEnvironment environment, IOptionsMonitor standbyOptions, IOptions options, + public FlexConsumptionMetricsPublisher(IEnvironment environment, IConfiguration configuration, IOptionsMonitor standbyOptions, IOptions options, ILogger logger, IFileSystem fileSystem, IHostMetricsProvider metricsProvider) { _standbyOptions = standbyOptions ?? throw new ArgumentNullException(nameof(standbyOptions)); _environment = environment ?? throw new ArgumentNullException(nameof(environment)); + _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _fileSystem = fileSystem ?? new FileSystem(); @@ -61,6 +65,8 @@ public FlexConsumptionMetricsPublisher(IEnvironment environment, IOptionsMonitor internal bool IsAlwaysReady { get; set; } + internal bool IsPrescaled { get; set; } + internal LegionMetricsFileManager MetricsFileManager => _metricsFileManager; private bool IsStarted => _lifecycle is not null; @@ -82,10 +88,20 @@ public void Start() IsAlwaysReady = _environment .GetEnvironmentVariable(EnvironmentSettingNames.FunctionsAlwaysReadyInstance) == "1"; + IsPrescaled = string.Equals( + _configuration[EnvironmentSettingNames.FunctionsPrescaledInstance], + "1", + StringComparison.Ordinal); + _logger.LogInformation( $"Starting metrics publisher (AlwaysReady={IsAlwaysReady}," + $" MetricsPath='{_metricsFileManager.MetricsFilePath}')."); + if (IsPrescaled) + { + _logger.LogInformation("Metrics publisher instance is prescaled; publishing AlwaysReady baseline until first execution."); + } + _lifecycle = new( this, TimeSpan.FromMilliseconds(_options.InitialPublishDelayMS), @@ -103,6 +119,7 @@ internal async Task OnPublishMetrics(DateTime now, ValueStopwatch stopwatch) { try { + bool shouldPublishAsAlwaysReady; lock (_lock) { if (ActiveFunctionCount > 0) @@ -110,14 +127,16 @@ internal async Task OnPublishMetrics(DateTime now, ValueStopwatch stopwatch) // at the end of an interval, we'll meter any outstanding activity up to the end of the interval MeterCurrentActiveInterval(now); } + + shouldPublishAsAlwaysReady = IsAlwaysReady || (IsPrescaled && !_firstExecutionObserved); } bool hasActivity = FunctionExecutionCount > 0 || FunctionExecutionTimeMS > 0 || _metricsProvider.HasMetrics(); bool shouldForcePublish = (now - _lastPublishTime) >= TimeSpan.FromMilliseconds(_options.KeepAliveIntervalMS); - if (!hasActivity && !shouldForcePublish && !IsAlwaysReady) + if (!hasActivity && !shouldForcePublish && !shouldPublishAsAlwaysReady) { - // No activity and not time for keep-alive publish & not always ready + // No activity and not time for keep-alive publish or AlwaysReady baseline return; } @@ -126,12 +145,13 @@ internal async Task OnPublishMetrics(DateTime now, ValueStopwatch stopwatch) Metrics metrics = null; lock (_lock) { + bool publishAsAlwaysReady = IsAlwaysReady || (IsPrescaled && !_firstExecutionObserved); metrics = new Metrics { TotalTimeMS = (long)stopwatch.GetElapsedTime().TotalMilliseconds, ExecutionCount = FunctionExecutionCount, ExecutionTimeMS = FunctionExecutionTimeMS, - IsAlwaysReady = IsAlwaysReady, + IsAlwaysReady = publishAsAlwaysReady, InstanceId = _metricsProvider.InstanceId, FunctionGroup = _metricsProvider.FunctionGroup }; @@ -179,6 +199,8 @@ internal void OnFunctionStarted(string functionName, string invocationId, DateTi lock (_lock) { + _firstExecutionObserved = true; + if (ActiveFunctionCount == 0) { // we're transitioning from inactive to active diff --git a/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs b/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs index db36b217e..76ce14f5a 100644 --- a/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs +++ b/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs @@ -342,11 +342,12 @@ internal static void AddLinuxContainerServices(this IServiceCollection services, } else if (environment.IsFlexConsumptionSku()) { + var configuration = s.GetService(); var options = s.GetService>(); var standbyOptions = s.GetService>(); var logger = s.GetService>(); var metricsProvider = s.GetService(); - return new FlexConsumptionMetricsPublisher(environment, standbyOptions, options, logger, new FileSystem(), metricsProvider); + return new FlexConsumptionMetricsPublisher(environment, configuration, standbyOptions, options, logger, new FileSystem(), metricsProvider); } else if (environment.IsLinuxMetricsPublishingEnabled()) { diff --git a/src/WebJobs.Script/Environment/EnvironmentSettingNames.cs b/src/WebJobs.Script/Environment/EnvironmentSettingNames.cs index 0c8db2d3d..a4458c636 100644 --- a/src/WebJobs.Script/Environment/EnvironmentSettingNames.cs +++ b/src/WebJobs.Script/Environment/EnvironmentSettingNames.cs @@ -83,6 +83,7 @@ public static class EnvironmentSettingNames public const string WebsiteNodeDefaultVersion = "WEBSITE_NODE_DEFAULT_VERSION"; public const string FunctionsMetricsPublishPath = "FUNCTIONS_METRICS_PUBLISH_PATH"; public const string FunctionsAlwaysReadyInstance = "FUNCTIONS_ALWAYSREADY_INSTANCE"; + public const string FunctionsPrescaledInstance = "FUNCTIONS_PRESCALED_INSTANCE"; public const string FunctionsTimeZone = "TZ"; public const string FunctionsWebsiteTimeZone = "WEBSITE_TIME_ZONE"; public const string FunctionsTargetGroup = "FUNCTIONS_TARGET_GROUP"; diff --git a/test/WebJobs.Script.Tests/Metrics/FlexConsumptionMetricsPublisherTests.cs b/test/WebJobs.Script.Tests/Metrics/FlexConsumptionMetricsPublisherTests.cs index c909a6a8c..dee01acb9 100644 --- a/test/WebJobs.Script.Tests/Metrics/FlexConsumptionMetricsPublisherTests.cs +++ b/test/WebJobs.Script.Tests/Metrics/FlexConsumptionMetricsPublisherTests.cs @@ -12,6 +12,7 @@ using Microsoft.Azure.WebJobs.Script.WebHost; using Microsoft.Azure.WebJobs.Script.WebHost.Configuration; using Microsoft.Azure.WebJobs.Script.WebHost.Metrics; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.WebJobs.Script.Tests; @@ -26,16 +27,17 @@ public class FlexConsumptionMetricsPublisherTests { private string _metricsFilePath; private IEnvironment _environment; + private IConfiguration _configuration; private StandbyOptions _standbyOptions; private TestOptionsMonitor _standbyOptionsMonitor; private FlexConsumptionMetricsPublisherOptions _options; private TestLogger _logger; - private HostMetricsProvider _metricsProvider; public FlexConsumptionMetricsPublisherTests() { _metricsFilePath = Path.Combine(Path.GetTempPath(), "metrics"); _environment = new TestEnvironment(); + _configuration = new ConfigurationBuilder().AddInMemoryCollection().Build(); } [Theory] @@ -107,6 +109,8 @@ public async Task OnPublishMetrics_WritesFileAndResetsCounts(bool isAlwaysReadyI } int executionDurationMS = 5700; + publisher.OnFunctionStarted("foo", "111"); + publisher.OnFunctionCompleted("foo", "111"); publisher.FunctionExecutionCount = 123; publisher.FunctionExecutionTimeMS = executionDurationMS; @@ -129,6 +133,85 @@ public async Task OnPublishMetrics_WritesFileAndResetsCounts(bool isAlwaysReadyI Assert.Equal(0, publisher.FunctionExecutionTimeMS); } + [Fact] + public async Task OnPublishMetrics_PrescaledInstance_PublishesAlwaysReadyBaselineUntilFirstExecution() + { + CleanupMetricsFiles(); + _configuration[EnvironmentSettingNames.FunctionsPrescaledInstance] = "1"; + using var publisher = CreatePublisher(); + + // Prescaled + unused: a baseline file is written even at zero activity, stamped AlwaysReady. + await Task.Delay(100); + await publisher.OnPublishMetrics(DateTime.UtcNow, ValueStopwatch.StartNew()); + var files = GetMetricsFilesSafe(_metricsFilePath); + Assert.Equal(1, files.Length); + var metrics = await ReadMetricsAsync(files[0].FullName); + Assert.True(metrics.IsAlwaysReady); + Assert.Equal(0, metrics.ExecutionCount); + Assert.Equal(0, metrics.ExecutionTimeMS); + files[0].Delete(); + + // First execution flips the instance to OnDemand. + publisher.OnFunctionStarted("foo", "111"); + publisher.OnFunctionCompleted("foo", "111"); + await Task.Delay(100); + await publisher.OnPublishMetrics(DateTime.UtcNow, ValueStopwatch.StartNew()); + files = GetMetricsFilesSafe(_metricsFilePath); + Assert.Equal(1, files.Length); + metrics = await ReadMetricsAsync(files[0].FullName); + Assert.False(metrics.IsAlwaysReady); + Assert.Equal(1, metrics.ExecutionCount); + files[0].Delete(); + + // Once served, an idle interval publishes nothing (no baseline). + await publisher.OnPublishMetrics(DateTime.UtcNow, ValueStopwatch.StartNew()); + files = GetMetricsFilesSafe(_metricsFilePath); + Assert.Equal(0, files.Length); + } + + [Fact] + public async Task OnPublishMetrics_PrescaledInstance_FirstExecutionDuringPublish_PublishesOnDemand() + { + CleanupMetricsFiles(); + _configuration[EnvironmentSettingNames.FunctionsPrescaledInstance] = "1"; + + var hasMetricsEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var continuePublish = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var metricsProvider = new Mock(); + metricsProvider.SetupGet(p => p.InstanceId).Returns(string.Empty); + metricsProvider.SetupGet(p => p.FunctionGroup).Returns(string.Empty); + metricsProvider.Setup(p => p.HasMetrics()).Returns(() => + { + hasMetricsEntered.TrySetResult(true); + continuePublish.Task.GetAwaiter().GetResult(); + return false; + }); + metricsProvider.Setup(p => p.GetHostMetricsOrNull()).Returns((IReadOnlyDictionary)null); + + using var publisher = CreatePublisher(metricsProvider: metricsProvider.Object); + Task publishTask = Task.Run(() => publisher.OnPublishMetrics(DateTime.UtcNow, ValueStopwatch.StartNew())); + + try + { + await hasMetricsEntered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + publisher.OnFunctionStarted("foo", "111"); + publisher.OnFunctionCompleted("foo", "111"); + } + finally + { + continuePublish.TrySetResult(true); + } + + await publishTask; + + var files = GetMetricsFilesSafe(_metricsFilePath); + Assert.Equal(1, files.Length); + var metrics = await ReadMetricsAsync(files[0].FullName); + Assert.False(metrics.IsAlwaysReady); + Assert.Equal(1, metrics.ExecutionCount); + files[0].Delete(); + } + [Fact] public async Task OnPublishMetrics_PurgesOldFiles() { @@ -571,7 +654,7 @@ public void CleanupMetricsFiles() } } - private FlexConsumptionMetricsPublisher CreatePublisher(TimeSpan? metricsPublishInterval = null, bool inStandbyMode = false) + private FlexConsumptionMetricsPublisher CreatePublisher(TimeSpan? metricsPublishInterval = null, bool inStandbyMode = false, IHostMetricsProvider metricsProvider = null) { _standbyOptions = new StandbyOptions { InStandbyMode = inStandbyMode }; _standbyOptionsMonitor = new TestOptionsMonitor(_standbyOptions); @@ -589,10 +672,14 @@ private FlexConsumptionMetricsPublisher CreatePublisher(TimeSpan? metricsPublish } var optionsWrapper = new OptionsWrapper(_options); _logger = new TestLogger(); - var serviceProvider = new Mock(); - var hostMetricsLogger = new TestLogger(); - _metricsProvider = new HostMetricsProvider(serviceProvider.Object, _standbyOptionsMonitor, hostMetricsLogger, _environment); - var publisher = new FlexConsumptionMetricsPublisher(_environment, _standbyOptionsMonitor, optionsWrapper, _logger, new FileSystem(), _metricsProvider); + if (metricsProvider is null) + { + var serviceProvider = new Mock(); + var hostMetricsLogger = new TestLogger(); + metricsProvider = new HostMetricsProvider(serviceProvider.Object, _standbyOptionsMonitor, hostMetricsLogger, _environment); + } + + var publisher = new FlexConsumptionMetricsPublisher(_environment, _configuration, _standbyOptionsMonitor, optionsWrapper, _logger, new FileSystem(), metricsProvider); return publisher; }