[Flex] Bill HTTP-prescaled instances as AlwaysReady baseline until first execution

src/WebJobs.Script.WebHost/Metrics/FlexConsumptionMetricsPublisher.cs

@@ -8,6 +8,7 @@
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Metrics;
 using Microsoft.Azure.WebJobs.Script.WebHost.Configuration;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
@@ -18,6 +19,7 @@ public sealed partial class FlexConsumptionMetricsPublisher : IMetricsPublisher,
         private readonly IOptionsMonitor<StandbyOptions> _standbyOptions;
         private readonly FlexConsumptionMetricsPublisherOptions _options;
         private readonly IEnvironment _environment;
+        private readonly IConfiguration _configuration;
         private readonly ILogger<FlexConsumptionMetricsPublisher> _logger;
         private readonly IHostMetricsProvider _metricsProvider;
         private readonly object _lock = new();
@@ -28,13 +30,15 @@ public sealed partial class FlexConsumptionMetricsPublisher : IMetricsPublisher,
         private DateTime _activityIntervalHighWatermark = DateTime.MinValue;
         private IDisposable _standbyOptionsOnChangeSubscription;
         private DateTime _lastPublishTime = DateTime.UtcNow;
+        private bool _firstExecutionObserved;
         private Lifecycle _lifecycle;
 
-        public FlexConsumptionMetricsPublisher(IEnvironment environment, IOptionsMonitor<StandbyOptions> standbyOptions, IOptions<FlexConsumptionMetricsPublisherOptions> options,
+        public FlexConsumptionMetricsPublisher(IEnvironment environment, IConfiguration configuration, IOptionsMonitor<StandbyOptions> standbyOptions, IOptions<FlexConsumptionMetricsPublisherOptions> options,
             ILogger<FlexConsumptionMetricsPublisher> logger, IFileSystem fileSystem, IHostMetricsProvider metricsProvider)
         {
             _standbyOptions = standbyOptions ?? throw new ArgumentNullException(nameof(standbyOptions));
             _environment = environment ?? throw new ArgumentNullException(nameof(environment));
+            _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
             _options = options?.Value ?? throw new ArgumentNullException(nameof(options));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
             _fileSystem = fileSystem ?? new FileSystem();
@@ -61,6 +65,8 @@ public FlexConsumptionMetricsPublisher(IEnvironment environment, IOptionsMonitor
 
         internal bool IsAlwaysReady { get; set; }
 
+        internal bool IsPrescaled { get; set; }
+
         internal LegionMetricsFileManager MetricsFileManager => _metricsFileManager;
 
         private bool IsStarted => _lifecycle is not null;
@@ -82,10 +88,20 @@ public void Start()
                 IsAlwaysReady = _environment
                     .GetEnvironmentVariable(EnvironmentSettingNames.FunctionsAlwaysReadyInstance) == "1";
 
+                IsPrescaled = string.Equals(
+                    _configuration[EnvironmentSettingNames.FunctionsPrescaledInstance],
+                    "1",
+                    StringComparison.Ordinal);
+
                 _logger.LogInformation(
                     $"Starting metrics publisher (AlwaysReady={IsAlwaysReady},"
                     + $" MetricsPath='{_metricsFileManager.MetricsFilePath}').");
 
+                if (IsPrescaled)
+                {
+                    _logger.LogInformation("Metrics publisher instance is prescaled; publishing AlwaysReady baseline until first execution.");
+                }
+
                 _lifecycle = new(
                     this,
                     TimeSpan.FromMilliseconds(_options.InitialPublishDelayMS),
@@ -103,21 +119,24 @@ internal async Task OnPublishMetrics(DateTime now, ValueStopwatch stopwatch)
         {
             try
             {
+                bool shouldPublishAsAlwaysReady;
                 lock (_lock)
                 {
                     if (ActiveFunctionCount > 0)
                     {
                         // at the end of an interval, we'll meter any outstanding activity up to the end of the interval
                         MeterCurrentActiveInterval(now);
                     }
+
+                    shouldPublishAsAlwaysReady = IsAlwaysReady || (IsPrescaled && !_firstExecutionObserved);
                 }
 
                 bool hasActivity = FunctionExecutionCount > 0 || FunctionExecutionTimeMS > 0 || _metricsProvider.HasMetrics();
                 bool shouldForcePublish = (now - _lastPublishTime) >= TimeSpan.FromMilliseconds(_options.KeepAliveIntervalMS);
 
-                if (!hasActivity && !shouldForcePublish && !IsAlwaysReady)
+                if (!hasActivity && !shouldForcePublish && !shouldPublishAsAlwaysReady)
                 {
-                    // No activity and not time for keep-alive publish & not always ready
+                    // No activity and not time for keep-alive publish or AlwaysReady baseline
                     return;
                 }
 
@@ -126,12 +145,13 @@ internal async Task OnPublishMetrics(DateTime now, ValueStopwatch stopwatch)
                 Metrics metrics = null;
                 lock (_lock)
                 {
+                    bool publishAsAlwaysReady = IsAlwaysReady || (IsPrescaled && !_firstExecutionObserved);
                     metrics = new Metrics
                     {
                         TotalTimeMS = (long)stopwatch.GetElapsedTime().TotalMilliseconds,
                         ExecutionCount = FunctionExecutionCount,
                         ExecutionTimeMS = FunctionExecutionTimeMS,
-                        IsAlwaysReady = IsAlwaysReady,
+                        IsAlwaysReady = publishAsAlwaysReady,
                         InstanceId = _metricsProvider.InstanceId,
                         FunctionGroup = _metricsProvider.FunctionGroup
                     };
@@ -179,6 +199,8 @@ internal void OnFunctionStarted(string functionName, string invocationId, DateTi
 
             lock (_lock)
             {
+                _firstExecutionObserved = true;
+
                 if (ActiveFunctionCount == 0)
                 {
                     // we're transitioning from inactive to active

src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs

@@ -342,11 +342,12 @@ internal static void AddLinuxContainerServices(this IServiceCollection services,
                 }
                 else if (environment.IsFlexConsumptionSku())
                 {
+                    var configuration = s.GetService<IConfiguration>();
                     var options = s.GetService<IOptions<FlexConsumptionMetricsPublisherOptions>>();
                     var standbyOptions = s.GetService<IOptionsMonitor<StandbyOptions>>();
                     var logger = s.GetService<ILogger<FlexConsumptionMetricsPublisher>>();
                     var metricsProvider = s.GetService<IHostMetricsProvider>();
-                    return new FlexConsumptionMetricsPublisher(environment, standbyOptions, options, logger, new FileSystem(), metricsProvider);
+                    return new FlexConsumptionMetricsPublisher(environment, configuration, standbyOptions, options, logger, new FileSystem(), metricsProvider);
                 }
                 else if (environment.IsLinuxMetricsPublishingEnabled())
                 {

src/WebJobs.Script/Environment/EnvironmentSettingNames.cs

@@ -83,6 +83,7 @@ public static class EnvironmentSettingNames
         public const string WebsiteNodeDefaultVersion = "WEBSITE_NODE_DEFAULT_VERSION";
         public const string FunctionsMetricsPublishPath = "FUNCTIONS_METRICS_PUBLISH_PATH";
         public const string FunctionsAlwaysReadyInstance = "FUNCTIONS_ALWAYSREADY_INSTANCE";
+        public const string FunctionsPrescaledInstance = "FUNCTIONS_PRESCALED_INSTANCE";
         public const string FunctionsTimeZone = "TZ";
         public const string FunctionsWebsiteTimeZone = "WEBSITE_TIME_ZONE";
         public const string FunctionsTargetGroup = "FUNCTIONS_TARGET_GROUP";

test/WebJobs.Script.Tests/Metrics/FlexConsumptionMetricsPublisherTests.cs

@@ -12,6 +12,7 @@
 using Microsoft.Azure.WebJobs.Script.WebHost;
 using Microsoft.Azure.WebJobs.Script.WebHost.Configuration;
 using Microsoft.Azure.WebJobs.Script.WebHost.Metrics;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Microsoft.WebJobs.Script.Tests;
@@ -26,16 +27,17 @@ public class FlexConsumptionMetricsPublisherTests
     {
         private string _metricsFilePath;
         private IEnvironment _environment;
+        private IConfiguration _configuration;
         private StandbyOptions _standbyOptions;
         private TestOptionsMonitor<StandbyOptions> _standbyOptionsMonitor;
         private FlexConsumptionMetricsPublisherOptions _options;
         private TestLogger<FlexConsumptionMetricsPublisher> _logger;
-        private HostMetricsProvider _metricsProvider;
 
         public FlexConsumptionMetricsPublisherTests()
         {
             _metricsFilePath = Path.Combine(Path.GetTempPath(), "metrics");
             _environment = new TestEnvironment();
+            _configuration = new ConfigurationBuilder().AddInMemoryCollection().Build();
         }
 
         [Theory]
@@ -107,6 +109,8 @@ public async Task OnPublishMetrics_WritesFileAndResetsCounts(bool isAlwaysReadyI
             }
 
             int executionDurationMS = 5700;
+            publisher.OnFunctionStarted("foo", "111");
+            publisher.OnFunctionCompleted("foo", "111");
             publisher.FunctionExecutionCount = 123;
             publisher.FunctionExecutionTimeMS = executionDurationMS;
 
@@ -129,6 +133,85 @@ public async Task OnPublishMetrics_WritesFileAndResetsCounts(bool isAlwaysReadyI
             Assert.Equal(0, publisher.FunctionExecutionTimeMS);
         }
 
+        [Fact]
+        public async Task OnPublishMetrics_PrescaledInstance_PublishesAlwaysReadyBaselineUntilFirstExecution()
+        {
+            CleanupMetricsFiles();
+            _configuration[EnvironmentSettingNames.FunctionsPrescaledInstance] = "1";
+            using var publisher = CreatePublisher();
+
+            // Prescaled + unused: a baseline file is written even at zero activity, stamped AlwaysReady.
+            await Task.Delay(100);
+            await publisher.OnPublishMetrics(DateTime.UtcNow, ValueStopwatch.StartNew());
+            var files = GetMetricsFilesSafe(_metricsFilePath);
+            Assert.Equal(1, files.Length);
+            var metrics = await ReadMetricsAsync(files[0].FullName);
+            Assert.True(metrics.IsAlwaysReady);
+            Assert.Equal(0, metrics.ExecutionCount);
+            Assert.Equal(0, metrics.ExecutionTimeMS);
+            files[0].Delete();
+
+            // First execution flips the instance to OnDemand.
+            publisher.OnFunctionStarted("foo", "111");
+            publisher.OnFunctionCompleted("foo", "111");
+            await Task.Delay(100);
+            await publisher.OnPublishMetrics(DateTime.UtcNow, ValueStopwatch.StartNew());
+            files = GetMetricsFilesSafe(_metricsFilePath);
+            Assert.Equal(1, files.Length);
+            metrics = await ReadMetricsAsync(files[0].FullName);
+            Assert.False(metrics.IsAlwaysReady);
+            Assert.Equal(1, metrics.ExecutionCount);
+            files[0].Delete();
+
+            // Once served, an idle interval publishes nothing (no baseline).
+            await publisher.OnPublishMetrics(DateTime.UtcNow, ValueStopwatch.StartNew());
+            files = GetMetricsFilesSafe(_metricsFilePath);
+            Assert.Equal(0, files.Length);
+        }
+
+        [Fact]
+        public async Task OnPublishMetrics_PrescaledInstance_FirstExecutionDuringPublish_PublishesOnDemand()
+        {
+            CleanupMetricsFiles();
+            _configuration[EnvironmentSettingNames.FunctionsPrescaledInstance] = "1";
+
+            var hasMetricsEntered = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
+            var continuePublish = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
+            var metricsProvider = new Mock<IHostMetricsProvider>();
+            metricsProvider.SetupGet(p => p.InstanceId).Returns(string.Empty);
+            metricsProvider.SetupGet(p => p.FunctionGroup).Returns(string.Empty);
+            metricsProvider.Setup(p => p.HasMetrics()).Returns(() =>
+            {
+                hasMetricsEntered.TrySetResult(true);
+                continuePublish.Task.GetAwaiter().GetResult();
+                return false;
+            });
+            metricsProvider.Setup(p => p.GetHostMetricsOrNull()).Returns((IReadOnlyDictionary<string, long>)null);
+
+            using var publisher = CreatePublisher(metricsProvider: metricsProvider.Object);
+            Task publishTask = Task.Run(() => publisher.OnPublishMetrics(DateTime.UtcNow, ValueStopwatch.StartNew()));
+
+            try
+            {
+                await hasMetricsEntered.Task.WaitAsync(TimeSpan.FromSeconds(5));
+                publisher.OnFunctionStarted("foo", "111");
+                publisher.OnFunctionCompleted("foo", "111");
+            }
+            finally
+            {
+                continuePublish.TrySetResult(true);
+            }
+
+            await publishTask;
+
+            var files = GetMetricsFilesSafe(_metricsFilePath);
+            Assert.Equal(1, files.Length);
+            var metrics = await ReadMetricsAsync(files[0].FullName);
+            Assert.False(metrics.IsAlwaysReady);
+            Assert.Equal(1, metrics.ExecutionCount);
+            files[0].Delete();
+        }
+
         [Fact]
         public async Task OnPublishMetrics_PurgesOldFiles()
         {
@@ -571,7 +654,7 @@ public void CleanupMetricsFiles()
             }
         }
 
-        private FlexConsumptionMetricsPublisher CreatePublisher(TimeSpan? metricsPublishInterval = null, bool inStandbyMode = false)
+        private FlexConsumptionMetricsPublisher CreatePublisher(TimeSpan? metricsPublishInterval = null, bool inStandbyMode = false, IHostMetricsProvider metricsProvider = null)
         {
             _standbyOptions = new StandbyOptions { InStandbyMode = inStandbyMode };
             _standbyOptionsMonitor = new TestOptionsMonitor<StandbyOptions>(_standbyOptions);
@@ -589,10 +672,14 @@ private FlexConsumptionMetricsPublisher CreatePublisher(TimeSpan? metricsPublish
             }
             var optionsWrapper = new OptionsWrapper<FlexConsumptionMetricsPublisherOptions>(_options);
             _logger = new TestLogger<FlexConsumptionMetricsPublisher>();
-            var serviceProvider = new Mock<IServiceProvider>();
-            var hostMetricsLogger = new TestLogger<HostMetricsProvider>();
-            _metricsProvider = new HostMetricsProvider(serviceProvider.Object, _standbyOptionsMonitor, hostMetricsLogger, _environment);
-            var publisher = new FlexConsumptionMetricsPublisher(_environment, _standbyOptionsMonitor, optionsWrapper, _logger, new FileSystem(), _metricsProvider);
+            if (metricsProvider is null)
+            {
+                var serviceProvider = new Mock<IServiceProvider>();
+                var hostMetricsLogger = new TestLogger<HostMetricsProvider>();
+                metricsProvider = new HostMetricsProvider(serviceProvider.Object, _standbyOptionsMonitor, hostMetricsLogger, _environment);
+            }
+
+            var publisher = new FlexConsumptionMetricsPublisher(_environment, _configuration, _standbyOptionsMonitor, optionsWrapper, _logger, new FileSystem(), metricsProvider);
 
             return publisher;
         }