From 468f34f17600f2f22f1b58142d7158407683db5f Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Tue, 16 Jun 2026 16:41:14 +1000 Subject: [PATCH 1/2] Default polling workers to CPU-based connection count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Polling Tentacle workers previously defaulted to 1 TCP connection, serialising all RPC calls. Under load (e.g. DRB), RPCs queue in Halibut's Pending Request Queue and time out — causing deployment failures. This PR implements Option B from the RFC: the default connection count scales with available CPU cores (clamped to [2, 8]). A 4-core machine gets 4 connections; a 1-core machine gets 2 (minimum); machines with many cores are capped at 8. The TentaclePollingConnectionCount env var still overrides this default. Co-Authored-By: Claude Sonnet 4.6 --- .../Communications/HalibutInitializer.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/Octopus.Tentacle/Communications/HalibutInitializer.cs b/source/Octopus.Tentacle/Communications/HalibutInitializer.cs index 92ca8982e..86181eed3 100644 --- a/source/Octopus.Tentacle/Communications/HalibutInitializer.cs +++ b/source/Octopus.Tentacle/Communications/HalibutInitializer.cs @@ -115,19 +115,19 @@ void AddPollingEndpoints() } } - const int MaximumPollingConnectionCount = 8; - + const uint MinimumPollingConnectionCount = 2; + const uint MaximumPollingConnectionCount = 8; + uint GetPollingConnectionCount() { - //Open multiple polling connections if the env var is set to a non-zero/negative number - var connectionCount = 1u; + // Default to one connection per CPU core, clamped to [min, max] + var connectionCount = (uint)Math.Clamp(Environment.ProcessorCount, MinimumPollingConnectionCount, MaximumPollingConnectionCount); if (uint.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariables.TentaclePollingConnectionCount), out var count)) { log.InfoFormat("Requested polling connection count: {0}", count); connectionCount = count; } - //Coerce the requested value as it might be outside our max & min switch (connectionCount) { case > MaximumPollingConnectionCount: From 1092a2808691d8d0b7ff4d547754a41549438a3f Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Wed, 17 Jun 2026 11:03:28 +1000 Subject: [PATCH 2/2] Only apply multi-connection default when Tentacle is a worker Persist IsWorker=true to config on register-worker and false on register-with. HalibutInitializer reads this flag so deployment targets keep their existing single-connection behaviour. Co-Authored-By: Claude Sonnet 4.6 --- .../Commands/RegisterMachineCommandBase.cs | 3 +++ .../Commands/RegisterWorkerCommandBase.cs | 2 ++ .../Octopus.Tentacle/Communications/HalibutInitializer.cs | 6 ++++-- .../Configuration/ITentacleConfiguration.cs | 4 ++++ .../Configuration/TentacleConfiguration.cs | 8 ++++++++ 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/source/Octopus.Tentacle/Commands/RegisterMachineCommandBase.cs b/source/Octopus.Tentacle/Commands/RegisterMachineCommandBase.cs index 02a96e28c..72f07f6db 100644 --- a/source/Octopus.Tentacle/Commands/RegisterMachineCommandBase.cs +++ b/source/Octopus.Tentacle/Commands/RegisterMachineCommandBase.cs @@ -177,11 +177,14 @@ async Task RegisterMachine(IOctopusSystemAsyncRepository systemRepository, IOcto } configuration.Value.AddOrUpdateTrustedOctopusServer(server); + configuration.Value.SetIsWorker(IsRegisteredAsWorker); VoteForRestart(); log.Info("Machine registered successfully"); } + protected virtual bool IsRegisteredAsWorker => false; + protected abstract void CheckArgs(); protected abstract void EnhanceOperation(TRegistrationOperationType registerOperation); diff --git a/source/Octopus.Tentacle/Commands/RegisterWorkerCommandBase.cs b/source/Octopus.Tentacle/Commands/RegisterWorkerCommandBase.cs index add9a363d..bfb92c810 100644 --- a/source/Octopus.Tentacle/Commands/RegisterWorkerCommandBase.cs +++ b/source/Octopus.Tentacle/Commands/RegisterWorkerCommandBase.cs @@ -29,6 +29,8 @@ public RegisterWorkerCommandBase(Lazy lazyRegisterMach Options.Add("workerpool=", "The worker pool name, slug or Id to add the machine to - e.g., 'Windows Pool'; specify this argument multiple times to add to multiple pools", s => workerpools.Add(s)); } + protected override bool IsRegisteredAsWorker => true; + protected override void CheckArgs() { if (workerpools.Count == 0 || string.IsNullOrWhiteSpace(workerpools.First())) diff --git a/source/Octopus.Tentacle/Communications/HalibutInitializer.cs b/source/Octopus.Tentacle/Communications/HalibutInitializer.cs index 86181eed3..180f983fc 100644 --- a/source/Octopus.Tentacle/Communications/HalibutInitializer.cs +++ b/source/Octopus.Tentacle/Communications/HalibutInitializer.cs @@ -120,8 +120,10 @@ void AddPollingEndpoints() uint GetPollingConnectionCount() { - // Default to one connection per CPU core, clamped to [min, max] - var connectionCount = (uint)Math.Clamp(Environment.ProcessorCount, MinimumPollingConnectionCount, MaximumPollingConnectionCount); + // Default to one connection per CPU core (clamped to [min, max]) for workers; deployment targets stay at 1 + var connectionCount = configuration.IsWorker + ? (uint)Math.Clamp(Environment.ProcessorCount, MinimumPollingConnectionCount, MaximumPollingConnectionCount) + : 1u; if (uint.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariables.TentaclePollingConnectionCount), out var count)) { log.InfoFormat("Requested polling connection count: {0}", count); diff --git a/source/Octopus.Tentacle/Configuration/ITentacleConfiguration.cs b/source/Octopus.Tentacle/Configuration/ITentacleConfiguration.cs index 675b657bb..63907eb02 100644 --- a/source/Octopus.Tentacle/Configuration/ITentacleConfiguration.cs +++ b/source/Octopus.Tentacle/Configuration/ITentacleConfiguration.cs @@ -75,6 +75,8 @@ public interface ITentacleConfiguration bool IsRegistered { get; } + bool IsWorker { get; } + void WriteTo(IWritableKeyValueStore outputStore, IEnumerable excluding); } @@ -92,6 +94,8 @@ public interface IWritableTentacleConfiguration : ITentacleConfiguration bool SetIsRegistered(bool isRegistered = true); + bool SetIsWorker(bool isWorker = true); + /// /// Sets the IP address to listen on. /// diff --git a/source/Octopus.Tentacle/Configuration/TentacleConfiguration.cs b/source/Octopus.Tentacle/Configuration/TentacleConfiguration.cs index e74380aa8..a358a5e56 100644 --- a/source/Octopus.Tentacle/Configuration/TentacleConfiguration.cs +++ b/source/Octopus.Tentacle/Configuration/TentacleConfiguration.cs @@ -19,6 +19,7 @@ namespace Octopus.Tentacle.Configuration internal class TentacleConfiguration : ITentacleConfiguration { internal const string IsRegisteredSettingName = "Tentacle.Services.IsRegistered"; + internal const string IsWorkerSettingName = "Tentacle.Services.IsWorker"; internal const string ServicesPortSettingName = "Tentacle.Services.PortNumber"; internal const string ServicesListenIPSettingName = "Tentacle.Services.ListenIP"; internal const string ServicesNoListenSettingName = "Tentacle.Services.NoListen"; @@ -70,6 +71,8 @@ public IEnumerable TrustedOctopusThumbprints public bool IsRegistered => settings.Get(IsRegisteredSettingName, false); + public bool IsWorker => settings.Get(IsWorkerSettingName, false); + public void WriteTo(IWritableKeyValueStore outputStore, IEnumerable excluding) { excluding = new HashSet(excluding); @@ -191,6 +194,11 @@ public bool SetIsRegistered(bool isRegistered = true) return settings.Set(IsRegisteredSettingName, isRegistered); } + public bool SetIsWorker(bool isWorker = true) + { + return settings.Set(IsWorkerSettingName, isWorker); + } + public bool SetListenIpAddress(string? address) { return settings.Set(ServicesListenIPSettingName, address);