[PROVES EFT-3295 #1226] Abandon tests fail on main as it stands — do not merge

source/Octopus.Tentacle.Client/ITentacleClient.cs

@@ -8,6 +8,7 @@
 using Octopus.Tentacle.Client.Scripts.Models;
 using Octopus.Tentacle.Contracts;
 using Octopus.Tentacle.Contracts.Logging;
+using Octopus.Tentacle.Contracts.ScriptServiceV2;
 
 namespace Octopus.Tentacle.Client
 {
@@ -59,6 +60,11 @@ Task<ScriptOperationExecutionResult> StartScript(ExecuteScriptCommand command,
         /// <returns>The result, which includes the CommandContext for the next command</returns>
         Task<ScriptOperationExecutionResult> CancelScript(CommandContext commandContext, ITentacleClientTaskLog logger);
 
+        /// <summary>
+        /// Abandon the script (V2 only). Stub on this branch — see EFT-3295.
+        /// </summary>
+        Task<ScriptStatusResponseV2> AbandonScript(ScriptTicket scriptTicket, ITentacleClientTaskLog logger, CancellationToken cancellationToken);
+
         /// <summary>
         /// Complete the script.
         /// </summary>

source/Octopus.Tentacle.Client/TentacleClient.cs

@@ -16,6 +16,7 @@
 using Octopus.Tentacle.Contracts.Capabilities;
 using Octopus.Tentacle.Contracts.Logging;
 using Octopus.Tentacle.Contracts.Observability;
+using Octopus.Tentacle.Contracts.ScriptServiceV2;
 using ITentacleClientObserver = Octopus.Tentacle.Contracts.Observability.ITentacleClientObserver;
 
 namespace Octopus.Tentacle.Client
@@ -260,6 +261,36 @@ public async Task<ScriptOperationExecutionResult> CancelScript(CommandContext co
             return await scriptExecutor.CancelScript(commandContext);
         }
 
+        public async Task<ScriptStatusResponseV2> AbandonScript(ScriptTicket scriptTicket, ITentacleClientTaskLog logger, CancellationToken cancellationToken)
+        {
+            using var activity = ActivitySource.StartActivity($"{nameof(TentacleClient)}.{nameof(AbandonScript)}");
+            activity?.AddTag("octopus.tentacle.script.ticket", scriptTicket.TaskId);
+
+            var operationMetricsBuilder = ClientOperationMetricsBuilder.Start();
+
+            async Task<ScriptStatusResponseV2> AbandonScriptAction(CancellationToken ct)
+            {
+                var request = new AbandonScriptCommandV2(scriptTicket, lastLogSequence: 0);
+                return await allClients.ScriptServiceV2.AbandonScriptAsync(request, new HalibutProxyRequestOptions(ct));
+            }
+
+            try
+            {
+                return await rpcCallExecutor.Execute(
+                    retriesEnabled: clientOptions.RpcRetrySettings.RetriesEnabled,
+                    RpcCall.Create<IScriptServiceV2>(nameof(IScriptServiceV2.AbandonScript)),
+                    AbandonScriptAction,
+                    logger,
+                    operationMetricsBuilder,
+                    cancellationToken).ConfigureAwait(false);
+            }
+            catch (Exception e)
+            {
+                operationMetricsBuilder.Failure(e, cancellationToken);
+                throw;
+            }
+        }
+
         public async Task<ScriptStatus?> CompleteScript(CommandContext commandContext, ITentacleClientTaskLog logger, CancellationToken scriptExecutionCancellationToken)
         {
             using var activity = ActivitySource.StartActivity($"{nameof(TentacleClient)}.{nameof(CompleteScript)}");

source/Octopus.Tentacle.Contracts/ClientServices/IAsyncClientScriptServiceV2.cs

@@ -10,6 +10,7 @@ public interface IAsyncClientScriptServiceV2
         Task<ScriptStatusResponseV2> StartScriptAsync(StartScriptCommandV2 command, HalibutProxyRequestOptions proxyRequestOptions);
         Task<ScriptStatusResponseV2> GetStatusAsync(ScriptStatusRequestV2 request, HalibutProxyRequestOptions proxyRequestOptions);
         Task<ScriptStatusResponseV2> CancelScriptAsync(CancelScriptCommandV2 command, HalibutProxyRequestOptions proxyRequestOptions);
+        Task<ScriptStatusResponseV2> AbandonScriptAsync(AbandonScriptCommandV2 command, HalibutProxyRequestOptions proxyRequestOptions);
         Task CompleteScriptAsync(CompleteScriptCommandV2 command, HalibutProxyRequestOptions proxyRequestOptions);
     }
 }

source/Octopus.Tentacle.Contracts/ScriptExitCodes.cs

@@ -12,6 +12,7 @@ public static class ScriptExitCodes
         public const int UnknownScriptExitCode = -45;
         public const int UnknownResultExitCode = -46;
         public const int PowerShellNeverStartedExitCode = -47;
+        public const int AbandonedExitCode = -48;
 
         //Kubernetes Agent
         public const int KubernetesScriptPodNotFound = -81;

source/Octopus.Tentacle.Contracts/ScriptServiceV2/AbandonScriptCommandV2.cs

@@ -0,0 +1,17 @@
+using System;
+
+namespace Octopus.Tentacle.Contracts.ScriptServiceV2
+{
+    public class AbandonScriptCommandV2
+    {
+        public AbandonScriptCommandV2(ScriptTicket ticket, long lastLogSequence)
+        {
+            Ticket = ticket;
+            LastLogSequence = lastLogSequence;
+        }
+
+        public ScriptTicket Ticket { get; }
+
+        public long LastLogSequence { get; }
+    }
+}

source/Octopus.Tentacle.Contracts/ScriptServiceV2/IScriptServiceV2.cs

@@ -7,6 +7,7 @@ public interface IScriptServiceV2
         ScriptStatusResponseV2 StartScript(StartScriptCommandV2 command);
         ScriptStatusResponseV2 GetStatus(ScriptStatusRequestV2 request);
         ScriptStatusResponseV2 CancelScript(CancelScriptCommandV2 command);
+        ScriptStatusResponseV2 AbandonScript(AbandonScriptCommandV2 command);
         void CompleteScript(CompleteScriptCommandV2 command);
     }
 }

source/Octopus.Tentacle.Contracts/Services/Scripts/IAsyncScriptServiceV2.cs

@@ -12,6 +12,7 @@ public interface IAsyncScriptServiceV2
         Task<ScriptStatusResponseV2> StartScriptAsync(StartScriptCommandV2 command, CancellationToken cancellationToken);
         Task<ScriptStatusResponseV2> GetStatusAsync(ScriptStatusRequestV2 request, CancellationToken cancellationToken);
         Task<ScriptStatusResponseV2> CancelScriptAsync(CancelScriptCommandV2 command, CancellationToken cancellationToken);
+        Task<ScriptStatusResponseV2> AbandonScriptAsync(AbandonScriptCommandV2 command, CancellationToken cancellationToken);
         Task CompleteScriptAsync(CompleteScriptCommandV2 command, CancellationToken cancellationToken);
     }
 }

source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs

@@ -140,6 +140,16 @@ public async Task<ScriptStatusResponseV2> CancelScriptAsync(CancelScriptCommandV
             return GetResponse(command.Ticket, command.LastLogSequence, runningScript?.Process);
         }
 
+        public Task<ScriptStatusResponseV2> AbandonScriptAsync(AbandonScriptCommandV2 command, CancellationToken cancellationToken)
+        {
+            // EFT-3295 stub: this returns a status snapshot without firing any abandon
+            // token. The script process keeps running until it exits naturally or the
+            // cancel kills it, so the mutex is NOT released here. This is the behaviour
+            // gap the abandon integration tests demonstrate.
+            runningScripts.TryGetValue(command.Ticket, out var runningScript);
+            return Task.FromResult(GetResponse(command.Ticket, command.LastLogSequence, runningScript?.Process));
+        }
+
         public async Task CompleteScriptAsync(CompleteScriptCommandV2 command, CancellationToken cancellationToken)
         {
             await Task.CompletedTask;

source/Octopus.Tentacle.Core/Util/CommandLine/SilentProcessRunner.cs

@@ -9,6 +9,7 @@
 using System.Text;
 using System.Threading;
 using Octopus.Tentacle.Core.Diagnostics;
+using Octopus.Tentacle.Core.Util;
 
 namespace Octopus.Tentacle.Util
 {
@@ -251,11 +252,18 @@ class Hitman
         {
             public static void TryKillProcessAndChildrenRecursively(Process process)
             {
+                if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable(EnvironmentVariables.TentacleDebugDisableProcessKill_UNSAFE_FOR_PRODUCTION)))
+                {
+                    // Test-only no-op: simulate "kill was attempted but didn't terminate the process".
+                    // Only activated when the test harness sets this env var on the Tentacle process.
+                    return;
+                }
+
 #if NETFRAMEWORK
                 TryKillWindowsProcessAndChildrenRecursively(process.Id);
 #endif
 #if !NETFRAMEWORK
-                // Since .NET Core 3.0 there is support for killing a process and it's children 
+                // Since .NET Core 3.0 there is support for killing a process and it's children
                 process.Kill(true);
 #endif
             }

source/Octopus.Tentacle.Core/Util/EnvironmentVariables.cs

@@ -29,6 +29,7 @@ public static class EnvironmentVariables
         public const string TentacleMachineConfigurationHomeDirectory = "TentacleMachineConfigurationHomeDirectory";
         public const string TentaclePollingConnectionCount = "TentaclePollingConnectionCount";
         public const string TentaclePowerShellStartupTimeout = "TentaclePowerShellStartupTimeout";
+        public const string TentacleDebugDisableProcessKill_UNSAFE_FOR_PRODUCTION = "TentacleDebugDisableProcessKill_UNSAFE_FOR_PRODUCTION";
         public const string NfsWatchdogDirectory = "watchdog_directory";
         public static string TentacleUseTcpNoDelay = "TentacleUseTcpNoDelay";
         public static string TentacleUseAsyncListener = "TentacleUseAsyncListener";

source/Octopus.Tentacle.Tests.Integration.Common/Builders/Decorators/ScriptServiceV2DecoratorBuilder.cs

@@ -188,6 +188,12 @@ public async Task<ScriptStatusResponseV2> CancelScriptAsync(CancelScriptCommandV
                 return await cancelScriptFunc(inner, command, options);
             }
 
+            public async Task<ScriptStatusResponseV2> AbandonScriptAsync(AbandonScriptCommandV2 command, HalibutProxyRequestOptions options)
+            {
+                // Pass-through: AbandonScript decoration is not parameterised on this branch.
+                return await inner.AbandonScriptAsync(command, options);
+            }
+
             public async Task CompleteScriptAsync(CompleteScriptCommandV2 command, HalibutProxyRequestOptions options)
             {
                 await completeScriptAction(inner, command, options);

source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionAbandon.cs

@@ -0,0 +1,131 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using FluentAssertions;
+using NUnit.Framework;
+using Octopus.Tentacle.Contracts;
+using Octopus.Tentacle.Core.Util;
+using Octopus.Tentacle.Tests.Integration.Support;
+using Octopus.Tentacle.Tests.Integration.Util;
+using Octopus.Tentacle.Tests.Integration.Util.Builders;
+
+namespace Octopus.Tentacle.Tests.Integration
+{
+    [IntegrationTestTimeout]
+    public class ClientScriptExecutionAbandon : IntegrationTest
+    {
+        [Test]
+        [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.Version2)]
+        public async Task AbandonScript_WhenCancelFailsToKillProcess_ReturnsAbandonedExitCode(TentacleConfigurationTestCase tentacleConfigurationTestCase)
+        {
+            // TentacleDebugDisableProcessKill_UNSAFE_FOR_PRODUCTION=1 makes Hitman a no-op, so
+            // CancelScript cannot actually terminate the underlying script process. The script
+            // becomes genuinely "stuck" from Tentacle's perspective. AbandonScript should then
+            // return promptly with AbandonedExitCode without waiting for the process to exit.
+            await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder()
+                .WithTentacle(x => x.WithRunTentacleEnvironmentVariable(EnvironmentVariables.TentacleDebugDisableProcessKill_UNSAFE_FOR_PRODUCTION, "1"))
+                .Build(CancellationToken);
+
+            var startFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "start");
+            var releaseFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "release");
+
+            var firstCommand = new TestExecuteShellScriptCommandBuilder()
+                .SetScriptBody(new ScriptBuilder()
+                    .CreateFile(startFile)
+                    .WaitForFileToExist(releaseFile))
+                .WithIsolationLevel(ScriptIsolationLevel.NoIsolation)
+                .Build();
+
+            var tentacleClient = clientTentacle.TentacleClient;
+
+            var scriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(firstCommand, CancellationToken));
+
+            await Wait.For(() => File.Exists(startFile),
+                TimeSpan.FromSeconds(30),
+                () => throw new Exception("Script did not start"),
+                CancellationToken);
+
+            // Cancel: Hitman is a no-op so the process keeps running.
+            await tentacleClient.CancelScript(firstCommand.ScriptTicket);
+            await Task.Delay(TimeSpan.FromSeconds(1));
+
+            // Abandon: fires the abandon token. The RPC returns the current status snapshot
+            // immediately, so we poll GetStatus until the script reaches Complete state.
+            await tentacleClient.AbandonScript(firstCommand.ScriptTicket, CancellationToken);
+
+            ScriptStatus abandonResponse = null!;
+            await Wait.For(async () =>
+                {
+                    abandonResponse = await tentacleClient.GetStatus(firstCommand.ScriptTicket, CancellationToken);
+                    return abandonResponse.State == ProcessState.Complete;
+                },
+                TimeSpan.FromSeconds(30),
+                () => throw new Exception("Abandoned script did not reach Complete state within 30s"),
+                CancellationToken);
+            abandonResponse.ExitCode.Should().Be(ScriptExitCodes.AbandonedExitCode);
+
+            // Release the script process so it exits cleanly and stops leaking.
+            File.WriteAllText(releaseFile, "");
+            await scriptExecution;
+        }
+
+        [Test]
+        [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.Version2)]
+        public async Task AbandonScript_ReleasesIsolationMutexEvenWhileProcessIsStillRunning(TentacleConfigurationTestCase tentacleConfigurationTestCase)
+        {
+            // The whole reason Tentacle needs an abandon RPC is to release the isolation mutex
+            // when CancelScript can't unstick the script. This test proves that contract: a
+            // FullIsolation script gets stuck (because TentacleDebugDisableProcessKill_UNSAFE_FOR_PRODUCTION
+            // makes cancel a no-op), abandon is called, and a second FullIsolation script with
+            // the same mutex name must then be able to acquire the mutex and run.
+            await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder()
+                .WithTentacle(x => x.WithRunTentacleEnvironmentVariable(EnvironmentVariables.TentacleDebugDisableProcessKill_UNSAFE_FOR_PRODUCTION, "1"))
+                .Build(CancellationToken);
+
+            var startFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "start");
+            var releaseFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "release");
+
+            const string sharedMutex = "abandon-test-mutex";
+
+            var firstCommand = new TestExecuteShellScriptCommandBuilder()
+                .SetScriptBody(new ScriptBuilder()
+                    .CreateFile(startFile)
+                    .WaitForFileToExist(releaseFile))
+                .WithIsolationLevel(ScriptIsolationLevel.FullIsolation)
+                .WithIsolationMutexName(sharedMutex)
+                .Build();
+
+            var tentacleClient = clientTentacle.TentacleClient;
+
+            var firstScriptExecution = Task.Run(async () => await tentacleClient.ExecuteScript(firstCommand, CancellationToken));
+
+            await Wait.For(() => File.Exists(startFile),
+                TimeSpan.FromSeconds(30),
+                () => throw new Exception("First script did not start"),
+                CancellationToken);
+
+            await tentacleClient.CancelScript(firstCommand.ScriptTicket);
+            await Task.Delay(TimeSpan.FromSeconds(1));
+
+            await tentacleClient.AbandonScript(firstCommand.ScriptTicket, CancellationToken);
+
+            // Second FullIsolation script with the SAME mutex name. If the abandon released
+            // the mutex, this script can acquire it and run to completion. Otherwise it would
+            // block waiting for the (still-alive) first script's mutex hold.
+            var secondStartFile = Path.Combine(clientTentacle.TemporaryDirectory.DirectoryPath, "second-start");
+            var secondCommand = new TestExecuteShellScriptCommandBuilder()
+                .SetScriptBody(new ScriptBuilder().CreateFile(secondStartFile))
+                .WithIsolationLevel(ScriptIsolationLevel.FullIsolation)
+                .WithIsolationMutexName(sharedMutex)
+                .Build();
+
+            var (secondResult, _) = await tentacleClient.ExecuteScript(secondCommand, CancellationToken);
+            secondResult.ExitCode.Should().Be(0);
+            File.Exists(secondStartFile).Should().BeTrue("second script should have run after the mutex was released");
+
+            File.WriteAllText(releaseFile, "");
+            await firstScriptExecution;
+        }
+    }
+}

source/Octopus.Tentacle.Tests.Integration/Support/TestDecoratorsAreCalledInTheCorrectOrder.cs

@@ -105,6 +105,11 @@ public Task<ScriptStatusResponseV2> CancelScriptAsync(CancelScriptCommandV2 comm
                 throw new NotImplementedException();
             }
 
+            public Task<ScriptStatusResponseV2> AbandonScriptAsync(AbandonScriptCommandV2 command, HalibutProxyRequestOptions proxyRequestOptions)
+            {
+                throw new NotImplementedException();
+            }
+
             public async Task CompleteScriptAsync(CompleteScriptCommandV2 command, HalibutProxyRequestOptions proxyRequestOptions)
             {
                 await Task.CompletedTask;

source/Octopus.Tentacle.Tests.Integration/Util/Builders/TentacleClientExtensionMethods.cs

@@ -5,10 +5,12 @@
 using System.Threading.Tasks;
 using Halibut;
 using Octopus.Tentacle.Client;
+using Octopus.Tentacle.Client.EventDriven;
 using Octopus.Tentacle.Client.Scripts;
 using Octopus.Tentacle.Client.Scripts.Models;
 using Octopus.Tentacle.Contracts;
 using Octopus.Tentacle.Contracts.Logging;
+using Octopus.Tentacle.Contracts.ScriptServiceV2;
 using Octopus.Tentacle.Tests.Integration.Support;
 using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods;
 
@@ -56,6 +58,46 @@ public static async Task<UploadResult> UploadFile(
             return result;
         }
 
+        public static async Task<ScriptStatusResponseV2> AbandonScript(
+            this TentacleClient tentacleClient,
+            ScriptTicket scriptTicket,
+            CancellationToken token,
+            ITentacleClientTaskLog? log = null)
+        {
+            return await tentacleClient.AbandonScript(scriptTicket,
+                new SerilogLoggerBuilder().Build().ForContext<TentacleClient>().ToITentacleTaskLog().Chain(log),
+                token).ConfigureAwait(false);
+        }
+
+        // Some integration tests need to invoke CancelScript / GetStatus directly against an
+        // already-running ScriptServiceV2 script without going through ExecuteScript. They have
+        // a ScriptTicket but not a CommandContext (which TentacleClient's high-level methods
+        // expect). These helpers synthesize a CommandContext from the ticket.
+        public static async Task<ScriptStatus> CancelScript(
+            this TentacleClient tentacleClient,
+            ScriptTicket scriptTicket,
+            ITentacleClientTaskLog? log = null)
+        {
+            var commandContext = new CommandContext(scriptTicket, 0, ScriptServiceVersion.ScriptServiceVersion2);
+            var result = await tentacleClient.CancelScript(commandContext,
+                new SerilogLoggerBuilder().Build().ForContext<TentacleClient>().ToITentacleTaskLog().Chain(log))
+                .ConfigureAwait(false);
+            return result.ScriptStatus;
+        }
+
+        public static async Task<ScriptStatus> GetStatus(
+            this TentacleClient tentacleClient,
+            ScriptTicket scriptTicket,
+            CancellationToken token,
+            ITentacleClientTaskLog? log = null)
+        {
+            var commandContext = new CommandContext(scriptTicket, 0, ScriptServiceVersion.ScriptServiceVersion2);
+            var result = await tentacleClient.GetStatus(commandContext,
+                new SerilogLoggerBuilder().Build().ForContext<TentacleClient>().ToITentacleTaskLog().Chain(log),
+                token).ConfigureAwait(false);
+            return result.ScriptStatus;
+        }
+
         public static async Task<DataStream> DownloadFile(
             this TentacleClient tentacleClient,
             string remotePath,