Migrate from xUnit v2 to xUnit v3 with MTP support

.config/dotnet-tools.json

@@ -24,7 +24,7 @@
       "rollForward": false
     },
     "dotnet-stryker": {
-      "version": "4.14.0",
+      "version": "4.14.1",
       "commands": [
         "dotnet-stryker"
       ],

.github/workflows/mutation-tests.yml

@@ -24,7 +24,7 @@ jobs:
   mutations:
     name: 'mutations-${{ matrix.name }}'
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    timeout-minutes: 120
 
     strategy:
       fail-fast: false

.gitignore

@@ -5,6 +5,12 @@
 # mstest test results
 TestResults
 
+# Stryker mutation testing output
+StrykerOutput*/
+
+# JUnit test result files
+*.junit.xml
+
 ## Ignore Visual Studio temporary files, build results, and
 ## files generated by popular Visual Studio add-ons.
 

Directory.Packages.props

@@ -4,7 +4,7 @@
     <PackageVersion Include="Cake.FileHelpers" Version="7.0.0" />
     <PackageVersion Include="coverlet.msbuild" Version="6.0.4" />
     <PackageVersion Include="Flurl.Http.Signed" Version="4.0.2" />
-    <PackageVersion Include="FsCheck.Xunit" Version="3.3.2" />
+    <PackageVersion Include="FsCheck.Xunit.v3" Version="3.3.2" />
     <PackageVersion Include="FSharp.Core" Version="8.0.200" />
     <PackageVersion Include="GitHubActionsTestLogger" Version="3.0.3" />
     <PackageVersion Include="IcedTasks" Version="0.11.4" />
@@ -20,6 +20,7 @@
     <PackageVersion Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="10.0.5" />
     <PackageVersion Include="Microsoft.Extensions.TimeProvider.Testing" Version="10.4.0" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.4.0" />
+    <PackageVersion Include="Microsoft.Testing.Platform" Version="2.2.1" />
     <PackageVersion Include="MinVer" Version="7.0.0" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.4" />
     <PackageVersion Include="NSubstitute" Version="5.3.0" />
@@ -38,8 +39,8 @@
     <PackageVersion Include="System.ComponentModel.Annotations" Version="4.5.0" />
     <PackageVersion Include="System.Threading.Tasks.Extensions" Version="4.5.4" />
     <PackageVersion Include="System.ValueTuple" Version="4.5.0" />
-    <PackageVersion Include="xunit" Version="2.9.3" />
-    <PackageVersion Include="xunit.runner.visualstudio" Version="3.1.5" />
+    <PackageVersion Include="xunit.v3.assert" Version="3.2.2" />
+    <PackageVersion Include="xunit.v3.mtp-v2" Version="3.2.2" />
   </ItemGroup>
   <!-- Dependencies below are pinned for the libraries we ship to NuGet.org -->
   <ItemGroup>

cake.cs

@@ -136,28 +136,50 @@
 Task("__RunTests")
     .Does(() =>
 {
-    var loggers = Array.Empty<string>();
-
-    if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("GITHUB_SHA")))
-    {
-        loggers =
-        [
-            "junit;LogFilePath=junit.xml",
-            "GitHubActions;report-warnings=false;summary-include-passed=false",
-        ];
-    }
-
     var projects = GetFiles("./test/**/*.csproj");
 
     foreach (var proj in projects)
     {
-        DotNetTest(proj.FullPath, new DotNetTestSettings
+        var projectName = proj.GetFilenameWithoutExtension().ToString();
+        var configLower = configuration.ToLowerInvariant();
+        var outputBase = MakeAbsolute(Directory($"./artifacts/bin/{projectName}"));
+
+        foreach (var tfmDir in GetDirectories($"{outputBase}/{configLower}_*"))
         {
-            Configuration = configuration,
-            Loggers = loggers,
-            NoBuild = true,
-            ToolTimeout = System.TimeSpan.FromMinutes(10),
-        });
+            var dll = tfmDir.CombineWithFilePath($"{projectName}.dll");
+            var runtimeConfig = tfmDir.CombineWithFilePath($"{projectName}.runtimeconfig.json");
+            if (!FileExists(dll) || !FileExists(runtimeConfig))
+                continue;
+
+            var tfmName = tfmDir.GetDirectoryName().Substring(configLower.Length + 1);
+            Information($"Testing {projectName} ({tfmName})");
+
+            var args = new ProcessArgumentBuilder();
+            FilePath executable;
+
+            if (tfmName.StartsWith("net4"))
+            {
+                executable = tfmDir.CombineWithFilePath($"{projectName}.exe");
+            }
+            else
+            {
+                executable = Context.Tools.Resolve("dotnet") ?? new FilePath("dotnet");
+                args.Append("exec");
+                args.AppendQuoted(dll.FullPath);
+            }
+
+            if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("GITHUB_SHA")))
+            {
+                args.Append("-jUnit");
+                args.AppendQuoted($"{projectName}-{tfmName}.junit.xml");
+            }
+
+            var result = StartProcess(executable, new ProcessSettings { Arguments = args });
+            if (result != 0)
+            {
+                throw new InvalidOperationException($"Tests failed for '{projectName}' ({tfmName}).");
+            }
+        }
     }
 });
 
@@ -232,36 +254,47 @@
 // MUTATION TESTING TARGETS
 ///////////////////////////////////////////////////////////////////////////////
 
+Task("PatchStryker")
+    .Does((_) =>
+{
+    PatchStrykerMtpRunner();
+});
+
 Task("MutationTestsCore")
     .IsDependentOn("__Setup")
+    .IsDependentOn("PatchStryker")
     .Does((_) =>
 {
     RunMutationTests(File("./src/Polly.Core/Polly.Core.csproj"), File("./test/Polly.Core.Tests/Polly.Core.Tests.csproj"));
 });
 
 Task("MutationTestsRateLimiting")
     .IsDependentOn("__Setup")
+    .IsDependentOn("PatchStryker")
     .Does((_) =>
 {
     RunMutationTests(File("./src/Polly.RateLimiting/Polly.RateLimiting.csproj"), File("./test/Polly.RateLimiting.Tests/Polly.RateLimiting.Tests.csproj"));
 });
 
 Task("MutationTestsExtensions")
     .IsDependentOn("__Setup")
+    .IsDependentOn("PatchStryker")
     .Does((_) =>
 {
     RunMutationTests(File("./src/Polly.Extensions/Polly.Extensions.csproj"), File("./test/Polly.Extensions.Tests/Polly.Extensions.Tests.csproj"));
 });
 
 Task("MutationTestsTesting")
     .IsDependentOn("__Setup")
+    .IsDependentOn("PatchStryker")
     .Does((_) =>
 {
     RunMutationTests(File("./src/Polly.Testing/Polly.Testing.csproj"), File("./test/Polly.Testing.Tests/Polly.Testing.Tests.csproj"));
 });
 
 Task("MutationTestsLegacy")
     .IsDependentOn("__Setup")
+    .IsDependentOn("PatchStryker")
     .Does((_) =>
 {
     RunMutationTests(File("./src/Polly/Polly.csproj"), File("./test/Polly.Specs/Polly.Specs.csproj"));
@@ -300,6 +333,92 @@ string PatchStrykerConfig(string path, Action<Newtonsoft.Json.Linq.JObject> patc
     return tempPath;
 }
 
+void PatchStrykerMtpRunner()
+{
+    // Patches Stryker's MTP test runner to fix three bugs:
+    // 1. "error" execution state not counted as test failure (only "failed" was checked)
+    // 2. EveryTest() sentinel not properly accumulated when server crashes
+    // 3. Static field initializer mutations not killed due to MTP process reuse
+    // See: https://github.com/stryker-mutator/stryker-net/issues/3117
+    // This patch can be removed once Stryker fixes these issues upstream.
+
+    var strykerVersion = "4.14.1";
+    var strykerTag = $"dotnet-stryker@{strykerVersion}";
+    // Resolve relative to the directory containing cake.cs, not the process working directory
+    var scriptDir = System.IO.Path.GetDirectoryName(System.IO.Path.GetFullPath("cake.cs")) ?? ".";
+    var patchFile = System.IO.Path.Combine(scriptDir, "eng", "stryker-mtp-runner.patch");
+    var tempDir = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"stryker-patch-{strykerVersion}");
+    var targetDll = System.IO.Path.Combine(
+        Environment.GetFolderPath(Environment.SpecialFolder.UserProfile),
+        ".nuget", "packages", "dotnet-stryker", strykerVersion, "tools", "net8.0", "any",
+        "Stryker.TestRunner.MicrosoftTestPlatform.dll");
+
+    // Check if already patched (by presence of a marker file)
+    var markerFile = targetDll + ".patched";
+    if (System.IO.File.Exists(markerFile))
+    {
+        Information("Stryker MTP runner already patched.");
+        return;
+    }
+
+    Information("Patching Stryker MTP runner...");
+
+    // Clone stryker-net at the correct tag
+    if (!System.IO.Directory.Exists(tempDir))
+    {
+        var cloneResult = StartProcess("git", new ProcessSettings
+        {
+            Arguments = $"clone --depth 1 --branch {strykerTag} https://github.com/stryker-mutator/stryker-net.git {tempDir}",
+        });
+        if (cloneResult != 0)
+        {
+            throw new InvalidOperationException("Failed to clone stryker-net repository.");
+        }
+    }
+    else
+    {
+        // Reset any leftover changes from a previous failed run
+        StartProcess("git", new ProcessSettings
+        {
+            Arguments = "checkout -- .",
+            WorkingDirectory = tempDir,
+        });
+    }
+
+    // Apply the patch
+    var applyResult = StartProcess("git", new ProcessSettings
+    {
+        Arguments = $"apply {patchFile}",
+        WorkingDirectory = tempDir,
+    });
+    if (applyResult != 0)
+    {
+        throw new InvalidOperationException("Failed to apply Stryker MTP runner patch.");
+    }
+
+    // Build the patched project (must run from tempDir so NuGet packages resolve correctly)
+    var projectPath = System.IO.Path.Combine("src", "Stryker.TestRunner.MicrosoftTestPlatform",
+        "Stryker.TestRunner.MicrosoftTestPlatform.csproj");
+    var buildResult = StartProcess("dotnet", new ProcessSettings
+    {
+        Arguments = $"build \"{projectPath}\" -c Release",
+        WorkingDirectory = tempDir,
+    });
+    if (buildResult != 0)
+    {
+        throw new InvalidOperationException("Failed to build patched Stryker MTP runner.");
+    }
+
+    // Copy the patched DLL
+    var builtDll = System.IO.Path.Combine(tempDir, "src", "Stryker.TestRunner.MicrosoftTestPlatform",
+        "bin", "Release", "net8.0", "Stryker.TestRunner.MicrosoftTestPlatform.dll");
+
+    System.IO.File.Copy(builtDll, targetDll, overwrite: true);
+    System.IO.File.WriteAllText(markerFile, $"Patched from {System.IO.Path.GetFileName(patchFile)} at {DateTime.UtcNow:O}");
+
+    Information("Stryker MTP runner patched successfully.");
+}
+
 void RunMutationTests(FilePath target, FilePath testProject)
 {
     var mutationScore = XmlPeek(target, "/Project/PropertyGroup/MutationScore/text()", new XmlPeekSettings { SuppressWarning = true });

eng/AssemblyVersion.cs

@@ -0,0 +1,4 @@
+// Stryker mutation testing skips auto-generated files (those with <auto-generated> headers),
+// which causes the mutated assembly to lose its AssemblyVersion. This file ensures the
+// version is preserved. The SDK-generated attribute is disabled via Library.targets.
+[assembly: System.Reflection.AssemblyVersion("8.0.0.0")]

eng/Library.targets

@@ -5,6 +5,7 @@
     <Copyright>Copyright (c) 2015-$([System.DateTime]::Now.ToString(yyyy)), App vNext</Copyright>
     <DefaultLanguage>en-US</DefaultLanguage>
     <GenerateAssemblyInfo>true</GenerateAssemblyInfo>
+    <GenerateAssemblyVersionAttribute>false</GenerateAssemblyVersionAttribute>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <Authors>Michael Wolfenden, App vNext</Authors>
     <AllowedOutputExtensionsInPackageBuildOutputFolder>$(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb</AllowedOutputExtensionsInPackageBuildOutputFolder>
@@ -38,6 +39,10 @@
     <None Include="$(MsBuildThisFileDirectory)..\LICENSE" Pack="true" PackagePath="" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(Language)' == 'C#'">
+    <Compile Include="$(MSBuildThisFileDirectory)AssemblyVersion.cs" LinkBase="Properties" />
+  </ItemGroup>
+
   <ItemGroup>
     <InternalsVisibleTo Include="DynamicProxyGenAssembly2" PublicKey="0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7" />
   </ItemGroup>

eng/Test.targets

@@ -15,11 +15,11 @@
     <PackageReference Include="GitHubActionsTestLogger" />
     <PackageReference Include="JunitXml.TestLogger" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="Microsoft.Testing.Platform" />
     <PackageReference Include="NSubstitute" />
     <PackageReference Include="ReportGenerator" PrivateAssets="all" />
     <PackageReference Include="Shouldly" />
-    <PackageReference Include="xunit" />
-    <PackageReference Include="xunit.runner.visualstudio" PrivateAssets="all" />
+    <PackageReference Include="xunit.v3.mtp-v2" />
   </ItemGroup>
 
   <PropertyGroup Condition="$([MSBuild]::GetTargetFrameworkIdentifier('$(TargetFramework)')) != '.NETFramework'">

eng/stryker-config.json

@@ -22,7 +22,7 @@
     "concurrency": 4,
     "configuration": "Debug",
     "language-version": "Preview",
-    "target-framework": "net10.0",
+    "test-runner": "mtp",
     "thresholds": {
       "high": 100,
       "low": 100

eng/stryker-mtp-runner.patch

@@ -0,0 +1,77 @@
+diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs
+index cfc34d6..ee4b3b1 100644
+--- a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs
++++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs
+@@ -86,6 +86,21 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable
+         _logger.LogDebug("{RunnerId}: Testing mutant(s) [{Mutants}] with active mutation ID: {MutantId}",
+             RunnerId, string.Join(",", mutants.Select(m => m.Id)), mutantId);
+
++        // Static mutations (e.g. static field initializers) require a fresh process
++        // because the static value is set once during class loading and cannot change.
++        if (mutants.Any(m => m.IsStaticValue))
++        {
++            _logger.LogDebug("{RunnerId}: Resetting servers for static mutation(s)", RunnerId);
++            lock (_serverLock)
++            {
++                foreach (var server in _assemblyServers.Values)
++                {
++                    server.Dispose();
++                }
++                _assemblyServers.Clear();
++            }
++        }
++
+         return RunAllTestsAsync(assemblies, mutantId, mutants, update, timeoutCalc);
+     }
+
+@@ -351,6 +366,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable
+     {
+         private readonly List<string> _executedTests = [];
+         private readonly List<string> _failedTests = [];
++        private bool _allTestsFailed;
+         private readonly List<string> _messages = [];
+         private readonly List<string> _errorMessages = [];
+         private int _totalDiscoveredTests;
+@@ -373,7 +389,14 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable
+                 _totalExecutedTests += executedIds.Count;
+             }
+
+-            _failedTests.AddRange(result.FailingTests.GetIdentifiers());
++            if (result.FailingTests.IsEveryTest)
++            {
++                _allTestsFailed = true;
++            }
++            else
++            {
++                _failedTests.AddRange(result.FailingTests.GetIdentifiers());
++            }
+             TotalDuration += result.Duration;
+             _messages.AddRange(result.Messages ?? []);
+
+@@ -390,7 +413,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable
+                 ? TestIdentifierList.EveryTest()
+                 : new TestIdentifierList(_executedTests);
+
+-        public ITestIdentifiers BuildFailedTests() => new TestIdentifierList(_failedTests);
++        public ITestIdentifiers BuildFailedTests() => _allTestsFailed ? TestIdentifierList.EveryTest() : new TestIdentifierList(_failedTests);
+
+         public ITestIdentifiers BuildTimedOutTests() => new TestIdentifierList(TimedOutTests);
+
+@@ -525,7 +548,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable
+
+             var duration = DateTime.UtcNow - startTime;
+             var finishedTests = testResults.Where(x => x.Node.ExecutionState is not "in-progress").ToList();
+-            var failedTests = finishedTests.Where(x => x.Node.ExecutionState is "failed").Select(x => x.Node.Uid).ToList();
++            var failedTests = finishedTests.Where(x => x.Node.ExecutionState is "failed" or "error").Select(x => x.Node.Uid).ToList();
+
+             lock (_discoveryLock)
+             {
+@@ -542,7 +565,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable
+             }
+
+             var errorMessagesStr = string.Join(Environment.NewLine,
+-                finishedTests.Where(x => x.Node.ExecutionState is "failed")
++                finishedTests.Where(x => x.Node.ExecutionState is "failed" or "error")
+                     .Select(x => $"{x.Node.DisplayName}{Environment.NewLine}{Environment.NewLine}Test failed"));
+
+             var messages = finishedTests.Select(x =>