diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index f948172fd44..f8b1de41787 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -24,7 +24,7 @@ "rollForward": false }, "dotnet-stryker": { - "version": "4.14.0", + "version": "4.14.1", "commands": [ "dotnet-stryker" ], diff --git a/.github/workflows/mutation-tests.yml b/.github/workflows/mutation-tests.yml index 5d0948a4cb4..4b16a62c966 100644 --- a/.github/workflows/mutation-tests.yml +++ b/.github/workflows/mutation-tests.yml @@ -24,7 +24,7 @@ jobs: mutations: name: 'mutations-${{ matrix.name }}' runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 120 strategy: fail-fast: false diff --git a/.gitignore b/.gitignore index dbc94ceb5ea..60efcebcdc8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,12 @@ # mstest test results TestResults +# Stryker mutation testing output +StrykerOutput*/ + +# JUnit test result files +*.junit.xml + ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. diff --git a/Directory.Packages.props b/Directory.Packages.props index c62c59ffde9..46a2a62a188 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,7 +4,7 @@ - + @@ -20,6 +20,7 @@ + @@ -38,8 +39,8 @@ - - + + diff --git a/cake.cs b/cake.cs index a2b8d897ada..dbf7b829bda 100644 --- a/cake.cs +++ b/cake.cs @@ -136,28 +136,50 @@ Task("__RunTests") .Does(() => { - var loggers = Array.Empty(); - - if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("GITHUB_SHA"))) - { - loggers = - [ - "junit;LogFilePath=junit.xml", - "GitHubActions;report-warnings=false;summary-include-passed=false", - ]; - } - var projects = GetFiles("./test/**/*.csproj"); foreach (var proj in projects) { - DotNetTest(proj.FullPath, new DotNetTestSettings + var projectName = proj.GetFilenameWithoutExtension().ToString(); + var configLower = configuration.ToLowerInvariant(); + var outputBase = MakeAbsolute(Directory($"./artifacts/bin/{projectName}")); + + foreach (var tfmDir in GetDirectories($"{outputBase}/{configLower}_*")) { - Configuration = configuration, - Loggers = loggers, - NoBuild = true, - ToolTimeout = System.TimeSpan.FromMinutes(10), - }); + var dll = tfmDir.CombineWithFilePath($"{projectName}.dll"); + var runtimeConfig = tfmDir.CombineWithFilePath($"{projectName}.runtimeconfig.json"); + if (!FileExists(dll) || !FileExists(runtimeConfig)) + continue; + + var tfmName = tfmDir.GetDirectoryName().Substring(configLower.Length + 1); + Information($"Testing {projectName} ({tfmName})"); + + var args = new ProcessArgumentBuilder(); + FilePath executable; + + if (tfmName.StartsWith("net4")) + { + executable = tfmDir.CombineWithFilePath($"{projectName}.exe"); + } + else + { + executable = Context.Tools.Resolve("dotnet") ?? new FilePath("dotnet"); + args.Append("exec"); + args.AppendQuoted(dll.FullPath); + } + + if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("GITHUB_SHA"))) + { + args.Append("-jUnit"); + args.AppendQuoted($"{projectName}-{tfmName}.junit.xml"); + } + + var result = StartProcess(executable, new ProcessSettings { Arguments = args }); + if (result != 0) + { + throw new InvalidOperationException($"Tests failed for '{projectName}' ({tfmName})."); + } + } } }); @@ -232,8 +254,15 @@ // MUTATION TESTING TARGETS /////////////////////////////////////////////////////////////////////////////// +Task("PatchStryker") + .Does((_) => +{ + PatchStrykerMtpRunner(); +}); + Task("MutationTestsCore") .IsDependentOn("__Setup") + .IsDependentOn("PatchStryker") .Does((_) => { RunMutationTests(File("./src/Polly.Core/Polly.Core.csproj"), File("./test/Polly.Core.Tests/Polly.Core.Tests.csproj")); @@ -241,6 +270,7 @@ Task("MutationTestsRateLimiting") .IsDependentOn("__Setup") + .IsDependentOn("PatchStryker") .Does((_) => { RunMutationTests(File("./src/Polly.RateLimiting/Polly.RateLimiting.csproj"), File("./test/Polly.RateLimiting.Tests/Polly.RateLimiting.Tests.csproj")); @@ -248,6 +278,7 @@ Task("MutationTestsExtensions") .IsDependentOn("__Setup") + .IsDependentOn("PatchStryker") .Does((_) => { RunMutationTests(File("./src/Polly.Extensions/Polly.Extensions.csproj"), File("./test/Polly.Extensions.Tests/Polly.Extensions.Tests.csproj")); @@ -255,6 +286,7 @@ Task("MutationTestsTesting") .IsDependentOn("__Setup") + .IsDependentOn("PatchStryker") .Does((_) => { RunMutationTests(File("./src/Polly.Testing/Polly.Testing.csproj"), File("./test/Polly.Testing.Tests/Polly.Testing.Tests.csproj")); @@ -262,6 +294,7 @@ Task("MutationTestsLegacy") .IsDependentOn("__Setup") + .IsDependentOn("PatchStryker") .Does((_) => { RunMutationTests(File("./src/Polly/Polly.csproj"), File("./test/Polly.Specs/Polly.Specs.csproj")); @@ -300,6 +333,92 @@ string PatchStrykerConfig(string path, Action patc return tempPath; } +void PatchStrykerMtpRunner() +{ + // Patches Stryker's MTP test runner to fix three bugs: + // 1. "error" execution state not counted as test failure (only "failed" was checked) + // 2. EveryTest() sentinel not properly accumulated when server crashes + // 3. Static field initializer mutations not killed due to MTP process reuse + // See: https://github.com/stryker-mutator/stryker-net/issues/3117 + // This patch can be removed once Stryker fixes these issues upstream. + + var strykerVersion = "4.14.1"; + var strykerTag = $"dotnet-stryker@{strykerVersion}"; + // Resolve relative to the directory containing cake.cs, not the process working directory + var scriptDir = System.IO.Path.GetDirectoryName(System.IO.Path.GetFullPath("cake.cs")) ?? "."; + var patchFile = System.IO.Path.Combine(scriptDir, "eng", "stryker-mtp-runner.patch"); + var tempDir = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"stryker-patch-{strykerVersion}"); + var targetDll = System.IO.Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + ".nuget", "packages", "dotnet-stryker", strykerVersion, "tools", "net8.0", "any", + "Stryker.TestRunner.MicrosoftTestPlatform.dll"); + + // Check if already patched (by presence of a marker file) + var markerFile = targetDll + ".patched"; + if (System.IO.File.Exists(markerFile)) + { + Information("Stryker MTP runner already patched."); + return; + } + + Information("Patching Stryker MTP runner..."); + + // Clone stryker-net at the correct tag + if (!System.IO.Directory.Exists(tempDir)) + { + var cloneResult = StartProcess("git", new ProcessSettings + { + Arguments = $"clone --depth 1 --branch {strykerTag} https://github.com/stryker-mutator/stryker-net.git {tempDir}", + }); + if (cloneResult != 0) + { + throw new InvalidOperationException("Failed to clone stryker-net repository."); + } + } + else + { + // Reset any leftover changes from a previous failed run + StartProcess("git", new ProcessSettings + { + Arguments = "checkout -- .", + WorkingDirectory = tempDir, + }); + } + + // Apply the patch + var applyResult = StartProcess("git", new ProcessSettings + { + Arguments = $"apply {patchFile}", + WorkingDirectory = tempDir, + }); + if (applyResult != 0) + { + throw new InvalidOperationException("Failed to apply Stryker MTP runner patch."); + } + + // Build the patched project (must run from tempDir so NuGet packages resolve correctly) + var projectPath = System.IO.Path.Combine("src", "Stryker.TestRunner.MicrosoftTestPlatform", + "Stryker.TestRunner.MicrosoftTestPlatform.csproj"); + var buildResult = StartProcess("dotnet", new ProcessSettings + { + Arguments = $"build \"{projectPath}\" -c Release", + WorkingDirectory = tempDir, + }); + if (buildResult != 0) + { + throw new InvalidOperationException("Failed to build patched Stryker MTP runner."); + } + + // Copy the patched DLL + var builtDll = System.IO.Path.Combine(tempDir, "src", "Stryker.TestRunner.MicrosoftTestPlatform", + "bin", "Release", "net8.0", "Stryker.TestRunner.MicrosoftTestPlatform.dll"); + + System.IO.File.Copy(builtDll, targetDll, overwrite: true); + System.IO.File.WriteAllText(markerFile, $"Patched from {System.IO.Path.GetFileName(patchFile)} at {DateTime.UtcNow:O}"); + + Information("Stryker MTP runner patched successfully."); +} + void RunMutationTests(FilePath target, FilePath testProject) { var mutationScore = XmlPeek(target, "/Project/PropertyGroup/MutationScore/text()", new XmlPeekSettings { SuppressWarning = true }); diff --git a/eng/AssemblyVersion.cs b/eng/AssemblyVersion.cs new file mode 100644 index 00000000000..574f2882a07 --- /dev/null +++ b/eng/AssemblyVersion.cs @@ -0,0 +1,4 @@ +// Stryker mutation testing skips auto-generated files (those with headers), +// which causes the mutated assembly to lose its AssemblyVersion. This file ensures the +// version is preserved. The SDK-generated attribute is disabled via Library.targets. +[assembly: System.Reflection.AssemblyVersion("8.0.0.0")] diff --git a/eng/Library.targets b/eng/Library.targets index 4a8e68da6c5..1266798e671 100644 --- a/eng/Library.targets +++ b/eng/Library.targets @@ -5,6 +5,7 @@ Copyright (c) 2015-$([System.DateTime]::Now.ToString(yyyy)), App vNext en-US true + false true Michael Wolfenden, App vNext $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb @@ -38,6 +39,10 @@ + + + + diff --git a/eng/Test.targets b/eng/Test.targets index 4058f40c299..5065acc7235 100644 --- a/eng/Test.targets +++ b/eng/Test.targets @@ -15,11 +15,11 @@ + - - + diff --git a/eng/stryker-config.json b/eng/stryker-config.json index 8113fef7a0f..b36637e5ff9 100644 --- a/eng/stryker-config.json +++ b/eng/stryker-config.json @@ -22,7 +22,7 @@ "concurrency": 4, "configuration": "Debug", "language-version": "Preview", - "target-framework": "net10.0", + "test-runner": "mtp", "thresholds": { "high": 100, "low": 100 diff --git a/eng/stryker-mtp-runner.patch b/eng/stryker-mtp-runner.patch new file mode 100644 index 00000000000..e52d1f69418 --- /dev/null +++ b/eng/stryker-mtp-runner.patch @@ -0,0 +1,77 @@ +diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs +index cfc34d6..ee4b3b1 100644 +--- a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs ++++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs +@@ -86,6 +86,21 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable + _logger.LogDebug("{RunnerId}: Testing mutant(s) [{Mutants}] with active mutation ID: {MutantId}", + RunnerId, string.Join(",", mutants.Select(m => m.Id)), mutantId); + ++ // Static mutations (e.g. static field initializers) require a fresh process ++ // because the static value is set once during class loading and cannot change. ++ if (mutants.Any(m => m.IsStaticValue)) ++ { ++ _logger.LogDebug("{RunnerId}: Resetting servers for static mutation(s)", RunnerId); ++ lock (_serverLock) ++ { ++ foreach (var server in _assemblyServers.Values) ++ { ++ server.Dispose(); ++ } ++ _assemblyServers.Clear(); ++ } ++ } ++ + return RunAllTestsAsync(assemblies, mutantId, mutants, update, timeoutCalc); + } + +@@ -351,6 +366,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable + { + private readonly List _executedTests = []; + private readonly List _failedTests = []; ++ private bool _allTestsFailed; + private readonly List _messages = []; + private readonly List _errorMessages = []; + private int _totalDiscoveredTests; +@@ -373,7 +389,14 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable + _totalExecutedTests += executedIds.Count; + } + +- _failedTests.AddRange(result.FailingTests.GetIdentifiers()); ++ if (result.FailingTests.IsEveryTest) ++ { ++ _allTestsFailed = true; ++ } ++ else ++ { ++ _failedTests.AddRange(result.FailingTests.GetIdentifiers()); ++ } + TotalDuration += result.Duration; + _messages.AddRange(result.Messages ?? []); + +@@ -390,7 +413,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable + ? TestIdentifierList.EveryTest() + : new TestIdentifierList(_executedTests); + +- public ITestIdentifiers BuildFailedTests() => new TestIdentifierList(_failedTests); ++ public ITestIdentifiers BuildFailedTests() => _allTestsFailed ? TestIdentifierList.EveryTest() : new TestIdentifierList(_failedTests); + + public ITestIdentifiers BuildTimedOutTests() => new TestIdentifierList(TimedOutTests); + +@@ -525,7 +548,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable + + var duration = DateTime.UtcNow - startTime; + var finishedTests = testResults.Where(x => x.Node.ExecutionState is not "in-progress").ToList(); +- var failedTests = finishedTests.Where(x => x.Node.ExecutionState is "failed").Select(x => x.Node.Uid).ToList(); ++ var failedTests = finishedTests.Where(x => x.Node.ExecutionState is "failed" or "error").Select(x => x.Node.Uid).ToList(); + + lock (_discoveryLock) + { +@@ -542,7 +565,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable + } + + var errorMessagesStr = string.Join(Environment.NewLine, +- finishedTests.Where(x => x.Node.ExecutionState is "failed") ++ finishedTests.Where(x => x.Node.ExecutionState is "failed" or "error") + .Select(x => $"{x.Node.DisplayName}{Environment.NewLine}{Environment.NewLine}Test failed")); + + var messages = finishedTests.Select(x => diff --git a/global.json b/global.json index 80027b468d9..3168c262f1b 100644 --- a/global.json +++ b/global.json @@ -6,5 +6,8 @@ }, "msbuild-sdks": { "Cake.Sdk": "6.1.1" + }, + "test": { + "runner": "Microsoft.Testing.Platform" } } \ No newline at end of file diff --git a/src/Snippets/Snippets.csproj b/src/Snippets/Snippets.csproj index f9743050c13..7acf86aad48 100644 --- a/src/Snippets/Snippets.csproj +++ b/src/Snippets/Snippets.csproj @@ -1,4 +1,4 @@ - + false @@ -32,7 +32,7 @@ - + diff --git a/test/Polly.Core.Tests/Hedging/HedgingResilienceStrategyTests.cs b/test/Polly.Core.Tests/Hedging/HedgingResilienceStrategyTests.cs index f196b730cce..2c6b1aab850 100644 --- a/test/Polly.Core.Tests/Hedging/HedgingResilienceStrategyTests.cs +++ b/test/Polly.Core.Tests/Hedging/HedgingResilienceStrategyTests.cs @@ -3,7 +3,6 @@ using Polly.Hedging.Utils; using Polly.Telemetry; using Polly.Testing; -using Xunit.Abstractions; namespace Polly.Core.Tests.Hedging; diff --git a/test/Polly.Core.Tests/Polly.Core.Tests.csproj b/test/Polly.Core.Tests/Polly.Core.Tests.csproj index 5105ea208e1..73d00757cc2 100644 --- a/test/Polly.Core.Tests/Polly.Core.Tests.csproj +++ b/test/Polly.Core.Tests/Polly.Core.Tests.csproj @@ -1,4 +1,4 @@ - + net10.0;net9.0;net8.0 $(TargetFrameworks);net481 @@ -6,6 +6,7 @@ Test enable 100 + Exe $(NoWarn);S6966 [Polly.Core]* true @@ -15,7 +16,7 @@ - + diff --git a/test/Polly.Core.Tests/ResiliencePipelineTests.cs b/test/Polly.Core.Tests/ResiliencePipelineTests.cs index c47604b7908..614c3f2ba14 100644 --- a/test/Polly.Core.Tests/ResiliencePipelineTests.cs +++ b/test/Polly.Core.Tests/ResiliencePipelineTests.cs @@ -11,7 +11,7 @@ public partial class ResiliencePipelineTests #pragma warning disable IDE0028 public static TheoryData ResilienceContextPools = new() { - null, + null as ResilienceContextPool, ResilienceContextPool.Shared, }; #pragma warning restore IDE0028 diff --git a/test/Polly.Core.Tests/Utils/Pipeline/PipelineComponentFactoryTests.cs b/test/Polly.Core.Tests/Utils/Pipeline/PipelineComponentFactoryTests.cs index 40eb7f2b26d..7383d19159d 100644 --- a/test/Polly.Core.Tests/Utils/Pipeline/PipelineComponentFactoryTests.cs +++ b/test/Polly.Core.Tests/Utils/Pipeline/PipelineComponentFactoryTests.cs @@ -1,4 +1,4 @@ -using NSubstitute; +using NSubstitute; using Polly.Utils.Pipeline; namespace Polly.Core.Tests.Utils.Pipeline; @@ -8,17 +8,17 @@ public class PipelineComponentFactoryTests #pragma warning disable IDE0028 public static TheoryData> EmptyCallbacks = new() { - Array.Empty(), + Array.Empty() as IEnumerable, Enumerable.Empty(), - new List(), - new EmptyActionEnumerable(), // Explicitly does not provide TryGetNonEnumeratedCount() + new List() as IEnumerable, + new EmptyActionEnumerable() as IEnumerable, // Explicitly does not provide TryGetNonEnumeratedCount() }; public static TheoryData> NonEmptyCallbacks = new() { - new[] { () => { } }, + new[] { () => { } } as IEnumerable, Enumerable.TakeWhile(Enumerable.Repeat(() => { }, 50), (_, i) => i < 1), // Defeat optimisation for TryGetNonEnumeratedCount() - new List { () => { } }, + new List { () => { } } as IEnumerable, }; #pragma warning restore IDE0028 diff --git a/test/Polly.Extensions.Tests/Polly.Extensions.Tests.csproj b/test/Polly.Extensions.Tests/Polly.Extensions.Tests.csproj index 344d5001a12..d98987fafb2 100644 --- a/test/Polly.Extensions.Tests/Polly.Extensions.Tests.csproj +++ b/test/Polly.Extensions.Tests/Polly.Extensions.Tests.csproj @@ -1,10 +1,11 @@ - + net10.0;net9.0;net8.0 $(TargetFrameworks);net481 Test enable 100 + Exe [Polly.Extensions]* diff --git a/test/Polly.RateLimiting.Tests/Polly.RateLimiting.Tests.csproj b/test/Polly.RateLimiting.Tests/Polly.RateLimiting.Tests.csproj index 88fd0658026..5e55a7b8b0f 100644 --- a/test/Polly.RateLimiting.Tests/Polly.RateLimiting.Tests.csproj +++ b/test/Polly.RateLimiting.Tests/Polly.RateLimiting.Tests.csproj @@ -1,10 +1,11 @@ - + net10.0;net9.0;net8.0 $(TargetFrameworks);net481 Test enable 100 + Exe [Polly.RateLimiting]* diff --git a/test/Polly.Specs/Helpers/Bulkhead/AnnotatedOutputHelper.cs b/test/Polly.Specs/Helpers/Bulkhead/AnnotatedOutputHelper.cs index fa61f84806e..92812c94f59 100644 --- a/test/Polly.Specs/Helpers/Bulkhead/AnnotatedOutputHelper.cs +++ b/test/Polly.Specs/Helpers/Bulkhead/AnnotatedOutputHelper.cs @@ -1,4 +1,4 @@ -using System.Globalization; +using System.Globalization; namespace Polly.Specs.Helpers.Bulkhead; @@ -29,6 +29,8 @@ public Item(string format, object[] args) private readonly ITestOutputHelper _innerOutputHelper; + public string Output => _innerOutputHelper.Output; + public AnnotatedOutputHelper(ITestOutputHelper innerOutputHelper) => _innerOutputHelper = innerOutputHelper ?? throw new ArgumentNullException(nameof(innerOutputHelper)); diff --git a/test/Polly.Specs/Helpers/Bulkhead/SilentOutputHelper.cs b/test/Polly.Specs/Helpers/Bulkhead/SilentOutputHelper.cs index 2aa50df2883..fc280ae0917 100644 --- a/test/Polly.Specs/Helpers/Bulkhead/SilentOutputHelper.cs +++ b/test/Polly.Specs/Helpers/Bulkhead/SilentOutputHelper.cs @@ -1,7 +1,19 @@ -namespace Polly.Specs.Helpers.Bulkhead; +namespace Polly.Specs.Helpers.Bulkhead; public class SilentOutputHelper : ITestOutputHelper { + public string Output => string.Empty; + + public void Write(string message) + { + // Do nothing: intentionally silent. + } + + public void Write(string format, params object[] args) + { + // Do nothing: intentionally silent. + } + public void WriteLine(string message) { // Do nothing: intentionally silent. diff --git a/test/Polly.Specs/Polly.Specs.csproj b/test/Polly.Specs/Polly.Specs.csproj index a7d836659d8..6e19fa42da5 100644 --- a/test/Polly.Specs/Polly.Specs.csproj +++ b/test/Polly.Specs/Polly.Specs.csproj @@ -1,8 +1,9 @@ - + net10.0;net9.0;net8.0 $(TargetFrameworks);net481 enable + Exe Test 94,94,91 [Polly]* @@ -22,7 +23,6 @@ - diff --git a/test/Polly.Testing.Tests/Polly.Testing.Tests.csproj b/test/Polly.Testing.Tests/Polly.Testing.Tests.csproj index 646329abf83..860a2d756aa 100644 --- a/test/Polly.Testing.Tests/Polly.Testing.Tests.csproj +++ b/test/Polly.Testing.Tests/Polly.Testing.Tests.csproj @@ -1,10 +1,11 @@ - + net10.0;net9.0;net8.0 $(TargetFrameworks);net481 Test enable 100 + Exe [Polly.Testing]* diff --git a/test/Shared/TestCancellation.cs b/test/Shared/TestCancellation.cs index be9cd8b0678..5396df4d754 100644 --- a/test/Shared/TestCancellation.cs +++ b/test/Shared/TestCancellation.cs @@ -1,6 +1,8 @@ +using Xunit; + namespace Polly; internal static class TestCancellation { - public static CancellationToken Token => CancellationToken.None; + public static CancellationToken Token => TestContext.Current.CancellationToken; }