diff --git a/VoiceCraft.Tools/AllocHarness/AllocHarness.csproj b/VoiceCraft.Tools/AllocHarness/AllocHarness.csproj new file mode 100644 index 00000000..b67aacdf --- /dev/null +++ b/VoiceCraft.Tools/AllocHarness/AllocHarness.csproj @@ -0,0 +1,14 @@ + + + Exe + net10.0 + enable + enable + + + + + + + + diff --git a/VoiceCraft.Tools/AllocHarness/BenchmarkCatalog.cs b/VoiceCraft.Tools/AllocHarness/BenchmarkCatalog.cs new file mode 100644 index 00000000..7a386c72 --- /dev/null +++ b/VoiceCraft.Tools/AllocHarness/BenchmarkCatalog.cs @@ -0,0 +1,176 @@ +internal static class BenchmarkCatalog +{ + public static Dictionary Build() + { + return new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["visibility"] = new( + Name: "visibility", + Description: "Measures allocations and timing for VisibilitySystem.Update().", + ParameterNames: ["Entities", "Effects", "Updates"], + DefaultScenarios: + [ + new Scenario(40, 1, 20), + new Scenario(40, 8, 20), + new Scenario(64, 8, 10) + ], + MeasurementDescription: "Steady-state allocations and elapsed time across repeated update calls after one warm-up run.", + SupportsLegacyComparison: true, + CreateSampleResult: Measurements.MeasureVisibilitySample), + + ["audio-effects"] = new( + Name: "audio-effects", + Description: "Measures allocations and timing for audio effect collection reads.", + ParameterNames: ["Effects", "Reads", "Unused"], + DefaultScenarios: + [ + new Scenario(1, 10_000, 0), + new Scenario(4, 10_000, 0), + new Scenario(8, 10_000, 0) + ], + MeasurementDescription: "Steady-state allocations and elapsed time across repeated effect collection reads after one warm-up run.", + SupportsLegacyComparison: true, + CreateSampleResult: Measurements.MeasureAudioEffectsSample), + + ["tcp-frame-read"] = new( + Name: "tcp-frame-read", + Description: "Measures allocations and timing for TCP frame read and payload parsing.", + ParameterNames: ["Packets", "PacketBytes", "Frames"], + DefaultScenarios: + [ + new Scenario(4, 64, 1_000), + new Scenario(8, 256, 1_000), + new Scenario(16, 512, 500) + ], + MeasurementDescription: "Steady-state allocations and elapsed time across repeated TCP frame read/parse operations.", + SupportsLegacyComparison: true, + CreateSampleResult: Measurements.MeasureTcpFrameReadSample), + + ["tcp-write-payload"] = new( + Name: "tcp-write-payload", + Description: "Measures allocations and timing for TCP response frame formatting.", + ParameterNames: ["Packets", "PacketBytes", "Frames"], + DefaultScenarios: + [ + new Scenario(4, 64, 1_000), + new Scenario(8, 256, 1_000), + new Scenario(16, 512, 500) + ], + MeasurementDescription: "Steady-state allocations and elapsed time across repeated TCP response frame writes into memory.", + SupportsLegacyComparison: true, + CreateSampleResult: Measurements.MeasureTcpWritePayloadSample), + + ["http-packed-packets"] = new( + Name: "http-packed-packets", + Description: "Measures allocations and timing for HTTP packed packet decode and encode.", + ParameterNames: ["Packets", "PacketBytes", "Requests"], + DefaultScenarios: + [ + new Scenario(4, 64, 1_000), + new Scenario(8, 256, 500), + new Scenario(16, 512, 250) + ], + MeasurementDescription: "Steady-state allocations and elapsed time across repeated HTTP packed-packet decode/encode operations.", + SupportsLegacyComparison: false, + CreateSampleResult: Measurements.MeasureHttpPackedPacketsSample), + + ["http-auth-path"] = new( + Name: "http-auth-path", + Description: "Measures allocations and timing for HTTP bearer auth, packet decode, validation, and in-memory dispatch.", + ParameterNames: ["Packets", "PacketBytes", "Requests"], + DefaultScenarios: + [ + new Scenario(1, 32, 5_000), + new Scenario(4, 64, 2_000), + new Scenario(8, 128, 1_000) + ], + MeasurementDescription: "Steady-state allocations and elapsed time across repeated HTTP auth-path request handling without sockets.", + SupportsLegacyComparison: false, + CreateSampleResult: Measurements.MeasureHttpAuthPathSample), + + ["wss-data-tunnel"] = new( + Name: "wss-data-tunnel", + Description: "Measures allocations and timing for WSS data tunnel decode, packet slicing, and response encoding.", + ParameterNames: ["Packets", "PacketBytes", "Commands"], + DefaultScenarios: + [ + new Scenario(4, 64, 1_000), + new Scenario(8, 256, 500), + new Scenario(16, 512, 250) + ], + MeasurementDescription: "Steady-state allocations and elapsed time across repeated WSS data-tunnel command handling.", + SupportsLegacyComparison: false, + CreateSampleResult: Measurements.MeasureWssDataTunnelSample), + + ["mcapi-broadcast-fanout"] = new( + Name: "mcapi-broadcast-fanout", + Description: "Measures allocations and timing for McApi packet serialization and broadcast fanout across peers.", + ParameterNames: ["Peers", "PacketBytes", "Broadcasts"], + DefaultScenarios: + [ + new Scenario(16, 64, 2_000), + new Scenario(64, 256, 1_000), + new Scenario(256, 512, 250) + ], + MeasurementDescription: "Steady-state allocations and elapsed time across repeated McApi broadcast calls.", + SupportsLegacyComparison: false, + CreateSampleResult: Measurements.MeasureMcApiBroadcastFanoutSample), + + ["event-handler-burst"] = new( + Name: "event-handler-burst", + Description: "Measures allocations and timing for EventHandlerSystem task bursts drained through Update().", + ParameterNames: ["QueuedTasks", "VisiblePeers", "Bursts"], + DefaultScenarios: + [ + new Scenario(32, 8, 500), + new Scenario(64, 32, 250), + new Scenario(128, 64, 100) + ], + MeasurementDescription: "Steady-state allocations and elapsed time across repeated event-enqueue bursts and queue draining.", + SupportsLegacyComparison: false, + CreateSampleResult: Measurements.MeasureEventHandlerBurstSample), + + ["entity-create-sync"] = new( + Name: "entity-create-sync", + Description: "Measures allocations and timing for initial McApi peer sync of effects and entity snapshots.", + ParameterNames: ["Entities", "Effects", "Connects"], + DefaultScenarios: + [ + new Scenario(16, 4, 500), + new Scenario(64, 8, 250), + new Scenario(128, 16, 100) + ], + MeasurementDescription: "Steady-state allocations and elapsed time across repeated simulated peer-connect sync runs.", + SupportsLegacyComparison: false, + CreateSampleResult: Measurements.MeasureEntityCreateSyncSample), + + ["audio-effect-process"] = new( + Name: "audio-effect-process", + Description: "Measures allocations and timing for applying audio effects across entities and audio buffers.", + ParameterNames: ["Effects", "Entities", "Runs"], + DefaultScenarios: + [ + new Scenario(4, 8, 2_000), + new Scenario(8, 32, 1_000), + new Scenario(16, 64, 250) + ], + MeasurementDescription: "Steady-state allocations and elapsed time across repeated audio effect processing passes.", + SupportsLegacyComparison: false, + CreateSampleResult: Measurements.MeasureAudioEffectProcessSample), + + ["jitter-buffer"] = new( + Name: "jitter-buffer", + Description: "Measures allocations and timing for JitterBuffer add and drain cycles.", + ParameterNames: ["Packets", "PacketBytes", "Runs"], + DefaultScenarios: + [ + new Scenario(128, 160, 100), + new Scenario(512, 160, 50), + new Scenario(1_024, 320, 25) + ], + MeasurementDescription: "Steady-state allocations and elapsed time across repeated JitterBuffer fill/drain runs.", + SupportsLegacyComparison: false, + CreateSampleResult: Measurements.MeasureJitterBufferSample) + }; + } +} diff --git a/VoiceCraft.Tools/AllocHarness/BenchmarkRunner.cs b/VoiceCraft.Tools/AllocHarness/BenchmarkRunner.cs new file mode 100644 index 00000000..d5dadaa2 --- /dev/null +++ b/VoiceCraft.Tools/AllocHarness/BenchmarkRunner.cs @@ -0,0 +1,39 @@ +internal static class BenchmarkRunner +{ + public static BenchmarkRunResult Run(BenchmarkDefinition benchmark, Options options) + { + if (options.Mode == MeasurementMode.Legacy && !benchmark.SupportsLegacyComparison) + throw new ArgumentException($"Benchmark '{benchmark.Name}' does not have a legacy-simulated comparison path."); + + var results = new List(); + foreach (var scenario in options.Scenarios) + { + if (options.Mode is MeasurementMode.CheckedOut or MeasurementMode.Both) + results.Add(MeasureScenario(benchmark, scenario, ScenarioMode.CheckedOut, options.Samples)); + + if (benchmark.SupportsLegacyComparison && options.Mode is (MeasurementMode.Legacy or MeasurementMode.Both)) + results.Add(MeasureScenario(benchmark, scenario, ScenarioMode.LegacySimulated, options.Samples)); + } + + ConsoleOutput.PrintHeader(options, benchmark); + ConsoleOutput.PrintScenarioTable(results, benchmark); + ConsoleOutput.PrintComparisonTable(results, benchmark); + ConsoleOutput.PrintFooter(benchmark); + + return new BenchmarkRunResult(benchmark, options, results); + } + + private static ScenarioResult MeasureScenario(BenchmarkDefinition benchmark, Scenario scenario, ScenarioMode mode, int samples) + { + var sampleResults = new List(samples); + for (var sampleIndex = 0; sampleIndex < samples; sampleIndex++) + sampleResults.Add(benchmark.CreateSampleResult(scenario, mode)); + + return new ScenarioResult( + scenario, + mode, + sampleResults, + Stats.BuildLongStats(sampleResults.Select(x => x.AllocatedBytes).ToArray()), + Stats.BuildDoubleStats(sampleResults.Select(x => x.Elapsed.TotalMilliseconds).ToArray())); + } +} diff --git a/VoiceCraft.Tools/AllocHarness/ConsoleOutput.cs b/VoiceCraft.Tools/AllocHarness/ConsoleOutput.cs new file mode 100644 index 00000000..9353a01e --- /dev/null +++ b/VoiceCraft.Tools/AllocHarness/ConsoleOutput.cs @@ -0,0 +1,272 @@ +using System.Diagnostics; +using System.Globalization; +using System.Runtime.InteropServices; + +internal static class ConsoleOutput +{ + public static void PrintBenchmarkList(IReadOnlyDictionary benchmarkDefinitions) + { + Console.WriteLine("Available benchmarks:"); + foreach (var benchmark in benchmarkDefinitions.Values.OrderBy(x => x.Name, StringComparer.OrdinalIgnoreCase)) + Console.WriteLine($"- {benchmark.Name}: {benchmark.Description}"); + } + + public static void PrintHeader(Options options, BenchmarkDefinition benchmark) + { + Console.WriteLine("Allocation harness"); + Console.WriteLine($"Git: {RuntimeMetadata.GetGitDescription()}"); + Console.WriteLine($".NET: {RuntimeMetadata.GetDotNetVersion()}"); + Console.WriteLine($"OS: {RuntimeMetadata.GetOsDescription()}"); + Console.WriteLine($"Benchmark: {benchmark.Name}"); + Console.WriteLine($"Description: {benchmark.Description}"); + Console.WriteLine($"Mode: {FormatMode(options.Mode)}"); + Console.WriteLine($"Samples per scenario: {options.Samples}"); + Console.WriteLine($"Measurement: {benchmark.MeasurementDescription}"); + Console.WriteLine(); + } + + public static void PrintScenarioTable(IReadOnlyList results, BenchmarkDefinition benchmark) + { + var p1Name = benchmark.ParameterNames[0]; + var p2Name = benchmark.ParameterNames[1]; + var p3Name = benchmark.ParameterNames[2]; + var rows = new List + { + new[] + { + "Mode", + p1Name, + p2Name, + p3Name, + "Samples", + "Alloc min", + "Alloc med", + "Alloc avg", + "Alloc max", + "B/op med", + "Time med (ms)" + } + }; + + rows.AddRange(results.Select(result => new[] + { + FormatScenarioMode(result.Mode), + result.Scenario.P1.ToString(CultureInfo.InvariantCulture), + result.Scenario.P2.ToString(CultureInfo.InvariantCulture), + result.Scenario.P3.ToString(CultureInfo.InvariantCulture), + result.SampleResults.Count.ToString(CultureInfo.InvariantCulture), + FormatWholeNumber(result.AllocationStats.Min), + FormatWholeNumber(result.AllocationStats.Median), + FormatWholeNumber(result.AllocationStats.Average), + FormatWholeNumber(result.AllocationStats.Max), + FormatWholeNumber(result.AllocationStats.Median / GetOperationCount(result.Scenario, benchmark)), + result.ElapsedStats.Median.ToString("0.###", CultureInfo.InvariantCulture) + })); + + PrintTable(rows); + } + + public static void PrintComparisonTable(IReadOnlyList results, BenchmarkDefinition benchmark) + { + var grouped = results + .GroupBy(x => x.Scenario) + .Select(group => new + { + Scenario = group.Key, + CheckedOut = group.FirstOrDefault(x => x.Mode == ScenarioMode.CheckedOut), + Legacy = group.FirstOrDefault(x => x.Mode == ScenarioMode.LegacySimulated) + }) + .Where(x => x.CheckedOut is not null && x.Legacy is not null) + .ToArray(); + + if (grouped.Length == 0) return; + + var p1Name = benchmark.ParameterNames[0]; + var p2Name = benchmark.ParameterNames[1]; + var p3Name = benchmark.ParameterNames[2]; + + Console.WriteLine(); + Console.WriteLine("Legacy-simulated vs checked-out"); + + var rows = new List + { + new[] + { + p1Name, + p2Name, + p3Name, + "Alloc ratio (L/C)", + "Time ratio (L/C)" + } + }; + + rows.AddRange(grouped.Select(group => new[] + { + group.Scenario.P1.ToString(CultureInfo.InvariantCulture), + group.Scenario.P2.ToString(CultureInfo.InvariantCulture), + group.Scenario.P3.ToString(CultureInfo.InvariantCulture), + (group.Legacy!.AllocationStats.Median / group.CheckedOut!.AllocationStats.Median).ToString("0.##", CultureInfo.InvariantCulture) + "x", + (group.Legacy.ElapsedStats.Median / group.CheckedOut.ElapsedStats.Median).ToString("0.##", CultureInfo.InvariantCulture) + "x" + })); + + PrintTable(rows); + } + + public static void PrintFooter(BenchmarkDefinition benchmark) + { + Console.WriteLine(); + Console.WriteLine("Notes:"); + Console.WriteLine("- Alloc values are total bytes allocated across the measured loop."); + Console.WriteLine("- 'checked-out' means the code that is currently checked out in your worktree."); + Console.WriteLine(benchmark.SupportsLegacyComparison + ? "- 'legacy-simulated' is the comparison path for the selected benchmark." + : "- This benchmark has no legacy-simulated path; '--mode both' runs the checked-out path only."); + if (benchmark.Name == "visibility") + Console.WriteLine("- visibility legacy-simulated mirrors world.Entities.OfType(...) per entity and AudioEffects materialization inside visibility checks."); + else if (benchmark.Name == "audio-effects") + Console.WriteLine("- audio-effects legacy-simulated reads the AudioEffects getter, while checked-out reads AudioEffectsSnapshot."); + else if (benchmark.Name == "tcp-frame-read") + Console.WriteLine("- tcp-frame-read legacy-simulated uses per-frame header/payload arrays; checked-out uses a reused header buffer and pooled payload buffer."); + else if (benchmark.Name == "tcp-write-payload") + Console.WriteLine("- tcp-write-payload legacy-simulated builds an intermediate payload array before the frame; checked-out writes directly into a pooled frame buffer."); + else if (benchmark.Name == "http-packed-packets") + Console.WriteLine("- http-packed-packets measures Z85 decode, packet copies, Z85 encode, and UTF-8 response bytes."); + else if (benchmark.Name == "http-auth-path") + Console.WriteLine("- http-auth-path measures bearer parsing, request-size validation, Z85 decode, packet deserialization, auth check, and lightweight in-memory dispatch."); + else if (benchmark.Name == "wss-data-tunnel") + Console.WriteLine("- wss-data-tunnel measures Z85 command decode, packet extraction, outbound packing, and response command encoding."); + else if (benchmark.Name == "mcapi-broadcast-fanout") + Console.WriteLine("- mcapi-broadcast-fanout measures the in-memory cost of serializing one McApi packet and fanning it out across many peers."); + else if (benchmark.Name == "event-handler-burst") + Console.WriteLine("- event-handler-burst measures actual EventHandlerSystem task enqueue bursts followed by Update() queue draining."); + else if (benchmark.Name == "entity-create-sync") + Console.WriteLine("- entity-create-sync measures the initial McApi sync path that sends current effects and entity snapshots to a newly connected peer."); + else if (benchmark.Name == "audio-effect-process") + Console.WriteLine("- audio-effect-process measures repeated effect.Process(...) passes across many target entities and a stereo buffer."); + else if (benchmark.Name == "jitter-buffer") + Console.WriteLine("- jitter-buffer pre-creates packet payloads and measures buffer add/drain work."); + Console.WriteLine("- Use '--list' to see available benchmarks."); + } + + public static void PrintReportLocation(string reportPath) + { + Console.WriteLine(); + Console.WriteLine($"HTML report: {reportPath}"); + } + + private static int GetOperationCount(Scenario scenario, BenchmarkDefinition benchmark) + { + return benchmark.Name switch + { + "visibility" => scenario.P3, + "audio-effects" => scenario.P2, + "tcp-frame-read" => scenario.P3, + "tcp-write-payload" => scenario.P3, + "http-packed-packets" => scenario.P3, + "http-auth-path" => scenario.P3, + "wss-data-tunnel" => scenario.P3, + "mcapi-broadcast-fanout" => scenario.P3, + "event-handler-burst" => scenario.P3, + "entity-create-sync" => scenario.P3, + "audio-effect-process" => scenario.P3, + "jitter-buffer" => scenario.P3, + _ => 1 + }; + } + + private static void PrintTable(IReadOnlyList rows) + { + var widths = new int[rows[0].Length]; + foreach (var row in rows) + { + for (var i = 0; i < row.Length; i++) + widths[i] = Math.Max(widths[i], row[i].Length); + } + + for (var rowIndex = 0; rowIndex < rows.Count; rowIndex++) + { + var row = rows[rowIndex]; + Console.WriteLine("| " + string.Join(" | ", row.Select((cell, i) => cell.PadLeft(widths[i]))) + " |"); + + if (rowIndex != 0) continue; + Console.WriteLine("|-" + string.Join("-|-", widths.Select(width => new string('-', width))) + "-|"); + } + } + + private static string FormatMode(MeasurementMode mode) + { + return mode switch + { + MeasurementMode.CheckedOut => "checked-out", + MeasurementMode.Legacy => "legacy-simulated", + MeasurementMode.Both => "both", + _ => throw new ArgumentOutOfRangeException(nameof(mode), mode, null) + }; + } + + private static string FormatScenarioMode(ScenarioMode mode) + { + return mode switch + { + ScenarioMode.CheckedOut => "checked-out", + ScenarioMode.LegacySimulated => "legacy-sim", + _ => throw new ArgumentOutOfRangeException(nameof(mode), mode, null) + }; + } + + private static string FormatWholeNumber(double value) + { + return Math.Round(value).ToString("N0", CultureInfo.InvariantCulture); + } + +} + +internal static class RuntimeMetadata +{ + public static string GetGitDescription() + { + try + { + var commit = RunGit("rev-parse --short HEAD"); + var branch = RunGit("branch --show-current"); + return string.IsNullOrWhiteSpace(branch) + ? $"{commit} (detached HEAD)" + : $"{commit} ({branch})"; + } + catch + { + return "unknown"; + } + } + + public static string GetDotNetVersion() + { + return Environment.Version.ToString(); + } + + public static string GetOsDescription() + { + return RuntimeInformation.OSDescription; + } + + private static string RunGit(string arguments) + { + using var process = Process.Start(new ProcessStartInfo("git", arguments) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }); + + if (process is null) + throw new InvalidOperationException("Failed to start git process."); + + var output = process.StandardOutput.ReadToEnd().Trim(); + process.WaitForExit(); + if (process.ExitCode != 0) + throw new InvalidOperationException(process.StandardError.ReadToEnd()); + + return output; + } +} diff --git a/VoiceCraft.Tools/AllocHarness/HtmlReportWriter.cs b/VoiceCraft.Tools/AllocHarness/HtmlReportWriter.cs new file mode 100644 index 00000000..eada47af --- /dev/null +++ b/VoiceCraft.Tools/AllocHarness/HtmlReportWriter.cs @@ -0,0 +1,675 @@ +using System.Globalization; +using System.Net; +using System.Text; + +internal static class HtmlReportWriter +{ + private const string ReportFileName = "report.html"; + private const string EntriesMarker = ""; + private const string TemplateVersionMarker = "data-report-version=\"4\""; + + public static string Append(HarnessRunReport report) + { + var reportPath = GetReportPath(); + var document = LoadOrCreateDocument(reportPath); + var entry = BuildReportEntry(report); + var updated = document.Replace(EntriesMarker, entry + Environment.NewLine + EntriesMarker, StringComparison.Ordinal); + File.WriteAllText(reportPath, updated, Encoding.UTF8); + return reportPath; + } + + private static string GetReportPath() + { + var projectDirectory = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..")); + return Path.Combine(projectDirectory, ReportFileName); + } + + private static string LoadOrCreateDocument(string reportPath) + { + if (!File.Exists(reportPath)) + return BuildDocumentSkeleton(); + + var document = File.ReadAllText(reportPath, Encoding.UTF8); + return document.Contains(TemplateVersionMarker, StringComparison.Ordinal) + ? document + : BuildDocumentSkeleton(); + } + + private static string BuildDocumentSkeleton() + { + return """ + + + + + + AllocHarness Report + + + + +
+
+

AllocHarness Report

+ Append-only run log for allocation and timing measurements, with filters, history, and run-vs-run comparisons. +
+ +
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+ +
+
+
+
+
Visible Runs
+
0
+
+
+
Visible Benchmarks
+
0
+
+
+
Unique Commits
+
0
+
+
+
Latest Visible Run
+
-
+
+
+
+ +
+
+
+

VS Mode

+
Choose two runs and optionally a benchmark. Below, each shared benchmark renders as its own comparison block.
+
+
+
+
+ + +
+
+ + +
+
+ + +
+
+
+
Need at least two runs with shared benchmark scenarios to compare.
+
+ +
+ +
+ +
+
No results match the current filters.
+
Try clearing the search, switching the commit filter, or disabling latest-only mode.
+
+
+ + + + +"""; + } + + private static string BuildReportEntry(HarnessRunReport report) + { + var runId = report.StartedAtUtc.ToString("O", CultureInfo.InvariantCulture); + var timestampLabel = report.StartedAtUtc.ToLocalTime().ToString("yyyy-MM-dd HH:mm:ss zzz", CultureInfo.InvariantCulture); + var runSearch = $"{report.GitDescription} {report.DotNetVersion} {report.OsDescription}".ToLowerInvariant(); + + var builder = new StringBuilder(); + builder.AppendLine($"""
"""); + builder.AppendLine("""
"""); + builder.AppendLine("""
"""); + builder.AppendLine($"""

Run {Encode(timestampLabel)}

"""); + builder.AppendLine("""
"""); + builder.AppendLine($""" Commit: {Encode(report.GitDescription)}"""); + builder.AppendLine($""" .NET: {Encode(report.DotNetVersion)}"""); + builder.AppendLine($""" OS: {Encode(report.OsDescription)}"""); + builder.AppendLine($""" Benchmarks: {report.Benchmarks.Count.ToString(CultureInfo.InvariantCulture)}"""); + builder.AppendLine("""
"""); + builder.AppendLine("""
"""); + builder.AppendLine("""
"""); + + foreach (var benchmarkRun in report.Benchmarks) + builder.Append(BuildBenchmarkSection(benchmarkRun, report.GitDescription, timestampLabel)); + + builder.AppendLine("
"); + return builder.ToString(); + } + + private static string BuildBenchmarkSection(BenchmarkRunResult run, string commit, string timestampLabel) + { + var builder = new StringBuilder(); + var benchmark = run.Benchmark; + var mode = FormatMode(run.Options.Mode); + var searchText = string.Join(' ', + benchmark.Name, + benchmark.Description, + mode, + commit, + timestampLabel, + benchmark.MeasurementDescription).ToLowerInvariant(); + + builder.AppendLine($"""
"""); + builder.AppendLine("""
"""); + builder.AppendLine("""
"""); + builder.AppendLine($"""

{Encode(benchmark.Name)}

"""); + builder.AppendLine($"""

{Encode(benchmark.Description)}

"""); + builder.AppendLine("""
"""); + builder.AppendLine($""" Mode: {Encode(mode)}"""); + builder.AppendLine($""" Samples: {run.Options.Samples.ToString(CultureInfo.InvariantCulture)}"""); + builder.AppendLine($""" Measurement: {Encode(benchmark.MeasurementDescription)}"""); + builder.AppendLine("""
"""); + builder.AppendLine("""
"""); + builder.AppendLine("""
"""); + + builder.AppendLine("""
"""); + builder.AppendLine(""" """); + builder.AppendLine(""" """); + builder.AppendLine($""" """); + builder.AppendLine(""" """); + builder.AppendLine(""" """); + + foreach (var result in run.ScenarioResults) + { + var bytesPerOp = result.AllocationStats.Median / GetOperationCount(result.Scenario, benchmark); + var scenarioKey = $"{result.Scenario.P1}|{result.Scenario.P2}|{result.Scenario.P3}"; + var scenarioLabel = $"{benchmark.ParameterNames[0]}={result.Scenario.P1}, {benchmark.ParameterNames[1]}={result.Scenario.P2}, {benchmark.ParameterNames[2]}={result.Scenario.P3}"; + builder.AppendLine($""" """); + builder.AppendLine($""" """); + builder.AppendLine($""" """); + builder.AppendLine($""" """); + builder.AppendLine($""" """); + builder.AppendLine($""" """); + builder.AppendLine($""" """); + builder.AppendLine($""" """); + builder.AppendLine($""" """); + builder.AppendLine($""" """); + builder.AppendLine($""" """); + builder.AppendLine($""" """); + builder.AppendLine(""" """); + } + + builder.AppendLine(""" """); + builder.AppendLine("""
Mode{Encode(benchmark.ParameterNames[0])}{Encode(benchmark.ParameterNames[1])}{Encode(benchmark.ParameterNames[2])}SamplesAlloc minAlloc medAlloc avgAlloc maxB/op medTime med (ms)
{Encode(FormatScenarioMode(result.Mode))}{result.Scenario.P1.ToString(CultureInfo.InvariantCulture)}{result.Scenario.P2.ToString(CultureInfo.InvariantCulture)}{result.Scenario.P3.ToString(CultureInfo.InvariantCulture)}{result.SampleResults.Count.ToString(CultureInfo.InvariantCulture)}{FormatWholeNumber(result.AllocationStats.Min)}{FormatWholeNumber(result.AllocationStats.Median)}{FormatWholeNumber(result.AllocationStats.Average)}{FormatWholeNumber(result.AllocationStats.Max)}{FormatWholeNumber(bytesPerOp)}{result.ElapsedStats.Median.ToString("0.###", CultureInfo.InvariantCulture)}
"""); + builder.AppendLine("""
"""); + + var comparisons = BuildComparisonRows(run.ScenarioResults); + if (comparisons.Count > 0) + { + builder.AppendLine("""
"""); + builder.AppendLine("""
Legacy-simulated vs checked-out
"""); + builder.AppendLine("""
"""); + builder.AppendLine(""" """); + builder.AppendLine(""" """); + builder.AppendLine($""" """); + builder.AppendLine(""" """); + builder.AppendLine(""" """); + foreach (var comparison in comparisons) + { + builder.AppendLine(""" """); + builder.AppendLine($""" """); + builder.AppendLine($""" """); + builder.AppendLine($""" """); + builder.AppendLine($""" """); + builder.AppendLine($""" """); + builder.AppendLine(""" """); + } + builder.AppendLine(""" """); + builder.AppendLine("""
{Encode(benchmark.ParameterNames[0])}{Encode(benchmark.ParameterNames[1])}{Encode(benchmark.ParameterNames[2])}Alloc ratio (L/C)Time ratio (L/C)
{comparison.Scenario.P1.ToString(CultureInfo.InvariantCulture)}{comparison.Scenario.P2.ToString(CultureInfo.InvariantCulture)}{comparison.Scenario.P3.ToString(CultureInfo.InvariantCulture)}{comparison.AllocationRatio.ToString("0.##", CultureInfo.InvariantCulture)}x{comparison.ElapsedRatio.ToString("0.##", CultureInfo.InvariantCulture)}x
"""); + builder.AppendLine("""
"""); + builder.AppendLine("""
"""); + } + + builder.AppendLine("""
"""); + return builder.ToString(); + } + + private static List BuildComparisonRows(IReadOnlyList results) + { + return results + .GroupBy(x => x.Scenario) + .Select(group => new + { + Scenario = group.Key, + CheckedOut = group.FirstOrDefault(x => x.Mode == ScenarioMode.CheckedOut), + Legacy = group.FirstOrDefault(x => x.Mode == ScenarioMode.LegacySimulated) + }) + .Where(x => x.CheckedOut is not null && x.Legacy is not null) + .Select(x => new ComparisonRow( + x.Scenario, + x.Legacy!.AllocationStats.Median / x.CheckedOut!.AllocationStats.Median, + x.Legacy.ElapsedStats.Median / x.CheckedOut.ElapsedStats.Median)) + .ToList(); + } + + private static string GetRatioClass(double ratio) + { + return ratio >= 1.0d ? "ratio-warn" : "ratio-good"; + } + + private static string FormatMode(MeasurementMode mode) + { + return mode switch + { + MeasurementMode.CheckedOut => "checked-out", + MeasurementMode.Legacy => "legacy-simulated", + MeasurementMode.Both => "both", + _ => throw new ArgumentOutOfRangeException(nameof(mode), mode, null) + }; + } + + private static string FormatScenarioMode(ScenarioMode mode) + { + return mode switch + { + ScenarioMode.CheckedOut => "checked-out", + ScenarioMode.LegacySimulated => "legacy-sim", + _ => throw new ArgumentOutOfRangeException(nameof(mode), mode, null) + }; + } + + private static string FormatWholeNumber(double value) + { + return Math.Round(value).ToString("N0", CultureInfo.InvariantCulture); + } + + private static double GetOperationCount(Scenario scenario, BenchmarkDefinition benchmark) + { + return benchmark.Name switch + { + "visibility" => scenario.P3, + "audio-effects" => scenario.P2, + "tcp-frame-read" => scenario.P3, + "tcp-write-payload" => scenario.P3, + "http-packed-packets" => scenario.P3, + "http-auth-path" => scenario.P3, + "wss-data-tunnel" => scenario.P3, + "mcapi-broadcast-fanout" => scenario.P3, + "event-handler-burst" => scenario.P3, + "entity-create-sync" => scenario.P3, + "audio-effect-process" => scenario.P3, + "jitter-buffer" => scenario.P3, + _ => 1 + }; + } + + private static string Encode(string value) + { + return WebUtility.HtmlEncode(value); + } + + private sealed record ComparisonRow(Scenario Scenario, double AllocationRatio, double ElapsedRatio); +} diff --git a/VoiceCraft.Tools/AllocHarness/Measurements.cs b/VoiceCraft.Tools/AllocHarness/Measurements.cs new file mode 100644 index 00000000..a3ffd495 --- /dev/null +++ b/VoiceCraft.Tools/AllocHarness/Measurements.cs @@ -0,0 +1,753 @@ +using System.Buffers; +using System.Buffers.Binary; +using System.Diagnostics; +using System.Numerics; +using System.Text; +using LiteNetLib.Utils; +using VoiceCraft.Core.Interfaces; +using VoiceCraft.Core.World; +using VoiceCraft.Network; +using VoiceCraft.Network.Audio; +using VoiceCraft.Network.Interfaces; +using VoiceCraft.Network.NetPeers; +using VoiceCraft.Network.Packets.McApiPackets; +using VoiceCraft.Network.Packets.McApiPackets.Request; +using VoiceCraft.Network.Systems; +using VoiceCraft.Network.World; +using VoiceCraft.Server.Systems; + +internal static class Measurements +{ + public static SampleResult MeasureVisibilitySample(Scenario scenario, ScenarioMode mode) + { + using var world = new VoiceCraftWorld(); + using var effectSystem = new AudioEffectSystem(); + var visibilitySystem = new VisibilitySystem(world, effectSystem); + + for (var i = 0; i < scenario.P1; i++) + world.AddEntity(CreateNetworkEntity(i + 1)); + + for (var i = 0; i < scenario.P2; i++) + effectSystem.SetEffect((ushort)(1 << i), new FakeVisibleEffect(true)); + + Action action = mode switch + { + ScenarioMode.CheckedOut => visibilitySystem.Update, + ScenarioMode.LegacySimulated => () => _ = RunLegacyVisibilityPass(world, effectSystem), + _ => throw new ArgumentOutOfRangeException(nameof(mode), mode, null) + }; + + return MeasureAction(action, scenario.P3); + } + + public static SampleResult MeasureAudioEffectsSample(Scenario scenario, ScenarioMode mode) + { + using var effectSystem = new AudioEffectSystem(); + for (var i = 0; i < scenario.P1; i++) + effectSystem.SetEffect((ushort)(1 << i), new FakeVisibleEffect(true)); + + Action action = mode switch + { + ScenarioMode.CheckedOut => () => GC.KeepAlive(effectSystem.AudioEffectsSnapshot), + ScenarioMode.LegacySimulated => () => GC.KeepAlive(effectSystem.AudioEffects), + _ => throw new ArgumentOutOfRangeException(nameof(mode), mode, null) + }; + + return MeasureAction(action, scenario.P2); + } + + public static SampleResult MeasureTcpFrameReadSample(Scenario scenario, ScenarioMode mode) + { + var packets = BuildPackets(scenario.P1, scenario.P2); + var payload = BuildTcpPayload(string.Empty, packets); + var frame = BuildTcpFrame(payload, ProtocolConstants.TcpRequestKind); + var headerBuffer = ArrayPool.Shared.Rent(ProtocolConstants.TcpFrameHeaderSize); + + try + { + Action action = mode switch + { + ScenarioMode.CheckedOut => () => RunTcpFrameReadCheckedOut(frame, headerBuffer), + ScenarioMode.LegacySimulated => () => RunTcpFrameReadLegacy(frame), + _ => throw new ArgumentOutOfRangeException(nameof(mode), mode, null) + }; + + return MeasureAction(action, scenario.P3); + } + finally + { + ArrayPool.Shared.Return(headerBuffer); + } + } + + public static SampleResult MeasureTcpWritePayloadSample(Scenario scenario, ScenarioMode mode) + { + var packets = BuildPackets(scenario.P1, scenario.P2); + + Action action = mode switch + { + ScenarioMode.CheckedOut => () => RunTcpWritePayloadCheckedOut(packets), + ScenarioMode.LegacySimulated => () => RunTcpWritePayloadLegacy(packets), + _ => throw new ArgumentOutOfRangeException(nameof(mode), mode, null) + }; + + return MeasureAction(action, scenario.P3); + } + + public static SampleResult MeasureHttpPackedPacketsSample(Scenario scenario, ScenarioMode mode) + { + if (mode != ScenarioMode.CheckedOut) + throw new ArgumentOutOfRangeException(nameof(mode), mode, "HTTP packed packets has no legacy comparison path."); + + var packets = BuildPackets(scenario.P1, scenario.P2); + var encodedRequest = BuildHttpPackedString(packets); + var reader = new NetDataReader(); + var writer = new NetDataWriter(); + + return MeasureAction(() => RunHttpPackedPacketsCheckedOut(encodedRequest, reader, writer), scenario.P3); + } + + public static SampleResult MeasureHttpAuthPathSample(Scenario scenario, ScenarioMode mode) + { + if (mode != ScenarioMode.CheckedOut) + throw new ArgumentOutOfRangeException(nameof(mode), mode, "HTTP auth path has no legacy comparison path."); + + var requestBody = BuildHttpAuthRequestBody(scenario.P1, scenario.P2); + var authorizationHeader = "Bearer session-token"; + var reader = new NetDataReader(); + var packet = new McApiSetEntityPositionRequestPacket(); + var peer = new FakeMcApiPeer("session-token"); + var entity = new VoiceCraftEntity(100); + + return MeasureAction(() => RunHttpAuthPathCheckedOut(authorizationHeader, requestBody, reader, packet, peer, entity), scenario.P3); + } + + public static SampleResult MeasureWssDataTunnelSample(Scenario scenario, ScenarioMode mode) + { + if (mode != ScenarioMode.CheckedOut) + throw new ArgumentOutOfRangeException(nameof(mode), mode, "WSS data tunnel has no legacy comparison path."); + + var packets = BuildPackets(scenario.P1, scenario.P2); + var encodedCommand = BuildHttpPackedString(packets); + var reader = new NetDataReader(); + var writer = new NetDataWriter(); + + return MeasureAction(() => RunWssDataTunnelCheckedOut(encodedCommand, packets, reader, writer), scenario.P3); + } + + public static SampleResult MeasureMcApiBroadcastFanoutSample(Scenario scenario, ScenarioMode mode) + { + if (mode != ScenarioMode.CheckedOut) + throw new ArgumentOutOfRangeException(nameof(mode), mode, "McApi broadcast fanout has no legacy comparison path."); + + using var world = new VoiceCraftWorld(); + using var effectSystem = new AudioEffectSystem(); + var server = new FakeMcApiServer(world, effectSystem); + for (var i = 0; i < scenario.P1; i++) + server.AddPeer(new FakeMcApiPeer($"peer-{i}")); + + return MeasureAction(() => + { + server.ResetCounters(); + server.Broadcast(new FakeMcApiPayloadPacket(scenario.P2)); + GC.KeepAlive(server.TotalBytesWritten); + }, scenario.P3); + } + + public static SampleResult MeasureEventHandlerBurstSample(Scenario scenario, ScenarioMode mode) + { + if (mode != ScenarioMode.CheckedOut) + throw new ArgumentOutOfRangeException(nameof(mode), mode, "Event handler burst has no legacy comparison path."); + + using var world = new VoiceCraftWorld(); + using var effectSystem = new AudioEffectSystem(); + using var liteNetServer = new FakeLiteNetVoiceCraftServer(world); + using var mcApiServer = new FakeMcApiServer(world, effectSystem); + using var eventHandlerSystem = new EventHandlerSystem(liteNetServer, [mcApiServer], effectSystem, world); + + var sourceEntity = new VoiceCraftEntity(1); + world.AddEntity(sourceEntity); + + for (var i = 0; i < scenario.P2; i++) + { + var visible = CreateNetworkEntity(i + 1000); + world.AddEntity(visible); + sourceEntity.AddVisibleEntity(visible); + } + + eventHandlerSystem.Update(); + + var positionSeed = 0f; + return MeasureAction(() => + { + liteNetServer.ResetCounters(); + mcApiServer.ResetCounters(); + + for (var i = 0; i < scenario.P1; i++) + { + positionSeed += 1f; + sourceEntity.Position = new Vector3(positionSeed, i, 0f); + } + + eventHandlerSystem.Update(); + GC.KeepAlive(liteNetServer.TotalBytesWritten + mcApiServer.TotalBytesWritten); + }, scenario.P3); + } + + public static SampleResult MeasureEntityCreateSyncSample(Scenario scenario, ScenarioMode mode) + { + if (mode != ScenarioMode.CheckedOut) + throw new ArgumentOutOfRangeException(nameof(mode), mode, "Entity create sync has no legacy comparison path."); + + using var world = new VoiceCraftWorld(); + using var effectSystem = new AudioEffectSystem(); + using var liteNetServer = new FakeLiteNetVoiceCraftServer(world); + using var mcApiServer = new FakeMcApiServer(world, effectSystem); + using var eventHandlerSystem = new EventHandlerSystem(liteNetServer, [mcApiServer], effectSystem, world); + + for (var i = 0; i < scenario.P1; i++) + world.AddEntity(CreateEntityForSync(i)); + + for (var i = 0; i < scenario.P2; i++) + effectSystem.SetEffect((ushort)(1 << i), new FakeVisibleEffect(true)); + + eventHandlerSystem.Update(); + + var peer = new FakeMcApiPeer("entity-create-sync"); + return MeasureAction(() => + { + liteNetServer.ResetCounters(); + mcApiServer.ResetCounters(); + mcApiServer.RaisePeerConnected(peer, peer.SessionToken); + eventHandlerSystem.Update(); + GC.KeepAlive(mcApiServer.TotalBytesWritten + liteNetServer.TotalBytesWritten); + }, scenario.P3); + } + + public static SampleResult MeasureAudioEffectProcessSample(Scenario scenario, ScenarioMode mode) + { + if (mode != ScenarioMode.CheckedOut) + throw new ArgumentOutOfRangeException(nameof(mode), mode, "Audio effect process has no legacy comparison path."); + + using var effectSystem = new AudioEffectSystem(); + var source = new VoiceCraftEntity(1) + { + CaveFactor = 0.4f, + MuffleFactor = 0.1f + }; + var targets = Enumerable.Range(0, scenario.P2) + .Select(i => new VoiceCraftEntity(i + 2) + { + CaveFactor = (i % 5) / 5f, + MuffleFactor = (i % 7) / 7f + }) + .ToArray(); + + for (var i = 0; i < scenario.P1; i++) + effectSystem.SetEffect((ushort)(1 << i), new FakeProcessingEffect((i % 4) + 1)); + + var buffer = new float[960]; + + return MeasureAction(() => + { + Array.Fill(buffer, 0.25f); + foreach (var target in targets) + foreach (var effect in effectSystem.AudioEffectsSnapshot) + effect.Value.Process(source, target, effect.Key, buffer); + + GC.KeepAlive(buffer[0]); + }, scenario.P3); + } + + public static SampleResult MeasureJitterBufferSample(Scenario scenario, ScenarioMode mode) + { + if (mode != ScenarioMode.CheckedOut) + throw new ArgumentOutOfRangeException(nameof(mode), mode, "Jitter buffer has no legacy comparison path."); + + var jitterBuffer = new JitterBuffer(TimeSpan.Zero); + var packets = BuildJitterPackets(scenario.P1, scenario.P2); + + return MeasureAction(() => RunJitterBufferCycle(jitterBuffer, packets), scenario.P3); + } + + private static SampleResult MeasureAction(Action action, int iterations) + { + action(); + + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + var before = GC.GetTotalAllocatedBytes(true); + var stopwatch = Stopwatch.StartNew(); + for (var i = 0; i < iterations; i++) + action(); + stopwatch.Stop(); + var after = GC.GetTotalAllocatedBytes(true); + + return new SampleResult(after - before, stopwatch.Elapsed); + } + + private static int RunLegacyVisibilityPass(VoiceCraftWorld world, AudioEffectSystem effectSystem) + { + var visibleCount = 0; + foreach (var entity in world.Entities) + { + entity.TrimDeadEntities(); + + var visibleNetworkEntities = world.Entities.OfType(); + foreach (var possibleEntity in visibleNetworkEntities) + { + if (possibleEntity.Id == entity.Id) continue; + if ((entity.TalkBitmask & possibleEntity.ListenBitmask) == 0) + { + entity.RemoveVisibleEntity(possibleEntity); + continue; + } + + var visible = true; + foreach (var effect in effectSystem.AudioEffects) + { + if (effect.Value is not IVisible visibleEffect) continue; + if (visibleEffect.Visibility(entity, possibleEntity, effect.Key)) continue; + + visible = false; + break; + } + + if (!visible) + { + entity.RemoveVisibleEntity(possibleEntity); + continue; + } + + entity.AddVisibleEntity(possibleEntity); + visibleCount++; + } + } + + return visibleCount; + } + + private static byte[][] BuildPackets(int packetCount, int packetBytes) + { + var packets = new byte[packetCount][]; + for (var packetIndex = 0; packetIndex < packets.Length; packetIndex++) + { + var packet = new byte[packetBytes]; + for (var byteIndex = 0; byteIndex < packet.Length; byteIndex++) + packet[byteIndex] = (byte)((packetIndex + byteIndex) & 0xFF); + packets[packetIndex] = packet; + } + + return packets; + } + + private static JitterPacket[] BuildJitterPackets(int packetCount, int packetBytes) + { + var packets = new JitterPacket[packetCount]; + for (var i = 0; i < packets.Length; i++) + packets[i] = new JitterPacket((ushort)i, BuildPacketPayload(i, packetBytes)); + return packets; + } + + private static byte[] BuildPacketPayload(int seed, int packetBytes) + { + var data = new byte[packetBytes]; + for (var i = 0; i < data.Length; i++) + data[i] = (byte)((seed + i) & 0xFF); + return data; + } + + private static byte[] BuildTcpPayload(string token, IReadOnlyList packets) + { + var tokenLength = string.IsNullOrEmpty(token) ? 0 : Encoding.UTF8.GetByteCount(token); + var payloadLength = 4 + tokenLength + 4; + foreach (var packet in packets) + payloadLength += 4 + packet.Length; + + var payload = new byte[payloadLength]; + var offset = 0; + WriteInt32(payload, ref offset, tokenLength); + if (tokenLength > 0) + offset += Encoding.UTF8.GetBytes(token, payload.AsSpan(offset, tokenLength)); + + WriteInt32(payload, ref offset, packets.Count); + foreach (var packet in packets) + { + WriteInt32(payload, ref offset, packet.Length); + packet.CopyTo(payload.AsSpan(offset)); + offset += packet.Length; + } + + return payload; + } + + private static byte[] BuildTcpFrame(ReadOnlySpan payload, ushort kind) + { + var frame = new byte[ProtocolConstants.TcpFrameHeaderSize + payload.Length]; + BinaryPrimitives.WriteInt32BigEndian(frame.AsSpan(0, 4), ProtocolConstants.TcpFrameMagic); + BinaryPrimitives.WriteUInt16BigEndian(frame.AsSpan(4, 2), ProtocolConstants.TcpFrameVersion); + BinaryPrimitives.WriteUInt16BigEndian(frame.AsSpan(6, 2), kind); + BinaryPrimitives.WriteInt32BigEndian(frame.AsSpan(8, 4), payload.Length); + payload.CopyTo(frame.AsSpan(ProtocolConstants.TcpFrameHeaderSize)); + return frame; + } + + private static void RunTcpFrameReadCheckedOut(ReadOnlySpan frame, byte[] headerBuffer) + { + frame[..ProtocolConstants.TcpFrameHeaderSize].CopyTo(headerBuffer); + if (!TryReadTcpFrameHeader(headerBuffer, ProtocolConstants.TcpRequestKind, out var payloadLength)) + throw new InvalidOperationException("Invalid TCP frame header."); + + var payloadBuffer = ArrayPool.Shared.Rent(payloadLength); + try + { + frame.Slice(ProtocolConstants.TcpFrameHeaderSize, payloadLength).CopyTo(payloadBuffer); + if (!TryReadTcpPayload(payloadBuffer.AsSpan(0, payloadLength), out var token, out var packets)) + throw new InvalidOperationException("Invalid TCP payload."); + + GC.KeepAlive(token); + GC.KeepAlive(packets); + } + finally + { + ArrayPool.Shared.Return(payloadBuffer); + } + } + + private static void RunTcpFrameReadLegacy(ReadOnlySpan frame) + { + var headerBuffer = new byte[ProtocolConstants.TcpFrameHeaderSize]; + frame[..ProtocolConstants.TcpFrameHeaderSize].CopyTo(headerBuffer); + if (!TryReadTcpFrameHeader(headerBuffer, ProtocolConstants.TcpRequestKind, out var payloadLength)) + throw new InvalidOperationException("Invalid TCP frame header."); + + var payloadBuffer = new byte[payloadLength]; + frame.Slice(ProtocolConstants.TcpFrameHeaderSize, payloadLength).CopyTo(payloadBuffer); + if (!TryReadTcpPayload(payloadBuffer, out var token, out var packets)) + throw new InvalidOperationException("Invalid TCP payload."); + + GC.KeepAlive(token); + GC.KeepAlive(packets); + } + + private static void RunTcpWritePayloadCheckedOut(IReadOnlyList packets) + { + var payloadLength = CalculateTcpPayloadLength(tokenLength: 0, packets); + var frameLength = ProtocolConstants.TcpFrameHeaderSize + payloadLength; + var frameBuffer = ArrayPool.Shared.Rent(frameLength); + + try + { + WriteTcpFrameHeader(frameBuffer, ProtocolConstants.TcpResponseKind, payloadLength); + + var offset = ProtocolConstants.TcpFrameHeaderSize; + WriteInt32(frameBuffer, ref offset, 0); + WriteInt32(frameBuffer, ref offset, packets.Count); + foreach (var packet in packets) + { + WriteInt32(frameBuffer, ref offset, packet.Length); + packet.CopyTo(frameBuffer.AsSpan(offset)); + offset += packet.Length; + } + + GC.KeepAlive(frameBuffer[0]); + } + finally + { + ArrayPool.Shared.Return(frameBuffer); + } + } + + private static void RunTcpWritePayloadLegacy(IReadOnlyList packets) + { + var payloadLength = CalculateTcpPayloadLength(tokenLength: 0, packets); + var payload = new byte[payloadLength]; + var payloadOffset = 0; + WriteInt32(payload, ref payloadOffset, 0); + WriteInt32(payload, ref payloadOffset, packets.Count); + foreach (var packet in packets) + { + WriteInt32(payload, ref payloadOffset, packet.Length); + packet.CopyTo(payload.AsSpan(payloadOffset)); + payloadOffset += packet.Length; + } + + var frameBuffer = new byte[ProtocolConstants.TcpFrameHeaderSize + payload.Length]; + WriteTcpFrameHeader(frameBuffer, ProtocolConstants.TcpResponseKind, payload.Length); + payload.CopyTo(frameBuffer.AsSpan(ProtocolConstants.TcpFrameHeaderSize)); + GC.KeepAlive(frameBuffer); + } + + private static string BuildHttpPackedString(IReadOnlyList packets) + { + var writer = new NetDataWriter(); + foreach (var packet in packets) + { + writer.Put((ushort)packet.Length); + writer.Put(packet); + } + + return Z85.GetStringWithPadding(writer.AsReadOnlySpan()); + } + + private static void RunHttpPackedPacketsCheckedOut(string encodedRequest, NetDataReader reader, NetDataWriter writer) + { + var packets = new List(); + if (!TryReadHttpPackedPackets(encodedRequest, packets, reader)) + throw new InvalidOperationException("Invalid HTTP packed packets."); + + writer.Reset(); + foreach (var packet in packets) + { + writer.Put((ushort)packet.Length); + writer.Put(packet); + } + + var encodedResponse = Z85.GetStringWithPadding(writer.AsReadOnlySpan()); + var responseBuffer = Encoding.UTF8.GetBytes(encodedResponse); + GC.KeepAlive(responseBuffer); + } + + private static string BuildHttpAuthRequestBody(int packetCount, int packetBytes) + { + var packets = new byte[packetCount][]; + for (var i = 0; i < packetCount; i++) + { + var writer = new NetDataWriter(); + writer.Put((byte)McApiPacketType.SetEntityPositionRequest); + var packet = new McApiSetEntityPositionRequestPacket() + .Set(i + 1, new Vector3(i, i + 1, i + 2)); + writer.Put(packet); + + if (packetBytes > writer.Length) + writer.Put(new byte[packetBytes - writer.Length]); + + packets[i] = writer.CopyData(); + } + + return BuildHttpPackedString(packets); + } + + private static void RunHttpAuthPathCheckedOut( + string authorizationHeader, + string encodedRequest, + NetDataReader reader, + McApiSetEntityPositionRequestPacket packet, + FakeMcApiPeer peer, + VoiceCraftEntity entity) + { + if (!TryGetBearerToken(authorizationHeader, out var token)) + throw new InvalidOperationException("Missing bearer token."); + + var contentLength = Encoding.UTF8.GetByteCount(encodedRequest); + if (contentLength is <= 0 or > 1_000_000) + throw new InvalidOperationException("Invalid request length."); + + var packets = new List(); + if (!TryReadHttpPackedPackets(encodedRequest, packets, reader)) + throw new InvalidOperationException("Invalid packed request."); + + foreach (var packetBytes in packets) + { + reader.Clear(); + reader.SetSource(packetBytes); + var packetType = (McApiPacketType)reader.GetByte(); + if (packetType != McApiPacketType.SetEntityPositionRequest) + continue; + + packet.Deserialize(reader); + if (!AuthorizePacket(packet, peer, token)) + continue; + + entity.Position = packet.Value; + } + } + + private static void RunWssDataTunnelCheckedOut(string encodedCommand, IReadOnlyList outboundPackets, NetDataReader reader, NetDataWriter writer) + { + var packets = new List(); + if (!TryReadHttpPackedPackets(encodedCommand, packets, reader)) + throw new InvalidOperationException("Invalid WSS command payload."); + + writer.Reset(); + foreach (var packet in outboundPackets) + { + writer.Put((ushort)packet.Length); + writer.Put(packet); + } + + var encodedResponse = Z85.GetStringWithPadding(writer.AsReadOnlySpan()); + GC.KeepAlive(packets); + GC.KeepAlive(encodedResponse); + } + + private static void RunJitterBufferCycle(JitterBuffer jitterBuffer, IReadOnlyList packets) + { + jitterBuffer.Reset(); + foreach (var packet in packets) + jitterBuffer.Add(packet); + + while (jitterBuffer.Get(out var packet)) + GC.KeepAlive(packet); + } + + private static bool TryReadHttpPackedPackets(string encodedRequest, List packets, NetDataReader reader) + { + try + { + var packedPackets = Z85.GetBytesWithPadding(encodedRequest); + reader.Clear(); + reader.SetSource(packedPackets); + while (!reader.EndOfData) + { + var packetSize = reader.GetUShort(); + if (packetSize <= 0) continue; + if (reader.AvailableBytes < packetSize) + return false; + + var packet = new byte[packetSize]; + reader.GetBytes(packet, packetSize); + packets.Add(packet); + } + + return true; + } + catch (ArgumentException) + { + return false; + } + } + + private static bool AuthorizePacket(IMcApiPacket packet, FakeMcApiPeer netPeer, string token) + { + return packet switch + { + McApiLoginRequestPacket => true, + McApiLogoutRequestPacket => true, + _ => netPeer.ConnectionState == McApiConnectionState.Connected && token == netPeer.SessionToken + }; + } + + private static bool TryGetBearerToken(string? authorizationHeader, out string token) + { + const string bearerPrefix = "Bearer "; + + token = string.Empty; + if (string.IsNullOrWhiteSpace(authorizationHeader) || + !authorizationHeader.StartsWith(bearerPrefix, StringComparison.OrdinalIgnoreCase)) + return false; + + token = authorizationHeader[bearerPrefix.Length..].Trim(); + return !string.IsNullOrWhiteSpace(token); + } + + private static bool TryReadTcpFrameHeader(ReadOnlySpan header, ushort expectedKind, out int payloadLength) + { + payloadLength = 0; + if (header.Length < ProtocolConstants.TcpFrameHeaderSize) + return false; + + var magic = BinaryPrimitives.ReadInt32BigEndian(header[..4]); + var version = BinaryPrimitives.ReadUInt16BigEndian(header.Slice(4, 2)); + var kind = BinaryPrimitives.ReadUInt16BigEndian(header.Slice(6, 2)); + payloadLength = BinaryPrimitives.ReadInt32BigEndian(header.Slice(8, 4)); + + return magic == ProtocolConstants.TcpFrameMagic && + version == ProtocolConstants.TcpFrameVersion && + kind == expectedKind && + payloadLength is >= 0 and <= ProtocolConstants.MaxTcpFramePayloadLength; + } + + private static bool TryReadTcpPayload(ReadOnlySpan payload, out string token, out List packets) + { + token = string.Empty; + packets = []; + if (payload.Length < 8) + return false; + + var offset = 0; + if (!TryReadInt32(payload, ref offset, out var tokenLength) || tokenLength < 0 || + payload.Length - offset < tokenLength) + return false; + + token = tokenLength == 0 ? string.Empty : Encoding.UTF8.GetString(payload.Slice(offset, tokenLength)); + offset += tokenLength; + + if (!TryReadInt32(payload, ref offset, out var packetCount) || packetCount < 0) + return false; + if (packetCount > (payload.Length - offset) / 5) + return false; + + packets = new List(packetCount); + for (var i = 0; i < packetCount; i++) + { + if (!TryReadInt32(payload, ref offset, out var packetLength) || packetLength <= 0 || + payload.Length - offset < packetLength) + return false; + + var packet = new byte[packetLength]; + payload.Slice(offset, packetLength).CopyTo(packet); + packets.Add(packet); + offset += packetLength; + } + + return offset == payload.Length; + } + + private static bool TryReadInt32(ReadOnlySpan payload, ref int offset, out int value) + { + value = 0; + if (payload.Length - offset < 4) + return false; + + value = BinaryPrimitives.ReadInt32BigEndian(payload.Slice(offset, 4)); + offset += 4; + return true; + } + + private static int CalculateTcpPayloadLength(int tokenLength, IReadOnlyList packets) + { + var payloadLength = 4 + tokenLength + 4; + foreach (var packet in packets) + payloadLength += 4 + packet.Length; + return payloadLength; + } + + private static void WriteTcpFrameHeader(byte[] frameBuffer, ushort kind, int payloadLength) + { + BinaryPrimitives.WriteInt32BigEndian(frameBuffer.AsSpan(0, 4), ProtocolConstants.TcpFrameMagic); + BinaryPrimitives.WriteUInt16BigEndian(frameBuffer.AsSpan(4, 2), ProtocolConstants.TcpFrameVersion); + BinaryPrimitives.WriteUInt16BigEndian(frameBuffer.AsSpan(6, 2), kind); + BinaryPrimitives.WriteInt32BigEndian(frameBuffer.AsSpan(8, 4), payloadLength); + } + + private static void WriteInt32(byte[] payload, ref int offset, int value) + { + BinaryPrimitives.WriteInt32BigEndian(payload.AsSpan(offset, 4), value); + offset += 4; + } + + private static VoiceCraftNetworkEntity CreateNetworkEntity(int id) + { + return new VoiceCraftNetworkEntity( + new FakeNetPeer(Guid.NewGuid(), Guid.NewGuid(), "en-US", PositioningType.Client), + id); + } + + private static VoiceCraftEntity CreateEntityForSync(int id) + { + return id % 3 == 0 + ? CreateNetworkEntity(id + 1) + : new VoiceCraftEntity(id + 1) + { + Name = $"Entity {id + 1}", + WorldId = $"world-{id % 4}", + Position = new Vector3(id, id * 0.5f, id * 0.25f), + Rotation = new Vector2(id % 360, (id * 2) % 360), + CaveFactor = (id % 5) / 5f, + MuffleFactor = (id % 7) / 7f + }; + } +} diff --git a/VoiceCraft.Tools/AllocHarness/Models.cs b/VoiceCraft.Tools/AllocHarness/Models.cs new file mode 100644 index 00000000..850826c9 --- /dev/null +++ b/VoiceCraft.Tools/AllocHarness/Models.cs @@ -0,0 +1,365 @@ +using LiteNetLib.Utils; +using VoiceCraft.Core.Interfaces; +using VoiceCraft.Core.World; +using VoiceCraft.Network; +using VoiceCraft.Network.Interfaces; +using VoiceCraft.Network.NetPeers; +using VoiceCraft.Network.Packets.McApiPackets; +using VoiceCraft.Network.Packets.VcPackets; +using VoiceCraft.Network.Servers; +using VoiceCraft.Network.World; + +internal sealed class FakeNetPeer(Guid userGuid, Guid serverUserGuid, string locale, PositioningType positioningType) + : VoiceCraftNetPeer(userGuid, serverUserGuid, locale, positioningType) +{ + public override VcConnectionState ConnectionState => VcConnectionState.Connected; +} + +internal sealed class FakeVisibleEffect(bool result) : IAudioEffect, IVisible +{ + public EffectType EffectType => EffectType.Visibility; + + public bool Visibility(VoiceCraftEntity from, VoiceCraftEntity to, ushort effectBitmask) + { + return result; + } + + public void Process(VoiceCraftEntity from, VoiceCraftEntity to, ushort effectBitmask, Span buffer) + { + } + + public void Reset() + { + } + + public void Serialize(NetDataWriter writer) + { + } + + public void Deserialize(NetDataReader reader) + { + } + + public void Dispose() + { + } +} + +internal sealed class FakeProcessingEffect(int stride) : IAudioEffect +{ + public EffectType EffectType => EffectType.Echo; + + public void Process(VoiceCraftEntity from, VoiceCraftEntity to, ushort effectBitmask, Span buffer) + { + var scale = 0.02f * ((effectBitmask & 0xF) + 1); + for (var i = 0; i < buffer.Length; i += stride) + buffer[i] = (buffer[i] + from.CaveFactor + to.MuffleFactor) * scale; + } + + public void Reset() + { + } + + public void Serialize(NetDataWriter writer) + { + writer.Put(stride); + } + + public void Deserialize(NetDataReader reader) + { + _ = reader.GetInt(); + } + + public void Dispose() + { + } +} + +internal sealed class FakeMcApiPeer(string sessionToken, McApiConnectionState connectionState = McApiConnectionState.Connected) + : McApiNetPeer +{ + public override McApiConnectionState ConnectionState { get; } = connectionState; + public override string SessionToken { get; } = sessionToken; +} + +internal sealed class FakeMcApiPayloadPacket(int payloadBytes) : IMcApiPacket +{ + private readonly byte[] _payload = Enumerable.Repeat((byte)0x5A, Math.Max(0, payloadBytes)).ToArray(); + + public McApiPacketType PacketType => McApiPacketType.OnEntityAudioReceived; + + public void Serialize(NetDataWriter writer) + { + writer.Put((ushort)_payload.Length); + writer.Put(_payload); + } + + public void Deserialize(NetDataReader reader) + { + var length = reader.GetUShort(); + var buffer = new byte[length]; + reader.GetBytes(buffer, length); + } +} + +internal sealed class FakeVcPayloadPacket(int payloadBytes) : IVoiceCraftPacket +{ + private readonly byte[] _payload = Enumerable.Repeat((byte)0x47, Math.Max(0, payloadBytes)).ToArray(); + + public VcPacketType PacketType => VcPacketType.OnEntityAudioReceived; + + public void Serialize(NetDataWriter writer) + { + writer.Put((ushort)_payload.Length); + writer.Put(_payload); + } + + public void Deserialize(NetDataReader reader) + { + var length = reader.GetUShort(); + var buffer = new byte[length]; + reader.GetBytes(buffer, length); + } +} + +internal sealed class FakeMcApiServer(VoiceCraftWorld world, VoiceCraft.Network.Systems.AudioEffectSystem audioEffectSystem) + : McApiServer(world, audioEffectSystem) +{ + private readonly NetDataWriter _writer = new(); + private readonly List _peers = []; + + public long TotalBytesWritten { get; private set; } + public int TotalPacketsWritten { get; private set; } + + public override string LoginToken => string.Empty; + public override uint MaxClients => 10_000; + public override int ConnectedPeers => _peers.Count; + public override event Action? OnPeerConnected; + public override event Action? OnPeerDisconnected; + + public void AddPeer(FakeMcApiPeer peer) + { + peer.Tag = this; + _peers.Add(peer); + } + + public void RaisePeerConnected(FakeMcApiPeer peer, string token) + { + peer.Tag = this; + OnPeerConnected?.Invoke(peer, token); + } + + public void ResetCounters() + { + TotalBytesWritten = 0; + TotalPacketsWritten = 0; + } + + public override void Start() + { + } + + public override void Update() + { + } + + public override void Stop() + { + } + + public override void SendPacket(McApiNetPeer netPeer, T packet) + { + try + { + _writer.Reset(); + _writer.Put((byte)packet.PacketType); + _writer.Put(packet); + TotalBytesWritten += _writer.Length; + TotalPacketsWritten++; + } + finally + { + PacketPool.Return(packet); + } + } + + public override void Broadcast(T packet, params McApiNetPeer?[] excludes) + { + try + { + _writer.Reset(); + _writer.Put((byte)packet.PacketType); + _writer.Put(packet); + foreach (var peer in _peers) + { + if (excludes.Contains(peer)) continue; + TotalBytesWritten += _writer.Length; + TotalPacketsWritten++; + } + } + finally + { + PacketPool.Return(packet); + } + } + + public override void Disconnect(McApiNetPeer netPeer, bool force = false) + { + OnPeerDisconnected?.Invoke(netPeer, netPeer.SessionToken); + } + + protected override void AcceptRequest(VoiceCraft.Network.Packets.McApiPackets.Request.McApiLoginRequestPacket packet, object? data) + { + } + + protected override void RejectRequest(VoiceCraft.Network.Packets.McApiPackets.Request.McApiLoginRequestPacket packet, string reason, object? data) + { + } +} + +internal sealed class FakeLiteNetVoiceCraftServer(VoiceCraftWorld world) : LiteNetVoiceCraftServer(world) +{ + private readonly NetDataWriter _writer = new(); + + public long TotalBytesWritten { get; private set; } + public int TotalPacketsWritten { get; private set; } + + public void ResetCounters() + { + TotalBytesWritten = 0; + TotalPacketsWritten = 0; + } + + public override void Start() + { + } + + public override void Update() + { + } + + public override void Stop() + { + } + + public override void SendUnconnectedPacket(System.Net.IPEndPoint endPoint, T packet) + { + try + { + _writer.Reset(); + _writer.Put((byte)packet.PacketType); + _writer.Put(packet); + TotalBytesWritten += _writer.Length; + TotalPacketsWritten++; + } + finally + { + PacketPool.Return(packet); + } + } + + public override void SendPacket(VoiceCraftNetPeer vcNetPeer, T packet, VcDeliveryMethod deliveryMethod = VcDeliveryMethod.Reliable) + { + try + { + _writer.Reset(); + _writer.Put((byte)packet.PacketType); + _writer.Put(packet); + TotalBytesWritten += _writer.Length; + TotalPacketsWritten++; + } + finally + { + PacketPool.Return(packet); + } + } + + public override void Broadcast(T packet, VcDeliveryMethod deliveryMethod = VcDeliveryMethod.Reliable, params VoiceCraftNetPeer?[] excludes) + { + try + { + _writer.Reset(); + _writer.Put((byte)packet.PacketType); + _writer.Put(packet); + TotalBytesWritten += _writer.Length; + TotalPacketsWritten++; + } + finally + { + PacketPool.Return(packet); + } + } + + public override void Disconnect(VoiceCraftNetPeer vcNetPeer, string reason, bool force = false) + { + } + + public override void DisconnectAll(string? reason = null) + { + } +} + +internal sealed record Options( + IReadOnlyList Scenarios, + int Samples, + MeasurementMode Mode, + string BenchmarkName, + bool ListBenchmarks, + bool RunAllBenchmarks); + +internal sealed record BenchmarkDefinition( + string Name, + string Description, + string[] ParameterNames, + IReadOnlyList DefaultScenarios, + string MeasurementDescription, + bool SupportsLegacyComparison, + Func CreateSampleResult); + +internal readonly record struct Scenario(int P1, int P2, int P3); + +internal sealed record ScenarioResult( + Scenario Scenario, + ScenarioMode Mode, + IReadOnlyList SampleResults, + NumericStats AllocationStats, + NumericStats ElapsedStats); + +internal sealed record BenchmarkRunResult( + BenchmarkDefinition Benchmark, + Options Options, + IReadOnlyList ScenarioResults); + +internal sealed record HarnessRunReport( + DateTimeOffset StartedAtUtc, + string GitDescription, + string DotNetVersion, + string OsDescription, + IReadOnlyList Benchmarks); + +internal readonly record struct SampleResult(long AllocatedBytes, TimeSpan Elapsed); + +internal readonly record struct NumericStats(double Min, double Median, double Average, double Max); + +internal enum MeasurementMode +{ + CheckedOut, + Legacy, + Both +} + +internal enum ScenarioMode +{ + CheckedOut, + LegacySimulated +} + +internal static class ProtocolConstants +{ + public const int TcpFrameHeaderSize = 12; + public const int MaxTcpFramePayloadLength = 1024 * 1024; + public const int TcpFrameMagic = 0x4D435450; + public const ushort TcpFrameVersion = 1; + public const ushort TcpRequestKind = 1; + public const ushort TcpResponseKind = 2; +} diff --git a/VoiceCraft.Tools/AllocHarness/OptionsParser.cs b/VoiceCraft.Tools/AllocHarness/OptionsParser.cs new file mode 100644 index 00000000..7294f8de --- /dev/null +++ b/VoiceCraft.Tools/AllocHarness/OptionsParser.cs @@ -0,0 +1,101 @@ +internal static class OptionsParser +{ + public static Options Parse(string[] args, IReadOnlyDictionary benchmarkDefinitions) + { + var samples = 7; + var mode = MeasurementMode.CheckedOut; + var benchmarkName = "visibility"; + var listBenchmarks = false; + var runAllBenchmarks = false; + var positionalValues = new List(); + + for (var i = 0; i < args.Length; i++) + { + switch (args[i]) + { + case "--samples": + case "-s": + samples = ParsePositiveInt(args, ref i, "samples"); + break; + case "--mode": + case "-m": + mode = ParseMode(args, ref i); + break; + case "--benchmark": + case "-b": + i++; + if (i >= args.Length) + throw new ArgumentException(GetUsage()); + benchmarkName = args[i]; + break; + case "--list": + listBenchmarks = true; + break; + case "--all": + runAllBenchmarks = true; + break; + default: + if (!int.TryParse(args[i], out var value)) + throw new ArgumentException(GetUsage()); + + positionalValues.Add(value); + break; + } + } + + if (listBenchmarks) + return new Options([], samples, mode, benchmarkName, ListBenchmarks: true, RunAllBenchmarks: false); + + if (runAllBenchmarks) + return new Options([], samples, mode, benchmarkName, ListBenchmarks: false, RunAllBenchmarks: true); + + if (!benchmarkDefinitions.TryGetValue(benchmarkName, out var benchmark)) + throw new ArgumentException($"Unknown benchmark '{benchmarkName}'. Use --list to see available benchmarks."); + + if (samples <= 0) + throw new ArgumentOutOfRangeException(nameof(samples), "Samples must be greater than zero."); + + if (positionalValues.Count == 0) + return new Options(benchmark.DefaultScenarios, samples, mode, benchmark.Name, ListBenchmarks: false, RunAllBenchmarks: false); + + if (positionalValues.Count % 3 != 0) + throw new ArgumentException(GetUsage()); + + var scenarios = new List(positionalValues.Count / 3); + for (var i = 0; i < positionalValues.Count; i += 3) + scenarios.Add(new Scenario(positionalValues[i], positionalValues[i + 1], positionalValues[i + 2])); + + return new Options(scenarios, samples, mode, benchmark.Name, ListBenchmarks: false, RunAllBenchmarks: false); + } + + private static int ParsePositiveInt(string[] args, ref int index, string name) + { + index++; + if (index >= args.Length || !int.TryParse(args[index], out var value) || value <= 0) + throw new ArgumentException(GetUsage(), name); + + return value; + } + + private static MeasurementMode ParseMode(string[] args, ref int index) + { + index++; + if (index >= args.Length) + throw new ArgumentException(GetUsage(), "mode"); + + return args[index].ToLowerInvariant() switch + { + "current" => MeasurementMode.CheckedOut, + "checked-out" => MeasurementMode.CheckedOut, + "legacy" => MeasurementMode.Legacy, + "both" => MeasurementMode.Both, + _ => throw new ArgumentException(GetUsage(), "mode") + }; + } + + private static string GetUsage() + { + return "Usage: dotnet run --project .\\VoiceCraft.Tools\\AllocHarness\\AllocHarness.csproj -c Release -- " + + "[--list] [--all] [--benchmark NAME] [--samples N] [--mode checked-out|legacy|both] [p1 p2 p3]..."; + } +} diff --git a/VoiceCraft.Tools/AllocHarness/Program.cs b/VoiceCraft.Tools/AllocHarness/Program.cs new file mode 100644 index 00000000..1f3b10b7 --- /dev/null +++ b/VoiceCraft.Tools/AllocHarness/Program.cs @@ -0,0 +1,45 @@ +var benchmarkDefinitions = BenchmarkCatalog.Build(); +var options = OptionsParser.Parse(args, benchmarkDefinitions); + +if (options.ListBenchmarks) +{ + ConsoleOutput.PrintBenchmarkList(benchmarkDefinitions); + return; +} + +if (options.RunAllBenchmarks) +{ + var orderedBenchmarks = benchmarkDefinitions.Values.OrderBy(x => x.Name, StringComparer.OrdinalIgnoreCase).ToArray(); + var runResults = new List(orderedBenchmarks.Length); + for (var index = 0; index < orderedBenchmarks.Length; index++) + { + var selectedBenchmark = orderedBenchmarks[index]; + var selectedOptions = options with + { + BenchmarkName = selectedBenchmark.Name, + Scenarios = selectedBenchmark.DefaultScenarios + }; + + runResults.Add(BenchmarkRunner.Run(selectedBenchmark, selectedOptions)); + if (index < orderedBenchmarks.Length - 1) + Console.WriteLine(); + } + + var reportPath = HtmlReportWriter.Append(new HarnessRunReport( + DateTimeOffset.UtcNow, + RuntimeMetadata.GetGitDescription(), + RuntimeMetadata.GetDotNetVersion(), + RuntimeMetadata.GetOsDescription(), + runResults)); + ConsoleOutput.PrintReportLocation(reportPath); + return; +} + +var benchmarkRun = BenchmarkRunner.Run(benchmarkDefinitions[options.BenchmarkName], options); +var singleReportPath = HtmlReportWriter.Append(new HarnessRunReport( + DateTimeOffset.UtcNow, + RuntimeMetadata.GetGitDescription(), + RuntimeMetadata.GetDotNetVersion(), + RuntimeMetadata.GetOsDescription(), + [benchmarkRun])); +ConsoleOutput.PrintReportLocation(singleReportPath); diff --git a/VoiceCraft.Tools/AllocHarness/Stats.cs b/VoiceCraft.Tools/AllocHarness/Stats.cs new file mode 100644 index 00000000..2594d19c --- /dev/null +++ b/VoiceCraft.Tools/AllocHarness/Stats.cs @@ -0,0 +1,30 @@ +internal static class Stats +{ + public static NumericStats BuildLongStats(long[] values) + { + Array.Sort(values); + var min = values[0]; + var max = values[^1]; + var average = values.Average(); + var median = CalculateMedian(values.Select(x => (double)x).ToArray()); + return new NumericStats(min, median, average, max); + } + + public static NumericStats BuildDoubleStats(double[] values) + { + Array.Sort(values); + var min = values[0]; + var max = values[^1]; + var average = values.Average(); + var median = CalculateMedian(values); + return new NumericStats(min, median, average, max); + } + + private static double CalculateMedian(double[] values) + { + var middle = values.Length / 2; + return values.Length % 2 == 0 + ? (values[middle - 1] + values[middle]) / 2d + : values[middle]; + } +}