Report held-out evaluation during training

IssueLabeler/IssueLabeler.sln

@@ -33,6 +33,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "workflows", "workflows", "{
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Common.Tests", "tests\Common.Tests\Common.Tests.csproj", "{D3F816D3-5CAE-4CF1-8977-F92AE96B481B}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Trainer.Tests", "tests\Trainer.Tests\Trainer.Tests.csproj", "{BA177CD9-81D1-4143-A433-665B2699B361}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -67,6 +69,10 @@ Global
 		{D3F816D3-5CAE-4CF1-8977-F92AE96B481B}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{D3F816D3-5CAE-4CF1-8977-F92AE96B481B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{D3F816D3-5CAE-4CF1-8977-F92AE96B481B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BA177CD9-81D1-4143-A433-665B2699B361}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BA177CD9-81D1-4143-A433-665B2699B361}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BA177CD9-81D1-4143-A433-665B2699B361}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BA177CD9-81D1-4143-A433-665B2699B361}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

IssueLabeler/src/Tester/Tester.cs

@@ -17,6 +17,12 @@
 var action = provider.GetRequiredService<ICoreService>();
 var config = Args.Parse(args, action);
 if (config is not Args argsData) return 1;
+
+const string disclaimer = "This test downloads currently labeled GitHub items and may include training data. It is not an independent held-out evaluation.";
+action.WriteWarning(disclaimer);
+action.Summary.AddPersistent(summary => summary.AddAlert(disclaimer, AlertType.Warning));
+await action.Summary.WritePersistentAsync();
+
 string[] excludedLabels = argsData.ExcludedLabels ?? [];
 
 List<Task<(Type ItemType, TestStats Stats)>> tasks = [];

IssueLabeler/src/Trainer/HeldOutResult.cs

@@ -0,0 +1,88 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+internal sealed class HeldOutPrediction
+{
+    public string Label { get; set; } = string.Empty;
+    public float[] Score { get; set; } = [];
+}
+
+internal sealed record PrecisionTargetResult(int TargetPrecision, float? Threshold, int Correct, int Accepted)
+{
+    public double? Precision => Accepted == 0 ? null : (double)Correct / Accepted;
+}
+
+internal sealed record HeldOutResult(int Total, PrecisionTargetResult[] Targets)
+{
+    internal static HeldOutResult Evaluate(IEnumerable<HeldOutPrediction> predictions, string[] labels)
+    {
+        List<(float Score, bool Correct)> ranked = [];
+        foreach (var prediction in predictions)
+        {
+            if (string.IsNullOrWhiteSpace(prediction.Label) || labels.Length == 0 || prediction.Score.Length != labels.Length ||
+                prediction.Score.Any(score => !float.IsFinite(score)))
+            {
+                throw new InvalidDataException("A held-out row has an invalid reference label or score vector.");
+            }
+
+            int index = Array.IndexOf(prediction.Score, prediction.Score.Max());
+            ranked.Add((prediction.Score[index], labels[index].Equals(prediction.Label, StringComparison.OrdinalIgnoreCase)));
+        }
+
+        if (ranked.Count == 0)
+        {
+            throw new InvalidOperationException("The held-out test set is empty. Download more data before training.");
+        }
+
+        int[] precisionTargets = [60, 70, 80, 90, 95];
+        PrecisionTargetResult[] results = [.. precisionTargets.Select(target => new PrecisionTargetResult(target, null, 0, 0))];
+        ranked.Sort((left, right) => right.Score.CompareTo(left.Score));
+        int correct = 0;
+        for (int index = 0; index < ranked.Count; index++)
+        {
+            if (ranked[index].Correct)
+            {
+                correct++;
+            }
+
+            // A cutoff must accept all predictions with the same score.
+            if (index + 1 < ranked.Count && ranked[index].Score == ranked[index + 1].Score)
+            {
+                continue;
+            }
+
+            int accepted = index + 1;
+            for (int targetIndex = 0; targetIndex < results.Length; targetIndex++)
+            {
+                int target = results[targetIndex].TargetPrecision;
+                if ((long)correct * 100 >= (long)target * accepted)
+                {
+                    results[targetIndex] = new(target, ranked[index].Score, correct, accepted);
+                }
+            }
+        }
+
+        return new(ranked.Count, results);
+    }
+
+    public string ToMarkdown()
+    {
+        var rows = Targets.Select(target => target.Threshold is float threshold
+            ? FormattableString.Invariant($"| {target.TargetPrecision}% | {threshold:R} | {target.Precision:P2} | {(double)target.Accepted / Total:P2} | {target.Accepted} |")
+            : FormattableString.Invariant($"| {target.TargetPrecision}% | Not reached | N/A | N/A | N/A |"));
+        return FormattableString.Invariant($"""
+            ### Held-out evaluation
+
+            Evaluated {Total} items from the random 20% test split, excluded from model fitting.
+            Each row aims to label as many test items as possible, with at least the target percentage labeled correctly.
+            Threshold is the minimum score required to use a prediction. Accepted is the number of predictions kept at this cutoff, including incorrect predictions.
+
+            | Target precision | Threshold | Observed precision | Coverage | Accepted |
+            |---:|---:|---:|---:|---:|
+            {string.Join(Environment.NewLine, rows)}
+
+            Precision is the percentage of accepted predictions that are correct. Coverage is the percentage of all test items accepted.
+            Thresholds are selected and evaluated on this same split.
+            """);
+    }
+}

IssueLabeler/src/Trainer/Trainer.cs

@@ -108,39 +108,46 @@ static async Task CreateModel(string dataPath, string modelPath, ModelType type,
     var testModel = trainedModel.Transform(split.TestSet);
 
     await action.WriteStatusAsync("Evaluating against the test set...");
+    VBuffer<ReadOnlyMemory<char>> slots = default;
+    testModel.Schema[nameof(HeldOutPrediction.Score)].GetSlotNames(ref slots);
+    string[] labels = [.. slots.DenseValues().Select(label => label.ToString())];
+    var evaluation = HeldOutResult.Evaluate(
+        mlContext.Data.CreateEnumerable<HeldOutPrediction>(testModel, reuseRowObject: false), labels);
     var metrics = mlContext.MulticlassClassification.Evaluate(testModel, labelColumnName: "LabelKey");
+    var classLosses = metrics.PerClassLogLoss.Take(3)
+        .Select((loss, index) => FormattableString.Invariant($"| Class {index + 1} | {loss:0.####} |"));
+    string metricsReport = FormattableString.Invariant($"""
+        These metrics use all held-out items, without a score cutoff.
+
+        | Metric | Value | Meaning |
+        |---|---:|---|
+        | MacroAccuracy | {metrics.MacroAccuracy:0.####} | Average accuracy across labels, each weighted equally (higher is better). |
+        | MicroAccuracy | {metrics.MicroAccuracy:0.####} | Fraction of all items predicted correctly (higher is better). |
+        | LogLoss | {metrics.LogLoss:0.####} | Penalizes low scores for the correct label (lower is better). |
+
+        **Per-class LogLoss** (first three label categories, lower is better)
+
+        | Class | LogLoss |
+        |---|---:|
+        {string.Join(Environment.NewLine, classLosses)}
+        """);
 
     action.Summary.AddPersistent(summary =>
     {
         summary.AddMarkdownHeading($"Finished Training {(type == ModelType.Issue ? "Issues" : "Pull Requests")} Model", 2);
 
-        summary.AddRawMarkdown($"""
-            * MacroAccuracy: {metrics.MacroAccuracy:0.####} (a value between 0 and 1; the closer to 1, the better)
-            * MicroAccuracy: {metrics.MicroAccuracy:0.####} (a value between 0 and 1; the closer to 1, the better)
-            * LogLoss: {metrics.LogLoss:0.####} (the closer to 0, the better)
-            {(metrics.PerClassLogLoss.Count() > 0 ? $"    * Class 1: {metrics.PerClassLogLoss[0]:0.####}" : "")}
-            {(metrics.PerClassLogLoss.Count() > 1 ? $"    * Class 2: {metrics.PerClassLogLoss[1]:0.####}" : "")}
-            {(metrics.PerClassLogLoss.Count() > 2 ? $"    * Class 3: {metrics.PerClassLogLoss[2]:0.####}" : "")}
-            """);
+        summary.AddRawMarkdown(metricsReport);
+        summary.AddNewLine();
+        summary.AddNewLine();
+        summary.AddRawMarkdown(evaluation.ToMarkdown(), true);
     });
 
     await action.Summary.WriteAsync();
 
     action.WriteInfo($"************************************************************");
-    action.WriteInfo($"MacroAccuracy = {metrics.MacroAccuracy:0.####}, a value between 0 and 1, the closer to 1, the better");
-    action.WriteInfo($"MicroAccuracy = {metrics.MicroAccuracy:0.####}, a value between 0 and 1, the closer to 1, the better");
-    action.WriteInfo($"LogLoss = {metrics.LogLoss:0.####}, the closer to 0, the better");
-
-    if (metrics.PerClassLogLoss.Count() > 0)
-        action.WriteInfo($"LogLoss for class 1 = {metrics.PerClassLogLoss[0]:0.####}, the closer to 0, the better");
-
-    if (metrics.PerClassLogLoss.Count() > 1)
-        action.WriteInfo($"LogLoss for class 2 = {metrics.PerClassLogLoss[1]:0.####}, the closer to 0, the better");
-
-    if (metrics.PerClassLogLoss.Count() > 2)
-        action.WriteInfo($"LogLoss for class 3 = {metrics.PerClassLogLoss[2]:0.####}, the closer to 0, the better");
-
+    action.WriteInfo(metricsReport);
     action.WriteInfo($"************************************************************");
+    action.WriteInfo(evaluation.ToMarkdown());
 
     action.WriteInfo($"Saving model to '{modelPath}'...");
     EnsureOutputDirectory(modelPath);

IssueLabeler/src/Trainer/Trainer.csproj

@@ -18,6 +18,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\Common\Common.csproj" />
+    <InternalsVisibleTo Include="Trainer.Tests" />
   </ItemGroup>
 
 </Project>

IssueLabeler/tests/Trainer.Tests/Trainer.Tests.csproj

@@ -0,0 +1,11 @@
+<Project Sdk="MSTest.Sdk/4.5.1">
+  <PropertyGroup>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <ManagePackageVersionsCentrally>false</ManagePackageVersionsCentrally>
+    <UseVSTest>true</UseVSTest>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Trainer\Trainer.csproj" />
+  </ItemGroup>
+</Project>

IssueLabeler/tests/Trainer.Tests/TrainerTests.cs

@@ -0,0 +1,153 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+using Microsoft.ML;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+[assembly: Parallelize(Scope = ExecutionScope.MethodLevel)]
+
+namespace IssueLabeler.Tests;
+
+[TestClass]
+public sealed class TrainerTests
+{
+    private string directory = string.Empty;
+
+    [TestInitialize]
+    public void Initialize() => directory = Directory.CreateTempSubdirectory("issue-labeler-training-").FullName;
+
+    [TestCleanup]
+    public void Cleanup() => Directory.Delete(directory, recursive: true);
+
+    [TestMethod]
+    public void PrecisionTargetsChooseMaximumCoverageEvenWhenPrecisionRecovers()
+    {
+        int[] wrongRanks = [2, 11, 12, 16, 17, 18, 21, 22, 23, 24];
+        HeldOutPrediction[] predictions = [.. Enumerable.Range(1, 25).Reverse().Select(rank => new HeldOutPrediction
+        {
+            Label = wrongRanks.Contains(rank) ? "area-B" : "area-A",
+            Score = [(100 - rank) / 100f, rank / 100f]
+        })];
+        var result = HeldOutResult.Evaluate(predictions, ["area-A", "area-B"]);
+
+        Assert.AreEqual(25, result.Total);
+        PrecisionTargetResult[] expected =
+        [
+            new(60, 0.75f, 15, 25),
+            new(70, 0.8f, 14, 20),
+            new(80, 0.85f, 12, 15),
+            new(90, 0.9f, 9, 10),
+            new(95, 0.99f, 1, 1)
+        ];
+        CollectionAssert.AreEqual(expected, result.Targets);
+        Assert.AreEqual(0.6, result.Targets[0].Precision);
+    }
+
+    [TestMethod]
+    public void TiedPredictionScoresAreAcceptedTogetherAtTheThreshold()
+    {
+        var result = HeldOutResult.Evaluate(
+            [new() { Label = "AREA-A", Score = [0.9f, 0.1f] },
+             new() { Label = "area-B", Score = [0.9f, 0.1f] },
+             new() { Label = "area-A", Score = [0.8f, 0.2f] }],
+            ["area-A", "area-B"]);
+        Assert.AreEqual(new PrecisionTargetResult(60, 0.8f, 2, 3), result.Targets[0]);
+        Assert.IsTrue(result.Targets.Skip(1).All(target => target.Threshold is null));
+    }
+
+    [TestMethod]
+    public void UnreachableTargetsAreNotReportedAsPerfectPrecisionWithNoCoverage()
+    {
+        var result = HeldOutResult.Evaluate(
+            [new() { Label = "area-B", Score = [1f, 0f] },
+             new() { Label = "area-B", Score = [0.7f, 0.3f] }],
+            ["area-A", "area-B"]);
+        Assert.AreEqual(2, result.Total);
+        Assert.HasCount(5, result.Targets);
+        foreach (var target in result.Targets)
+        {
+            Assert.IsNull(target.Threshold);
+            Assert.IsNull(target.Precision);
+            Assert.AreEqual(0, target.Accepted);
+        }
+    }
+
+    [DataRow(ModelType.Issue)]
+    [DataRow(ModelType.PullRequest)]
+    [TestMethod]
+    public async Task TrainingProducesUsableModel(ModelType type)
+    {
+        string dataPath = Path.Combine(directory, "data.tsv");
+        string modelPath = Path.Combine(directory, "model.zip");
+        string[] labels = ["area-A", "area-B", "area-C"];
+        string[] records = [type == ModelType.Issue ? "Label\tTitle\tBody" : "Label\tTitle\tBody\tFileNames\tFolderNames",
+            .. Enumerable.Range(0, 300).Select(index => type == ModelType.Issue
+                ? DataFileUtils.FormatIssueRecord(labels[index % 3], $"Item {index}", $"Content for {labels[index % 3]}")
+                : DataFileUtils.FormatPullRequestRecord(labels[index % 3], $"Item {index}", $"Content for {labels[index % 3]}",
+                    [$"File{index % 3}.cs"], [$"Folder{index % 3}"]))];
+        File.WriteAllLines(dataPath, records);
+        var result = await RunTrainer(
+            type == ModelType.Issue ? "--issues-data" : "--pulls-data", dataPath,
+            type == ModelType.Issue ? "--issues-model" : "--pulls-model", modelPath);
+        Assert.AreEqual(0, result.ExitCode, result.Output);
+        var context = new MLContext();
+        using var predictor = context.Model.CreatePredictionEngine<ModelInput, ModelOutput>(context.Model.Load(modelPath, out _));
+        var input = new ModelInput { Title = "Item 0", Body = "Content", FileNames = "File0.cs", FolderNames = "Folder0" };
+        float[] scores = predictor.Predict(input).Score;
+        Assert.HasCount(3, scores);
+        Assert.IsTrue(scores.All(float.IsFinite));
+    }
+
+    private async Task<(int ExitCode, string Output)> RunTrainer(params string[] arguments)
+    {
+        string summaryPath = Path.Combine(directory, "summary.md");
+        File.WriteAllText(summaryPath, string.Empty);
+        using var process = new Process
+        {
+            StartInfo = new("dotnet")
+            {
+                WorkingDirectory = directory,
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false
+            }
+        };
+        process.StartInfo.ArgumentList.Add(typeof(Args).Assembly.Location);
+        foreach (string argument in arguments)
+        {
+            process.StartInfo.ArgumentList.Add(argument);
+        }
+        process.StartInfo.Environment.Remove("GITHUB_TOKEN");
+        process.StartInfo.Environment["GITHUB_STEP_SUMMARY"] = summaryPath;
+        process.Start();
+        Task<string> output = process.StandardOutput.ReadToEndAsync();
+        Task<string> error = process.StandardError.ReadToEndAsync();
+        try
+        {
+            await process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(60));
+            return (process.ExitCode, await output + await error);
+        }
+        finally
+        {
+            if (!process.HasExited)
+            {
+                process.Kill(entireProcessTree: true);
+            }
+        }
+    }
+
+    public sealed class ModelInput
+    {
+        public string? Label { get; set; }
+        public string Title { get; set; } = string.Empty;
+        public string Body { get; set; } = string.Empty;
+        public string FileNames { get; set; } = string.Empty;
+        public string FolderNames { get; set; } = string.Empty;
+    }
+
+    public sealed class ModelOutput
+    {
+        public float[] Score { get; set; } = [];
+    }
+}

train/action.yml

@@ -1,5 +1,5 @@
 name: "Train Model"
-description: "Train the Issues, Pull Requests, or Discussions model for label prediction. Each type writes its own cached model file."
+description: "Train the Issues, Pull Requests, or Discussions model for label prediction and report held-out evaluation. Each type writes its own cached model file."
 
 inputs:
   type: