Report held-out evaluation during training

IssueLabeler/IssueLabeler.sln

@@ -33,6 +33,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "workflows", "workflows", "{
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Common.Tests", "tests\Common.Tests\Common.Tests.csproj", "{D3F816D3-5CAE-4CF1-8977-F92AE96B481B}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Trainer.Tests", "tests\Trainer.Tests\Trainer.Tests.csproj", "{BA177CD9-81D1-4143-A433-665B2699B361}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -67,6 +69,10 @@ Global
 		{D3F816D3-5CAE-4CF1-8977-F92AE96B481B}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{D3F816D3-5CAE-4CF1-8977-F92AE96B481B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{D3F816D3-5CAE-4CF1-8977-F92AE96B481B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BA177CD9-81D1-4143-A433-665B2699B361}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BA177CD9-81D1-4143-A433-665B2699B361}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BA177CD9-81D1-4143-A433-665B2699B361}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BA177CD9-81D1-4143-A433-665B2699B361}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

IssueLabeler/src/Tester/Tester.cs

@@ -17,6 +17,12 @@
 var action = provider.GetRequiredService<ICoreService>();
 var config = Args.Parse(args, action);
 if (config is not Args argsData) return 1;
+
+const string disclaimer = "This test downloads currently labeled GitHub items and may include training data. It is not an independent held-out evaluation.";
+action.WriteWarning(disclaimer);
+action.Summary.AddPersistent(summary => summary.AddAlert(disclaimer, AlertType.Warning));
+await action.Summary.WritePersistentAsync();
+
 string[] excludedLabels = argsData.ExcludedLabels ?? [];
 
 List<Task<(Type ItemType, TestStats Stats)>> tasks = [];

IssueLabeler/src/Trainer/Trainer.cs

@@ -108,41 +108,134 @@ static async Task CreateModel(string dataPath, string modelPath, ModelType type,
     var testModel = trainedModel.Transform(split.TestSet);
 
     await action.WriteStatusAsync("Evaluating against the test set...");
+    VBuffer<ReadOnlyMemory<char>> slots = default;
+    testModel.Schema[nameof(HeldOutPrediction.Score)].GetSlotNames(ref slots);
+    string[] labels = [.. slots.DenseValues().Select(label => label.ToString())];
+    var evaluation = HeldOutResult.Evaluate(
+        mlContext.Data.CreateEnumerable<HeldOutPrediction>(testModel, reuseRowObject: false), labels);
     var metrics = mlContext.MulticlassClassification.Evaluate(testModel, labelColumnName: "LabelKey");
+    var classLosses = metrics.PerClassLogLoss.Take(3)
+        .Select((loss, index) => FormattableString.Invariant($"| Class {index + 1} | {loss:0.####} |"));
+    string metricsReport = FormattableString.Invariant($"""
+        These metrics use all held-out items, without a score cutoff.
+
+        | Metric | Value | Meaning |
+        |---|---:|---|
+        | MacroAccuracy | {metrics.MacroAccuracy:0.####} | Average accuracy across labels, each weighted equally (higher is better). |
+        | MicroAccuracy | {metrics.MicroAccuracy:0.####} | Fraction of all items predicted correctly (higher is better). |
+        | LogLoss | {metrics.LogLoss:0.####} | Penalizes low scores for the correct label (lower is better). |
+
+        **Per-class LogLoss** (first three label categories, lower is better)
+
+        | Class | LogLoss |
+        |---|---:|
+        {string.Join(Environment.NewLine, classLosses)}
+        """);
 
     action.Summary.AddPersistent(summary =>
     {
         summary.AddMarkdownHeading($"Finished Training {(type == ModelType.Issue ? "Issues" : "Pull Requests")} Model", 2);
 
-        summary.AddRawMarkdown($"""
-            * MacroAccuracy: {metrics.MacroAccuracy:0.####} (a value between 0 and 1; the closer to 1, the better)
-            * MicroAccuracy: {metrics.MicroAccuracy:0.####} (a value between 0 and 1; the closer to 1, the better)
-            * LogLoss: {metrics.LogLoss:0.####} (the closer to 0, the better)
-            {(metrics.PerClassLogLoss.Count() > 0 ? $"    * Class 1: {metrics.PerClassLogLoss[0]:0.####}" : "")}
-            {(metrics.PerClassLogLoss.Count() > 1 ? $"    * Class 2: {metrics.PerClassLogLoss[1]:0.####}" : "")}
-            {(metrics.PerClassLogLoss.Count() > 2 ? $"    * Class 3: {metrics.PerClassLogLoss[2]:0.####}" : "")}
-            """);
+        summary.AddRawMarkdown(metricsReport);
+        summary.AddNewLine();
+        summary.AddNewLine();
+        summary.AddRawMarkdown(evaluation.ToMarkdown(), true);
     });
 
     await action.Summary.WriteAsync();
 
     action.WriteInfo($"************************************************************");
-    action.WriteInfo($"MacroAccuracy = {metrics.MacroAccuracy:0.####}, a value between 0 and 1, the closer to 1, the better");
-    action.WriteInfo($"MicroAccuracy = {metrics.MicroAccuracy:0.####}, a value between 0 and 1, the closer to 1, the better");
-    action.WriteInfo($"LogLoss = {metrics.LogLoss:0.####}, the closer to 0, the better");
-
-    if (metrics.PerClassLogLoss.Count() > 0)
-        action.WriteInfo($"LogLoss for class 1 = {metrics.PerClassLogLoss[0]:0.####}, the closer to 0, the better");
-
-    if (metrics.PerClassLogLoss.Count() > 1)
-        action.WriteInfo($"LogLoss for class 2 = {metrics.PerClassLogLoss[1]:0.####}, the closer to 0, the better");
-
-    if (metrics.PerClassLogLoss.Count() > 2)
-        action.WriteInfo($"LogLoss for class 3 = {metrics.PerClassLogLoss[2]:0.####}, the closer to 0, the better");
-
+    action.WriteInfo(metricsReport);
     action.WriteInfo($"************************************************************");
+    action.WriteInfo(evaluation.ToMarkdown());
 
     action.WriteInfo($"Saving model to '{modelPath}'...");
     EnsureOutputDirectory(modelPath);
     mlContext.Model.Save(trainedModel, split.TrainSet.Schema, modelPath);
 }
+
+internal sealed class HeldOutPrediction
+{
+    public string Label { get; set; } = string.Empty;
+    public float[] Score { get; set; } = [];
+}
+
+internal sealed record PrecisionTargetResult(int TargetPrecision, float? Threshold, int Correct, int Accepted)
+{
+    public double? Precision => Accepted == 0 ? null : (double)Correct / Accepted;
+}
+
+internal sealed record HeldOutResult(int Total, PrecisionTargetResult[] Targets)
+{
+    internal static HeldOutResult Evaluate(IEnumerable<HeldOutPrediction> predictions, string[] labels)
+    {
+        List<(float Score, bool Correct)> ranked = [];
+        foreach (var prediction in predictions)
+        {
+            if (string.IsNullOrWhiteSpace(prediction.Label) || labels.Length == 0 || prediction.Score.Length != labels.Length ||
+                prediction.Score.Any(score => !float.IsFinite(score)))
+            {
+                throw new InvalidDataException("A held-out row has an invalid reference label or score vector.");
+            }
+
+            int index = Array.IndexOf(prediction.Score, prediction.Score.Max());
+            ranked.Add((prediction.Score[index], labels[index].Equals(prediction.Label, StringComparison.OrdinalIgnoreCase)));
+        }
+
+        if (ranked.Count == 0)
+        {
+            throw new InvalidOperationException("The held-out test set is empty. Download more data before training.");
+        }
+
+        int[] precisionTargets = [60, 70, 80, 90, 95];
+        PrecisionTargetResult[] results = [.. precisionTargets.Select(target => new PrecisionTargetResult(target, null, 0, 0))];
+        ranked.Sort((left, right) => right.Score.CompareTo(left.Score));
+        int correct = 0;
+        for (int index = 0; index < ranked.Count; index++)
+        {
+            if (ranked[index].Correct)
+            {
+                correct++;
+            }
+
+            // A cutoff must accept all predictions with the same score.
+            if (index + 1 < ranked.Count && ranked[index].Score == ranked[index + 1].Score)
+            {
+                continue;
+            }
+
+            int accepted = index + 1;
+            for (int targetIndex = 0; targetIndex < results.Length; targetIndex++)
+            {
+                int target = results[targetIndex].TargetPrecision;
+                if ((long)correct * 100 >= (long)target * accepted)
+                {
+                    results[targetIndex] = new(target, ranked[index].Score, correct, accepted);
+                }
+            }
+        }
+
+        return new(ranked.Count, results);
+    }
+
+    public string ToMarkdown()
+    {
+        var rows = Targets.Select(target => target.Threshold is float threshold
+            ? FormattableString.Invariant($"| {target.TargetPrecision}% | {threshold:R} | {target.Precision:P2} | {(double)target.Accepted / Total:P2} | {target.Accepted} |")
+            : FormattableString.Invariant($"| {target.TargetPrecision}% | Not reached | N/A | N/A | N/A |"));
+        return FormattableString.Invariant($"""
+            ### Held-out evaluation
+
+            Evaluated {Total} items from the random 20% test split, excluded from model fitting.
+            Each row aims to label as many test items as possible, with at least the target percentage labeled correctly.
+            Threshold is the minimum score required to use a prediction. Accepted is the number of predictions kept at this cutoff, including incorrect predictions.
+
+            | Target precision | Threshold | Observed precision | Coverage | Accepted |
+            |---:|---:|---:|---:|---:|
+            {string.Join(Environment.NewLine, rows)}
+
+            Precision is the percentage of accepted predictions that are correct. Coverage is the percentage of all test items accepted.
+            Thresholds are selected and evaluated on this same split.
+            """);
+    }
+}

IssueLabeler/src/Trainer/Trainer.csproj

@@ -18,6 +18,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\Common\Common.csproj" />
+    <InternalsVisibleTo Include="Trainer.Tests" />
   </ItemGroup>
 
 </Project>

IssueLabeler/tests/Trainer.Tests/Trainer.Tests.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="MSTest.Sdk/4.5.1">
+  <PropertyGroup>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <ManagePackageVersionsCentrally>false</ManagePackageVersionsCentrally>
+    <UseVSTest>true</UseVSTest>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Trainer\Trainer.csproj" />
+    <ProjectReference Include="..\..\src\Tester\Tester.csproj" Aliases="Tester" />
+  </ItemGroup>
+</Project>