Report held-out evaluation during training

.github/workflows/labeler-cache-retention.yml

@@ -1,4 +1,4 @@
-# Regularly restore the prediction models from cache to prevent cache eviction
+# Regularly restore models and their held-out datasets to prevent cache eviction
 name: "Labeler: Cache Retention"
 
 # For more information about GitHub's action cache limits and eviction policy, see:
@@ -31,7 +31,22 @@ jobs:
         type: ["issues", "pulls", "discussions"]
     steps:
       - uses: dotnet/issue-labeler/restore@160b6b1e1e8d36da09beb34ef1e4806d2c0520a3 # v2.2.0
+        id: restore-model
         with:
           type: ${{ matrix.type }}
           cache_key: ${{ env.CACHE_KEY }}
           fail-on-cache-miss: true
+
+      - name: "Restore Held-Out Data for Retention"
+        if: ${{ steps.restore-model.outputs.cache-hit == 'true' }}
+        id: restore-test-data
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: labeler-cache/${{ matrix.type }}-model.test-data.bin
+          key: issue-labeler/test-data/${{ matrix.type }}/${{ hashFiles(format('labeler-cache/{0}-model.zip', matrix.type)) }}
+
+      - name: "Report Missing Held-Out Data"
+        if: ${{ steps.restore-model.outputs.cache-hit == 'true' && steps.restore-test-data.outputs.cache-hit != 'true' }}
+        shell: bash
+        run: |
+          echo "::warning::The held-out dataset for this model is unavailable. Predictions are unaffected, but the model must be retrained before held-out evaluation."

IssueLabeler/IssueLabeler.sln

@@ -33,6 +33,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "workflows", "workflows", "{
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Common.Tests", "tests\Common.Tests\Common.Tests.csproj", "{D3F816D3-5CAE-4CF1-8977-F92AE96B481B}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tester.Tests", "tests\Tester.Tests\Tester.Tests.csproj", "{BA177CD9-81D1-4143-A433-665B2699B361}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -67,6 +69,10 @@ Global
 		{D3F816D3-5CAE-4CF1-8977-F92AE96B481B}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{D3F816D3-5CAE-4CF1-8977-F92AE96B481B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{D3F816D3-5CAE-4CF1-8977-F92AE96B481B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BA177CD9-81D1-4143-A433-665B2699B361}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BA177CD9-81D1-4143-A433-665B2699B361}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BA177CD9-81D1-4143-A433-665B2699B361}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BA177CD9-81D1-4143-A433-665B2699B361}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

IssueLabeler/src/Common/App.cs

@@ -30,13 +30,13 @@ public async static Task<bool> RunTasks(List<Task> tasks, ICoreService action)
     /// <typeparam name="TResult">The Task result type.</typeparam>
     /// <param name="tasks">The list of tasks to run, waiting for all tasks to complete.</param>
     /// <param name="action">The GitHub action service.</param>
-    /// <returns>A tuple containing the results of the tasks and a boolean indicating whether all tasks were completed successfully.</returns>
+    /// <returns>A tuple containing the results of successfully completed tasks and a boolean indicating whether all tasks were completed successfully.</returns>
     public async static Task<(TResult[], bool)> RunTasks<TResult>(List<Task<TResult>> tasks, ICoreService action)
     {
         var allTasks = Task.WhenAll(tasks);
         var success = await RunTasks(allTasks, action);
 
-        return (allTasks.Result, success);
+        return (tasks.Where(task => task.IsCompletedSuccessfully).Select(task => task.Result).ToArray(), success);
     }
 
     /// <summary>

IssueLabeler/src/Tester/Args.cs

@@ -5,58 +5,34 @@
 
 public struct Args
 {
-    public readonly string GitHubToken => Environment.GetEnvironmentVariable("GITHUB_TOKEN")!;
-    public string Org { get; set; }
-    public List<string> Repos { get; set; }
     public float Threshold { get; set; }
-    public Predicate<string> LabelPredicate { get; set; }
-    public string[]? ExcludedAuthors { get; set; }
     public string[]? ExcludedLabels { get; set; }
     public string? IssuesModelPath { get; set; }
     public bool TestDiscussions { get; set; }
-    public int? IssuesLimit { get; set; }
-    public int? DiscussionsLimit { get; set; }
     public string? PullsModelPath { get; set; }
-    public int? PullsLimit { get; set; }
-    public int? PageSize { get; set; }
-    public int? PageLimit { get; set; }
-    public int[] Retries { get; set; }
     public bool Verbose { get; set; }
 
     static void ShowUsage(string? message, ICoreService action)
     {
         action.WriteNotice($$"""
             ERROR: Invalid or missing arguments.{{(message is null ? "" : " " + message)}}
 
-            Required environment variables:
-              GITHUB_TOKEN            GitHub token to be used for API calls.
+            Supply at least one model:
+              --issues-model          Path to the issue prediction model (ZIP file).
+              --pulls-model           Path to the pull request prediction model (ZIP file).
 
-            Required arguments:
-              --repo                  The GitHub repositories in format org/repo (comma separated for multiple).
-              --label-prefix          Prefix for label predictions. Must end with a character other than a letter or number.
-
-            Required for testing the issues model:
-              --issues-model          Path to existing issue prediction model file (ZIP file).
-              --discussions           Also evaluate discussions using the issues prediction model.
-
-            Required for testing the pull requests model:
-              --pulls-model           Path to existing pull request prediction model file (ZIP file).
+            Each model requires its saved held-out dataset beside it:
+              example-model.zip       Model file.
+              example-model.test-data.bin
+                                      The exact test split saved during training.
 
             Optional arguments:
-              --excluded-authors      Comma-separated list of authors to exclude.
-              --excluded-labels       Comma-separated list of labels to exclude.
-              --threshold             Minimum prediction confidence threshold. Range (0,1].
+              --discussions           Treat --issues-model as a discussion model instead.
+              --excluded-labels       Comma-separated labels to report if detected.
+                                      Does not change the held-out dataset.
+              --threshold             Minimum prediction confidence threshold. Range [0,1].
                                       Defaults to: 0.4.
-              --issues-limit          Maximum number of issues to download. Defaults to: No limit.
-              --discussions-limit     Maximum number of discussions to download. Defaults to: No limit.
-              --pulls-limit           Maximum number of pull requests to download. Defaults to: No limit.
-              --page-size             Number of items per page in GitHub API requests.
-                                      Defaults to: 100 for issues, 25 for pull requests.
-              --page-limit            Maximum number of pages to retrieve.
-                                      Defaults to: 1000 for issues, 4000 for pull requests.
-              --retries               Comma-separated retry delays in seconds.
-                                      Defaults to: 30,30,300,300,3000,3000.
-              --verbose               Enable verbose output.
+              --verbose               Log predictions for individual held-out rows.
             """);
 
         Environment.Exit(1);
@@ -66,132 +42,59 @@ static void ShowUsage(string? message, ICoreService action)
     {
         Queue<string> arguments = new(args);
         ArgUtils argUtils = new(action, ShowUsage, arguments);
-
         Args argsData = new()
         {
-            Threshold = 0.4f,
-            Retries = [30, 30, 300, 300, 3000, 3000]
+            Threshold = 0.4f
         };
 
-        if (string.IsNullOrEmpty(argsData.GitHubToken))
-        {
-            ShowUsage("Environment variable GITHUB_TOKEN is empty.", action);
-            return null;
-        }
-
         while (arguments.Count > 0)
         {
             string argument = arguments.Dequeue();
 
             switch (argument)
             {
-                case "--repo":
-                    if (!argUtils.TryGetRepoList("--repo", out string? org, out List<string>? repos))
+                case "--issues-model":
+                    if (!argUtils.TryGetPath(argument, out string? issuesModelPath))
                     {
+                        ShowUsage($"Input '{argument}' requires a path.", action);
                         return null;
                     }
-                    argsData.Org = org;
-                    argsData.Repos = repos;
+                    argsData.IssuesModelPath = issuesModelPath;
                     break;
 
-                case "--label-prefix":
-                    if (!argUtils.TryGetLabelPrefix("--label-prefix", out Func<string, bool>? labelPredicate))
+                case "--pulls-model":
+                    if (!argUtils.TryGetPath(argument, out string? pullsModelPath))
                     {
+                        ShowUsage($"Input '{argument}' requires a path.", action);
                         return null;
                     }
-                    argsData.LabelPredicate = new(labelPredicate);
+                    argsData.PullsModelPath = pullsModelPath;
                     break;
 
-                case "--excluded-authors":
-                    if (!argUtils.TryGetStringArray("--excluded-authors", out string[]? excludedAuthors))
-                    {
-                        return null;
-                    }
-                    argsData.ExcludedAuthors = excludedAuthors;
+                case "--discussions":
+                    argsData.TestDiscussions = true;
                     break;
 
                 case "--excluded-labels":
-                    if (!argUtils.TryGetStringArray("--excluded-labels", out string[]? excludedLabels))
+                    if (!argUtils.TryGetStringArray(argument, out string[]? excludedLabels))
                     {
+                        ShowUsage($"Input '{argument}' requires a label list.", action);
                         return null;
                     }
                     argsData.ExcludedLabels = LabelUtils.NormalizeLabels(excludedLabels);
                     break;
 
                 case "--threshold":
-                    if (!argUtils.TryGetFloat("--threshold", out float? threshold))
-                    {
-                        return null;
-                    }
-                    argsData.Threshold = threshold.Value;
-                    break;
-
-                case "--issues-model":
-                    if (!argUtils.TryGetPath("--issues-model", out string? IssuesModelPath))
+                    if (!argUtils.TryGetFloat(argument, out float? threshold))
                     {
                         return null;
                     }
-                    argsData.IssuesModelPath = IssuesModelPath;
-                    break;
-
-                case "--discussions":
-                    argsData.TestDiscussions = true;
-                    break;
-
-                case "--issues-limit":
-                    if (!argUtils.TryGetInt("--issues-limit", out int? IssuesLimit))
+                    if (!float.IsFinite(threshold.Value) || threshold.Value is < 0 or > 1)
                     {
+                        ShowUsage("The threshold must be between 0 and 1, inclusive.", action);
                         return null;
                     }
-                    argsData.IssuesLimit = IssuesLimit;
-                    break;
-
-                case "--discussions-limit":
-                    if (!argUtils.TryGetInt("--discussions-limit", out int? discussionsLimit))
-                    {
-                        return null;
-                    }
-                    argsData.DiscussionsLimit = discussionsLimit;
-                    break;
-
-                case "--pulls-model":
-                    if (!argUtils.TryGetPath("--pulls-model", out string? PullsModelPath))
-                    {
-                        return null;
-                    }
-                    argsData.PullsModelPath = PullsModelPath;
-                    break;
-
-                case "--pulls-limit":
-                    if (!argUtils.TryGetInt("--pulls-limit", out int? PullsLimit))
-                    {
-                        return null;
-                    }
-                    argsData.PullsLimit = PullsLimit;
-                    break;
-
-                case "--page-size":
-                    if (!argUtils.TryGetInt("--page-size", out int? pageSize))
-                    {
-                        return null;
-                    }
-                    argsData.PageSize = pageSize;
-                    break;
-
-                case "--page-limit":
-                    if (!argUtils.TryGetInt("--page-limit", out int? pageLimit))
-                    {
-                        return null;
-                    }
-                    argsData.PageLimit = pageLimit;
-                    break;
-
-                case "--retries":
-                    if (!argUtils.TryGetIntArray("--retries", out int[]? retries))
-                    {
-                        return null;
-                    }
-                    argsData.Retries = retries;
+                    argsData.Threshold = threshold.Value;
                     break;
 
                 case "--verbose":
@@ -204,10 +107,9 @@ static void ShowUsage(string? message, ICoreService action)
             }
         }
 
-        if (argsData.Org is null || argsData.Repos.Count == 0 || argsData.LabelPredicate is null ||
-            (argsData.IssuesModelPath is null && argsData.PullsModelPath is null))
+        if (argsData.IssuesModelPath is null && argsData.PullsModelPath is null)
         {
-            ShowUsage(null, action);
+            ShowUsage("At least one model path is required.", action);
             return null;
         }
 

IssueLabeler/src/Tester/HeldOutModelTester.cs

@@ -0,0 +1,57 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Microsoft.ML;
+using Microsoft.ML.Data;
+
+public static class HeldOutModelTester
+{
+    public static IEnumerable<(string? Label, string? PredictedLabel, float? Score)> Predict(string modelPath, float threshold)
+    {
+        string testDataPath = Path.ChangeExtension(modelPath, ".test-data.bin");
+        if (!File.Exists(testDataPath))
+        {
+            throw new FileNotFoundException(
+                "The saved held-out dataset is missing. Retrain the model to create it; live GitHub data cannot replace it.",
+                testDataPath);
+        }
+
+        var context = new MLContext();
+        var data = context.Data.LoadFromBinary(testDataPath);
+        using var dataLifetime = data as IDisposable;
+        var model = context.Model.Load(modelPath, out _);
+        var predictions = model.Transform(data);
+
+        VBuffer<ReadOnlyMemory<char>> labelNames = default;
+        predictions.Schema[nameof(LabelPrediction.Score)].GetSlotNames(ref labelNames);
+        string[] labels = labelNames.DenseValues().Select(label => label.ToString()).ToArray();
+        bool hasRows = false;
+
+        foreach (var prediction in context.Data.CreateEnumerable<LabelPrediction>(predictions, reuseRowObject: false))
+        {
+            hasRows = true;
+            var (predictedLabel, score) = SelectPrediction(prediction.Score, labels, threshold);
+            yield return (string.IsNullOrEmpty(prediction.Label) ? null : prediction.Label, predictedLabel, score);
+        }
+
+        if (!hasRows)
+        {
+            throw new InvalidOperationException("The saved held-out dataset is empty. Retrain the model with more data.");
+        }
+    }
+
+    internal static (string? Label, float? Score) SelectPrediction(float[] scores, string[] labels, float threshold)
+    {
+        if (scores.Length != labels.Length)
+        {
+            throw new InvalidOperationException("The model's score and label counts do not match.");
+        }
+
+        var best = scores
+            .Select((score, index) => (Score: score, Label: labels[index]))
+            .OrderByDescending(prediction => prediction.Score)
+            .FirstOrDefault(prediction => prediction.Score >= threshold);
+
+        return best.Label is not null ? (best.Label, best.Score) : (null, null);
+    }
+}