diff --git a/IssueLabeler/IssueLabeler.sln b/IssueLabeler/IssueLabeler.sln index 14cbb47..eb28387 100644 --- a/IssueLabeler/IssueLabeler.sln +++ b/IssueLabeler/IssueLabeler.sln @@ -33,6 +33,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "workflows", "workflows", "{ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Common.Tests", "tests\Common.Tests\Common.Tests.csproj", "{D3F816D3-5CAE-4CF1-8977-F92AE96B481B}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Trainer.Tests", "tests\Trainer.Tests\Trainer.Tests.csproj", "{BA177CD9-81D1-4143-A433-665B2699B361}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -67,6 +69,10 @@ Global {D3F816D3-5CAE-4CF1-8977-F92AE96B481B}.Debug|Any CPU.Build.0 = Debug|Any CPU {D3F816D3-5CAE-4CF1-8977-F92AE96B481B}.Release|Any CPU.ActiveCfg = Release|Any CPU {D3F816D3-5CAE-4CF1-8977-F92AE96B481B}.Release|Any CPU.Build.0 = Release|Any CPU + {BA177CD9-81D1-4143-A433-665B2699B361}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BA177CD9-81D1-4143-A433-665B2699B361}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BA177CD9-81D1-4143-A433-665B2699B361}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BA177CD9-81D1-4143-A433-665B2699B361}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/IssueLabeler/src/Tester/Tester.cs b/IssueLabeler/src/Tester/Tester.cs index 1e88bef..70a5c44 100644 --- a/IssueLabeler/src/Tester/Tester.cs +++ b/IssueLabeler/src/Tester/Tester.cs @@ -17,6 +17,12 @@ var action = provider.GetRequiredService(); var config = Args.Parse(args, action); if (config is not Args argsData) return 1; + +const string disclaimer = "This test downloads currently labeled GitHub items and may include training data. It is not an independent held-out evaluation."; +action.WriteWarning(disclaimer); +action.Summary.AddPersistent(summary => summary.AddAlert(disclaimer, AlertType.Warning)); +await action.Summary.WritePersistentAsync(); + string[] excludedLabels = argsData.ExcludedLabels ?? []; List> tasks = []; diff --git a/IssueLabeler/src/Trainer/HeldOutResult.cs b/IssueLabeler/src/Trainer/HeldOutResult.cs new file mode 100644 index 0000000..e219709 --- /dev/null +++ b/IssueLabeler/src/Trainer/HeldOutResult.cs @@ -0,0 +1,88 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +internal sealed class HeldOutPrediction +{ + public string Label { get; set; } = string.Empty; + public float[] Score { get; set; } = []; +} + +internal sealed record PrecisionTargetResult(int TargetPrecision, float? Threshold, int Correct, int Accepted) +{ + public double? Precision => Accepted == 0 ? null : (double)Correct / Accepted; +} + +internal sealed record HeldOutResult(int Total, PrecisionTargetResult[] Targets) +{ + internal static HeldOutResult Evaluate(IEnumerable predictions, string[] labels) + { + List<(float Score, bool Correct)> ranked = []; + foreach (var prediction in predictions) + { + if (string.IsNullOrWhiteSpace(prediction.Label) || labels.Length == 0 || prediction.Score.Length != labels.Length || + prediction.Score.Any(score => !float.IsFinite(score))) + { + throw new InvalidDataException("A held-out row has an invalid reference label or score vector."); + } + + int index = Array.IndexOf(prediction.Score, prediction.Score.Max()); + ranked.Add((prediction.Score[index], labels[index].Equals(prediction.Label, StringComparison.OrdinalIgnoreCase))); + } + + if (ranked.Count == 0) + { + throw new InvalidOperationException("The held-out test set is empty. Download more data before training."); + } + + int[] precisionTargets = [60, 70, 80, 90, 95]; + PrecisionTargetResult[] results = [.. precisionTargets.Select(target => new PrecisionTargetResult(target, null, 0, 0))]; + ranked.Sort((left, right) => right.Score.CompareTo(left.Score)); + int correct = 0; + for (int index = 0; index < ranked.Count; index++) + { + if (ranked[index].Correct) + { + correct++; + } + + // A cutoff must accept all predictions with the same score. + if (index + 1 < ranked.Count && ranked[index].Score == ranked[index + 1].Score) + { + continue; + } + + int accepted = index + 1; + for (int targetIndex = 0; targetIndex < results.Length; targetIndex++) + { + int target = results[targetIndex].TargetPrecision; + if ((long)correct * 100 >= (long)target * accepted) + { + results[targetIndex] = new(target, ranked[index].Score, correct, accepted); + } + } + } + + return new(ranked.Count, results); + } + + public string ToMarkdown() + { + var rows = Targets.Select(target => target.Threshold is float threshold + ? FormattableString.Invariant($"| {target.TargetPrecision}% | {threshold:R} | {target.Precision:P2} | {(double)target.Accepted / Total:P2} | {target.Accepted} |") + : FormattableString.Invariant($"| {target.TargetPrecision}% | Not reached | N/A | N/A | N/A |")); + return FormattableString.Invariant($""" + ### Held-out evaluation + + Evaluated {Total} items from the random 20% test split, excluded from model fitting. + Each row aims to label as many test items as possible, with at least the target percentage labeled correctly. + Threshold is the minimum score required to use a prediction. Accepted is the number of predictions kept at this cutoff, including incorrect predictions. + + | Target precision | Threshold | Observed precision | Coverage | Accepted | + |---:|---:|---:|---:|---:| + {string.Join(Environment.NewLine, rows)} + + Precision is the percentage of accepted predictions that are correct. Coverage is the percentage of all test items accepted. + Thresholds are selected and evaluated on this same split. + """); + } +} diff --git a/IssueLabeler/src/Trainer/Trainer.cs b/IssueLabeler/src/Trainer/Trainer.cs index d74e694..091d0c8 100644 --- a/IssueLabeler/src/Trainer/Trainer.cs +++ b/IssueLabeler/src/Trainer/Trainer.cs @@ -108,39 +108,46 @@ static async Task CreateModel(string dataPath, string modelPath, ModelType type, var testModel = trainedModel.Transform(split.TestSet); await action.WriteStatusAsync("Evaluating against the test set..."); + VBuffer> slots = default; + testModel.Schema[nameof(HeldOutPrediction.Score)].GetSlotNames(ref slots); + string[] labels = [.. slots.DenseValues().Select(label => label.ToString())]; + var evaluation = HeldOutResult.Evaluate( + mlContext.Data.CreateEnumerable(testModel, reuseRowObject: false), labels); var metrics = mlContext.MulticlassClassification.Evaluate(testModel, labelColumnName: "LabelKey"); + var classLosses = metrics.PerClassLogLoss.Take(3) + .Select((loss, index) => FormattableString.Invariant($"| Class {index + 1} | {loss:0.####} |")); + string metricsReport = FormattableString.Invariant($""" + These metrics use all held-out items, without a score cutoff. + + | Metric | Value | Meaning | + |---|---:|---| + | MacroAccuracy | {metrics.MacroAccuracy:0.####} | Average accuracy across labels, each weighted equally (higher is better). | + | MicroAccuracy | {metrics.MicroAccuracy:0.####} | Fraction of all items predicted correctly (higher is better). | + | LogLoss | {metrics.LogLoss:0.####} | Penalizes low scores for the correct label (lower is better). | + + **Per-class LogLoss** (first three label categories, lower is better) + + | Class | LogLoss | + |---|---:| + {string.Join(Environment.NewLine, classLosses)} + """); action.Summary.AddPersistent(summary => { summary.AddMarkdownHeading($"Finished Training {(type == ModelType.Issue ? "Issues" : "Pull Requests")} Model", 2); - summary.AddRawMarkdown($""" - * MacroAccuracy: {metrics.MacroAccuracy:0.####} (a value between 0 and 1; the closer to 1, the better) - * MicroAccuracy: {metrics.MicroAccuracy:0.####} (a value between 0 and 1; the closer to 1, the better) - * LogLoss: {metrics.LogLoss:0.####} (the closer to 0, the better) - {(metrics.PerClassLogLoss.Count() > 0 ? $" * Class 1: {metrics.PerClassLogLoss[0]:0.####}" : "")} - {(metrics.PerClassLogLoss.Count() > 1 ? $" * Class 2: {metrics.PerClassLogLoss[1]:0.####}" : "")} - {(metrics.PerClassLogLoss.Count() > 2 ? $" * Class 3: {metrics.PerClassLogLoss[2]:0.####}" : "")} - """); + summary.AddRawMarkdown(metricsReport); + summary.AddNewLine(); + summary.AddNewLine(); + summary.AddRawMarkdown(evaluation.ToMarkdown(), true); }); await action.Summary.WriteAsync(); action.WriteInfo($"************************************************************"); - action.WriteInfo($"MacroAccuracy = {metrics.MacroAccuracy:0.####}, a value between 0 and 1, the closer to 1, the better"); - action.WriteInfo($"MicroAccuracy = {metrics.MicroAccuracy:0.####}, a value between 0 and 1, the closer to 1, the better"); - action.WriteInfo($"LogLoss = {metrics.LogLoss:0.####}, the closer to 0, the better"); - - if (metrics.PerClassLogLoss.Count() > 0) - action.WriteInfo($"LogLoss for class 1 = {metrics.PerClassLogLoss[0]:0.####}, the closer to 0, the better"); - - if (metrics.PerClassLogLoss.Count() > 1) - action.WriteInfo($"LogLoss for class 2 = {metrics.PerClassLogLoss[1]:0.####}, the closer to 0, the better"); - - if (metrics.PerClassLogLoss.Count() > 2) - action.WriteInfo($"LogLoss for class 3 = {metrics.PerClassLogLoss[2]:0.####}, the closer to 0, the better"); - + action.WriteInfo(metricsReport); action.WriteInfo($"************************************************************"); + action.WriteInfo(evaluation.ToMarkdown()); action.WriteInfo($"Saving model to '{modelPath}'..."); EnsureOutputDirectory(modelPath); diff --git a/IssueLabeler/src/Trainer/Trainer.csproj b/IssueLabeler/src/Trainer/Trainer.csproj index 9fe4e21..31e96e5 100644 --- a/IssueLabeler/src/Trainer/Trainer.csproj +++ b/IssueLabeler/src/Trainer/Trainer.csproj @@ -18,6 +18,7 @@ + diff --git a/IssueLabeler/tests/Trainer.Tests/Trainer.Tests.csproj b/IssueLabeler/tests/Trainer.Tests/Trainer.Tests.csproj new file mode 100644 index 0000000..ab48f59 --- /dev/null +++ b/IssueLabeler/tests/Trainer.Tests/Trainer.Tests.csproj @@ -0,0 +1,11 @@ + + + enable + enable + false + true + + + + + diff --git a/IssueLabeler/tests/Trainer.Tests/TrainerTests.cs b/IssueLabeler/tests/Trainer.Tests/TrainerTests.cs new file mode 100644 index 0000000..34fdf1a --- /dev/null +++ b/IssueLabeler/tests/Trainer.Tests/TrainerTests.cs @@ -0,0 +1,153 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using Microsoft.ML; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[assembly: Parallelize(Scope = ExecutionScope.MethodLevel)] + +namespace IssueLabeler.Tests; + +[TestClass] +public sealed class TrainerTests +{ + private string directory = string.Empty; + + [TestInitialize] + public void Initialize() => directory = Directory.CreateTempSubdirectory("issue-labeler-training-").FullName; + + [TestCleanup] + public void Cleanup() => Directory.Delete(directory, recursive: true); + + [TestMethod] + public void PrecisionTargetsChooseMaximumCoverageEvenWhenPrecisionRecovers() + { + int[] wrongRanks = [2, 11, 12, 16, 17, 18, 21, 22, 23, 24]; + HeldOutPrediction[] predictions = [.. Enumerable.Range(1, 25).Reverse().Select(rank => new HeldOutPrediction + { + Label = wrongRanks.Contains(rank) ? "area-B" : "area-A", + Score = [(100 - rank) / 100f, rank / 100f] + })]; + var result = HeldOutResult.Evaluate(predictions, ["area-A", "area-B"]); + + Assert.AreEqual(25, result.Total); + PrecisionTargetResult[] expected = + [ + new(60, 0.75f, 15, 25), + new(70, 0.8f, 14, 20), + new(80, 0.85f, 12, 15), + new(90, 0.9f, 9, 10), + new(95, 0.99f, 1, 1) + ]; + CollectionAssert.AreEqual(expected, result.Targets); + Assert.AreEqual(0.6, result.Targets[0].Precision); + } + + [TestMethod] + public void TiedPredictionScoresAreAcceptedTogetherAtTheThreshold() + { + var result = HeldOutResult.Evaluate( + [new() { Label = "AREA-A", Score = [0.9f, 0.1f] }, + new() { Label = "area-B", Score = [0.9f, 0.1f] }, + new() { Label = "area-A", Score = [0.8f, 0.2f] }], + ["area-A", "area-B"]); + Assert.AreEqual(new PrecisionTargetResult(60, 0.8f, 2, 3), result.Targets[0]); + Assert.IsTrue(result.Targets.Skip(1).All(target => target.Threshold is null)); + } + + [TestMethod] + public void UnreachableTargetsAreNotReportedAsPerfectPrecisionWithNoCoverage() + { + var result = HeldOutResult.Evaluate( + [new() { Label = "area-B", Score = [1f, 0f] }, + new() { Label = "area-B", Score = [0.7f, 0.3f] }], + ["area-A", "area-B"]); + Assert.AreEqual(2, result.Total); + Assert.HasCount(5, result.Targets); + foreach (var target in result.Targets) + { + Assert.IsNull(target.Threshold); + Assert.IsNull(target.Precision); + Assert.AreEqual(0, target.Accepted); + } + } + + [DataRow(ModelType.Issue)] + [DataRow(ModelType.PullRequest)] + [TestMethod] + public async Task TrainingProducesUsableModel(ModelType type) + { + string dataPath = Path.Combine(directory, "data.tsv"); + string modelPath = Path.Combine(directory, "model.zip"); + string[] labels = ["area-A", "area-B", "area-C"]; + string[] records = [type == ModelType.Issue ? "Label\tTitle\tBody" : "Label\tTitle\tBody\tFileNames\tFolderNames", + .. Enumerable.Range(0, 300).Select(index => type == ModelType.Issue + ? DataFileUtils.FormatIssueRecord(labels[index % 3], $"Item {index}", $"Content for {labels[index % 3]}") + : DataFileUtils.FormatPullRequestRecord(labels[index % 3], $"Item {index}", $"Content for {labels[index % 3]}", + [$"File{index % 3}.cs"], [$"Folder{index % 3}"]))]; + File.WriteAllLines(dataPath, records); + var result = await RunTrainer( + type == ModelType.Issue ? "--issues-data" : "--pulls-data", dataPath, + type == ModelType.Issue ? "--issues-model" : "--pulls-model", modelPath); + Assert.AreEqual(0, result.ExitCode, result.Output); + var context = new MLContext(); + using var predictor = context.Model.CreatePredictionEngine(context.Model.Load(modelPath, out _)); + var input = new ModelInput { Title = "Item 0", Body = "Content", FileNames = "File0.cs", FolderNames = "Folder0" }; + float[] scores = predictor.Predict(input).Score; + Assert.HasCount(3, scores); + Assert.IsTrue(scores.All(float.IsFinite)); + } + + private async Task<(int ExitCode, string Output)> RunTrainer(params string[] arguments) + { + string summaryPath = Path.Combine(directory, "summary.md"); + File.WriteAllText(summaryPath, string.Empty); + using var process = new Process + { + StartInfo = new("dotnet") + { + WorkingDirectory = directory, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + } + }; + process.StartInfo.ArgumentList.Add(typeof(Args).Assembly.Location); + foreach (string argument in arguments) + { + process.StartInfo.ArgumentList.Add(argument); + } + process.StartInfo.Environment.Remove("GITHUB_TOKEN"); + process.StartInfo.Environment["GITHUB_STEP_SUMMARY"] = summaryPath; + process.Start(); + Task output = process.StandardOutput.ReadToEndAsync(); + Task error = process.StandardError.ReadToEndAsync(); + try + { + await process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(60)); + return (process.ExitCode, await output + await error); + } + finally + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + } + } + } + + public sealed class ModelInput + { + public string? Label { get; set; } + public string Title { get; set; } = string.Empty; + public string Body { get; set; } = string.Empty; + public string FileNames { get; set; } = string.Empty; + public string FolderNames { get; set; } = string.Empty; + } + + public sealed class ModelOutput + { + public float[] Score { get; set; } = []; + } +} diff --git a/train/action.yml b/train/action.yml index 7fe5f9e..24fe489 100644 --- a/train/action.yml +++ b/train/action.yml @@ -1,5 +1,5 @@ name: "Train Model" -description: "Train the Issues, Pull Requests, or Discussions model for label prediction. Each type writes its own cached model file." +description: "Train the Issues, Pull Requests, or Discussions model for label prediction and report held-out evaluation. Each type writes its own cached model file." inputs: type: