fix: old package from upstream during retention, DeletePrereleasesOfOlderMajors

docs/docs/configuration.md

@@ -180,12 +180,13 @@ downloaded if you know the package's id and version. You can override this behav
 If your build server generates many nuget packages, your BaGetter server can quickly run out of space. Bagetter leverages [SemVer 2](https://semver.org/) and has logic to keep a history of packages based on the version numbering such as `<major>.<minor>.<patch>-<prerelease tag>.<prerelease build number>`.
 
 There is an optional config section for `Retention` and the following parameters can be enabled to limit history for each level of the version. If none of these are set, there are no cleaning rules enforced. Each parameter is optional, e.g. if you specify only a `MaxPatchVersions`, the package limit will only enforced within each major and minor version combination.
-Packages deleted are always the oldest based on version numbers.
+Packages deleted are always the oldest based on version numbers. The package version that is currently being pushed or mirrored is never deleted by its own retention run; it will be cleaned up the next time a version of that package is indexed.
 
 - MaxMajorVersions: Maximum number of major versions for each package
 - MaxMinorVersions: Maximum number of minor versions for each major version
 - MaxPatchVersions: Maximum number of patch versions for each major + minor version
 - MaxPrereleaseVersions: Maximum number of prerelease builds for each major + minor + patch version and prerelease type. If you have `beta` and `alpha` this will keep `MaxPrereleaseVersions` versions for both `beta` and `alpha`. Suffixes incompatible with [SemVer 2](https://semver.org/) will be treated as a separate type.
+- DeletePrereleasesOfOlderMajors: If `true`, all prerelease versions whose major version is lower than the latest stable major version are deleted (e.g. with `3.0.0` published, `2.1.0-beta.1` is deleted while `3.1.0-beta.1` is kept).
 
 ```json
 {
@@ -195,6 +196,7 @@ Packages deleted are always the oldest based on version numbers.
         "MaxMinorVersions": 5,
         "MaxPatchVersions": 5,
         "MaxPrereleaseVersions": 5,
+        "DeletePrereleasesOfOlderMajors": true
     }
     ...
 }

src/BaGetter.Core/Configuration/RetentionOptions.cs

@@ -34,4 +34,10 @@ public class RetentionOptions
     /// For a limit of 5, if there are versions 1.0.0-alpha.1 through 1.0.0-alpha.5 and a package version 1.0.0-alpha.6 is pushed, version 1.0.0-alpha.0 will be deleted.  
     /// </summary>  
     public uint? MaxPrereleaseVersions { get; set; }
+
+    /// <summary>
+    /// If this is set to <c>true</c>, all prerelease versions whose major version is lower than the latest stable major version will be deleted.
+    /// For example, if version 3.0.0 exists, versions like 2.1.0-beta.1 or 1.0.0-alpha will be deleted, while 3.1.0-beta.1 or 4.0.0-alpha.1 are kept.
+    /// </summary>
+    public bool DeletePrereleasesOfOlderMajors { get; set; }
 }

src/BaGetter.Core/Indexing/IPackageDeletionService.cs

@@ -10,16 +10,18 @@ public interface IPackageDeletionService
     /// This method deletes old versions of a package.
     /// This leverages semver 2.0 - and assume a package is major.minor.patch-prerelease.build
     /// It can leverage the <see cref="IPackageDatabase"/> to list all versions of a package and then delete all but the last <paramref name="maxMajor"/> versions.
+    /// The version of <paramref name="package"/> itself is never deleted.
     /// It also takes into account the <paramref name="maxMinor"/>, <paramref name="maxPath"/> and <paramref name="maxPrerelease"/> parameters to further filter the versions to delete.
     /// </summary>
     /// <param name="package">Package name</param>
     /// <param name="maxMajor">Maximum of major versions to keep (optional)</param>
     /// <param name="maxMinor">Maximum of minor versions to keep (optional)</param>
     /// <param name="maxPatch">Maximum of patch versions to keep (optional)</param>
     /// <param name="maxPrerelease">Maximum of pre-release versions (optional)</param>
+    /// <param name="deletePrereleasesOfOlderMajors">Delete all pre-release versions whose major version is lower than the latest stable major version</param>
     /// <param name="cancellationToken">Cancel the operation</param>
     /// <returns>Number of packages deleted</returns>
-    Task<int> DeleteOldVersionsAsync(Package package, uint? maxMajor, uint? maxMinor, uint? maxPatch, uint? maxPrerelease, CancellationToken cancellationToken);
+    Task<int> DeleteOldVersionsAsync(Package package, uint? maxMajor, uint? maxMinor, uint? maxPatch, uint? maxPrerelease, bool deletePrereleasesOfOlderMajors, CancellationToken cancellationToken);
 
     /// <summary>
     /// Attempt to delete a package.

src/BaGetter.Core/Indexing/PackageDeletionService.cs

@@ -107,11 +107,11 @@ private static IList<NuGetVersion> GetValidVersions<S, T>(IEnumerable<NuGetVersi
         return versions.Where(k => validVersions.Any(v => getParent(k).Equals(v.parent) && getSelector(k).Equals(v.selector))).ToList();
     }
 
-    public async Task<int> DeleteOldVersionsAsync(Package package, uint? maxMajor, uint? maxMinor, uint? maxPatch, uint? maxPrerelease, CancellationToken cancellationToken)
+    public async Task<int> DeleteOldVersionsAsync(Package package, uint? maxMajor, uint? maxMinor, uint? maxPatch, uint? maxPrerelease, bool deletePrereleasesOfOlderMajors, CancellationToken cancellationToken)
     {
         // list all versions of the package
         var packages = await _packages.FindAsync(package.Id, includeUnlisted: true, cancellationToken);
-        if (packages is null || packages.Count <= maxMajor) return 0;
+        if (packages is null || (packages.Count <= maxMajor && !deletePrereleasesOfOlderMajors)) return 0;
 
         var goodVersions = new HashSet<NuGetVersion>();
 
@@ -142,6 +142,21 @@ public async Task<int> DeleteOldVersionsAsync(Package package, uint? maxMajor, u
             goodVersions.UnionWith(allPreReleaseValidVersions);
         }
 
+        if (deletePrereleasesOfOlderMajors)
+        {
+            // drop prereleases whose major is below the latest stable major
+            var latestReleaseMajor = packages
+                .Where(p => !p.Version.IsPrerelease)
+                .Select(p => p.Version.Major)
+                .DefaultIfEmpty(-1)
+                .Max();
+            goodVersions.RemoveWhere(v => v.IsPrerelease && v.Major < latestReleaseMajor);
+        }
+
+        // Never delete the package that is currently being indexed. When it was mirrored from an upstream feed,
+        // the caller still has to serve it; it will be cleaned up the next time a version of this package is indexed.
+        goodVersions.Add(package.Version);
+
         // sort by version and take everything except the last maxPackages
         var versionsToDelete = packages.Where(p => !goodVersions.Contains(p.Version)).ToList();
 

src/BaGetter.Core/Indexing/PackageIndexingService.cs

@@ -186,6 +186,7 @@ await _storage.SavePackageContentAsync(
                     _retentionOptions.Value.MaxMinorVersions,
                     _retentionOptions.Value.MaxPatchVersions,
                     _retentionOptions.Value.MaxPrereleaseVersions,
+                    _retentionOptions.Value.DeletePrereleasesOfOlderMajors,
                     cancellationToken);
                 if (deleted > 0)
                 {

tests/BaGetter.Core.Tests/Services/PackageDeletionServiceTests.cs

@@ -1,3 +1,4 @@
+using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
@@ -144,7 +145,7 @@ public async Task WhenAddNewPackage_DeleteOldPackages_Major(uint maxVersions, in
         // Act
         var deleted = await _target.DeleteOldVersionsAsync(
             new Package { Id = PackageId, Version = new NuGetVersion("4.0.0"), IsPrerelease = false },
-            maxMajor: maxVersions, maxMinor:null, maxPatch:null, maxPrerelease:null, cancellationToken);
+            maxMajor: maxVersions, maxMinor:null, maxPatch:null, maxPrerelease:null, deletePrereleasesOfOlderMajors: false, cancellationToken);
 
         // Assert - The database step MUST happen before the storage step.
         Assert.Equal(expectedCount, deleted);
@@ -186,7 +187,7 @@ public async Task WhenAddNewPackage_DeleteOldPackages_Minor(uint maxVersions, in
         // Act
         var deleted = await _target.DeleteOldVersionsAsync(
             new Package { Id = PackageId, Version = new NuGetVersion("4.0.0"), IsPrerelease = false },
-            maxMajor: null, maxMinor:maxVersions, maxPatch:null, maxPrerelease:null, cancellationToken);
+            maxMajor: null, maxMinor:maxVersions, maxPatch:null, maxPrerelease:null, deletePrereleasesOfOlderMajors: false, cancellationToken);
 
         // Assert - The database step MUST happen before the storage step.
         Assert.Equal(expectedCount, deleted);
@@ -228,7 +229,7 @@ public async Task WhenAddNewPackage_DeleteOldPackages_Patch(uint maxPrereleaseVe
         // Act
         var deleted = await _target.DeleteOldVersionsAsync(
             new Package { Id = PackageId, Version = new NuGetVersion("4.0.0"), IsPrerelease = false },
-            maxMajor: null, maxMinor:null, maxPatch:maxPrereleaseVersions, maxPrerelease:null, cancellationToken);
+            maxMajor: null, maxMinor:null, maxPatch:maxPrereleaseVersions, maxPrerelease:null, deletePrereleasesOfOlderMajors: false, cancellationToken);
 
         // Assert - The database step MUST happen before the storage step.
         Assert.Equal(expectedCount, deleted);
@@ -298,7 +299,7 @@ public async Task WhenAddNewPackage_DeleteOldPackages_Prerelease(uint maxVersion
         // Act
         var deleted = await _target.DeleteOldVersionsAsync(
             new Package { Id = PackageId, Version = new NuGetVersion("4.0.0"), IsPrerelease = false },
-            maxMajor: null, maxMinor:null, maxPatch:null, maxPrerelease:maxVersions, cancellationToken);
+            maxMajor: null, maxMinor:null, maxPatch:null, maxPrerelease:maxVersions, deletePrereleasesOfOlderMajors: false, cancellationToken);
 
         // Assert - The database step MUST happen before the storage step.
         Assert.Equal(expectedCount, deleted);
@@ -380,11 +381,80 @@ public async Task WhenAddNewPackage_DeleteOldPackages_CrossLimitsCheck(uint maxM
         // Act
         var deleted = await _target.DeleteOldVersionsAsync(
             new Package { Id = PackageId, Version = new NuGetVersion("4.4.5"), IsPrerelease = false },
-            maxMajor: maxMajorVersions, maxMinor:maxMinorVersions, maxPatch:maxPatchVersions, maxPrerelease:maxPrereleaseVersions, cancellationToken);
+            maxMajor: maxMajorVersions, maxMinor:maxMinorVersions, maxPatch:maxPatchVersions, maxPrerelease:maxPrereleaseVersions, deletePrereleasesOfOlderMajors: false, cancellationToken);
 
         // Assert - The database step MUST happen before the storage step.
         Assert.Equal(expectedCount, deleted);
         Assert.Equal(expectedCount, databaseStep);
         Assert.Equal(expectedCount, storageStep);
     }
+
+    [Theory]
+    [InlineData(false, 0)]
+    [InlineData(true, 3)]
+    public async Task WhenAddNewPackage_DeletePrereleasesOfOlderMajors(bool deletePrereleasesOfOlderMajors, int expectedCount)
+    {
+        // Arrange
+        var cancellationToken = CancellationToken.None;
+        var deletedVersions = new List<NuGetVersion>();
+        _packages
+            .Setup(p => p.FindAsync(PackageId, true, cancellationToken))
+            .ReturnsAsync([
+                new Package { Id = PackageId, Version = new NuGetVersion("1.0.0-alpha") },
+                new Package { Id = PackageId, Version = new NuGetVersion("1.0.0") },
+                new Package { Id = PackageId, Version = new NuGetVersion("2.0.0-beta.1") },
+                new Package { Id = PackageId, Version = new NuGetVersion("2.1.0-beta.1") },
+                new Package { Id = PackageId, Version = new NuGetVersion("2.1.0") },
+                new Package { Id = PackageId, Version = new NuGetVersion("3.0.0") },
+                new Package { Id = PackageId, Version = new NuGetVersion("3.1.0-beta.1") },
+                new Package { Id = PackageId, Version = new NuGetVersion("4.0.0-alpha.1") },
+            ]);
+        _storage
+            .Setup(s => s.DeleteAsync(PackageId, It.IsAny<NuGetVersion>(), cancellationToken))
+            .Returns(Task.CompletedTask);
+        _packages
+            .Setup(p => p.HardDeletePackageAsync(PackageId, It.IsAny<NuGetVersion>(), cancellationToken))
+            .Callback<string, NuGetVersion, CancellationToken>((_, v, _) => deletedVersions.Add(v))
+            .ReturnsAsync(true);
+
+        // Act
+        var deleted = await _target.DeleteOldVersionsAsync(
+            new Package { Id = PackageId, Version = new NuGetVersion("3.0.0"), IsPrerelease = false },
+            maxMajor: 5, maxMinor: null, maxPatch: null, maxPrerelease: null, deletePrereleasesOfOlderMajors, cancellationToken);
+
+        // Assert
+        Assert.Equal(expectedCount, deleted);
+        Assert.All(deletedVersions, v => Assert.True(v.IsPrerelease && v.Major < 3));
+    }
+
+    [Fact]
+    public async Task WhenAddOldPackage_DoesNotDeleteIndexedPackage()
+    {
+        // Arrange
+        var cancellationToken = CancellationToken.None;
+        var deletedVersions = new List<NuGetVersion>();
+        _packages
+            .Setup(p => p.FindAsync(PackageId, true, cancellationToken))
+            .ReturnsAsync([
+                new Package { Id = PackageId, Version = new NuGetVersion("1.0.0") },
+                new Package { Id = PackageId, Version = new NuGetVersion("2.0.0") },
+                new Package { Id = PackageId, Version = new NuGetVersion("3.0.0") },
+            ]);
+        _storage
+            .Setup(s => s.DeleteAsync(PackageId, It.IsAny<NuGetVersion>(), cancellationToken))
+            .Returns(Task.CompletedTask);
+        _packages
+            .Setup(p => p.HardDeletePackageAsync(PackageId, It.IsAny<NuGetVersion>(), cancellationToken))
+            .Callback<string, NuGetVersion, CancellationToken>((_, v, _) => deletedVersions.Add(v))
+            .ReturnsAsync(true);
+
+        // Act - e.g. an old version mirrored from an upstream feed
+        var deleted = await _target.DeleteOldVersionsAsync(
+            new Package { Id = PackageId, Version = new NuGetVersion("1.0.0"), IsPrerelease = false },
+            maxMajor: 1, maxMinor: null, maxPatch: null, maxPrerelease: null, deletePrereleasesOfOlderMajors: false, cancellationToken);
+
+        // Assert
+        Assert.Equal(1, deleted);
+        Assert.Equal(new NuGetVersion("2.0.0"), Assert.Single(deletedVersions));
+    }
 }