Mirrored package identification

src/BaGetter.Core/Entities/AbstractContext.cs

@@ -102,6 +102,7 @@ private void BuildPackageEntity(EntityTypeBuilder<Package> package)
         package.Property(p => p.Summary).HasMaxLength(DefaultMaxStringLength);
         package.Property(p => p.Title).HasMaxLength(MaxPackageTitleLength);
         package.Property(p => p.RepositoryType).HasMaxLength(MaxRepositoryTypeLength);
+        package.Property(p => p.CachedFrom).HasMaxLength(DefaultMaxStringLength);
 
         package.Ignore(p => p.Version);
         package.Ignore(p => p.IconUrlString);

src/BaGetter.Core/Entities/Package.cs

@@ -36,6 +36,7 @@ public NuGetVersion Version
     public bool HasReadme { get; set; }
     public bool HasEmbeddedIcon { get; set; }
     public bool IsPrerelease { get; set; }
+    public string CachedFrom { get; set; }
     public string ReleaseNotes { get; set; }
     public string Language { get; set; }
     public bool Listed { get; set; }

src/BaGetter.Core/Indexing/IPackageIndexingService.cs

@@ -6,7 +6,7 @@ namespace BaGetter.Core;
 
 /// <summary>
 /// The result of attempting to index a package.
-/// See <see cref="IPackageIndexingService.IndexAsync(Stream, CancellationToken)"/>.
+/// See <see cref="IPackageIndexingService.IndexAsync(Stream, string, CancellationToken)"/>.
 /// </summary>
 public enum PackageIndexingResult
 {
@@ -35,7 +35,16 @@ public interface IPackageIndexingService
     /// Attempt to index a new package.
     /// </summary>
     /// <param name="stream">The stream containing the package's content.</param>
+    /// <param name="cacheFeedUrl">The url of the feed the package has been cached from in case of pull through caching, else null.</param>
     /// <param name="cancellationToken"></param>
     /// <returns>The result of the attempted indexing operation.</returns>
-    Task<PackageIndexingResult> IndexAsync(Stream stream, CancellationToken cancellationToken);
+    Task<PackageIndexingResult> IndexAsync(Stream stream, string cacheFeedUrl = null, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Attempt to index a new package.
+    /// </summary>
+    /// <param name="stream">The stream containing the package's content.</param>
+    /// <param name="cancellationToken"></param>
+    /// <returns>The result of the attempted indexing operation.</returns>
+    Task<PackageIndexingResult> IndexAsync(Stream stream, CancellationToken cancellationToken = default);
 }

src/BaGetter.Core/Indexing/PackageIndexingService.cs

@@ -45,7 +45,12 @@ public PackageIndexingService(
 #pragma warning restore CS0618 // Type or member is obsolete
     }
 
-    public async Task<PackageIndexingResult> IndexAsync(Stream packageStream, CancellationToken cancellationToken)
+    public Task<PackageIndexingResult> IndexAsync(Stream stream, CancellationToken cancellationToken = default)
+    {
+        return IndexAsync(stream, null, cancellationToken);
+    }
+
+    public async Task<PackageIndexingResult> IndexAsync(Stream packageStream, string cacheFeedUrl = null, CancellationToken cancellationToken = default)
     {
         // Try to extract all the necessary information from the package.
         Package package;
@@ -57,6 +62,7 @@ public async Task<PackageIndexingResult> IndexAsync(Stream packageStream, Cancel
         {
             using var packageReader = new PackageArchiveReader(packageStream, leaveStreamOpen: true);
             package = packageReader.GetPackageMetadata();
+            package.CachedFrom = cacheFeedUrl;
             package.Published = _time.UtcNow;
 
             nuspecStream = await packageReader.GetNuspecAsync(cancellationToken);
@@ -167,7 +173,8 @@ await _storage.SavePackageContentAsync(
 
         if (_retentionOptions.Value.MaxMajorVersions.HasValue)
         {
-            try { 
+            try
+            {
                 _logger.LogInformation(
                     "Deleting older packages for package {PackageId} {PackageVersion}",
                     package.Id,

src/BaGetter.Core/PackageService.cs

@@ -100,10 +100,13 @@ private async Task<bool> MirrorAsync(string id, NuGetVersion version, Cancellati
             return true;
         }
 
+        var cacheFeedUrl = _upstream.GetServiceIndexUrl();
+
         _logger.LogInformation(
-            "Package {PackageId} {PackageVersion} does not exist locally. Checking upstream feed...",
+            "Package {PackageId} {PackageVersion} does not exist locally. Checking upstream feed ({cacheFeedUrl})...",
             id,
-            version);
+            version,
+            cacheFeedUrl);
 
         try
         {
@@ -122,7 +125,7 @@ private async Task<bool> MirrorAsync(string id, NuGetVersion version, Cancellati
                 id,
                 version);
 
-            var result = await _indexer.IndexAsync(packageStream, cancellationToken);
+            var result = await _indexer.IndexAsync(packageStream, cacheFeedUrl, cancellationToken);
 
             _logger.LogInformation(
                 "Finished indexing package {PackageId} {PackageVersion} from upstream feed with result {Result}",

src/BaGetter.Core/Upstream/Clients/DisabledUpstreamClient.cs

@@ -1,4 +1,4 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
@@ -31,4 +31,9 @@ public Task<Stream> DownloadPackageOrNullAsync(
     {
         return Task.FromResult<Stream>(null);
     }
+
+    public string GetServiceIndexUrl()
+    {
+        return null;
+    }
 }

src/BaGetter.Core/Upstream/Clients/V2UpstreamClient.cs

@@ -68,6 +68,8 @@ public V2UpstreamClient(
 
     }
 
+    public string GetServiceIndexUrl() => _repository.PackageSource.Source;
+
     public async Task<IReadOnlyList<NuGetVersion>> ListPackageVersionsAsync(string id, CancellationToken cancellationToken)
     {
         try

src/BaGetter.Core/Upstream/Clients/V3UpstreamClient.cs

@@ -29,6 +29,7 @@ public V3UpstreamClient(NuGetClient client, ILogger<V3UpstreamClient> logger)
         _client = client;
         _logger = logger;
     }
+    public string GetServiceIndexUrl() => _client.ServiceIndexUrl;
 
     public async Task<Stream> DownloadPackageOrNullAsync(
         string id,

src/BaGetter.Core/Upstream/IUpstreamClient.cs

@@ -1,4 +1,4 @@
-using NuGet.Versioning;
+using NuGet.Versioning;
 using System.Collections.Generic;
 using System.IO;
 using System.Threading;
@@ -46,4 +46,10 @@ public interface IUpstreamClient
     /// The stream is guaranteed to be seekable if not not null.
     /// </returns>
     Task<Stream> DownloadPackageOrNullAsync(string id, NuGetVersion version, CancellationToken cancellationToken);
+
+    /// <summary>
+    /// Get the service index url from the upstream package source.
+    /// </summary>
+    /// <returns>The service index url as a string</returns>
+    string GetServiceIndexUrl();
 }