From ac22227859200b4836f1a2bf8ea911a170403736 Mon Sep 17 00:00:00 2001 From: Stoia <23234609+StoiaCode@users.noreply.github.com> Date: Tue, 5 May 2026 16:08:48 +0200 Subject: [PATCH 1/6] Implement Container Peeking --- Penumbra/Config/Configuration.cs | 1 + Penumbra/Services/FileWatcher.cs | 376 ++++++++++++++++++++++-- Penumbra/UI/ManagementTab/CleanupTab.cs | 9 +- Penumbra/UI/Tabs/SettingsTab.cs | 3 + 4 files changed, 356 insertions(+), 33 deletions(-) diff --git a/Penumbra/Config/Configuration.cs b/Penumbra/Config/Configuration.cs index 71c81bb99..2efe4ddea 100644 --- a/Penumbra/Config/Configuration.cs +++ b/Penumbra/Config/Configuration.cs @@ -77,6 +77,7 @@ public bool EnableMods public bool DefaultTemporaryMode { get; set; } = false; public bool EnableDirectoryWatch { get; set; } = false; public bool EnableAutomaticModImport { get; set; } = false; + public bool EnableContainerPeeking { get; set; } = true; public bool AutoDismissModImportSuccessReports { get; set; } = true; public bool AlwaysShowDetailedModImport { get; set; } = false; public bool PreventExportLoopback { get; set; } = true; diff --git a/Penumbra/Services/FileWatcher.cs b/Penumbra/Services/FileWatcher.cs index b2eab1e1a..4c4ed2316 100644 --- a/Penumbra/Services/FileWatcher.cs +++ b/Penumbra/Services/FileWatcher.cs @@ -1,30 +1,49 @@ -using ImSharp; +using System.Diagnostics; +using ImSharp; using Luna; using Penumbra.Mods.Manager; +using SharpCompress.Archives; +using SharpCompress.Archives.Rar; +using SharpCompress.Archives.SevenZip; +using ZipArchive = SharpCompress.Archives.Zip.ZipArchive; namespace Penumbra.Services; public sealed class FileWatcher : IDisposable, IService { - private readonly ConcurrentSet _pending = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentSet _pending = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary _ignored = new(StringComparer.OrdinalIgnoreCase); - private readonly ModImportManager _modImportManager; - private readonly MessageService _messageService; - private readonly Configuration _config; + private readonly ConcurrentDictionary _extractedArchives = new(StringComparer.OrdinalIgnoreCase); + private readonly ModImportManager _modImportManager; + private readonly MessageService _messageService; + private readonly Configuration _config; - private bool _pausedConsumer; - private FileSystemWatcher? _fsw; + private bool _pausedConsumer; + private FileSystemWatcher? _fsw; private CancellationTokenSource? _cts = new(); - private Task? _consumer; + private Task? _consumer; /// The time-to-live of ignore entries, in the same unit as , namely milliseconds. private const long IgnoreTimeToLive = 60000L; + private static readonly HashSet ModExtensions = + new(StringComparer.OrdinalIgnoreCase) { ".pmp", ".pcp", ".ttmp", ".ttmp2" }; + + private static readonly HashSet ContainerExtensions = + new(StringComparer.OrdinalIgnoreCase) { ".zip", ".rar", ".7z" }; + + /// + /// Subdirectory under the system temp directory used for extracted archive entries. + /// + private static readonly string TempRoot = Path.Combine(Path.GetTempPath(), "Penumbra-FileWatcher"); + public FileWatcher(ModImportManager modImportManager, MessageService messageService, Configuration config) { _modImportManager = modImportManager; - _messageService = messageService; - _config = config; + _messageService = messageService; + _config = config; + + WipeTempRoot(); if (_config.EnableDirectoryWatch) { @@ -52,12 +71,38 @@ public void Toggle(bool value) } } + public void ToggleContainerPeeking(bool value) + { + if (_config.EnableContainerPeeking == value) + return; + + _config.EnableContainerPeeking = value; + _config.Save(); + + // Re-create the FSW so its filter list reflects the new state. + if (_config.EnableDirectoryWatch && _fsw is not null) + SetupFileWatcher(_config.WatchDirectory); + } + public void IgnoreFile(string fullPath) { if (_config.EnableDirectoryWatch) _ignored[fullPath] = Environment.TickCount64 + IgnoreTimeToLive; } + /// + /// Deletes every extracted archive directory tracked by this instance. Called from the debug drawer. + /// + public void CleanExtracted() + { + foreach (var dir in _extractedArchives.Keys.ToList()) + { + if (TryDeleteDirectory(dir)) + _extractedArchives.TryRemove(dir, out _); + } + Penumbra.Log.Verbose("[FileWatcher] Manual cleanup of extracted archives requested."); + } + private void EndFileWatcher() { if (_fsw is null) @@ -73,16 +118,23 @@ private void SetupFileWatcher(string directory) _fsw = new FileSystemWatcher { IncludeSubdirectories = false, - NotifyFilter = NotifyFilters.FileName | NotifyFilters.CreationTime, - InternalBufferSize = 32 * 1024, + NotifyFilter = NotifyFilters.FileName | NotifyFilters.CreationTime, + InternalBufferSize = 32 * 1024, }; - // Only wake us for the exact patterns we care about + // Only wake us for the exact patterns we care about. _fsw.Filters.Add("*.pmp"); _fsw.Filters.Add("*.pcp"); _fsw.Filters.Add("*.ttmp"); _fsw.Filters.Add("*.ttmp2"); + if (_config.EnableContainerPeeking) + { + _fsw.Filters.Add("*.zip"); + _fsw.Filters.Add("*.rar"); + _fsw.Filters.Add("*.7z"); + } + _fsw.Created += OnPath; _fsw.Renamed += OnPath; UpdateDirectory(directory); @@ -127,15 +179,21 @@ public void UpdateDirectory(string newPath) } else { - _fsw.Path = newPath; + _fsw.Path = newPath; _fsw.EnableRaisingEvents = true; } } private void OnPath(object? sender, FileSystemEventArgs e) { - if (!_ignored.TryRemove(e.FullPath, out var expiresAtTickCount) || expiresAtTickCount <= Environment.TickCount64) - _pending.TryAdd(e.FullPath); + if (_ignored.TryRemove(e.FullPath, out var expiresAtTickCount) && expiresAtTickCount > Environment.TickCount64) + { + Penumbra.Log.Verbose($"[FileWatcher] FSW event for '{e.FullPath}' suppressed by ignore list."); + return; + } + + if (_pending.TryAdd(e.FullPath)) + Penumbra.Log.Verbose($"[FileWatcher] FSW event for '{e.FullPath}' enqueued."); } private async Task ConsumerLoopAsync(CancellationToken token) @@ -143,6 +201,7 @@ private async Task ConsumerLoopAsync(CancellationToken token) while (true) { GarbageCollectIgnored(); + var path = _pending.FirstOrDefault(); if (path is null || _pausedConsumer) { @@ -150,9 +209,23 @@ private async Task ConsumerLoopAsync(CancellationToken token) continue; } + var totalSw = Stopwatch.StartNew(); + Penumbra.Log.Verbose($"[FileWatcher] Picked up '{path}' from queue."); + try { - await ProcessOneAsync(path, token).ConfigureAwait(false); + var ext = Path.GetExtension(path); + if (ContainerExtensions.Contains(ext)) + { + if (_config.EnableContainerPeeking) + await ProcessContainerAsync(path, token).ConfigureAwait(false); + // else: peeking was toggled off after the event was queued; drop silently. + } + else if (ModExtensions.Contains(ext)) + { + await ProcessOneAsync(path, token).ConfigureAwait(false); + } + // else: extension we don't recognise (shouldn't happen given filters); drop silently. } catch (OperationCanceledException) { @@ -165,6 +238,7 @@ private async Task ConsumerLoopAsync(CancellationToken token) finally { _pending.TryRemove(path); + Penumbra.Log.Verbose($"[FileWatcher] Finished '{path}' in {totalSw.ElapsedMilliseconds}ms."); } } } @@ -180,44 +254,255 @@ private void GarbageCollectIgnored() private async Task ProcessOneAsync(string path, CancellationToken token) { - // Downloads often finish via rename; file may be locked briefly. - // Wait until it exists and is readable; also require two stable size checks. + if (!await WaitForStableAsync(path, token).ConfigureAwait(false)) + return; + + Penumbra.Log.Verbose($"[FileWatcher] Triggering import for '{path}'."); + TriggerImport(path); + } + + /// + /// Polls the file until two consecutive size readings match, indicating the writer is done. + /// Returns false if the file never settled within the retry budget or the token was canceled. + /// + private static async Task WaitForStableAsync(string path, CancellationToken token) + { const int maxTries = 40; - long lastLen = -1; + long lastLen = -1; + var sw = Stopwatch.StartNew(); for (var i = 0; i < maxTries && !token.IsCancellationRequested; i++) { if (!File.Exists(path)) { - await Task.Delay(100, token); + await Task.Delay(100, token).ConfigureAwait(false); continue; } try { - var fi = new FileInfo(path); - var len = fi.Length; + var len = new FileInfo(path).Length; if (len > 0 && len == lastLen) { - if (_config.EnableAutomaticModImport) - _modImportManager.AddUnpack(path); - else - _messageService.AddMessage(new InstallNotification(_modImportManager, path), false); - return; + Penumbra.Log.Verbose( + $"[FileWatcher] '{path}' stable at {len} bytes after {sw.ElapsedMilliseconds}ms ({i + 1} polls)."); + return true; } lastLen = len; } catch (IOException) { - Penumbra.Log.Debug($"[FileWatcher] File is still being written to."); + Penumbra.Log.Debug("[FileWatcher] File is still being written to."); } catch (UnauthorizedAccessException) { - Penumbra.Log.Debug($"[FileWatcher] File is locked."); + Penumbra.Log.Debug("[FileWatcher] File is locked."); } - await Task.Delay(150, token); + await Task.Delay(150, token).ConfigureAwait(false); + } + + Penumbra.Log.Verbose($"[FileWatcher] '{path}' did not stabilize within {sw.ElapsedMilliseconds}ms."); + return false; + } + + private void TriggerImport(string path) + { + if (_config.EnableAutomaticModImport) + _modImportManager.AddUnpack(path); + else + _messageService.AddMessage(new InstallNotification(_modImportManager, path), false); + } + + /// + /// Opens an archive, scans entries for mod files (by entry-name extension only), extracts matches + /// into a per-archive subdirectory of , then queues each extracted file via + /// . Per-archive subdirectory keeps the original filename intact for the UI. + /// + private async Task ProcessContainerAsync(string path, CancellationToken token) + { + if (!await WaitForStableAsync(path, token).ConfigureAwait(false)) + return; + + var ext = Path.GetExtension(path); + string? archiveDir = null; + var extractedNow = new List(); + + try + { + var openSw = Stopwatch.StartNew(); + Penumbra.Log.Verbose($"[FileWatcher] Opening container '{path}'."); + using var archive = OpenArchive(path, ext); + if (archive is null) + return; + Penumbra.Log.Verbose( + $"[FileWatcher] Opened container '{path}' in {openSw.ElapsedMilliseconds}ms."); + + var enumSw = Stopwatch.StartNew(); + var candidates = archive.Entries + .Where(e => !e.IsDirectory + && e.Key is { Length: > 0 } key + && ModExtensions.Contains(Path.GetExtension(key))) + .ToList(); + Penumbra.Log.Verbose( + $"[FileWatcher] Enumerated entries of '{path}' in {enumSw.ElapsedMilliseconds}ms; {candidates.Count} mod entries found."); + + // Silent ignore for archives that contain nothing relevant + if (candidates.Count == 0) + return; + + Directory.CreateDirectory(TempRoot); + archiveDir = Path.Combine(TempRoot, Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(archiveDir); + + foreach (var entry in candidates) + { + token.ThrowIfCancellationRequested(); + + if (entry.IsEncrypted) + { + Penumbra.Log.Warning( + $"[FileWatcher] Skipping encrypted entry '{entry.Key}' in container '{path}'."); + continue; + } + + var safeName = Path.GetFileName(entry.Key!); + var tempPath = Path.Combine(archiveDir, safeName); + + if (File.Exists(tempPath)) + { + Penumbra.Log.Warning( + $"[FileWatcher] Duplicate entry name '{safeName}' in container '{path}'; skipping later occurrence."); + continue; + } + + try + { + var extractSw = Stopwatch.StartNew(); + await using (var input = entry.OpenEntryStream()) + await using (var output = new FileStream(tempPath, FileMode.CreateNew, FileAccess.Write, + FileShare.None, 81920, useAsync: true)) + { + await input.CopyToAsync(output, 81920, token).ConfigureAwait(false); + } + + Penumbra.Log.Verbose( + $"[FileWatcher] Extracted '{safeName}' ({new FileInfo(tempPath).Length} bytes) in {extractSw.ElapsedMilliseconds}ms."); + extractedNow.Add(tempPath); + } + catch (OperationCanceledException) + { + TryDelete(tempPath); + throw; + } + catch (Exception ex) + { + Penumbra.Log.Warning( + $"[FileWatcher] Failed to extract '{entry.Key}' from '{path}': {ex.Message}"); + TryDelete(tempPath); + } + + // Yield between entries so a large archive doesn't monopolise the worker thread + // while the game is rendering. + await Task.Yield(); + } + + if (extractedNow.Count > 0) + _extractedArchives[archiveDir] = Environment.TickCount64; + else + TryDeleteDirectory(archiveDir); + } + catch (OperationCanceledException) + { + if (archiveDir is not null) + TryDeleteDirectory(archiveDir); + throw; + } + catch (Exception ex) + { + Penumbra.Log.Warning($"[FileWatcher] Failed to read container '{path}': {ex.Message}"); + if (archiveDir is not null) + TryDeleteDirectory(archiveDir); + return; + } + + // Hand each extracted file off as if it were a fresh drop. The freshly-closed stream means + // we can skip WaitForStableAsync here and call TriggerImport directly. + foreach (var tempPath in extractedNow) + { + try + { + Penumbra.Log.Verbose($"[FileWatcher] Triggering import for extracted '{tempPath}'."); + TriggerImport(tempPath); + } + catch (Exception ex) + { + Penumbra.Log.Warning( + $"[FileWatcher] Failed to trigger import for extracted file '{tempPath}': {ex.Message}"); + } + } + } + + private static IArchive? OpenArchive(string path, string extension) + => extension.ToLowerInvariant() switch + { + ".zip" => ZipArchive.Open(path), + ".rar" => RarArchive.Open(path), + ".7z" => SevenZipArchive.Open(path), + _ => null, + }; + + private static void WipeTempRoot() + { + try + { + if (Directory.Exists(TempRoot)) + { + foreach (var entry in Directory.EnumerateFileSystemEntries(TempRoot)) + { + if (Directory.Exists(entry)) + TryDeleteDirectory(entry); + else + TryDelete(entry); + } + } + else + { + Directory.CreateDirectory(TempRoot); + } + } + catch (Exception ex) + { + Penumbra.Log.Warning($"[FileWatcher] Could not prepare temp root '{TempRoot}': {ex.Message}"); + } + } + + private static bool TryDelete(string path) + { + try + { + if (File.Exists(path)) + File.Delete(path); + return true; + } + catch + { + return false; + } + } + + private static bool TryDeleteDirectory(string path) + { + try + { + if (Directory.Exists(path)) + Directory.Delete(path, recursive: true); + return true; + } + catch + { + return false; } } @@ -226,6 +511,8 @@ public void Dispose() { EndConsumerTask(); EndFileWatcher(); + // Cleanup of extracted files is intentionally skipped here. WipeTempRoot() on the next + // construction handles leftovers without blocking shutdown. } public sealed class FileWatcherDrawer(Configuration config, FileWatcher fileWatcher) : IUiService @@ -246,9 +533,15 @@ public void Draw() table.DrawColumn("Automatic Import"u8); table.DrawColumn($"{config.EnableAutomaticModImport}"); + table.DrawColumn("Container Peeking"u8); + table.DrawColumn($"{config.EnableContainerPeeking}"); + table.DrawColumn("Watched Directory"u8); table.DrawColumn(config.WatchDirectory); + table.DrawColumn("Temp Root"u8); + table.DrawColumn(TempRoot); + table.DrawColumn("File Watcher Path"u8); table.DrawColumn(fileWatcher._fsw?.Path ?? ""); @@ -274,9 +567,28 @@ public void Draw() table.DrawColumn(StringU8.Join((byte)'\n', fileWatcher._ignored.Select(entry => (entry.Value - Environment.TickCount64) switch { - <= 0 => $" {entry.Key}", + <= 0 => $" {entry.Key}", var ttl => $"<{ttl}ms> {entry.Key}", }).ToList())); + + table.DrawColumn("Extracted Archives"u8); + table.DrawColumn(StringU8.Join((byte)'\n', fileWatcher._extractedArchives.Select(entry => + { + var ageSec = (Environment.TickCount64 - entry.Value) / 1000; + var fileCount = TryCountFiles(entry.Key); + return $"<{ageSec}s, {fileCount} files> {entry.Key}"; + }).ToList())); + + table.DrawColumn("Clean Extracted"u8); + table.NextColumn(); + if (Im.SmallButton("Delete All Extracted"u8)) + fileWatcher.CleanExtracted(); + } + + private static int TryCountFiles(string dir) + { + try { return Directory.Exists(dir) ? Directory.EnumerateFiles(dir).Count() : 0; } + catch { return 0; } } } } diff --git a/Penumbra/UI/ManagementTab/CleanupTab.cs b/Penumbra/UI/ManagementTab/CleanupTab.cs index b33618e0a..2133e2070 100644 --- a/Penumbra/UI/ManagementTab/CleanupTab.cs +++ b/Penumbra/UI/ManagementTab/CleanupTab.cs @@ -4,7 +4,7 @@ namespace Penumbra.UI.ManagementTab; -public sealed class CleanupTab(CleanupService cleanup, Configuration config) : ITab +public sealed class CleanupTab(CleanupService cleanup, FileWatcher fileWatcher, Configuration config) : ITab { public ReadOnlySpan Label => "General Cleanup"u8; @@ -52,5 +52,12 @@ public void DrawContent() cleanup.CleanupAllUnusedSettings(); if (!enabled) Im.Tooltip.OnHover(HoveredFlags.AllowWhenDisabled, $"Hold {config.DeleteModModifier} while clicking to remove settings."); + + if (ImEx.Button("Clear Extracted Archive Files"u8, default, + "Delete all temporary files extracted from archives by the File Watcher. Extracted files that have not yet been imported will be lost."u8, + !enabled || cleanup.IsRunning)) + fileWatcher.CleanExtracted(); + if (!enabled) + Im.Tooltip.OnHover(HoveredFlags.AllowWhenDisabled, $"Hold {config.DeleteModModifier} while clicking to delete files."); } } diff --git a/Penumbra/UI/Tabs/SettingsTab.cs b/Penumbra/UI/Tabs/SettingsTab.cs index 8cb903a4d..2f68b5cbd 100644 --- a/Penumbra/UI/Tabs/SettingsTab.cs +++ b/Penumbra/UI/Tabs/SettingsTab.cs @@ -624,6 +624,9 @@ private void DrawModHandlingSettings() Checkbox("Enable Directory Watcher"u8, "Enables a File Watcher that automatically listens for Mod files that enter a specified directory, causing Penumbra to open a popup to import these mods."u8, _config.EnableDirectoryWatch, _fileWatcher.Toggle); + Checkbox("Enable Archive Peeking"u8, + "Enables the File Watcher to Peek inside .rar .zip and .7z archives, extracting mods inside and causing Penumbra to open a popup to import these mods."u8, + _config.EnableContainerPeeking, _fileWatcher.ToggleContainerPeeking); Checkbox("Enable Fully Automatic Import"u8, "Uses the File Watcher in order to skip the query popup and automatically import any new mods."u8, _config.EnableAutomaticModImport, v => _config.EnableAutomaticModImport = v); From 91d63c437fc1d1de0770b328ff453df4c85b5bd8 Mon Sep 17 00:00:00 2001 From: Stoia <23234609+StoiaCode@users.noreply.github.com> Date: Tue, 5 May 2026 16:16:37 +0200 Subject: [PATCH 2/6] Fix Spacing --- Penumbra/Services/FileWatcher.cs | 39 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/Penumbra/Services/FileWatcher.cs b/Penumbra/Services/FileWatcher.cs index 4c4ed2316..ea1034fae 100644 --- a/Penumbra/Services/FileWatcher.cs +++ b/Penumbra/Services/FileWatcher.cs @@ -1,4 +1,3 @@ -using System.Diagnostics; using ImSharp; using Luna; using Penumbra.Mods.Manager; @@ -11,17 +10,17 @@ namespace Penumbra.Services; public sealed class FileWatcher : IDisposable, IService { - private readonly ConcurrentSet _pending = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentSet _pending = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary _ignored = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary _extractedArchives = new(StringComparer.OrdinalIgnoreCase); - private readonly ModImportManager _modImportManager; - private readonly MessageService _messageService; - private readonly Configuration _config; + private readonly ModImportManager _modImportManager; + private readonly MessageService _messageService; + private readonly Configuration _config; - private bool _pausedConsumer; - private FileSystemWatcher? _fsw; + private bool _pausedConsumer; + private FileSystemWatcher? _fsw; private CancellationTokenSource? _cts = new(); - private Task? _consumer; + private Task? _consumer; /// The time-to-live of ignore entries, in the same unit as , namely milliseconds. private const long IgnoreTimeToLive = 60000L; @@ -40,8 +39,8 @@ public sealed class FileWatcher : IDisposable, IService public FileWatcher(ModImportManager modImportManager, MessageService messageService, Configuration config) { _modImportManager = modImportManager; - _messageService = messageService; - _config = config; + _messageService = messageService; + _config = config; WipeTempRoot(); @@ -118,8 +117,8 @@ private void SetupFileWatcher(string directory) _fsw = new FileSystemWatcher { IncludeSubdirectories = false, - NotifyFilter = NotifyFilters.FileName | NotifyFilters.CreationTime, - InternalBufferSize = 32 * 1024, + NotifyFilter = NotifyFilters.FileName | NotifyFilters.CreationTime, + InternalBufferSize = 32 * 1024, }; // Only wake us for the exact patterns we care about. @@ -179,7 +178,7 @@ public void UpdateDirectory(string newPath) } else { - _fsw.Path = newPath; + _fsw.Path = newPath; _fsw.EnableRaisingEvents = true; } } @@ -268,8 +267,8 @@ private async Task ProcessOneAsync(string path, CancellationToken token) private static async Task WaitForStableAsync(string path, CancellationToken token) { const int maxTries = 40; - long lastLen = -1; - var sw = Stopwatch.StartNew(); + long lastLen = -1; + var sw = Stopwatch.StartNew(); for (var i = 0; i < maxTries && !token.IsCancellationRequested; i++) { @@ -325,9 +324,9 @@ private async Task ProcessContainerAsync(string path, CancellationToken token) if (!await WaitForStableAsync(path, token).ConfigureAwait(false)) return; - var ext = Path.GetExtension(path); + var ext = Path.GetExtension(path); string? archiveDir = null; - var extractedNow = new List(); + var extractedNow = new List(); try { @@ -449,7 +448,7 @@ private async Task ProcessContainerAsync(string path, CancellationToken token) { ".zip" => ZipArchive.Open(path), ".rar" => RarArchive.Open(path), - ".7z" => SevenZipArchive.Open(path), + ".7z" => SevenZipArchive.Open(path), _ => null, }; @@ -567,14 +566,14 @@ public void Draw() table.DrawColumn(StringU8.Join((byte)'\n', fileWatcher._ignored.Select(entry => (entry.Value - Environment.TickCount64) switch { - <= 0 => $" {entry.Key}", + <= 0 => $" {entry.Key}", var ttl => $"<{ttl}ms> {entry.Key}", }).ToList())); table.DrawColumn("Extracted Archives"u8); table.DrawColumn(StringU8.Join((byte)'\n', fileWatcher._extractedArchives.Select(entry => { - var ageSec = (Environment.TickCount64 - entry.Value) / 1000; + var ageSec = (Environment.TickCount64 - entry.Value) / 1000; var fileCount = TryCountFiles(entry.Key); return $"<{ageSec}s, {fileCount} files> {entry.Key}"; }).ToList())); From 1c3d54851d619b0c8a5f9acda410497d89f59da5 Mon Sep 17 00:00:00 2001 From: Stoia <23234609+StoiaCode@users.noreply.github.com> Date: Tue, 5 May 2026 16:29:01 +0200 Subject: [PATCH 3/6] Remove redundant FileInfo call --- Penumbra/Services/FileWatcher.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Penumbra/Services/FileWatcher.cs b/Penumbra/Services/FileWatcher.cs index ea1034fae..756ff6457 100644 --- a/Penumbra/Services/FileWatcher.cs +++ b/Penumbra/Services/FileWatcher.cs @@ -379,15 +379,17 @@ private async Task ProcessContainerAsync(string path, CancellationToken token) try { var extractSw = Stopwatch.StartNew(); + long bytesWritten; await using (var input = entry.OpenEntryStream()) await using (var output = new FileStream(tempPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, 81920, useAsync: true)) { await input.CopyToAsync(output, 81920, token).ConfigureAwait(false); + bytesWritten = output.Position; } Penumbra.Log.Verbose( - $"[FileWatcher] Extracted '{safeName}' ({new FileInfo(tempPath).Length} bytes) in {extractSw.ElapsedMilliseconds}ms."); + $"[FileWatcher] Extracted '{safeName}' ({bytesWritten} bytes) in {extractSw.ElapsedMilliseconds}ms."); extractedNow.Add(tempPath); } catch (OperationCanceledException) From 59b25f32f895eb80163ce9653fc9c09e2a011619 Mon Sep 17 00:00:00 2001 From: Exter-N Date: Sat, 9 May 2026 08:00:03 +0200 Subject: [PATCH 4/6] Make mod imports awaitable --- Penumbra/Import/TexToolsImport.cs | 21 +++++++++-------- Penumbra/Mods/Manager/ModImportManager.cs | 28 ++++++++++++++--------- Penumbra/Mods/Manager/ModImportResult.cs | 3 +++ Penumbra/Services/FileWatcher.cs | 10 +++++--- Penumbra/Services/InstallNotification.cs | 18 +++++++++++++-- Penumbra/UI/GlobalModImporter.cs | 2 +- Penumbra/UI/Tabs/Debug/DebugTab.cs | 2 +- 7 files changed, 57 insertions(+), 27 deletions(-) create mode 100644 Penumbra/Mods/Manager/ModImportResult.cs diff --git a/Penumbra/Import/TexToolsImport.cs b/Penumbra/Import/TexToolsImport.cs index 7981b4472..8061a5224 100644 --- a/Penumbra/Import/TexToolsImport.cs +++ b/Penumbra/Import/TexToolsImport.cs @@ -26,8 +26,8 @@ public partial class TexToolsImporter : IDisposable private readonly CancellationTokenSource _cancellation = new(); private readonly CancellationToken _token; - public ImporterState State { get; private set; } - public readonly List<(FileInfo File, DirectoryInfo? Mod, Exception? Error)> ExtractedMods; + public ImporterState State { get; private set; } + public readonly List ExtractedMods; private readonly Configuration _config; private readonly DuplicateManager _duplicates; @@ -37,8 +37,9 @@ public partial class TexToolsImporter : IDisposable private readonly MigrationManager _migrationManager; public TexToolsImporter(int count, IEnumerable modPackFiles, Action handler, - Configuration config, DuplicateManager duplicates, ModNormalizer modNormalizer, ModManager modManager, FileCompactor compactor, - MigrationManager migrationManager, TexToolsImporter? previous) + TaskCompletionSource taskCompletionSource, Configuration config, DuplicateManager duplicates, + ModNormalizer modNormalizer, ModManager modManager, FileCompactor compactor, MigrationManager migrationManager, + TexToolsImporter? previous) { if (previous is not null) { @@ -58,7 +59,7 @@ public TexToolsImporter(int count, IEnumerable modPackFiles, Action(count + _previousModPackCount); + ExtractedMods = new List(count + _previousModPackCount); _token = _cancellation.Token; if (previous is not null) ExtractedMods.AddRange(previous.ExtractedMods); @@ -68,7 +69,9 @@ public TexToolsImporter(int count, IEnumerable modPackFiles, Action { taskCompletionSource.SetResult(ExtractedMods.Skip(_previousModPackCount).ToArray()); }, + TaskScheduler.Default); } private void CloseStreams() @@ -100,14 +103,14 @@ private void ImportFiles() _currentModDirectory = null; if (_token.IsCancellationRequested) { - ExtractedMods.Add((file, null, new TaskCanceledException("Task canceled by user."))); + ExtractedMods.Add(new ModImportResult(file, null, new TaskCanceledException("Task canceled by user."))); continue; } try { var directory = VerifyVersionAndImport(file); - ExtractedMods.Add((file, directory, null)); + ExtractedMods.Add(new ModImportResult(file, directory, null)); if (_config.AutoDeduplicateOnImport) { State = ImporterState.DeduplicatingFiles; @@ -116,7 +119,7 @@ private void ImportFiles() } catch (Exception e) { - ExtractedMods.Add((file, _currentModDirectory, e)); + ExtractedMods.Add(new ModImportResult(file, _currentModDirectory, e)); _currentNumOptions = 0; _currentOptionIdx = 0; _currentFileIdx = 0; diff --git a/Penumbra/Mods/Manager/ModImportManager.cs b/Penumbra/Mods/Manager/ModImportManager.cs index 006fb473d..13ad17308 100644 --- a/Penumbra/Mods/Manager/ModImportManager.cs +++ b/Penumbra/Mods/Manager/ModImportManager.cs @@ -16,7 +16,7 @@ public class ModImportManager( FileCompactor compactor) : IDisposable, IService { private readonly Dictionary _uniqueModsToUnpack = new(StringComparer.OrdinalIgnoreCase); - internal readonly Queue> ModsToUnpack = new(); + internal readonly Queue ModsToUnpack = new(); /// Mods need to be added thread-safely outside of iteration. private readonly ConcurrentQueue _modsToAdd = new(); @@ -31,14 +31,14 @@ public void TryUnpacking() if (Importing && _import!.State is not ImporterState.Done) return; - List newMods; + UnpackRequest newMods; lock (ModsToUnpack) { - if (!ModsToUnpack.TryDequeue(out newMods!)) + if (!ModsToUnpack.TryDequeue(out newMods)) return; } - var files = newMods.Where(s => + var files = newMods.Paths.Where(s => { if (File.Exists(s)) return true; @@ -50,10 +50,13 @@ public void TryUnpacking() Penumbra.Log.Debug($"Unpacking mods: {string.Join("\n\t", files.Select(f => f.FullName))}."); if (files.Length == 0) + { + newMods.TaskCompletionSource.SetResult([]); return; + } - _import = new TexToolsImporter(files.Length, files, AddNewMod, config, duplicates, modNormalizer, modManager, compactor, - migrationManager, _import); + _import = new TexToolsImporter(files.Length, files, AddNewMod, newMods.TaskCompletionSource, config, duplicates, modNormalizer, + modManager, compactor, migrationManager, _import); } public bool Importing @@ -65,10 +68,7 @@ public bool IsImporting([NotNullWhen(true)] out TexToolsImporter? importer) return _import != null; } - public void AddUnpack(IEnumerable paths) - => AddUnpack(paths.ToList()); - - public void AddUnpack(params List paths) + public Task AddUnpack(params List paths) { lock (ModsToUnpack) { @@ -95,9 +95,13 @@ public void AddUnpack(params List paths) if (paths.Count > 0) { Penumbra.Log.Debug($"Adding mods to install: {string.Join("\n\t", paths)}"); - ModsToUnpack.Enqueue(paths); + var tcs = new TaskCompletionSource(); + ModsToUnpack.Enqueue(new UnpackRequest(paths, tcs)); + return tcs.Task; } } + + return Task.FromResult(Array.Empty()); } public void ClearImport() @@ -156,4 +160,6 @@ private void AddNewMod(FileInfo file, DirectoryInfo? dir, Exception? error) _modsToAdd.Enqueue(dir); } } + + internal readonly record struct UnpackRequest(List Paths, TaskCompletionSource TaskCompletionSource); } diff --git a/Penumbra/Mods/Manager/ModImportResult.cs b/Penumbra/Mods/Manager/ModImportResult.cs new file mode 100644 index 000000000..5350544cb --- /dev/null +++ b/Penumbra/Mods/Manager/ModImportResult.cs @@ -0,0 +1,3 @@ +namespace Penumbra.Mods.Manager; + +public readonly record struct ModImportResult(FileInfo File, DirectoryInfo? Mod, Exception? Error); diff --git a/Penumbra/Services/FileWatcher.cs b/Penumbra/Services/FileWatcher.cs index 756ff6457..638996e00 100644 --- a/Penumbra/Services/FileWatcher.cs +++ b/Penumbra/Services/FileWatcher.cs @@ -306,12 +306,16 @@ private static async Task WaitForStableAsync(string path, CancellationToke return false; } - private void TriggerImport(string path) + private Task TriggerImport(string path) { if (_config.EnableAutomaticModImport) - _modImportManager.AddUnpack(path); + return _modImportManager.AddUnpack(path); else - _messageService.AddMessage(new InstallNotification(_modImportManager, path), false); + { + var tcs = new TaskCompletionSource(); + _messageService.AddMessage(new InstallNotification(_modImportManager, path, tcs), false); + return tcs.Task; + } } /// diff --git a/Penumbra/Services/InstallNotification.cs b/Penumbra/Services/InstallNotification.cs index 71c6884f1..44c665ca0 100644 --- a/Penumbra/Services/InstallNotification.cs +++ b/Penumbra/Services/InstallNotification.cs @@ -6,8 +6,11 @@ namespace Penumbra.Services; -public class InstallNotification(ModImportManager modImportManager, string filePath) : Luna.IMessage +public class InstallNotification(ModImportManager modImportManager, string filePath, TaskCompletionSource tcs) + : Luna.INotificationAwareMessage { + private bool _reportCancellationOnDismissal = true; + public NotificationType NotificationType => NotificationType.Info; @@ -37,7 +40,9 @@ public void OnNotificationActions(INotificationDrawArgs args) var buttonSize = new Vector2((region.X - Im.Style.ItemSpacing.X) / 2, 0); if (Im.Button("Install"u8, buttonSize)) { - modImportManager.AddUnpack(filePath); + _reportCancellationOnDismissal = false; + modImportManager.AddUnpack(filePath) + .ContinueWith(tcs.SetFromTask); args.Notification.DismissNow(); } @@ -45,4 +50,13 @@ public void OnNotificationActions(INotificationDrawArgs args) if (Im.Button("Ignore"u8, buttonSize)) args.Notification.DismissNow(); } + + public void OnNotificationCreated(IActiveNotification notification) + { + notification.Dismiss += _ => + { + if (_reportCancellationOnDismissal) + tcs.SetResult([]); + }; + } } diff --git a/Penumbra/UI/GlobalModImporter.cs b/Penumbra/UI/GlobalModImporter.cs index 2e63cd238..4c99ecb12 100644 --- a/Penumbra/UI/GlobalModImporter.cs +++ b/Penumbra/UI/GlobalModImporter.cs @@ -46,7 +46,7 @@ public void Dispose() } private void ImportFiles(IReadOnlyList files, IReadOnlyList _) - => _importManager.AddUnpack(files.Where(f => ValidModExtensions.Contains(Path.GetExtension(f)))); + => _importManager.AddUnpack(files.Where(f => ValidModExtensions.Contains(Path.GetExtension(f))).ToList()); private static bool ValidExtension(IDragDropManager manager) => manager.Extensions.Any(ValidModExtensions.Contains); diff --git a/Penumbra/UI/Tabs/Debug/DebugTab.cs b/Penumbra/UI/Tabs/Debug/DebugTab.cs index c7146ae6d..9706e683d 100644 --- a/Penumbra/UI/Tabs/Debug/DebugTab.cs +++ b/Penumbra/UI/Tabs/Debug/DebugTab.cs @@ -384,7 +384,7 @@ private void DrawDebugTabGeneral() { foreach (var (index, batch) in _modImporter.ModsToUnpack.Index()) { - foreach (var mod in batch) + foreach (var mod in batch.Paths) table.DrawDataPair($"{index}", mod); } } From 92850f3d111a0c6d7913f2c54efcfdea7cd2a6db Mon Sep 17 00:00:00 2001 From: Stoia <23234609+StoiaCode@users.noreply.github.com> Date: Mon, 25 May 2026 12:56:12 +0200 Subject: [PATCH 5/6] Change Folder & Add Notification System for Extraction Progress --- .../Services/ArchiveExtractionNotification.cs | 91 +++++++++++++++++++ Penumbra/Services/FileWatcher.cs | 86 ++++++++++++++---- 2 files changed, 159 insertions(+), 18 deletions(-) create mode 100644 Penumbra/Services/ArchiveExtractionNotification.cs diff --git a/Penumbra/Services/ArchiveExtractionNotification.cs b/Penumbra/Services/ArchiveExtractionNotification.cs new file mode 100644 index 000000000..3fa7c8f62 --- /dev/null +++ b/Penumbra/Services/ArchiveExtractionNotification.cs @@ -0,0 +1,91 @@ +using Dalamud.Interface.ImGuiNotification; +using Dalamud.Interface.ImGuiNotification.EventArgs; +using ImSharp; +using Luna; + +namespace Penumbra.Services; + +public sealed class ArchiveExtractionNotification(MessageService messageService) + : AmassingNotification(messageService), IService +{ + public readonly record struct ArchiveInfo(string ArchiveName, int ModCount); + + private volatile bool _isExtracting; + private volatile int _currentEntryIndex; + private volatile int _totalEntries; + private volatile string? _currentEntryName; + + private bool _wasExtracting; + + public void AddArchive(string archiveName, int modCount) + => AddObject(new ArchiveInfo(archiveName, modCount)); + + public void SetProgress(int currentEntry, int totalEntries, string entryName) + { + _currentEntryIndex = currentEntry; + _totalEntries = totalEntries; + _currentEntryName = entryName; + _isExtracting = true; + } + + public void ClearProgress() + { + _isExtracting = false; + _currentEntryIndex = 0; + _totalEntries = 0; + _currentEntryName = null; + } + + public override NotificationType NotificationType + => NotificationType.Info; + + public override string NotificationTitle + => Count switch + { + 1 => $"Extracting mods from {GatheredObjects[0].ArchiveName}", + _ => $"Extracting mods from {Count} archives", + }; + + public override string NotificationMessage + => _isExtracting + ? $"Extracting '{_currentEntryName}'..." + : "Waiting..."; + + public override TimeSpan NotificationDuration + => TimeSpan.MaxValue; + + public override void NotificationActions(INotificationDrawArgs args) + { + if (_isExtracting) + { + _wasExtracting = true; + var total = _totalEntries; + if (total > 0 && CurrentNotification is { } notification) + { + notification.Progress = _currentEntryIndex / (float)total; + notification.Content = $"Extracting '{_currentEntryName}' ({_currentEntryIndex + 1} of {total})..."; + } + } + else if (_wasExtracting) + { + _wasExtracting = false; + if (CurrentNotification is { } notification) + { + notification.Progress = 1.0f; + notification.Content = "Extraction complete."; + notification.InitialDuration = TimeSpan.FromSeconds(15); + } + } + } + + protected override StoredNotification CreateStored(in ArchiveInfo @object) + => new Stored(this, @object); + + private sealed class Stored(ArchiveExtractionNotification parent, ArchiveInfo info) + : StoredNotification(parent, info) + { + public override string LogMessage { get; } = $"Extracted {info.ModCount} mod(s) from {info.ArchiveName}."; + public override StringU8 StoredMessage { get; } = new($"[{info.ArchiveName}] {info.ModCount} mod(s) extracted."); + public override StringU8 StoredTooltip { get; } = new($"Archive: {info.ArchiveName}\nMods found: {info.ModCount}"); + } +} diff --git a/Penumbra/Services/FileWatcher.cs b/Penumbra/Services/FileWatcher.cs index 638996e00..aa8a3209e 100644 --- a/Penumbra/Services/FileWatcher.cs +++ b/Penumbra/Services/FileWatcher.cs @@ -16,6 +16,7 @@ public sealed class FileWatcher : IDisposable, IService private readonly ModImportManager _modImportManager; private readonly MessageService _messageService; private readonly Configuration _config; + private readonly ArchiveExtractionNotification _archiveExtractionNotification; private bool _pausedConsumer; private FileSystemWatcher? _fsw; @@ -31,18 +32,22 @@ public sealed class FileWatcher : IDisposable, IService private static readonly HashSet ContainerExtensions = new(StringComparer.OrdinalIgnoreCase) { ".zip", ".rar", ".7z" }; + private static string GetTempPath(string basePath) + => Path.Combine(basePath, "Penumbra-FileWatcher"); + /// /// Subdirectory under the system temp directory used for extracted archive entries. /// - private static readonly string TempRoot = Path.Combine(Path.GetTempPath(), "Penumbra-FileWatcher"); - public FileWatcher(ModImportManager modImportManager, MessageService messageService, Configuration config) + public FileWatcher(ModImportManager modImportManager, MessageService messageService, Configuration config, + ArchiveExtractionNotification archiveExtractionNotification) { - _modImportManager = modImportManager; - _messageService = messageService; - _config = config; + _modImportManager = modImportManager; + _messageService = messageService; + _config = config; + _archiveExtractionNotification = archiveExtractionNotification; - WipeTempRoot(); + WipeTempRoot(_config.WatchDirectory); if (_config.EnableDirectoryWatch) { @@ -139,7 +144,6 @@ private void SetupFileWatcher(string directory) UpdateDirectory(directory); } - private void EndConsumerTask() { if (_cts is not null) @@ -320,7 +324,7 @@ private Task TriggerImport(string path) /// /// Opens an archive, scans entries for mod files (by entry-name extension only), extracts matches - /// into a per-archive subdirectory of , then queues each extracted file via + /// into a per-archive subdirectory of , then queues each extracted file via /// . Per-archive subdirectory keeps the original filename intact for the UI. /// private async Task ProcessContainerAsync(string path, CancellationToken token) @@ -355,8 +359,11 @@ private async Task ProcessContainerAsync(string path, CancellationToken token) if (candidates.Count == 0) return; - Directory.CreateDirectory(TempRoot); - archiveDir = Path.Combine(TempRoot, Guid.NewGuid().ToString("N")); + var archiveName = Path.GetFileName(path); + _archiveExtractionNotification.AddArchive(archiveName, candidates.Count); + + Directory.CreateDirectory(GetTempPath(_config.WatchDirectory)); + archiveDir = Path.Combine(GetTempPath(_config.WatchDirectory), Guid.NewGuid().ToString("N")); Directory.CreateDirectory(archiveDir); foreach (var entry in candidates) @@ -380,6 +387,8 @@ private async Task ProcessContainerAsync(string path, CancellationToken token) continue; } + _archiveExtractionNotification.SetProgress(extractedNow.Count, candidates.Count, safeName); + try { var extractSw = Stopwatch.StartNew(); @@ -413,6 +422,8 @@ private async Task ProcessContainerAsync(string path, CancellationToken token) await Task.Yield(); } + _archiveExtractionNotification.ClearProgress(); + if (extractedNow.Count > 0) _extractedArchives[archiveDir] = Environment.TickCount64; else @@ -420,12 +431,14 @@ private async Task ProcessContainerAsync(string path, CancellationToken token) } catch (OperationCanceledException) { + _archiveExtractionNotification.ClearProgress(); if (archiveDir is not null) TryDeleteDirectory(archiveDir); throw; } catch (Exception ex) { + _archiveExtractionNotification.ClearProgress(); Penumbra.Log.Warning($"[FileWatcher] Failed to read container '{path}': {ex.Message}"); if (archiveDir is not null) TryDeleteDirectory(archiveDir); @@ -434,17 +447,54 @@ private async Task ProcessContainerAsync(string path, CancellationToken token) // Hand each extracted file off as if it were a fresh drop. The freshly-closed stream means // we can skip WaitForStableAsync here and call TriggerImport directly. + var allSucceeded = true; foreach (var tempPath in extractedNow) { + var succeeded = false; try { Penumbra.Log.Verbose($"[FileWatcher] Triggering import for extracted '{tempPath}'."); - TriggerImport(tempPath); + var results = await TriggerImport(tempPath).ConfigureAwait(false); + succeeded = results.Length > 0 && results.All(r => r.Error is null); + + foreach (var result in results) + { + if (result.Error is not null) + Penumbra.Log.Warning( + $"[FileWatcher] Import of '{result.File.Name}' encountered an error: {result.Error.Message}"); + } } catch (Exception ex) { Penumbra.Log.Warning( - $"[FileWatcher] Failed to trigger import for extracted file '{tempPath}': {ex.Message}"); + $"[FileWatcher] Failed to import extracted file '{tempPath}': {ex.Message}"); + } + + if (succeeded) + { + if (TryDelete(tempPath)) + Penumbra.Log.Verbose($"[FileWatcher] Deleted extracted temp file '{tempPath}'."); + else + Penumbra.Log.Warning($"[FileWatcher] Could not delete extracted temp file '{tempPath}'."); + } + else + { + allSucceeded = false; + } + } + + if (archiveDir is not null && allSucceeded) + { + if (TryDeleteDirectory(archiveDir)) + { + _extractedArchives.TryRemove(archiveDir, out _); + Penumbra.Log.Verbose($"[FileWatcher] Cleaned up archive temp directory '{archiveDir}'."); + } + else + { + Penumbra.Log.Warning( + $"[FileWatcher] Could not fully clean archive temp directory '{archiveDir}'; " + + "it will remain tracked for manual cleanup."); } } } @@ -458,13 +508,13 @@ private async Task ProcessContainerAsync(string path, CancellationToken token) _ => null, }; - private static void WipeTempRoot() + private static void WipeTempRoot(string watchDir) { try { - if (Directory.Exists(TempRoot)) + if (Directory.Exists(GetTempPath(watchDir))) { - foreach (var entry in Directory.EnumerateFileSystemEntries(TempRoot)) + foreach (var entry in Directory.EnumerateFileSystemEntries(GetTempPath(watchDir))) { if (Directory.Exists(entry)) TryDeleteDirectory(entry); @@ -474,12 +524,12 @@ private static void WipeTempRoot() } else { - Directory.CreateDirectory(TempRoot); + Directory.CreateDirectory(GetTempPath(watchDir)); } } catch (Exception ex) { - Penumbra.Log.Warning($"[FileWatcher] Could not prepare temp root '{TempRoot}': {ex.Message}"); + Penumbra.Log.Warning($"[FileWatcher] Could not prepare temp root '{GetTempPath(watchDir)}': {ex.Message}"); } } @@ -545,7 +595,7 @@ public void Draw() table.DrawColumn(config.WatchDirectory); table.DrawColumn("Temp Root"u8); - table.DrawColumn(TempRoot); + table.DrawColumn(GetTempPath(config.WatchDirectory)); table.DrawColumn("File Watcher Path"u8); table.DrawColumn(fileWatcher._fsw?.Path ?? ""); From 692a63efdcb4037d9147a0bfb0518e210cfbdf36 Mon Sep 17 00:00:00 2001 From: Stoia <23234609+StoiaCode@users.noreply.github.com> Date: Wed, 27 May 2026 12:48:16 +0200 Subject: [PATCH 6/6] discard return task from TriggerImport after Nys suggestion Co-authored-by: N. Lo. --- Penumbra/Services/FileWatcher.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Penumbra/Services/FileWatcher.cs b/Penumbra/Services/FileWatcher.cs index aa8a3209e..28cb9f340 100644 --- a/Penumbra/Services/FileWatcher.cs +++ b/Penumbra/Services/FileWatcher.cs @@ -261,7 +261,7 @@ private async Task ProcessOneAsync(string path, CancellationToken token) return; Penumbra.Log.Verbose($"[FileWatcher] Triggering import for '{path}'."); - TriggerImport(path); + _ = TriggerImport(path); } ///