From 5cc5c57bd3f9b1e38a33971c509273ce10c7ae0f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Apr 2026 18:07:16 -0700 Subject: [PATCH 01/14] Replace native turbowalk with pure TypeScript workspace package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit turbowalk previously wrapped winapi-bindings' WalkDir() on Windows and fell back to a JS implementation on Linux. Replace the external git dependency with a local workspace package that uses Node's built-in fs.readdir + fs.lstat for all platforms. The new implementation supports all options used in the codebase: recurse, skipHidden, skipLinks, skipInaccessible, and details. No import changes needed — the workspace package shadows the old name. Resolves APP-405 --- etc/Dependency Report.md | 1 - extensions/collections/package.json | 2 +- extensions/extension-dashlet/package.json | 2 +- .../gamebryo-savegame-management/package.json | 2 +- .../games/game-stardewvalley/package.json | 2 +- extensions/gamestore-origin/package.json | 2 +- extensions/gamestore-xbox/package.json | 2 +- extensions/mod-content/package.json | 2 +- .../mod-dependency-manager/package.json | 2 +- extensions/mod-report/package.json | 2 +- extensions/new-file-monitor/package.json | 2 +- .../script-extender-error-check/package.json | 2 +- extensions/test-setup/package.json | 2 +- flatpak/generated-sources.json | 14 --- packages/turbowalk/package.json | 9 ++ packages/turbowalk/src/index.ts | 107 ++++++++++++++++++ pnpm-lock.yaml | 72 +++++------- pnpm-workspace.yaml | 1 - src/main/package.json | 2 +- src/renderer/package.json | 2 +- 20 files changed, 160 insertions(+), 72 deletions(-) create mode 100644 packages/turbowalk/package.json create mode 100644 packages/turbowalk/src/index.ts diff --git a/etc/Dependency Report.md b/etc/Dependency Report.md index cbd0081a3c..7c6113769b 100644 --- a/etc/Dependency Report.md +++ b/etc/Dependency Report.md @@ -116,7 +116,6 @@ This is a list of all modules leaked by Vortex to extensions. Any module listed | tailwindcss | 4.2.2 | | tmp | 0.1.0 | | tough-cookie | 6.0.1 | -| turbowalk | 3.1.1 | | universal-analytics | 0.4.23 | | uuid | 3.4.0 | | vortex-parse-ini | 0.4.0 | diff --git a/extensions/collections/package.json b/extensions/collections/package.json index 738566ae18..be7652fbb6 100644 --- a/extensions/collections/package.json +++ b/extensions/collections/package.json @@ -46,7 +46,7 @@ "shortid": "catalog:", "tmp": "catalog:", "ts-v-gen": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "typescript": "catalog:", "vortex-api": "workspace:*" } diff --git a/extensions/extension-dashlet/package.json b/extensions/extension-dashlet/package.json index 0ffa31980e..fcb5cbaf39 100644 --- a/extensions/extension-dashlet/package.json +++ b/extensions/extension-dashlet/package.json @@ -27,7 +27,7 @@ "react-i18next": "catalog:", "redux": "catalog:", "redux-thunk": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "typescript": "catalog:", "vortex-api": "workspace:*" } diff --git a/extensions/gamebryo-savegame-management/package.json b/extensions/gamebryo-savegame-management/package.json index 11acc4b683..7534fef802 100644 --- a/extensions/gamebryo-savegame-management/package.json +++ b/extensions/gamebryo-savegame-management/package.json @@ -33,7 +33,7 @@ "redux": "catalog:", "redux-act": "catalog:", "redux-thunk": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "vortex-api": "workspace:*", "vortex-parse-ini": "catalog:" }, diff --git a/extensions/games/game-stardewvalley/package.json b/extensions/games/game-stardewvalley/package.json index 0fb2dcb6f7..b98811505c 100644 --- a/extensions/games/game-stardewvalley/package.json +++ b/extensions/games/game-stardewvalley/package.json @@ -26,7 +26,7 @@ "redux-act": "catalog:", "relaxed-json": "catalog:", "semver": "catalog:", - "turbowalk": "catalog:" + "turbowalk": "workspace:*" }, "devDependencies": { "@nexusmods/nexus-api": "catalog:", diff --git a/extensions/gamestore-origin/package.json b/extensions/gamestore-origin/package.json index 863403128e..cd54c9b861 100644 --- a/extensions/gamestore-origin/package.json +++ b/extensions/gamestore-origin/package.json @@ -20,7 +20,7 @@ "@types/libxmljs": "catalog:", "bluebird": "catalog:", "copyfiles": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "typescript": "catalog:", "vortex-api": "workspace:*", "winapi-bindings": "catalog:", diff --git a/extensions/gamestore-xbox/package.json b/extensions/gamestore-xbox/package.json index ebb9a96cb1..e139f4ef89 100644 --- a/extensions/gamestore-xbox/package.json +++ b/extensions/gamestore-xbox/package.json @@ -23,7 +23,7 @@ "copyfiles": "catalog:", "redux": "catalog:", "redux-thunk": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "typescript": "catalog:", "vortex-api": "workspace:*", "winapi-bindings": "catalog:", diff --git a/extensions/mod-content/package.json b/extensions/mod-content/package.json index 361344cf23..d0e1355ede 100644 --- a/extensions/mod-content/package.json +++ b/extensions/mod-content/package.json @@ -26,7 +26,7 @@ "react": "catalog:", "react-bootstrap": "catalog:", "redux": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "vortex-api": "workspace:*" } } diff --git a/extensions/mod-dependency-manager/package.json b/extensions/mod-dependency-manager/package.json index c2d1a7a0fc..617d6386da 100644 --- a/extensions/mod-dependency-manager/package.json +++ b/extensions/mod-dependency-manager/package.json @@ -49,7 +49,7 @@ "reselect": "catalog:", "semver": "catalog:", "shortid": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "vortex-api": "workspace:*" } } diff --git a/extensions/mod-report/package.json b/extensions/mod-report/package.json index 6cbdc2732b..5b3040eb1f 100644 --- a/extensions/mod-report/package.json +++ b/extensions/mod-report/package.json @@ -15,7 +15,7 @@ "bs58": "catalog:", "copyfiles": "catalog:", "electron": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "typescript": "catalog:", "vortex-api": "workspace:*" } diff --git a/extensions/new-file-monitor/package.json b/extensions/new-file-monitor/package.json index 9782e2de85..dee4875b93 100644 --- a/extensions/new-file-monitor/package.json +++ b/extensions/new-file-monitor/package.json @@ -17,7 +17,7 @@ "bluebird": "catalog:", "copyfiles": "catalog:", "redux": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "typescript": "catalog:", "vortex-api": "workspace:*" } diff --git a/extensions/script-extender-error-check/package.json b/extensions/script-extender-error-check/package.json index 02490fece8..83b2afa344 100644 --- a/extensions/script-extender-error-check/package.json +++ b/extensions/script-extender-error-check/package.json @@ -23,7 +23,7 @@ "i18next": "catalog:", "react": "catalog:", "react-bootstrap": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "vortex-api": "workspace:*" } } diff --git a/extensions/test-setup/package.json b/extensions/test-setup/package.json index e865323bd1..7316865799 100644 --- a/extensions/test-setup/package.json +++ b/extensions/test-setup/package.json @@ -19,7 +19,7 @@ "copyfiles": "catalog:", "i18next": "catalog:", "react-bootstrap": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "vortex-api": "workspace:*", "winapi-bindings": "catalog:" } diff --git a/flatpak/generated-sources.json b/flatpak/generated-sources.json index 6e133961dc..c813289e3a 100644 --- a/flatpak/generated-sources.json +++ b/flatpak/generated-sources.json @@ -350,20 +350,6 @@ "dest-filename": "7c1b6f1d6437f2238be51316de823b0fbd63e4c0", "dest": "flatpak-node/yarn-mirror" }, - { - "type": "file", - "url": "https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/314f2cdb904a9a075c35261e8a1de10b0af20295", - "sha256": "c2be72ccc8122002afa064561db4b0d22f39a960e75ddc2c5e3272c399838dc6", - "dest-filename": "314f2cdb904a9a075c35261e8a1de10b0af20295", - "dest": "flatpak-node/yarn-mirror" - }, - { - "type": "file", - "url": "https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/cf4238a4d192763c8133500ce9b366a71b262a33", - "sha256": "85c82bb0151739fec38aa48eb276987ac62a7a0984283d1b3b457019de7f6574", - "dest-filename": "cf4238a4d192763c8133500ce9b366a71b262a33", - "dest": "flatpak-node/yarn-mirror" - }, { "type": "file", "url": "https://codeload.github.com/Nexus-Mods/node-vortexmt/tar.gz/5251ea012ce856742aeaf73a583073497aff773a", diff --git a/packages/turbowalk/package.json b/packages/turbowalk/package.json new file mode 100644 index 0000000000..293abf6b76 --- /dev/null +++ b/packages/turbowalk/package.json @@ -0,0 +1,9 @@ +{ + "name": "turbowalk", + "version": "4.0.0", + "description": "Pure TypeScript directory walking (no native addons)", + "main": "src/index.ts", + "types": "src/index.ts", + "private": true, + "type": "commonjs" +} diff --git a/packages/turbowalk/src/index.ts b/packages/turbowalk/src/index.ts new file mode 100644 index 0000000000..78bf9fbbd2 --- /dev/null +++ b/packages/turbowalk/src/index.ts @@ -0,0 +1,107 @@ +import * as fs from "fs"; +import * as path from "path"; + +export interface IEntry { + filePath: string; + isDirectory: boolean; + isReparsePoint: boolean; + size: number; + mtime: number; + isTerminator?: boolean; + id?: number; + idStr?: string; + linkCount?: number; +} + +export interface IWalkOptions { + terminators?: boolean; + details?: boolean; + threshold?: number; + recurse?: boolean; + skipHidden?: boolean; + skipLinks?: boolean; + skipInaccessible?: boolean; +} + +const DEFAULTS: Required = { + terminators: false, + details: false, + threshold: 1024, + recurse: true, + skipHidden: true, + skipLinks: true, + skipInaccessible: true, +}; + +async function walkDir( + dirPath: string, + progress: (entries: IEntry[]) => void, + opts: Required, +): Promise { + let names: string[]; + try { + names = await fs.promises.readdir(dirPath); + } catch (err: any) { + if (err?.code === "ENOENT") return; + if (opts.skipInaccessible && (err?.code === "EACCES" || err?.code === "EPERM")) return; + throw err; + } + + const entries: IEntry[] = []; + const subDirs: string[] = []; + + for (const name of names) { + if (opts.skipHidden && name.startsWith(".")) continue; + + const fullPath = path.join(dirPath, name); + let stats: fs.Stats; + try { + stats = await fs.promises.lstat(fullPath); + } catch (err: any) { + if (opts.skipInaccessible && (err?.code === "EACCES" || err?.code === "EPERM" || err?.code === "ENOENT")) { + continue; + } + throw err; + } + + const isLink = stats.isSymbolicLink(); + const isDir = stats.isDirectory(); + + entries.push({ + filePath: fullPath, + isDirectory: isDir, + isReparsePoint: isLink, + size: stats.size, + mtime: Math.floor(stats.mtimeMs / 1000), + ...(opts.details + ? { id: stats.ino, idStr: String(stats.ino), linkCount: stats.nlink } + : {}), + }); + + if (isDir && opts.recurse && !(opts.skipLinks && isLink)) { + subDirs.push(fullPath); + } + } + + if (entries.length > 0) { + progress(entries); + } + + for (const sub of subDirs) { + await walkDir(sub, progress, opts); + } +} + +async function turbowalk( + basePath: string, + progress: (entries: IEntry[]) => void, + options?: IWalkOptions, +): Promise { + if (basePath === undefined) { + throw new Error("expected at least one parameter"); + } + const opts = { ...DEFAULTS, ...options }; + await walkDir(path.normalize(basePath), progress, opts); +} + +export default turbowalk; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 85b5d1eb37..3a6d1c1fec 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -700,9 +700,6 @@ catalogs: tsdown: specifier: 0.21.0-beta.1 version: 0.21.0-beta.1 - turbowalk: - specifier: git+https://github.com/Nexus-Mods/node-turbowalk#3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f - version: 3.1.1 typed-binary: specifier: ^4.3.3 version: 4.3.3 @@ -1064,8 +1061,8 @@ importers: specifier: 'catalog:' version: 1.0.3 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk typescript: specifier: 'catalog:' version: 5.9.3 @@ -1181,8 +1178,8 @@ importers: specifier: 'catalog:' version: 2.4.2(redux@4.2.1) turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk typescript: specifier: 'catalog:' version: 5.9.3 @@ -1729,8 +1726,8 @@ importers: specifier: 'catalog:' version: 2.4.2(redux@4.2.1) turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk vortex-api: specifier: workspace:* version: link:../../packages/vortex-api @@ -2279,8 +2276,8 @@ importers: specifier: 'catalog:' version: 7.7.4 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../../packages/turbowalk devDependencies: '@nexusmods/nexus-api': specifier: 'catalog:' @@ -2487,8 +2484,8 @@ importers: specifier: 'catalog:' version: 2.4.1 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk typescript: specifier: 'catalog:' version: 5.9.3 @@ -2550,8 +2547,8 @@ importers: specifier: 'catalog:' version: 2.4.2(redux@4.2.1) turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk typescript: specifier: 'catalog:' version: 5.9.3 @@ -2872,8 +2869,8 @@ importers: specifier: 'catalog:' version: 4.2.1 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk vortex-api: specifier: workspace:* version: link:../../packages/vortex-api @@ -2992,8 +2989,8 @@ importers: specifier: 'catalog:' version: 2.2.17 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk vortex-api: specifier: workspace:* version: link:../../packages/vortex-api @@ -3049,8 +3046,8 @@ importers: specifier: 'catalog:' version: 41.3.0 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk typescript: specifier: 'catalog:' version: 5.9.3 @@ -3348,8 +3345,8 @@ importers: specifier: 'catalog:' version: 4.2.1 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk typescript: specifier: 'catalog:' version: 5.9.3 @@ -3537,8 +3534,8 @@ importers: specifier: 'catalog:' version: 0.33.1(react-dom@16.14.0)(react@16.14.0) turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk vortex-api: specifier: workspace:* version: link:../../packages/vortex-api @@ -3676,8 +3673,8 @@ importers: specifier: 'catalog:' version: 0.33.1(react-dom@16.14.0)(react@16.14.0) turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk vortex-api: specifier: workspace:* version: link:../../packages/vortex-api @@ -3851,6 +3848,8 @@ importers: specifier: 'catalog:' version: 2.0.0(typescript@5.9.3)(vite@8.0.3)(vitest@4.1.2) + packages/turbowalk: {} + packages/vortex-api: dependencies: '@types/bluebird': @@ -4272,8 +4271,8 @@ importers: specifier: 'catalog:' version: 6.0.1 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk universal-analytics: specifier: 'catalog:' version: 0.4.23 @@ -4913,8 +4912,8 @@ importers: specifier: 'catalog:' version: 0.1.0 turbowalk: - specifier: 'catalog:' - version: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f + specifier: workspace:* + version: link:../../packages/turbowalk universal-analytics: specifier: 'catalog:' version: 0.4.23 @@ -12231,10 +12230,6 @@ packages: tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} - turbowalk@https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f: - resolution: {tarball: https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f} - version: 3.1.1 - tweetnacl@0.14.5: resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==} @@ -21217,13 +21212,6 @@ snapshots: dependencies: safe-buffer: 5.2.1 - turbowalk@https://codeload.github.com/Nexus-Mods/node-turbowalk/tar.gz/3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f: - dependencies: - fs-extra: 8.1.0 - winapi-bindings: https://codeload.github.com/Nexus-Mods/node-winapi-bindings/tar.gz/faa92afe3320731e98abc15b3f5f19c60896d7c1 - transitivePeerDependencies: - - supports-color - tweetnacl@0.14.5: {} type-check@0.4.0: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index dd39906b23..a707a41161 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -266,7 +266,6 @@ catalog: ts-v-gen: ^1.0.1 tsconfig-paths-webpack-plugin: ^4.2.0 tsdown: 0.21.0-beta.1 - turbowalk: git+https://github.com/Nexus-Mods/node-turbowalk#3502f6ffc3f9eb55fe1c9c097b4e4772edce0c0f typed-binary: ^4.3.3 typescript: 5.9.3 typescript-eslint: 8.47.0 diff --git a/src/main/package.json b/src/main/package.json index 8591f17ec8..f4f56634b3 100644 --- a/src/main/package.json +++ b/src/main/package.json @@ -128,7 +128,7 @@ "tailwindcss": "catalog:", "tmp": "catalog:", "tough-cookie": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "universal-analytics": "catalog:", "uuid": "catalog:", "vortex-parse-ini": "catalog:", diff --git a/src/renderer/package.json b/src/renderer/package.json index fb840335cb..1cee94ebe6 100644 --- a/src/renderer/package.json +++ b/src/renderer/package.json @@ -171,7 +171,7 @@ "string-template": "catalog:", "tailwindcss": "catalog:", "tmp": "catalog:", - "turbowalk": "catalog:", + "turbowalk": "workspace:*", "universal-analytics": "catalog:", "uuid": "catalog:", "vortex-parse-ini": "catalog:", From 46ff372b81f99009c0343415dbd268878a50550d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Apr 2026 18:20:29 -0700 Subject: [PATCH 02/14] Fix Bluebird return type and add concurrent lstat for performance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Return Bluebird instead of Promise to match the type signature callers expect (fixes typecheck on Linux CI) - Add concurrent lstat batching (64 workers) to improve walk performance — roughly 2x faster than sequential on NVME - Add bluebird as devDependency for the workspace package --- packages/turbowalk/package.json | 6 +++- packages/turbowalk/src/index.ts | 63 ++++++++++++++++++++++++--------- pnpm-lock.yaml | 9 ++++- 3 files changed, 60 insertions(+), 18 deletions(-) diff --git a/packages/turbowalk/package.json b/packages/turbowalk/package.json index 293abf6b76..007db04bda 100644 --- a/packages/turbowalk/package.json +++ b/packages/turbowalk/package.json @@ -5,5 +5,9 @@ "main": "src/index.ts", "types": "src/index.ts", "private": true, - "type": "commonjs" + "type": "commonjs", + "devDependencies": { + "bluebird": "catalog:", + "@types/bluebird": "catalog:" + } } diff --git a/packages/turbowalk/src/index.ts b/packages/turbowalk/src/index.ts index 78bf9fbbd2..df7391a150 100644 --- a/packages/turbowalk/src/index.ts +++ b/packages/turbowalk/src/index.ts @@ -1,3 +1,4 @@ +import Bluebird from "bluebird"; import * as fs from "fs"; import * as path from "path"; @@ -33,6 +34,38 @@ const DEFAULTS: Required = { skipInaccessible: true, }; +const STAT_CONCURRENCY = 64; + +async function statBatch( + paths: string[], + opts: Required, +): Promise<(fs.Stats | null)[]> { + const results: (fs.Stats | null)[] = new Array(paths.length); + let cursor = 0; + + async function next(): Promise { + while (cursor < paths.length) { + const idx = cursor++; + try { + results[idx] = await fs.promises.lstat(paths[idx]); + } catch (err: any) { + if ( + opts.skipInaccessible && + (err?.code === "EACCES" || err?.code === "EPERM" || err?.code === "ENOENT") + ) { + results[idx] = null; + } else { + throw err; + } + } + } + } + + const workers = Math.min(STAT_CONCURRENCY, paths.length); + await Promise.all(Array.from({ length: workers }, () => next())); + return results; +} + async function walkDir( dirPath: string, progress: (entries: IEntry[]) => void, @@ -47,23 +80,21 @@ async function walkDir( throw err; } + if (opts.skipHidden) { + names = names.filter((n) => !n.startsWith(".")); + } + + const fullPaths = names.map((n) => path.join(dirPath, n)); + const statResults = await statBatch(fullPaths, opts); + const entries: IEntry[] = []; const subDirs: string[] = []; - for (const name of names) { - if (opts.skipHidden && name.startsWith(".")) continue; - - const fullPath = path.join(dirPath, name); - let stats: fs.Stats; - try { - stats = await fs.promises.lstat(fullPath); - } catch (err: any) { - if (opts.skipInaccessible && (err?.code === "EACCES" || err?.code === "EPERM" || err?.code === "ENOENT")) { - continue; - } - throw err; - } + for (let i = 0; i < names.length; i++) { + const stats = statResults[i]; + if (stats === null) continue; + const fullPath = fullPaths[i]; const isLink = stats.isSymbolicLink(); const isDir = stats.isDirectory(); @@ -92,16 +123,16 @@ async function walkDir( } } -async function turbowalk( +function turbowalk( basePath: string, progress: (entries: IEntry[]) => void, options?: IWalkOptions, -): Promise { +): Bluebird { if (basePath === undefined) { throw new Error("expected at least one parameter"); } const opts = { ...DEFAULTS, ...options }; - await walkDir(path.normalize(basePath), progress, opts); + return Bluebird.resolve(walkDir(path.normalize(basePath), progress, opts)); } export default turbowalk; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3a6d1c1fec..0a6ce05c4d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3848,7 +3848,14 @@ importers: specifier: 'catalog:' version: 2.0.0(typescript@5.9.3)(vite@8.0.3)(vitest@4.1.2) - packages/turbowalk: {} + packages/turbowalk: + devDependencies: + '@types/bluebird': + specifier: 'catalog:' + version: 3.5.20 + bluebird: + specifier: 'catalog:' + version: 3.7.2 packages/vortex-api: dependencies: From 20776e37f235d3ce6e4883e7b324ddbd5a0e21d7 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Apr 2026 18:23:30 -0700 Subject: [PATCH 03/14] Fix pre-existing Bluebird vs Bluebird type mismatch purgeLinks() returns PromiseBB but the .then() chain ended with PromiseBB.map() which returns Bluebird. Add .then(() => undefined) to satisfy the return type. This was masked by the old turbowalk type declarations. --- src/renderer/src/extensions/move_activator/index.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/renderer/src/extensions/move_activator/index.ts b/src/renderer/src/extensions/move_activator/index.ts index 97f8f333ec..ea7c204f8e 100644 --- a/src/renderer/src/extensions/move_activator/index.ts +++ b/src/renderer/src/extensions/move_activator/index.ts @@ -278,9 +278,11 @@ class DeploymentMethod extends LinkingDeployment { { details: true, }, - ).then(() => - PromiseBB.map(links, (entry) => this.restoreLink(entry.filePath)), - ); + ) + .then(() => + PromiseBB.map(links, (entry) => this.restoreLink(entry.filePath)), + ) + .then(() => undefined); } protected linkFile( From 2b01623871bc31407d7cf96d521398bec348253f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Apr 2026 18:27:13 -0700 Subject: [PATCH 04/14] Fix strict TypeScript errors in turbowalk implementation --- packages/turbowalk/src/index.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/turbowalk/src/index.ts b/packages/turbowalk/src/index.ts index df7391a150..39ae26324d 100644 --- a/packages/turbowalk/src/index.ts +++ b/packages/turbowalk/src/index.ts @@ -47,7 +47,7 @@ async function statBatch( while (cursor < paths.length) { const idx = cursor++; try { - results[idx] = await fs.promises.lstat(paths[idx]); + results[idx] = await fs.promises.lstat(paths[idx]!); } catch (err: any) { if ( opts.skipInaccessible && @@ -62,7 +62,9 @@ async function statBatch( } const workers = Math.min(STAT_CONCURRENCY, paths.length); - await Promise.all(Array.from({ length: workers }, () => next())); + const tasks: Promise[] = []; + for (let i = 0; i < workers; i++) tasks.push(next()); + await Promise.all(tasks); return results; } @@ -91,10 +93,10 @@ async function walkDir( const subDirs: string[] = []; for (let i = 0; i < names.length; i++) { - const stats = statResults[i]; + const stats = statResults[i]!; if (stats === null) continue; - const fullPath = fullPaths[i]; + const fullPath = fullPaths[i]!; const isLink = stats.isSymbolicLink(); const isDir = stats.isDirectory(); From a18c596763c6b4023400fc397913affff4e5e966 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Apr 2026 18:43:58 -0700 Subject: [PATCH 05/14] Add koffi FFI walk using FindFirstFileW/FindNextFileW on Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Uses koffi to call Win32 directory enumeration APIs directly, which returns attributes, size, and timestamps in a single call per entry — no separate lstat() needed. This also enables proper FILE_ATTRIBUTE_HIDDEN checking instead of dot-prefix convention. Falls back to the async fs.readdir + lstat path on Linux or if koffi is unavailable. Benchmark (Samsung 970 EVO NVMe, Windows 11): | Directory | Old Native | koffi FFI | Pure TS | |--------------|-----------|-----------|----------| | src/ (1.9k) | 8.8ms | 19.1ms | 62.0ms | | ext/ (3.9k) | 22.1ms | 48.9ms | 170.8ms | | n_m/ (89k) | 341ms | 1005ms | 3426ms | koffi is ~2-3x the old C++ addon vs 7-10x for pure TS — acceptable for a zero-compile solution. --- packages/turbowalk/package.json | 5 +- packages/turbowalk/src/index.ts | 41 ++++++- packages/turbowalk/src/walkWindows.ts | 160 ++++++++++++++++++++++++++ pnpm-lock.yaml | 8 ++ pnpm-workspace.yaml | 1 + 5 files changed, 210 insertions(+), 5 deletions(-) create mode 100644 packages/turbowalk/src/walkWindows.ts diff --git a/packages/turbowalk/package.json b/packages/turbowalk/package.json index 007db04bda..2a2052d364 100644 --- a/packages/turbowalk/package.json +++ b/packages/turbowalk/package.json @@ -1,13 +1,14 @@ { "name": "turbowalk", "version": "4.0.0", - "description": "Pure TypeScript directory walking (no native addons)", + "description": "Fast directory walking using koffi FFI on Windows, fs on Linux", "main": "src/index.ts", "types": "src/index.ts", "private": true, "type": "commonjs", "devDependencies": { "bluebird": "catalog:", - "@types/bluebird": "catalog:" + "@types/bluebird": "catalog:", + "koffi": "^2.9.0" } } diff --git a/packages/turbowalk/src/index.ts b/packages/turbowalk/src/index.ts index 39ae26324d..f24c9dc23d 100644 --- a/packages/turbowalk/src/index.ts +++ b/packages/turbowalk/src/index.ts @@ -34,6 +34,28 @@ const DEFAULTS: Required = { skipInaccessible: true, }; +// --- Windows: koffi FFI using FindFirstFileW/FindNextFileW --- + +let walkDirWindows: (( + dirPath: string, + progress: (entries: IEntry[]) => void, + opts: Required, +) => void) | undefined; + +if (process.platform === "win32") { + try { + // Dynamic require so it doesn't fail on Linux + const mod = require("./walkWindows") as { + walkDirWindows: typeof walkDirWindows; + }; + walkDirWindows = mod.walkDirWindows; + } catch { + // koffi not available — fall through to TS fallback + } +} + +// --- Linux/fallback: async fs.readdir + concurrent lstat --- + const STAT_CONCURRENCY = 64; async function statBatch( @@ -68,7 +90,7 @@ async function statBatch( return results; } -async function walkDir( +async function walkDirFallback( dirPath: string, progress: (entries: IEntry[]) => void, opts: Required, @@ -121,10 +143,12 @@ async function walkDir( } for (const sub of subDirs) { - await walkDir(sub, progress, opts); + await walkDirFallback(sub, progress, opts); } } +// --- Public API --- + function turbowalk( basePath: string, progress: (entries: IEntry[]) => void, @@ -134,7 +158,18 @@ function turbowalk( throw new Error("expected at least one parameter"); } const opts = { ...DEFAULTS, ...options }; - return Bluebird.resolve(walkDir(path.normalize(basePath), progress, opts)); + const normalized = path.normalize(basePath); + + if (walkDirWindows !== undefined) { + try { + walkDirWindows(normalized, progress, opts); + return Bluebird.resolve(); + } catch (err) { + return Bluebird.reject(err); + } + } + + return Bluebird.resolve(walkDirFallback(normalized, progress, opts)); } export default turbowalk; diff --git a/packages/turbowalk/src/walkWindows.ts b/packages/turbowalk/src/walkWindows.ts new file mode 100644 index 0000000000..789c1bd49b --- /dev/null +++ b/packages/turbowalk/src/walkWindows.ts @@ -0,0 +1,160 @@ +/** + * Fast directory walking on Windows using koffi FFI. + * + * Calls FindFirstFileW / FindNextFileW / FindClose directly, which returns + * file attributes, size, and timestamps in a single syscall per directory + * entry — no separate lstat() needed. + */ + +import * as path from "path"; +import type { IEntry, IWalkOptions } from "./index"; + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const koffi = require("koffi"); + +// --- Win32 constants --- + +const INVALID_HANDLE_VALUE = -1n; +const FILE_ATTRIBUTE_DIRECTORY = 0x10; +const FILE_ATTRIBUTE_HIDDEN = 0x2; +const FILE_ATTRIBUTE_REPARSE_POINT = 0x400; +const MAX_PATH = 260; + +// --- Win32 struct and function definitions --- + +// FILETIME: two 32-bit values representing 100ns intervals since 1601-01-01 +const FILETIME = koffi.struct("FILETIME", { + dwLowDateTime: "uint32", + dwHighDateTime: "uint32", +}); + +// WIN32_FIND_DATAW: the struct returned by FindFirstFileW / FindNextFileW +const WIN32_FIND_DATAW = koffi.struct("WIN32_FIND_DATAW", { + dwFileAttributes: "uint32", + ftCreationTime: FILETIME, + ftLastAccessTime: FILETIME, + ftLastWriteTime: FILETIME, + nFileSizeHigh: "uint32", + nFileSizeLow: "uint32", + dwReserved0: "uint32", + dwReserved1: "uint32", + cFileName: koffi.array("uint16", MAX_PATH), + cAlternateFileName: koffi.array("uint16", 14), +}); + +const kernel32 = koffi.load("kernel32.dll"); + +const FindFirstFileW = kernel32.func( + "intptr FindFirstFileW(const uint16 *lpFileName, _Out_ WIN32_FIND_DATAW *lpFindFileData)", +); +const FindNextFileW = kernel32.func( + "bool FindNextFileW(intptr hFindFile, _Out_ WIN32_FIND_DATAW *lpFindFileData)", +); +const FindClose = kernel32.func("bool FindClose(intptr hFindFile)"); + +// --- Helpers --- + +/** Convert a JS string to a null-terminated UTF-16LE buffer for Win32 wide APIs. */ +function toWideString(str: string): Buffer { + const buf = Buffer.alloc((str.length + 1) * 2); + for (let i = 0; i < str.length; i++) { + buf.writeUInt16LE(str.charCodeAt(i), i * 2); + } + // null terminator is already 0 from Buffer.alloc + return buf; +} + +/** Read a null-terminated UTF-16LE filename from the cFileName array. */ +function readFileName(arr: number[]): string { + const codes: number[] = []; + for (let i = 0; i < arr.length; i++) { + if (arr[i] === 0) break; + codes.push(arr[i]!); + } + return String.fromCharCode(...codes); +} + +/** Convert FILETIME to Unix epoch seconds. */ +function filetimeToUnix(ft: { dwLowDateTime: number; dwHighDateTime: number }): number { + // FILETIME is 100ns intervals since 1601-01-01 + // Unix epoch offset: 11644473600 seconds + const ticks = (BigInt(ft.dwHighDateTime) << 32n) | BigInt(ft.dwLowDateTime >>> 0); + return Number(ticks / 10000000n) - 11644473600; +} + +/** Combine nFileSizeHigh and nFileSizeLow into a single number. */ +function combineSize(high: number, low: number): number { + if (high === 0) return low >>> 0; + return Number((BigInt(high) << 32n) | BigInt(low >>> 0)); +} + +// --- Walk implementation --- + +export function walkDirWindows( + dirPath: string, + progress: (entries: IEntry[]) => void, + opts: Required, +): void { + // Append \* for the search pattern + const searchPath = path.join(dirPath, "*"); + const wideSearch = toWideString(searchPath); + + const findData: Record = {}; + const handle = FindFirstFileW(wideSearch, findData) as bigint; + + if (handle === INVALID_HANDLE_VALUE) { + // Directory doesn't exist or is inaccessible + return; + } + + try { + const entries: IEntry[] = []; + const subDirs: string[] = []; + + do { + const fd = findData as { + dwFileAttributes: number; + ftLastWriteTime: { dwLowDateTime: number; dwHighDateTime: number }; + nFileSizeHigh: number; + nFileSizeLow: number; + cFileName: number[]; + }; + + const name = readFileName(fd.cFileName); + + // Skip . and .. + if (name === "." || name === "..") continue; + + const attrs = fd.dwFileAttributes; + const isHidden = (attrs & FILE_ATTRIBUTE_HIDDEN) !== 0; + const isDir = (attrs & FILE_ATTRIBUTE_DIRECTORY) !== 0; + const isReparsePoint = (attrs & FILE_ATTRIBUTE_REPARSE_POINT) !== 0; + + if (opts.skipHidden && isHidden) continue; + + const fullPath = path.join(dirPath, name); + + entries.push({ + filePath: fullPath, + isDirectory: isDir, + isReparsePoint, + size: combineSize(fd.nFileSizeHigh, fd.nFileSizeLow), + mtime: filetimeToUnix(fd.ftLastWriteTime), + }); + + if (isDir && opts.recurse && !(opts.skipLinks && isReparsePoint)) { + subDirs.push(fullPath); + } + } while (FindNextFileW(handle, findData)); + + if (entries.length > 0) { + progress(entries); + } + + for (const sub of subDirs) { + walkDirWindows(sub, progress, opts); + } + } finally { + FindClose(handle); + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0a6ce05c4d..b203f73ea3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3856,6 +3856,9 @@ importers: bluebird: specifier: 'catalog:' version: 3.7.2 + koffi: + specifier: ^2.9.0 + version: 2.16.1 packages/vortex-api: dependencies: @@ -10128,6 +10131,9 @@ packages: resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} engines: {node: '>=0.10.0'} + koffi@2.16.1: + resolution: {integrity: sha512-0Ie6CfD026dNfWSosDw9dPxPzO9Rlyo0N8m5r05S8YjytIpuilzMFDMY4IDy/8xQsTwpuVinhncD+S8n3bcYZQ==} + layout-base@1.0.2: resolution: {integrity: sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg==} @@ -18801,6 +18807,8 @@ snapshots: kind-of@6.0.3: {} + koffi@2.16.1: {} + layout-base@1.0.2: {} lazy-val@1.0.5: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index a707a41161..9815413d23 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -21,6 +21,7 @@ allowBuilds: electron: true esbuild: true + koffi: true font-scanner: true gamebryo-savegame: true leveldown: true From b3640f0966cfec1fe24c6d19bb06e8177690f5a2 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Apr 2026 18:52:21 -0700 Subject: [PATCH 06/14] Switch from FindFirstFileW to NtQueryDirectoryFile for bulk iteration NtQueryDirectoryFile (the same NT API the old C++ addon used) returns multiple directory entries per syscall in a packed 64KB buffer, avoiding per-entry syscall overhead. Buffer parsing is done entirely in JS. Benchmark (Samsung 970 EVO NVMe, Windows 11): | Directory | Old Native | koffi NtQuery | koffi FindFirst | |--------------|-----------|---------------|-----------------| | src/ (1.9k) | 8.8ms | 13.0ms | 19.1ms | | ext/ (3.9k) | 22.1ms | 42.7ms | 48.9ms | | n_m/ (89k) | 341ms | 724ms | 1005ms | Now ~1.5-2.1x the old native addon, down from 2.2-2.9x with FindFirst. --- packages/turbowalk/src/walkWindows.ts | 297 +++++++++++++++++--------- 1 file changed, 195 insertions(+), 102 deletions(-) diff --git a/packages/turbowalk/src/walkWindows.ts b/packages/turbowalk/src/walkWindows.ts index 789c1bd49b..76748231f2 100644 --- a/packages/turbowalk/src/walkWindows.ts +++ b/packages/turbowalk/src/walkWindows.ts @@ -1,9 +1,9 @@ /** * Fast directory walking on Windows using koffi FFI. * - * Calls FindFirstFileW / FindNextFileW / FindClose directly, which returns - * file attributes, size, and timestamps in a single syscall per directory - * entry — no separate lstat() needed. + * Calls NtQueryDirectoryFile (the same NT API the old C++ addon used) which + * returns multiple directory entries per syscall in a packed buffer. This + * avoids the per-entry syscall overhead of FindFirstFileW/FindNextFileW. */ import * as path from "path"; @@ -12,149 +12,242 @@ import type { IEntry, IWalkOptions } from "./index"; // eslint-disable-next-line @typescript-eslint/no-var-requires const koffi = require("koffi"); -// --- Win32 constants --- +// --- NT constants --- -const INVALID_HANDLE_VALUE = -1n; +const STATUS_SUCCESS = 0; +const STATUS_NO_MORE_FILES = 0x80000006 | 0; // signed const FILE_ATTRIBUTE_DIRECTORY = 0x10; const FILE_ATTRIBUTE_HIDDEN = 0x2; const FILE_ATTRIBUTE_REPARSE_POINT = 0x400; -const MAX_PATH = 260; -// --- Win32 struct and function definitions --- +// CreateFileW constants +const GENERIC_READ = 0x80000000; +const FILE_SHARE_READ = 1; +const FILE_SHARE_WRITE = 2; +const FILE_SHARE_DELETE = 4; +const OPEN_EXISTING = 3; +const FILE_FLAG_BACKUP_SEMANTICS = 0x02000000; +const INVALID_HANDLE_VALUE = BigInt(-1); -// FILETIME: two 32-bit values representing 100ns intervals since 1601-01-01 -const FILETIME = koffi.struct("FILETIME", { - dwLowDateTime: "uint32", - dwHighDateTime: "uint32", -}); +// NtQueryDirectoryFile information class +const FileFullDirectoryInformation = 2; -// WIN32_FIND_DATAW: the struct returned by FindFirstFileW / FindNextFileW -const WIN32_FIND_DATAW = koffi.struct("WIN32_FIND_DATAW", { - dwFileAttributes: "uint32", - ftCreationTime: FILETIME, - ftLastAccessTime: FILETIME, - ftLastWriteTime: FILETIME, - nFileSizeHigh: "uint32", - nFileSizeLow: "uint32", - dwReserved0: "uint32", - dwReserved1: "uint32", - cFileName: koffi.array("uint16", MAX_PATH), - cAlternateFileName: koffi.array("uint16", 14), -}); +// FILETIME epoch offset: 100ns ticks between 1601-01-01 and 1970-01-01 +const UNIX_EPOCH_TICKS = 0x019DB1DED53E8000n; +const TICKS_PER_SECOND = 10000000n; + +// Buffer for NtQueryDirectoryFile — 64KB handles most directories in one call +const DIR_BUFFER_SIZE = 65536; + +// --- koffi function bindings --- const kernel32 = koffi.load("kernel32.dll"); +const ntdll = koffi.load("ntdll.dll"); + +// IO_STATUS_BLOCK — used by NtQueryDirectoryFile +const IO_STATUS_BLOCK = koffi.struct("IO_STATUS_BLOCK", { + Status: "int32", + Information: "uintptr", +}); -const FindFirstFileW = kernel32.func( - "intptr FindFirstFileW(const uint16 *lpFileName, _Out_ WIN32_FIND_DATAW *lpFindFileData)", +const CreateFileW = kernel32.func( + "intptr CreateFileW(const uint16 *lpFileName, uint32 dwDesiredAccess, uint32 dwShareMode, void *lpSecurityAttributes, uint32 dwCreationDisposition, uint32 dwFlagsAndAttributes, intptr hTemplateFile)", ); -const FindNextFileW = kernel32.func( - "bool FindNextFileW(intptr hFindFile, _Out_ WIN32_FIND_DATAW *lpFindFileData)", +const CloseHandle = kernel32.func("bool CloseHandle(intptr hObject)"); + +const NtQueryDirectoryFile = ntdll.func( + "int32 NtQueryDirectoryFile(intptr FileHandle, intptr Event, void *ApcRoutine, void *ApcContext, _Out_ IO_STATUS_BLOCK *IoStatusBlock, _Out_ uint8 *FileInformation, uint32 Length, int32 FileInformationClass, bool ReturnSingleEntry, void *FileName, bool RestartScan)", ); -const FindClose = kernel32.func("bool FindClose(intptr hFindFile)"); // --- Helpers --- -/** Convert a JS string to a null-terminated UTF-16LE buffer for Win32 wide APIs. */ -function toWideString(str: string): Buffer { +/** Convert a JS string to a null-terminated UTF-16LE buffer. */ +function toWide(str: string): Buffer { const buf = Buffer.alloc((str.length + 1) * 2); for (let i = 0; i < str.length; i++) { buf.writeUInt16LE(str.charCodeAt(i), i * 2); } - // null terminator is already 0 from Buffer.alloc return buf; } -/** Read a null-terminated UTF-16LE filename from the cFileName array. */ -function readFileName(arr: number[]): string { - const codes: number[] = []; - for (let i = 0; i < arr.length; i++) { - if (arr[i] === 0) break; - codes.push(arr[i]!); - } - return String.fromCharCode(...codes); +/** Open a directory handle for reading. */ +function openDirectory(dirPath: string): bigint { + // Use \\?\ prefix for long path support + const prefix = + dirPath.startsWith("\\\\") ? "" : "\\\\?\\"; + const widePath = toWide(prefix + dirPath + (dirPath.endsWith("\\") ? "" : "\\")); + return CreateFileW( + widePath, + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + null, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + 0n, + ) as bigint; } -/** Convert FILETIME to Unix epoch seconds. */ -function filetimeToUnix(ft: { dwLowDateTime: number; dwHighDateTime: number }): number { - // FILETIME is 100ns intervals since 1601-01-01 - // Unix epoch offset: 11644473600 seconds - const ticks = (BigInt(ft.dwHighDateTime) << 32n) | BigInt(ft.dwLowDateTime >>> 0); - return Number(ticks / 10000000n) - 11644473600; -} +/** + * Parse FILE_FULL_DIR_INFORMATION entries from the buffer returned by + * NtQueryDirectoryFile. Each entry is variable-length: + * + * ULONG NextEntryOffset (0) + * ULONG FileIndex (4) + * INT64 CreationTime (8) + * INT64 LastAccessTime (16) + * INT64 LastWriteTime (24) + * INT64 ChangeTime (32) + * INT64 EndOfFile (40) + * INT64 AllocationSize (48) + * ULONG FileAttributes (56) + * ULONG FileNameLength (60) + * ULONG EaSize (64) + * WCHAR FileName[1] (68) + */ +function parseEntries( + buf: Buffer, + dirPath: string, + opts: Required, + entries: IEntry[], + subDirs: string[], +): void { + let offset = 0; + + while (true) { + const nextEntryOffset = buf.readUInt32LE(offset); + const fileAttributes = buf.readUInt32LE(offset + 56); + const fileNameLength = buf.readUInt32LE(offset + 60); + + // Read the filename (UTF-16LE) + const nameStart = offset + 68; + const name = buf.toString("utf16le", nameStart, nameStart + fileNameLength); + + // Skip . and .. + if (name !== "." && name !== "..") { + const isHidden = (fileAttributes & FILE_ATTRIBUTE_HIDDEN) !== 0; + + if (!opts.skipHidden || !isHidden) { + const isDir = (fileAttributes & FILE_ATTRIBUTE_DIRECTORY) !== 0; + const isReparsePoint = (fileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) !== 0; + + // EndOfFile (actual file size) + const sizeLow = buf.readUInt32LE(offset + 40); + const sizeHigh = buf.readUInt32LE(offset + 44); + const size = sizeHigh === 0 ? sizeLow : Number((BigInt(sizeHigh) << 32n) | BigInt(sizeLow >>> 0)); + + // LastWriteTime → Unix seconds + const wtLow = buf.readUInt32LE(offset + 24); + const wtHigh = buf.readUInt32LE(offset + 28); + const ticks = (BigInt(wtHigh) << 32n) | BigInt(wtLow >>> 0); + const mtime = Number((ticks - UNIX_EPOCH_TICKS) / TICKS_PER_SECOND); + + const fullPath = dirPath + "\\" + name; + + entries.push({ + filePath: fullPath, + isDirectory: isDir, + isReparsePoint, + size, + mtime, + }); + + if (isDir && opts.recurse && !(opts.skipLinks && isReparsePoint)) { + subDirs.push(fullPath); + } + } + } -/** Combine nFileSizeHigh and nFileSizeLow into a single number. */ -function combineSize(high: number, low: number): number { - if (high === 0) return low >>> 0; - return Number((BigInt(high) << 32n) | BigInt(low >>> 0)); + if (nextEntryOffset === 0) break; + offset += nextEntryOffset; + } } // --- Walk implementation --- -export function walkDirWindows( +function walkDirInner( dirPath: string, - progress: (entries: IEntry[]) => void, + entries: IEntry[], + subDirs: string[], opts: Required, -): void { - // Append \* for the search pattern - const searchPath = path.join(dirPath, "*"); - const wideSearch = toWideString(searchPath); +): boolean { + const handle = openDirectory(dirPath); + if (handle === INVALID_HANDLE_VALUE) return true; // skip inaccessible - const findData: Record = {}; - const handle = FindFirstFileW(wideSearch, findData) as bigint; + const ioStatus: Record = {}; + const buf = Buffer.alloc(DIR_BUFFER_SIZE); - if (handle === INVALID_HANDLE_VALUE) { - // Directory doesn't exist or is inaccessible - return; + try { + while (true) { + const status = NtQueryDirectoryFile( + handle, + 0n, // Event + null, // ApcRoutine + null, // ApcContext + ioStatus, + buf, + DIR_BUFFER_SIZE, + FileFullDirectoryInformation, + false, // ReturnSingleEntry + null, // FileName (null = wildcard) + false, // RestartScan + ) as number; + + if (status === STATUS_SUCCESS) { + parseEntries(buf, dirPath, opts, entries, subDirs); + } else { + // STATUS_NO_MORE_FILES or any error — done with this directory + break; + } + } + } finally { + CloseHandle(handle); } - try { - const entries: IEntry[] = []; - const subDirs: string[] = []; + return true; +} - do { - const fd = findData as { - dwFileAttributes: number; - ftLastWriteTime: { dwLowDateTime: number; dwHighDateTime: number }; - nFileSizeHigh: number; - nFileSizeLow: number; - cFileName: number[]; - }; +export function walkDirWindows( + dirPath: string, + progress: (entries: IEntry[]) => void, + opts: Required, +): void { + const allEntries: IEntry[] = []; - const name = readFileName(fd.cFileName); + function walkRecursive(dir: string): void { + const entries: IEntry[] = []; + const subDirs: string[] = []; - // Skip . and .. - if (name === "." || name === "..") continue; + walkDirInner(dir, entries, subDirs, opts); - const attrs = fd.dwFileAttributes; - const isHidden = (attrs & FILE_ATTRIBUTE_HIDDEN) !== 0; - const isDir = (attrs & FILE_ATTRIBUTE_DIRECTORY) !== 0; - const isReparsePoint = (attrs & FILE_ATTRIBUTE_REPARSE_POINT) !== 0; + // Accumulate entries and flush when threshold is reached (matching native behavior) + for (const entry of entries) { + allEntries.push(entry); + } - if (opts.skipHidden && isHidden) continue; + if (allEntries.length >= opts.threshold) { + progress(allEntries.splice(0)); + } - const fullPath = path.join(dirPath, name); + for (const sub of subDirs) { + walkRecursive(sub); + } - entries.push({ - filePath: fullPath, - isDirectory: isDir, - isReparsePoint, - size: combineSize(fd.nFileSizeHigh, fd.nFileSizeLow), - mtime: filetimeToUnix(fd.ftLastWriteTime), + if (opts.terminators) { + allEntries.push({ + filePath: dir, + isDirectory: true, + isReparsePoint: false, + size: 0, + mtime: 0, + isTerminator: true, }); - - if (isDir && opts.recurse && !(opts.skipLinks && isReparsePoint)) { - subDirs.push(fullPath); - } - } while (FindNextFileW(handle, findData)); - - if (entries.length > 0) { - progress(entries); } + } - for (const sub of subDirs) { - walkDirWindows(sub, progress, opts); - } - } finally { - FindClose(handle); + walkRecursive(dirPath); + + // Flush remaining entries + if (allEntries.length > 0) { + progress(allEntries); } } From c9767094477037bf0166c86255c6e315f42a1310 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Apr 2026 19:37:03 -0700 Subject: [PATCH 07/14] Use 1KB buffer for NtQueryDirectoryFile (matches C++ addon) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Benchmarking revealed smaller buffers are faster because koffi marshals the entire buffer across the FFI boundary each call. A 1KB buffer minimizes that overhead while still fitting multiple entries per call. Results now match or beat the old C++ addon: src/ (1.9k): 8.8ms native → 7.0ms koffi ext/ (3.9k): 22.1ms native → 20.4ms koffi n_m/ (89k): 341ms native → 383ms koffi --- packages/turbowalk/src/walkWindows.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/turbowalk/src/walkWindows.ts b/packages/turbowalk/src/walkWindows.ts index 76748231f2..5487b5a727 100644 --- a/packages/turbowalk/src/walkWindows.ts +++ b/packages/turbowalk/src/walkWindows.ts @@ -36,8 +36,11 @@ const FileFullDirectoryInformation = 2; const UNIX_EPOCH_TICKS = 0x019DB1DED53E8000n; const TICKS_PER_SECOND = 10000000n; -// Buffer for NtQueryDirectoryFile — 64KB handles most directories in one call -const DIR_BUFFER_SIZE = 65536; +// Buffer for NtQueryDirectoryFile — 1KB matches the old C++ addon's buffer size. +// Counterintuitively, smaller buffers are faster because koffi marshals the +// entire buffer across the FFI boundary each call. 1KB minimizes that overhead +// while still fitting multiple entries per call. +const DIR_BUFFER_SIZE = 1024; // --- koffi function bindings --- From 71b32876aa7f7176005d4ac9c1475a62f5a4bae7 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Apr 2026 19:42:10 -0700 Subject: [PATCH 08/14] Add tests, proper error handling, and details option support - Add 9 tests covering recursive walk, skipHidden, recurse:false, non-existent dirs, file sizes, mtime, batched callbacks, and paths - Fix error handling: check GetLastError after CreateFileW, retry on sharing violations (matching C++ behavior), throw on unexpected errors when skipInaccessible is false - Implement details option: call NtQueryInformationFile with FileAllInformation to get linkCount, id, and idStr (same as C++) - Reuse buffer and ioStatus across directories to avoid per-dir allocs - Fix stale comment referencing FindFirstFileW --- packages/turbowalk/src/index.test.ts | 114 +++++++++++++ packages/turbowalk/src/index.ts | 2 +- packages/turbowalk/src/walkWindows.ts | 223 +++++++++++++++++++------- packages/turbowalk/vitest.config.ts | 8 + 4 files changed, 289 insertions(+), 58 deletions(-) create mode 100644 packages/turbowalk/src/index.test.ts create mode 100644 packages/turbowalk/vitest.config.ts diff --git a/packages/turbowalk/src/index.test.ts b/packages/turbowalk/src/index.test.ts new file mode 100644 index 0000000000..7d3d5b1dc0 --- /dev/null +++ b/packages/turbowalk/src/index.test.ts @@ -0,0 +1,114 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; + +import turbowalk, { type IEntry } from "./index"; + +let tmpDir: string; + +beforeAll(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "turbowalk-test-")); + + // Create a directory tree: + // root/ + // file1.txt (10 bytes) + // file2.dat (20 bytes) + // .hidden (0 bytes) + // subdir/ + // nested.txt (5 bytes) + // emptydir/ + // .hiddendir/ + // inside.txt (1 byte) + fs.writeFileSync(path.join(tmpDir, "file1.txt"), "0123456789"); + fs.writeFileSync(path.join(tmpDir, "file2.dat"), "01234567890123456789"); + fs.writeFileSync(path.join(tmpDir, ".hidden"), ""); + fs.mkdirSync(path.join(tmpDir, "subdir")); + fs.writeFileSync(path.join(tmpDir, "subdir", "nested.txt"), "hello"); + fs.mkdirSync(path.join(tmpDir, "emptydir")); + fs.mkdirSync(path.join(tmpDir, ".hiddendir")); + fs.writeFileSync(path.join(tmpDir, ".hiddendir", "inside.txt"), "x"); +}); + +afterAll(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +function collect(dir: string, options?: Parameters[2]): Promise { + const all: IEntry[] = []; + return turbowalk(dir, (entries) => { all.push(...entries); }, options).then(() => all); +} + +describe("turbowalk", () => { + it("walks all files and directories recursively", async () => { + const entries = await collect(tmpDir, { skipHidden: false }); + const names = entries.map((e) => path.basename(e.filePath)).sort(); + expect(names).toEqual([ + ".hidden", ".hiddendir", "emptydir", "file1.txt", "file2.dat", + "inside.txt", "nested.txt", "subdir", + ]); + }); + + it("skips hidden files and directories by default", async () => { + const entries = await collect(tmpDir); + const names = entries.map((e) => path.basename(e.filePath)).sort(); + // .hidden, .hiddendir, and inside.txt (inside .hiddendir) should be excluded + expect(names).toEqual(["emptydir", "file1.txt", "file2.dat", "nested.txt", "subdir"]); + }); + + it("reports correct isDirectory flag", async () => { + const entries = await collect(tmpDir, { skipHidden: false }); + const dirs = entries.filter((e) => e.isDirectory).map((e) => path.basename(e.filePath)).sort(); + const files = entries.filter((e) => !e.isDirectory).map((e) => path.basename(e.filePath)).sort(); + expect(dirs).toEqual([".hiddendir", "emptydir", "subdir"]); + expect(files).toEqual([".hidden", "file1.txt", "file2.dat", "inside.txt", "nested.txt"]); + }); + + it("reports correct file sizes", async () => { + const entries = await collect(tmpDir, { skipHidden: false }); + const file1 = entries.find((e) => path.basename(e.filePath) === "file1.txt"); + const file2 = entries.find((e) => path.basename(e.filePath) === "file2.dat"); + expect(file1?.size).toBe(10); + expect(file2?.size).toBe(20); + }); + + it("reports mtime as unix seconds", async () => { + const entries = await collect(tmpDir, { skipHidden: false }); + const file1 = entries.find((e) => path.basename(e.filePath) === "file1.txt"); + expect(file1?.mtime).toBeGreaterThan(0); + // Should be within a few seconds of now + const now = Math.floor(Date.now() / 1000); + expect(file1!.mtime).toBeGreaterThan(now - 60); + expect(file1!.mtime).toBeLessThanOrEqual(now + 1); + }); + + it("respects recurse: false", async () => { + const entries = await collect(tmpDir, { recurse: false, skipHidden: false }); + const names = entries.map((e) => path.basename(e.filePath)).sort(); + // Should only include top-level entries, not nested.txt or inside.txt + expect(names).toEqual([".hidden", ".hiddendir", "emptydir", "file1.txt", "file2.dat", "subdir"]); + }); + + it("handles non-existent directory gracefully", async () => { + const entries = await collect(path.join(tmpDir, "nonexistent")); + expect(entries).toEqual([]); + }); + + it("calls progress callback with batched entries", async () => { + const calls: number[] = []; + await turbowalk(tmpDir, (entries) => { calls.push(entries.length); }, { skipHidden: false }); + // At least one call should have been made + expect(calls.length).toBeGreaterThan(0); + // Total entries across all calls should match + const total = calls.reduce((a, b) => a + b, 0); + expect(total).toBe(8); // 5 files + 3 dirs + }); + + it("returns full absolute paths", async () => { + const entries = await collect(tmpDir, { skipHidden: false }); + for (const entry of entries) { + expect(path.isAbsolute(entry.filePath)).toBe(true); + expect(entry.filePath.startsWith(tmpDir)).toBe(true); + } + }); +}); diff --git a/packages/turbowalk/src/index.ts b/packages/turbowalk/src/index.ts index f24c9dc23d..dbbf2f8bfd 100644 --- a/packages/turbowalk/src/index.ts +++ b/packages/turbowalk/src/index.ts @@ -34,7 +34,7 @@ const DEFAULTS: Required = { skipInaccessible: true, }; -// --- Windows: koffi FFI using FindFirstFileW/FindNextFileW --- +// --- Windows: koffi FFI using NtQueryDirectoryFile --- let walkDirWindows: (( dirPath: string, diff --git a/packages/turbowalk/src/walkWindows.ts b/packages/turbowalk/src/walkWindows.ts index 5487b5a727..e7c64e8fa4 100644 --- a/packages/turbowalk/src/walkWindows.ts +++ b/packages/turbowalk/src/walkWindows.ts @@ -15,7 +15,6 @@ const koffi = require("koffi"); // --- NT constants --- const STATUS_SUCCESS = 0; -const STATUS_NO_MORE_FILES = 0x80000006 | 0; // signed const FILE_ATTRIBUTE_DIRECTORY = 0x10; const FILE_ATTRIBUTE_HIDDEN = 0x2; const FILE_ATTRIBUTE_REPARSE_POINT = 0x400; @@ -27,10 +26,19 @@ const FILE_SHARE_WRITE = 2; const FILE_SHARE_DELETE = 4; const OPEN_EXISTING = 3; const FILE_FLAG_BACKUP_SEMANTICS = 0x02000000; +const FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000; const INVALID_HANDLE_VALUE = BigInt(-1); +// GetLastError codes +const ERROR_FILE_NOT_FOUND = 2; +const ERROR_PATH_NOT_FOUND = 3; +const ERROR_ACCESS_DENIED = 5; +const ERROR_SHARING_VIOLATION = 32; +const ERROR_LOCK_VIOLATION = 33; + // NtQueryDirectoryFile information class const FileFullDirectoryInformation = 2; +const FileAllInformation = 18; // FILETIME epoch offset: 100ns ticks between 1601-01-01 and 1970-01-01 const UNIX_EPOCH_TICKS = 0x019DB1DED53E8000n; @@ -42,6 +50,10 @@ const TICKS_PER_SECOND = 10000000n; // while still fitting multiple entries per call. const DIR_BUFFER_SIZE = 1024; +// Max retries for sharing violations when opening directories +const OPEN_RETRIES = 3; +const RETRY_DELAY_MS = 100; + // --- koffi function bindings --- const kernel32 = koffi.load("kernel32.dll"); @@ -57,11 +69,17 @@ const CreateFileW = kernel32.func( "intptr CreateFileW(const uint16 *lpFileName, uint32 dwDesiredAccess, uint32 dwShareMode, void *lpSecurityAttributes, uint32 dwCreationDisposition, uint32 dwFlagsAndAttributes, intptr hTemplateFile)", ); const CloseHandle = kernel32.func("bool CloseHandle(intptr hObject)"); +const GetLastError = kernel32.func("uint32 GetLastError()"); +const Sleep = kernel32.func("void Sleep(uint32 dwMilliseconds)"); const NtQueryDirectoryFile = ntdll.func( "int32 NtQueryDirectoryFile(intptr FileHandle, intptr Event, void *ApcRoutine, void *ApcContext, _Out_ IO_STATUS_BLOCK *IoStatusBlock, _Out_ uint8 *FileInformation, uint32 Length, int32 FileInformationClass, bool ReturnSingleEntry, void *FileName, bool RestartScan)", ); +const NtQueryInformationFile = ntdll.func( + "int32 NtQueryInformationFile(intptr FileHandle, _Out_ IO_STATUS_BLOCK *IoStatusBlock, _Out_ uint8 *FileInformation, uint32 Length, int32 FileInformationClass)", +); + // --- Helpers --- /** Convert a JS string to a null-terminated UTF-16LE buffer. */ @@ -73,21 +91,106 @@ function toWide(str: string): Buffer { return buf; } -/** Open a directory handle for reading. */ +/** Apply \\?\ prefix for long path support. */ +function ensureLongPathPrefix(dirPath: string): string { + if (dirPath.startsWith("\\\\?\\") || dirPath.startsWith("\\\\")) { + return dirPath; + } + return "\\\\?\\" + dirPath; +} + +/** + * Open a directory handle for reading. + * Retries on sharing violations (matching C++ behavior). + * Returns INVALID_HANDLE_VALUE on failure. + */ function openDirectory(dirPath: string): bigint { - // Use \\?\ prefix for long path support - const prefix = - dirPath.startsWith("\\\\") ? "" : "\\\\?\\"; - const widePath = toWide(prefix + dirPath + (dirPath.endsWith("\\") ? "" : "\\")); - return CreateFileW( + const suffix = dirPath.endsWith("\\") ? "" : "\\"; + const widePath = toWide(ensureLongPathPrefix(dirPath) + suffix); + + let handle = INVALID_HANDLE_VALUE; + for (let tries = 0; tries < OPEN_RETRIES; tries++) { + handle = CreateFileW( + widePath, + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + null, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + 0n, + ) as bigint; + + if (handle !== INVALID_HANDLE_VALUE) return handle; + + const err = GetLastError() as number; + if (err !== ERROR_SHARING_VIOLATION && err !== ERROR_LOCK_VIOLATION) { + return INVALID_HANDLE_VALUE; + } + if (tries < OPEN_RETRIES - 1) { + Sleep(RETRY_DELAY_MS); + } + } + return handle; +} + +/** Check whether the last CreateFileW error is recoverable (skip the dir) or fatal. */ +function shouldSkipDirectory(skipInaccessible: boolean): boolean { + const err = GetLastError() as number; + if (err === ERROR_FILE_NOT_FOUND || err === ERROR_PATH_NOT_FOUND) return true; + if (skipInaccessible && err === ERROR_ACCESS_DENIED) return true; + return false; +} + +/** + * Get file details (linkCount, id) for a single file. + * Used only when opts.details is true. + */ +function getFileDetails( + filePath: string, + ioStatus: Record, +): { linkCount: number; id: number; idStr: string } | undefined { + const widePath = toWide(ensureLongPathPrefix(filePath)); + const handle = CreateFileW( widePath, - GENERIC_READ, + 0x80, // FILE_READ_ATTRIBUTES FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, null, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, 0n, ) as bigint; + + if (handle === INVALID_HANDLE_VALUE) return undefined; + + try { + // FILE_ALL_INFORMATION is large; we need at least up to InternalInformation + // BasicInformation (40) + StandardInformation (24) + InternalInformation (8) = 72 bytes + const infoBuf = Buffer.alloc(128); + const status = NtQueryInformationFile( + handle, + ioStatus, + infoBuf, + 128, + FileAllInformation, + ) as number; + + if (status !== STATUS_SUCCESS) return undefined; + + // StandardInformation starts at offset 40: NumberOfLinks at +16 (offset 56) + const numberOfLinks = infoBuf.readUInt32LE(56); + // InternalInformation starts at offset 64: IndexNumber (INT64) + const indexLow = infoBuf.readUInt32LE(64); + const indexHigh = infoBuf.readUInt32LE(68); + const indexNumber = (BigInt(indexHigh) << 32n) | BigInt(indexLow >>> 0); + + return { + linkCount: numberOfLinks, + id: Number(indexNumber), + idStr: String(indexNumber), + }; + } finally { + CloseHandle(handle); + } } /** @@ -113,6 +216,7 @@ function parseEntries( opts: Required, entries: IEntry[], subDirs: string[], + ioStatus: Record, ): void { let offset = 0; @@ -136,7 +240,10 @@ function parseEntries( // EndOfFile (actual file size) const sizeLow = buf.readUInt32LE(offset + 40); const sizeHigh = buf.readUInt32LE(offset + 44); - const size = sizeHigh === 0 ? sizeLow : Number((BigInt(sizeHigh) << 32n) | BigInt(sizeLow >>> 0)); + const size = + sizeHigh === 0 + ? sizeLow + : Number((BigInt(sizeHigh) << 32n) | BigInt(sizeLow >>> 0)); // LastWriteTime → Unix seconds const wtLow = buf.readUInt32LE(offset + 24); @@ -146,13 +253,24 @@ function parseEntries( const fullPath = dirPath + "\\" + name; - entries.push({ + const entry: IEntry = { filePath: fullPath, isDirectory: isDir, isReparsePoint, size, mtime, - }); + }; + + if (opts.details) { + const details = getFileDetails(fullPath, ioStatus); + if (details !== undefined) { + entry.linkCount = details.linkCount; + entry.id = details.id; + entry.idStr = details.idStr; + } + } + + entries.push(entry); if (isDir && opts.recurse && !(opts.skipLinks && isReparsePoint)) { subDirs.push(fullPath); @@ -167,62 +285,54 @@ function parseEntries( // --- Walk implementation --- -function walkDirInner( - dirPath: string, - entries: IEntry[], - subDirs: string[], - opts: Required, -): boolean { - const handle = openDirectory(dirPath); - if (handle === INVALID_HANDLE_VALUE) return true; // skip inaccessible - - const ioStatus: Record = {}; - const buf = Buffer.alloc(DIR_BUFFER_SIZE); - - try { - while (true) { - const status = NtQueryDirectoryFile( - handle, - 0n, // Event - null, // ApcRoutine - null, // ApcContext - ioStatus, - buf, - DIR_BUFFER_SIZE, - FileFullDirectoryInformation, - false, // ReturnSingleEntry - null, // FileName (null = wildcard) - false, // RestartScan - ) as number; - - if (status === STATUS_SUCCESS) { - parseEntries(buf, dirPath, opts, entries, subDirs); - } else { - // STATUS_NO_MORE_FILES or any error — done with this directory - break; - } - } - } finally { - CloseHandle(handle); - } - - return true; -} - export function walkDirWindows( dirPath: string, progress: (entries: IEntry[]) => void, opts: Required, ): void { const allEntries: IEntry[] = []; + // Reuse across all directories to avoid per-directory allocation + const ioStatus: Record = {}; + const buf = Buffer.alloc(DIR_BUFFER_SIZE); function walkRecursive(dir: string): void { + const handle = openDirectory(dir); + if (handle === INVALID_HANDLE_VALUE) { + if (!shouldSkipDirectory(opts.skipInaccessible)) { + throw new Error(`Failed to open directory: ${dir}`); + } + return; + } + const entries: IEntry[] = []; const subDirs: string[] = []; - walkDirInner(dir, entries, subDirs, opts); + try { + while (true) { + const status = NtQueryDirectoryFile( + handle, + 0n, + null, + null, + ioStatus, + buf, + DIR_BUFFER_SIZE, + FileFullDirectoryInformation, + false, + null, + false, + ) as number; + + if (status === STATUS_SUCCESS) { + parseEntries(buf, dir, opts, entries, subDirs, ioStatus); + } else { + break; + } + } + } finally { + CloseHandle(handle); + } - // Accumulate entries and flush when threshold is reached (matching native behavior) for (const entry of entries) { allEntries.push(entry); } @@ -249,7 +359,6 @@ export function walkDirWindows( walkRecursive(dirPath); - // Flush remaining entries if (allEntries.length > 0) { progress(allEntries); } diff --git a/packages/turbowalk/vitest.config.ts b/packages/turbowalk/vitest.config.ts new file mode 100644 index 0000000000..c1433e6ef3 --- /dev/null +++ b/packages/turbowalk/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + environment: "node", + include: ["src/**/*.test.ts"], + }, +}); From a1f3df6284002e18db96a9ec0153151abdba210e Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Apr 2026 19:45:50 -0700 Subject: [PATCH 09/14] Document zero-copy optimization path blocked by Electron --- packages/turbowalk/src/walkWindows.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/turbowalk/src/walkWindows.ts b/packages/turbowalk/src/walkWindows.ts index e7c64e8fa4..374b18f6f9 100644 --- a/packages/turbowalk/src/walkWindows.ts +++ b/packages/turbowalk/src/walkWindows.ts @@ -48,6 +48,11 @@ const TICKS_PER_SECOND = 10000000n; // Counterintuitively, smaller buffers are faster because koffi marshals the // entire buffer across the FFI boundary each call. 1KB minimizes that overhead // while still fitting multiple entries per call. +// +// A zero-copy approach using VirtualAlloc + koffi.view() with a 64KB buffer +// was benchmarked and beats even the old C++ addon. However, Electron forbids +// external ArrayBuffers (koffi.view throws), so we can't use it in production. +// If that restriction is ever lifted, switch to the zero-copy path. const DIR_BUFFER_SIZE = 1024; // Max retries for sharing violations when opening directories From 081e04f42fd33e91cb13bf9fd6af7ff489e8e73d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Apr 2026 20:05:58 -0700 Subject: [PATCH 10/14] Clarify Electron V8 sandbox is compile-time, no runtime workaround --- packages/turbowalk/src/walkWindows.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/turbowalk/src/walkWindows.ts b/packages/turbowalk/src/walkWindows.ts index 374b18f6f9..16353e3660 100644 --- a/packages/turbowalk/src/walkWindows.ts +++ b/packages/turbowalk/src/walkWindows.ts @@ -50,9 +50,11 @@ const TICKS_PER_SECOND = 10000000n; // while still fitting multiple entries per call. // // A zero-copy approach using VirtualAlloc + koffi.view() with a 64KB buffer -// was benchmarked and beats even the old C++ addon. However, Electron forbids -// external ArrayBuffers (koffi.view throws), so we can't use it in production. -// If that restriction is ever lifted, switch to the zero-copy path. +// was benchmarked and beats even the old C++ addon (~367ms vs 383ms on 89k +// entries). However, Electron's V8 sandbox (v8_enable_sandbox, compile-time +// flag since Electron 21) blocks external ArrayBuffers — koffi.view() throws. +// No runtime flag can disable this. If Vortex ever moves off Electron or the +// sandbox is lifted, switch to VirtualAlloc + koffi.view() + 64KB buffer. const DIR_BUFFER_SIZE = 1024; // Max retries for sharing violations when opening directories From cf044df176687ebf33ee5035fba50e4da0c16855 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 6 May 2026 14:48:48 -0700 Subject: [PATCH 11/14] Use .then(() => {}) convention in purgeLinks return type coercion --- src/renderer/src/extensions/move_activator/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/renderer/src/extensions/move_activator/index.ts b/src/renderer/src/extensions/move_activator/index.ts index 8e712f5c2e..d0630da260 100644 --- a/src/renderer/src/extensions/move_activator/index.ts +++ b/src/renderer/src/extensions/move_activator/index.ts @@ -252,7 +252,9 @@ class DeploymentMethod extends LinkingDeployment { { details: true, }, - ).then(() => PromiseBB.map(links, (entry) => this.restoreLink(entry.filePath))); + ) + .then(() => PromiseBB.map(links, (entry) => this.restoreLink(entry.filePath))) + .then(() => {}); } protected linkFile(linkPath: string, sourcePath: string, dirTags?: boolean): Promise { From 99ed6a037d74ad30e218f57d4af70fbf5e5ba8d4 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 6 May 2026 16:02:08 -0700 Subject: [PATCH 12/14] Update pnpm-lock.yaml for turbowalk tsdown dependency --- pnpm-lock.yaml | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 203de3e15f..7b8e9f9020 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3848,6 +3848,9 @@ importers: koffi: specifier: ^2.9.0 version: 2.16.1 + tsdown: + specifier: 'catalog:' + version: 0.21.0-beta.1(synckit@0.11.12)(typescript@5.9.3) packages/vortex-api: dependencies: @@ -9306,10 +9309,6 @@ packages: resolution: {integrity: sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==} engines: {node: '>=14.14'} - fs-extra@8.1.0: - resolution: {integrity: sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==} - engines: {node: '>=6 <7 || >=8'} - fs-extra@9.1.0: resolution: {integrity: sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==} engines: {node: '>=10'} @@ -10065,9 +10064,6 @@ packages: engines: {node: '>=6'} hasBin: true - jsonfile@4.0.0: - resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==} - jsonfile@6.2.0: resolution: {integrity: sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==} @@ -12414,10 +12410,6 @@ packages: universal-analytics@0.4.23: resolution: {integrity: sha512-lgMIH7XBI6OgYn1woDEmxhGdj8yDefMKg7GkWdeATAlQZFrMrNyxSkpDzY57iY0/6fdlzTbBV03OawvvzG+q7A==} - universalify@0.1.2: - resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} - engines: {node: '>= 4.0.0'} - universalify@0.2.0: resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==} engines: {node: '>= 4.0.0'} @@ -17927,12 +17919,6 @@ snapshots: jsonfile: 6.2.0 universalify: 2.0.1 - fs-extra@8.1.0: - dependencies: - graceful-fs: 4.2.11 - jsonfile: 4.0.0 - universalify: 0.1.2 - fs-extra@9.1.0: dependencies: at-least-node: 1.0.0 @@ -18710,10 +18696,6 @@ snapshots: json5@2.2.3: {} - jsonfile@4.0.0: - optionalDependencies: - graceful-fs: 4.2.11 - jsonfile@6.2.0: dependencies: universalify: 2.0.1 @@ -21412,8 +21394,6 @@ snapshots: transitivePeerDependencies: - supports-color - universalify@0.1.2: {} - universalify@0.2.0: optional: true From ac6d4c13a5d9ebc032a9410529e3743ff5b8f7d1 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 6 May 2026 16:06:10 -0700 Subject: [PATCH 13/14] Fix turbowalk build: rename tsdown config to .mts and format source files --- packages/turbowalk/src/index.test.ts | 48 +++++++++++++++---- packages/turbowalk/src/index.ts | 15 +++--- packages/turbowalk/src/walkWindows.ts | 7 ++- .../{tsdown.config.ts => tsdown.config.mts} | 0 4 files changed, 49 insertions(+), 21 deletions(-) rename packages/turbowalk/{tsdown.config.ts => tsdown.config.mts} (100%) diff --git a/packages/turbowalk/src/index.test.ts b/packages/turbowalk/src/index.test.ts index 7d3d5b1dc0..b70a263048 100644 --- a/packages/turbowalk/src/index.test.ts +++ b/packages/turbowalk/src/index.test.ts @@ -1,8 +1,9 @@ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; import * as fs from "fs"; import * as os from "os"; import * as path from "path"; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; + import turbowalk, { type IEntry } from "./index"; let tmpDir: string; @@ -36,7 +37,13 @@ afterAll(() => { function collect(dir: string, options?: Parameters[2]): Promise { const all: IEntry[] = []; - return turbowalk(dir, (entries) => { all.push(...entries); }, options).then(() => all); + return turbowalk( + dir, + (entries) => { + all.push(...entries); + }, + options, + ).then(() => all); } describe("turbowalk", () => { @@ -44,8 +51,14 @@ describe("turbowalk", () => { const entries = await collect(tmpDir, { skipHidden: false }); const names = entries.map((e) => path.basename(e.filePath)).sort(); expect(names).toEqual([ - ".hidden", ".hiddendir", "emptydir", "file1.txt", "file2.dat", - "inside.txt", "nested.txt", "subdir", + ".hidden", + ".hiddendir", + "emptydir", + "file1.txt", + "file2.dat", + "inside.txt", + "nested.txt", + "subdir", ]); }); @@ -58,8 +71,14 @@ describe("turbowalk", () => { it("reports correct isDirectory flag", async () => { const entries = await collect(tmpDir, { skipHidden: false }); - const dirs = entries.filter((e) => e.isDirectory).map((e) => path.basename(e.filePath)).sort(); - const files = entries.filter((e) => !e.isDirectory).map((e) => path.basename(e.filePath)).sort(); + const dirs = entries + .filter((e) => e.isDirectory) + .map((e) => path.basename(e.filePath)) + .sort(); + const files = entries + .filter((e) => !e.isDirectory) + .map((e) => path.basename(e.filePath)) + .sort(); expect(dirs).toEqual([".hiddendir", "emptydir", "subdir"]); expect(files).toEqual([".hidden", "file1.txt", "file2.dat", "inside.txt", "nested.txt"]); }); @@ -86,7 +105,14 @@ describe("turbowalk", () => { const entries = await collect(tmpDir, { recurse: false, skipHidden: false }); const names = entries.map((e) => path.basename(e.filePath)).sort(); // Should only include top-level entries, not nested.txt or inside.txt - expect(names).toEqual([".hidden", ".hiddendir", "emptydir", "file1.txt", "file2.dat", "subdir"]); + expect(names).toEqual([ + ".hidden", + ".hiddendir", + "emptydir", + "file1.txt", + "file2.dat", + "subdir", + ]); }); it("handles non-existent directory gracefully", async () => { @@ -96,7 +122,13 @@ describe("turbowalk", () => { it("calls progress callback with batched entries", async () => { const calls: number[] = []; - await turbowalk(tmpDir, (entries) => { calls.push(entries.length); }, { skipHidden: false }); + await turbowalk( + tmpDir, + (entries) => { + calls.push(entries.length); + }, + { skipHidden: false }, + ); // At least one call should have been made expect(calls.length).toBeGreaterThan(0); // Total entries across all calls should match diff --git a/packages/turbowalk/src/index.ts b/packages/turbowalk/src/index.ts index dbbf2f8bfd..32cede0041 100644 --- a/packages/turbowalk/src/index.ts +++ b/packages/turbowalk/src/index.ts @@ -1,7 +1,8 @@ -import Bluebird from "bluebird"; import * as fs from "fs"; import * as path from "path"; +import Bluebird from "bluebird"; + export interface IEntry { filePath: string; isDirectory: boolean; @@ -36,11 +37,9 @@ const DEFAULTS: Required = { // --- Windows: koffi FFI using NtQueryDirectoryFile --- -let walkDirWindows: (( - dirPath: string, - progress: (entries: IEntry[]) => void, - opts: Required, -) => void) | undefined; +let walkDirWindows: + | ((dirPath: string, progress: (entries: IEntry[]) => void, opts: Required) => void) + | undefined; if (process.platform === "win32") { try { @@ -128,9 +127,7 @@ async function walkDirFallback( isReparsePoint: isLink, size: stats.size, mtime: Math.floor(stats.mtimeMs / 1000), - ...(opts.details - ? { id: stats.ino, idStr: String(stats.ino), linkCount: stats.nlink } - : {}), + ...(opts.details ? { id: stats.ino, idStr: String(stats.ino), linkCount: stats.nlink } : {}), }); if (isDir && opts.recurse && !(opts.skipLinks && isLink)) { diff --git a/packages/turbowalk/src/walkWindows.ts b/packages/turbowalk/src/walkWindows.ts index 16353e3660..cffaaa62f4 100644 --- a/packages/turbowalk/src/walkWindows.ts +++ b/packages/turbowalk/src/walkWindows.ts @@ -7,6 +7,7 @@ */ import * as path from "path"; + import type { IEntry, IWalkOptions } from "./index"; // eslint-disable-next-line @typescript-eslint/no-var-requires @@ -41,7 +42,7 @@ const FileFullDirectoryInformation = 2; const FileAllInformation = 18; // FILETIME epoch offset: 100ns ticks between 1601-01-01 and 1970-01-01 -const UNIX_EPOCH_TICKS = 0x019DB1DED53E8000n; +const UNIX_EPOCH_TICKS = 0x019db1ded53e8000n; const TICKS_PER_SECOND = 10000000n; // Buffer for NtQueryDirectoryFile — 1KB matches the old C++ addon's buffer size. @@ -248,9 +249,7 @@ function parseEntries( const sizeLow = buf.readUInt32LE(offset + 40); const sizeHigh = buf.readUInt32LE(offset + 44); const size = - sizeHigh === 0 - ? sizeLow - : Number((BigInt(sizeHigh) << 32n) | BigInt(sizeLow >>> 0)); + sizeHigh === 0 ? sizeLow : Number((BigInt(sizeHigh) << 32n) | BigInt(sizeLow >>> 0)); // LastWriteTime → Unix seconds const wtLow = buf.readUInt32LE(offset + 24); diff --git a/packages/turbowalk/tsdown.config.ts b/packages/turbowalk/tsdown.config.mts similarity index 100% rename from packages/turbowalk/tsdown.config.ts rename to packages/turbowalk/tsdown.config.mts From e2c4cc540c924f0d4a1b5439222d6a5a71436bfb Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 6 May 2026 16:08:56 -0700 Subject: [PATCH 14/14] Replace deprecated external with deps.neverBundle in turbowalk tsdown config --- packages/turbowalk/tsdown.config.mts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/turbowalk/tsdown.config.mts b/packages/turbowalk/tsdown.config.mts index 07c00a858b..7dbd126f49 100644 --- a/packages/turbowalk/tsdown.config.mts +++ b/packages/turbowalk/tsdown.config.mts @@ -5,5 +5,5 @@ export default defineConfig({ format: ["cjs"], platform: "node", dts: true, - external: ["koffi"], + deps: { neverBundle: ["koffi"] }, });