From 97af0457a56cd5c61519fca8fd6f751b14215594 Mon Sep 17 00:00:00 2001 From: redwheelbarrow Date: Fri, 5 Dec 2025 18:03:31 -0700 Subject: [PATCH 1/3] fix(libzipimpl): Reduce WASM OOM issues with large dependencies (#6990) ## Summary Reduce the likelihood of large dependencies causing OOM crashes during install ## Problem Zip files are all loaded into WASM memory during pnp/pnpm install. The WASM libzip memory is limited to 2gigs. If there are many dependencies, or a few large ones, the limit will be reached and the runtime will crash. ## Solution Track the estimated memory usage per LibzipImpl, provide the current WASM or a new WASM depending on memory usage estimate. Cleanup WASMs that are no longer needed. ## Changes - Create WASM factory that tracks estimated heap usage in WASM - Construct new WASM for use by a LibzipImpl if heap usage is near max - Cleanup WASM upon discard ## Testing - Unit tests - Manually verified the reported issue #6990 is gone --- packages/yarnpkg-libzip/package.json | 3 +- packages/yarnpkg-libzip/sources/instance.ts | 4 + packages/yarnpkg-libzip/sources/libzipImpl.ts | 117 ++++++++++++++++-- 3 files changed, 116 insertions(+), 8 deletions(-) diff --git a/packages/yarnpkg-libzip/package.json b/packages/yarnpkg-libzip/package.json index 8989636e647d..4d072f121806 100644 --- a/packages/yarnpkg-libzip/package.json +++ b/packages/yarnpkg-libzip/package.json @@ -15,7 +15,8 @@ "build:libzip:wasm": "cd ./artifacts && ./build.sh", "postpack": "rm -rf lib", "prepack": "run build:compile \"$(pwd)\"", - "release": "yarn npm publish" + "release": "yarn npm publish", + "test": "run test:unit \"$(pwd)\"" }, "publishConfig": { "main": "./lib/sync.js", diff --git a/packages/yarnpkg-libzip/sources/instance.ts b/packages/yarnpkg-libzip/sources/instance.ts index 6fb5facc1909..0edeea8c2b14 100644 --- a/packages/yarnpkg-libzip/sources/instance.ts +++ b/packages/yarnpkg-libzip/sources/instance.ts @@ -6,6 +6,10 @@ let registeredFactory: () => Libzip = () => { throw new Error(`Assertion failed: No libzip instance is available, and no factory was configured`); }; +export function newInstance() { + return registeredFactory(); +} + export function setFactory(factory: () => Libzip) { registeredFactory = factory; } diff --git a/packages/yarnpkg-libzip/sources/libzipImpl.ts b/packages/yarnpkg-libzip/sources/libzipImpl.ts index 085bfbf46751..e31f3afb3074 100644 --- a/packages/yarnpkg-libzip/sources/libzipImpl.ts +++ b/packages/yarnpkg-libzip/sources/libzipImpl.ts @@ -2,7 +2,7 @@ import {PortablePath} from '@yarn import {Libzip} from '@yarnpkg/libzip'; import {ZipImplInput, type CompressionData, type Stat, type ZipImpl} from './ZipFS'; -import {getInstance} from './instance'; +import {newInstance} from './instance'; export class LibzipError extends Error { @@ -16,12 +16,110 @@ export class LibzipError extends Error { } } + +type LibzipInstance = {instance: Libzip, reserved: number, highWaterMark: number}; +type LibzipReservation = {byteLength: number, instanceIndex: number}; +/** + * Tracks the estimate of WASM memory usage by libzip to reduce the risk + * of OOM errors. + * + * Internally, favors the oldest WASM instances to minimize fragmentation. + * Cleans up instances when older instances have space to accomodate new zips. + */ +class ElasticLibzipFactory { + private static readonly LIBZIP_METADATA = 512 * 1024; // 500KB + private static readonly WASM_MEM_MAX = 2 * 1024 * 1024 * 1024 - (100 * 1024 * 1024); // 1.9GB + private static readonly WASM_HIGHWATER_CLEANUP = 1 * 1024 * 1024 * 1024; // 1GB + private static readonly WASM_CLEANUP_DELTA = 200 * 1024 * 1024; // 200MB + private static KEY = 1; + + /** + * The WASM instances, their currently reserved memory, and the high water mark, since + * WASM memory isn't usually shrinkable. + */ + private readonly instances: Array = []; + /** + * The reservations by unique ID, and the index into the {@link instances} array. + */ + private readonly reservations = new Map(); + + /** + * Provide (and possibly build new) a libzip WASM for the given ZIP byte length + * + * @param byteLength The size of the ZIP file + * @returns [unique ID, Libzip instance] + */ + getInstance(byteLength: number): [number, Libzip] { + let index = this.instances.findIndex(i => (i.reserved + byteLength + ElasticLibzipFactory.LIBZIP_METADATA) < ElasticLibzipFactory.WASM_MEM_MAX); + let instance; + if (index >= 0) { + instance = this.instances[index]; + instance.reserved += byteLength + ElasticLibzipFactory.LIBZIP_METADATA; + instance.highWaterMark = Math.max(instance.highWaterMark, instance.reserved); + } else { + index += 1; + instance = {instance: newInstance(), reserved: byteLength + ElasticLibzipFactory.LIBZIP_METADATA, highWaterMark: byteLength}; + this.instances.push(instance); + } + ElasticLibzipFactory.KEY += 1; + this.reservations.set(ElasticLibzipFactory.KEY, {byteLength: byteLength + ElasticLibzipFactory.LIBZIP_METADATA, instanceIndex: index}); + return [ElasticLibzipFactory.KEY, instance.instance]; + } + + /** + * Update the size without worrying about if there's enough space. + */ + update(key: number, delta: number) { + const reservation = this.reservations.get(key); + if (!reservation) + throw new Error(`Key ${key} not present in ${ElasticLibzipFactory.name}`); + + const instance = this.instances[reservation.instanceIndex]; + instance.reserved = instance.reserved + delta; + instance.highWaterMark = Math.max(instance.reserved, instance.highWaterMark); + } + + remove(key: number) { + const reservation = this.reservations.get(key); + if (!reservation) + return; + this.reservations.delete(key); + + const instance = this.instances[reservation.instanceIndex]; + instance.reserved -= reservation.byteLength; + + this.cleanup(reservation); + } + + /** + * Remove the reservation's instance if the previous one has enough space, + * or if the reservations instance is nearly out of memory. + * + * @param reservation + */ + private cleanup(reservation: LibzipReservation) { + // Delete the instance if the preceding one has some space to fill (to avoid creating and closing WASMS unnecessarily) + const instance = this.instances[reservation.instanceIndex]; + if (instance.reserved <= 0 && reservation.instanceIndex > 0) { + const precedingInstance = this.instances[reservation.instanceIndex - 1]; + const precedingWasmRemaining = ElasticLibzipFactory.WASM_MEM_MAX - precedingInstance.reserved; + if (precedingWasmRemaining >= ElasticLibzipFactory.WASM_CLEANUP_DELTA || + instance.highWaterMark >= ElasticLibzipFactory.WASM_HIGHWATER_CLEANUP) { + this.instances.pop(); + } + } + } +} + +const libzipFactory = new ElasticLibzipFactory(); + export class LibZipImpl implements ZipImpl { private readonly libzip: Libzip; private readonly lzSource: number; private readonly zip: number; private readonly listings: Array; private readonly symlinkCount: number; + private readonly key: number; public filesShouldBeCached = true; @@ -30,7 +128,7 @@ export class LibZipImpl implements ZipImpl { ? opts.buffer : opts.baseFs.readFileSync(opts.path); - this.libzip = getInstance(); + [this.key, this.libzip] = libzipFactory.getInstance(buffer.byteLength); const errPtr = this.libzip.malloc(4); try { @@ -50,20 +148,20 @@ export class LibZipImpl implements ZipImpl { if (this.zip === 0) { const error = this.libzip.struct.errorS(); this.libzip.error.initWithCode(error, this.libzip.getValue(errPtr, `i32`)); - throw this.makeLibzipError(error); } + } catch(error) { + libzipFactory.remove(this.key); + throw error; } finally { this.libzip.free(errPtr); } const entryCount = this.libzip.getNumEntries(this.zip, 0); - const listings = new Array(entryCount); + this.listings = new Array(entryCount); for (let t = 0; t < entryCount; ++t) - listings[t] = this.libzip.getName(this.zip, t, 0); - - this.listings = listings; + this.listings[t] = this.libzip.getName(this.zip, t, 0); this.symlinkCount = this.libzip.ext.countSymlinks(this.zip); if (this.symlinkCount === -1) { @@ -108,6 +206,7 @@ export class LibZipImpl implements ZipImpl { } setFileSource(target: PortablePath, compression: CompressionData, buffer: Buffer) { + libzipFactory.update(this.key, buffer.byteLength); const lzSource = this.allocateSource(buffer); try { @@ -121,9 +220,11 @@ export class LibZipImpl implements ZipImpl { throw this.makeLibzipError(this.libzip.getError(this.zip)); } } + return newIndex; } catch (error) { this.libzip.source.free(lzSource); + libzipFactory.update(this.key, -buffer.byteLength); throw error; } } @@ -261,6 +362,7 @@ export class LibZipImpl implements ZipImpl { } finally { this.libzip.source.close(this.lzSource); this.libzip.source.free(this.lzSource); + libzipFactory.remove(this.key); } } @@ -307,5 +409,6 @@ export class LibZipImpl implements ZipImpl { public discard(): void { this.libzip.discard(this.zip); + libzipFactory.remove(this.key); } } From 19be65855268733fca62fdbb068c6cc8049cb9d2 Mon Sep 17 00:00:00 2001 From: redwheelbarrow Date: Mon, 8 Dec 2025 18:29:21 -0700 Subject: [PATCH 2/3] chore: version check --- .yarn/versions/93e6f510.yml | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .yarn/versions/93e6f510.yml diff --git a/.yarn/versions/93e6f510.yml b/.yarn/versions/93e6f510.yml new file mode 100644 index 000000000000..9d2c15deaec2 --- /dev/null +++ b/.yarn/versions/93e6f510.yml @@ -0,0 +1,39 @@ +releases: + "@yarnpkg/builder": patch + "@yarnpkg/cli": patch + "@yarnpkg/core": patch + "@yarnpkg/doctor": patch + "@yarnpkg/extensions": patch + "@yarnpkg/fslib": patch + "@yarnpkg/libzip": patch + "@yarnpkg/nm": patch + "@yarnpkg/plugin-catalog": patch + "@yarnpkg/plugin-compat": patch + "@yarnpkg/plugin-constraints": patch + "@yarnpkg/plugin-dlx": patch + "@yarnpkg/plugin-essentials": patch + "@yarnpkg/plugin-exec": patch + "@yarnpkg/plugin-file": patch + "@yarnpkg/plugin-git": patch + "@yarnpkg/plugin-github": patch + "@yarnpkg/plugin-http": patch + "@yarnpkg/plugin-init": patch + "@yarnpkg/plugin-interactive-tools": patch + "@yarnpkg/plugin-jsr": patch + "@yarnpkg/plugin-link": patch + "@yarnpkg/plugin-nm": patch + "@yarnpkg/plugin-npm": patch + "@yarnpkg/plugin-npm-cli": patch + "@yarnpkg/plugin-pack": patch + "@yarnpkg/plugin-patch": patch + "@yarnpkg/plugin-pnp": patch + "@yarnpkg/plugin-pnpm": patch + "@yarnpkg/plugin-stage": patch + "@yarnpkg/plugin-typescript": patch + "@yarnpkg/plugin-version": patch + "@yarnpkg/plugin-workspace-tools": patch + "@yarnpkg/pnp": patch + "@yarnpkg/pnpify": patch + "@yarnpkg/sdks": patch + "@yarnpkg/shell": patch + vscode-zipfs: patch From 68f84c394724ea06d66af2e6df472e1c0264ad83 Mon Sep 17 00:00:00 2001 From: redwheelbarrow Date: Mon, 8 Dec 2025 20:50:52 -0700 Subject: [PATCH 3/3] fix: simplify tracker and fix a couple issues --- packages/yarnpkg-libzip/sources/libzipImpl.ts | 47 ++++++------------- 1 file changed, 14 insertions(+), 33 deletions(-) diff --git a/packages/yarnpkg-libzip/sources/libzipImpl.ts b/packages/yarnpkg-libzip/sources/libzipImpl.ts index e31f3afb3074..11eb50df36d3 100644 --- a/packages/yarnpkg-libzip/sources/libzipImpl.ts +++ b/packages/yarnpkg-libzip/sources/libzipImpl.ts @@ -17,7 +17,7 @@ export class LibzipError extends Error { } -type LibzipInstance = {instance: Libzip, reserved: number, highWaterMark: number}; +type LibzipInstance = {instance: Libzip | null, active: boolean, reserved: number, highWaterMark: number}; type LibzipReservation = {byteLength: number, instanceIndex: number}; /** * Tracks the estimate of WASM memory usage by libzip to reduce the risk @@ -29,8 +29,6 @@ type LibzipReservation = {byteLength: number, instanceIndex: number}; class ElasticLibzipFactory { private static readonly LIBZIP_METADATA = 512 * 1024; // 500KB private static readonly WASM_MEM_MAX = 2 * 1024 * 1024 * 1024 - (100 * 1024 * 1024); // 1.9GB - private static readonly WASM_HIGHWATER_CLEANUP = 1 * 1024 * 1024 * 1024; // 1GB - private static readonly WASM_CLEANUP_DELTA = 200 * 1024 * 1024; // 200MB private static KEY = 1; /** @@ -50,44 +48,33 @@ class ElasticLibzipFactory { * @returns [unique ID, Libzip instance] */ getInstance(byteLength: number): [number, Libzip] { - let index = this.instances.findIndex(i => (i.reserved + byteLength + ElasticLibzipFactory.LIBZIP_METADATA) < ElasticLibzipFactory.WASM_MEM_MAX); + const size = byteLength + ElasticLibzipFactory.LIBZIP_METADATA; + let index = this.instances.findIndex(i => i.active && (i.reserved + size) < ElasticLibzipFactory.WASM_MEM_MAX); let instance; + if (index >= 0) { instance = this.instances[index]; - instance.reserved += byteLength + ElasticLibzipFactory.LIBZIP_METADATA; + instance.reserved += size; instance.highWaterMark = Math.max(instance.highWaterMark, instance.reserved); } else { - index += 1; - instance = {instance: newInstance(), reserved: byteLength + ElasticLibzipFactory.LIBZIP_METADATA, highWaterMark: byteLength}; + index = this.instances.length; + instance = {instance: newInstance(), reserved: size, highWaterMark: size, active: true}; this.instances.push(instance); } ElasticLibzipFactory.KEY += 1; - this.reservations.set(ElasticLibzipFactory.KEY, {byteLength: byteLength + ElasticLibzipFactory.LIBZIP_METADATA, instanceIndex: index}); - return [ElasticLibzipFactory.KEY, instance.instance]; - } - - /** - * Update the size without worrying about if there's enough space. - */ - update(key: number, delta: number) { - const reservation = this.reservations.get(key); - if (!reservation) - throw new Error(`Key ${key} not present in ${ElasticLibzipFactory.name}`); - - const instance = this.instances[reservation.instanceIndex]; - instance.reserved = instance.reserved + delta; - instance.highWaterMark = Math.max(instance.reserved, instance.highWaterMark); + this.reservations.set(ElasticLibzipFactory.KEY, {byteLength: size, instanceIndex: index}); + return [ElasticLibzipFactory.KEY, instance.instance!]; } remove(key: number) { const reservation = this.reservations.get(key); if (!reservation) return; + this.reservations.delete(key); const instance = this.instances[reservation.instanceIndex]; instance.reserved -= reservation.byteLength; - this.cleanup(reservation); } @@ -98,15 +85,11 @@ class ElasticLibzipFactory { * @param reservation */ private cleanup(reservation: LibzipReservation) { - // Delete the instance if the preceding one has some space to fill (to avoid creating and closing WASMS unnecessarily) const instance = this.instances[reservation.instanceIndex]; - if (instance.reserved <= 0 && reservation.instanceIndex > 0) { - const precedingInstance = this.instances[reservation.instanceIndex - 1]; - const precedingWasmRemaining = ElasticLibzipFactory.WASM_MEM_MAX - precedingInstance.reserved; - if (precedingWasmRemaining >= ElasticLibzipFactory.WASM_CLEANUP_DELTA || - instance.highWaterMark >= ElasticLibzipFactory.WASM_HIGHWATER_CLEANUP) { - this.instances.pop(); - } + + if (instance.reserved <= 0) { + instance.active = false; + this.instances[reservation.instanceIndex].instance = null; } } } @@ -206,7 +189,6 @@ export class LibZipImpl implements ZipImpl { } setFileSource(target: PortablePath, compression: CompressionData, buffer: Buffer) { - libzipFactory.update(this.key, buffer.byteLength); const lzSource = this.allocateSource(buffer); try { @@ -224,7 +206,6 @@ export class LibZipImpl implements ZipImpl { return newIndex; } catch (error) { this.libzip.source.free(lzSource); - libzipFactory.update(this.key, -buffer.byteLength); throw error; } }