diff --git a/.yarn/versions/93e6f510.yml b/.yarn/versions/93e6f510.yml new file mode 100644 index 000000000000..9d2c15deaec2 --- /dev/null +++ b/.yarn/versions/93e6f510.yml @@ -0,0 +1,39 @@ +releases: + "@yarnpkg/builder": patch + "@yarnpkg/cli": patch + "@yarnpkg/core": patch + "@yarnpkg/doctor": patch + "@yarnpkg/extensions": patch + "@yarnpkg/fslib": patch + "@yarnpkg/libzip": patch + "@yarnpkg/nm": patch + "@yarnpkg/plugin-catalog": patch + "@yarnpkg/plugin-compat": patch + "@yarnpkg/plugin-constraints": patch + "@yarnpkg/plugin-dlx": patch + "@yarnpkg/plugin-essentials": patch + "@yarnpkg/plugin-exec": patch + "@yarnpkg/plugin-file": patch + "@yarnpkg/plugin-git": patch + "@yarnpkg/plugin-github": patch + "@yarnpkg/plugin-http": patch + "@yarnpkg/plugin-init": patch + "@yarnpkg/plugin-interactive-tools": patch + "@yarnpkg/plugin-jsr": patch + "@yarnpkg/plugin-link": patch + "@yarnpkg/plugin-nm": patch + "@yarnpkg/plugin-npm": patch + "@yarnpkg/plugin-npm-cli": patch + "@yarnpkg/plugin-pack": patch + "@yarnpkg/plugin-patch": patch + "@yarnpkg/plugin-pnp": patch + "@yarnpkg/plugin-pnpm": patch + "@yarnpkg/plugin-stage": patch + "@yarnpkg/plugin-typescript": patch + "@yarnpkg/plugin-version": patch + "@yarnpkg/plugin-workspace-tools": patch + "@yarnpkg/pnp": patch + "@yarnpkg/pnpify": patch + "@yarnpkg/sdks": patch + "@yarnpkg/shell": patch + vscode-zipfs: patch diff --git a/packages/yarnpkg-libzip/package.json b/packages/yarnpkg-libzip/package.json index 8989636e647d..4d072f121806 100644 --- a/packages/yarnpkg-libzip/package.json +++ b/packages/yarnpkg-libzip/package.json @@ -15,7 +15,8 @@ "build:libzip:wasm": "cd ./artifacts && ./build.sh", "postpack": "rm -rf lib", "prepack": "run build:compile \"$(pwd)\"", - "release": "yarn npm publish" + "release": "yarn npm publish", + "test": "run test:unit \"$(pwd)\"" }, "publishConfig": { "main": "./lib/sync.js", diff --git a/packages/yarnpkg-libzip/sources/instance.ts b/packages/yarnpkg-libzip/sources/instance.ts index 6fb5facc1909..0edeea8c2b14 100644 --- a/packages/yarnpkg-libzip/sources/instance.ts +++ b/packages/yarnpkg-libzip/sources/instance.ts @@ -6,6 +6,10 @@ let registeredFactory: () => Libzip = () => { throw new Error(`Assertion failed: No libzip instance is available, and no factory was configured`); }; +export function newInstance() { + return registeredFactory(); +} + export function setFactory(factory: () => Libzip) { registeredFactory = factory; } diff --git a/packages/yarnpkg-libzip/sources/libzipImpl.ts b/packages/yarnpkg-libzip/sources/libzipImpl.ts index 085bfbf46751..11eb50df36d3 100644 --- a/packages/yarnpkg-libzip/sources/libzipImpl.ts +++ b/packages/yarnpkg-libzip/sources/libzipImpl.ts @@ -2,7 +2,7 @@ import {PortablePath} from '@yarn import {Libzip} from '@yarnpkg/libzip'; import {ZipImplInput, type CompressionData, type Stat, type ZipImpl} from './ZipFS'; -import {getInstance} from './instance'; +import {newInstance} from './instance'; export class LibzipError extends Error { @@ -16,12 +16,93 @@ export class LibzipError extends Error { } } + +type LibzipInstance = {instance: Libzip | null, active: boolean, reserved: number, highWaterMark: number}; +type LibzipReservation = {byteLength: number, instanceIndex: number}; +/** + * Tracks the estimate of WASM memory usage by libzip to reduce the risk + * of OOM errors. + * + * Internally, favors the oldest WASM instances to minimize fragmentation. + * Cleans up instances when older instances have space to accomodate new zips. + */ +class ElasticLibzipFactory { + private static readonly LIBZIP_METADATA = 512 * 1024; // 500KB + private static readonly WASM_MEM_MAX = 2 * 1024 * 1024 * 1024 - (100 * 1024 * 1024); // 1.9GB + private static KEY = 1; + + /** + * The WASM instances, their currently reserved memory, and the high water mark, since + * WASM memory isn't usually shrinkable. + */ + private readonly instances: Array = []; + /** + * The reservations by unique ID, and the index into the {@link instances} array. + */ + private readonly reservations = new Map(); + + /** + * Provide (and possibly build new) a libzip WASM for the given ZIP byte length + * + * @param byteLength The size of the ZIP file + * @returns [unique ID, Libzip instance] + */ + getInstance(byteLength: number): [number, Libzip] { + const size = byteLength + ElasticLibzipFactory.LIBZIP_METADATA; + let index = this.instances.findIndex(i => i.active && (i.reserved + size) < ElasticLibzipFactory.WASM_MEM_MAX); + let instance; + + if (index >= 0) { + instance = this.instances[index]; + instance.reserved += size; + instance.highWaterMark = Math.max(instance.highWaterMark, instance.reserved); + } else { + index = this.instances.length; + instance = {instance: newInstance(), reserved: size, highWaterMark: size, active: true}; + this.instances.push(instance); + } + ElasticLibzipFactory.KEY += 1; + this.reservations.set(ElasticLibzipFactory.KEY, {byteLength: size, instanceIndex: index}); + return [ElasticLibzipFactory.KEY, instance.instance!]; + } + + remove(key: number) { + const reservation = this.reservations.get(key); + if (!reservation) + return; + + this.reservations.delete(key); + + const instance = this.instances[reservation.instanceIndex]; + instance.reserved -= reservation.byteLength; + this.cleanup(reservation); + } + + /** + * Remove the reservation's instance if the previous one has enough space, + * or if the reservations instance is nearly out of memory. + * + * @param reservation + */ + private cleanup(reservation: LibzipReservation) { + const instance = this.instances[reservation.instanceIndex]; + + if (instance.reserved <= 0) { + instance.active = false; + this.instances[reservation.instanceIndex].instance = null; + } + } +} + +const libzipFactory = new ElasticLibzipFactory(); + export class LibZipImpl implements ZipImpl { private readonly libzip: Libzip; private readonly lzSource: number; private readonly zip: number; private readonly listings: Array; private readonly symlinkCount: number; + private readonly key: number; public filesShouldBeCached = true; @@ -30,7 +111,7 @@ export class LibZipImpl implements ZipImpl { ? opts.buffer : opts.baseFs.readFileSync(opts.path); - this.libzip = getInstance(); + [this.key, this.libzip] = libzipFactory.getInstance(buffer.byteLength); const errPtr = this.libzip.malloc(4); try { @@ -50,20 +131,20 @@ export class LibZipImpl implements ZipImpl { if (this.zip === 0) { const error = this.libzip.struct.errorS(); this.libzip.error.initWithCode(error, this.libzip.getValue(errPtr, `i32`)); - throw this.makeLibzipError(error); } + } catch(error) { + libzipFactory.remove(this.key); + throw error; } finally { this.libzip.free(errPtr); } const entryCount = this.libzip.getNumEntries(this.zip, 0); - const listings = new Array(entryCount); + this.listings = new Array(entryCount); for (let t = 0; t < entryCount; ++t) - listings[t] = this.libzip.getName(this.zip, t, 0); - - this.listings = listings; + this.listings[t] = this.libzip.getName(this.zip, t, 0); this.symlinkCount = this.libzip.ext.countSymlinks(this.zip); if (this.symlinkCount === -1) { @@ -121,6 +202,7 @@ export class LibZipImpl implements ZipImpl { throw this.makeLibzipError(this.libzip.getError(this.zip)); } } + return newIndex; } catch (error) { this.libzip.source.free(lzSource); @@ -261,6 +343,7 @@ export class LibZipImpl implements ZipImpl { } finally { this.libzip.source.close(this.lzSource); this.libzip.source.free(this.lzSource); + libzipFactory.remove(this.key); } } @@ -307,5 +390,6 @@ export class LibZipImpl implements ZipImpl { public discard(): void { this.libzip.discard(this.zip); + libzipFactory.remove(this.key); } }