diff --git a/packages/dev/core/src/Loading/Plugins/babylonFileLoader.ts b/packages/dev/core/src/Loading/Plugins/babylonFileLoader.ts index 5dfd64096bc..3e452ebc2eb 100644 --- a/packages/dev/core/src/Loading/Plugins/babylonFileLoader.ts +++ b/packages/dev/core/src/Loading/Plugins/babylonFileLoader.ts @@ -464,6 +464,21 @@ const LoadAssetContainer = (scene: Scene, data: string | object, rootUrl: string instance._parentContainer = container; } } + // Load partProxies from GaussianSplattingMesh into AssetContainer + if (mesh.getClassName() === "GaussianSplattingMesh") { + const partProxies = (mesh as any)._partProxies as AbstractMesh[] | Map | undefined; + const proxies = Array.isArray(partProxies) ? partProxies : partProxies ? Array.from(partProxies.values()) : []; + for (const partProxy of proxies) { + if (!partProxy) { + continue; + } + container.meshes.push(partProxy); + partProxy._parentContainer = container; + if (partProxy._waitingParsedUniqueId != null) { + TempIndexContainer[partProxy._waitingParsedUniqueId] = partProxy; + } + } + } log += index === 0 ? "\n\tMeshes:" : ""; log += "\n\t\t" + mesh.toString(fullDetails); } @@ -1000,6 +1015,17 @@ RegisterSceneLoaderPlugin({ meshes.push(mesh); parsedIdToNodeMap.set(mesh._waitingParsedUniqueId!, mesh); mesh._waitingParsedUniqueId = null; + if (mesh.getClassName() === "GaussianSplattingMesh") { + const partProxies = (mesh as any)._partProxies as AbstractMesh[] | Map | undefined; + const proxies = Array.isArray(partProxies) ? partProxies : partProxies ? Array.from(partProxies.values()) : []; + for (const partProxy of proxies) { + if (!partProxy || partProxy._waitingParsedUniqueId == null) { + continue; + } + parsedIdToNodeMap.set(partProxy._waitingParsedUniqueId, partProxy); + partProxy._waitingParsedUniqueId = null; + } + } log += "\n\tMesh " + mesh.toString(fullDetails); } } diff --git a/packages/dev/core/src/Materials/GaussianSplatting/gaussianSplattingMaterial.ts b/packages/dev/core/src/Materials/GaussianSplatting/gaussianSplattingMaterial.ts index 4c1350c2e20..ad9c12e2062 100644 --- a/packages/dev/core/src/Materials/GaussianSplatting/gaussianSplattingMaterial.ts +++ b/packages/dev/core/src/Materials/GaussianSplatting/gaussianSplattingMaterial.ts @@ -597,6 +597,7 @@ export class GaussianSplattingMaterial extends PushMaterial { needAlphaBlending: alphaBlendedDepth, } ); + shaderMaterial.doNotSerialize = true; shaderMaterial.backFaceCulling = false; shaderMaterial.onBindObservable.add((mesh: AbstractMesh) => { const gsMaterial = mesh.material as GaussianSplattingMaterial; @@ -622,6 +623,7 @@ export class GaussianSplattingMaterial extends PushMaterial { shaderLanguage: shaderLanguage, } ); + shaderMaterial.doNotSerialize = true; shaderMaterial.backFaceCulling = false; const shadowDepthWrapper = new ShadowDepthWrapper(shaderMaterial, scene, { diff --git a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingCompoundMesh.ts b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingCompoundMesh.ts index 6979e1017e0..e4df3eb9ab7 100644 --- a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingCompoundMesh.ts +++ b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingCompoundMesh.ts @@ -2,6 +2,7 @@ import { type Nullable } from "core/types"; import { type Scene } from "core/scene"; import { GaussianSplattingMesh } from "./gaussianSplattingMesh"; import { type GaussianSplattingPartProxyMesh } from "./gaussianSplattingPartProxyMesh"; +import { Mesh } from "../mesh"; /** * Class used to compose multiple Gaussian Splatting meshes into a single draw call, @@ -26,7 +27,7 @@ export class GaussianSplattingCompoundMesh extends GaussianSplattingMesh { /** * Add another mesh to this compound mesh as a new part. - * The source mesh's splat data is read directly — no merged CPU buffer is constructed. + * The source mesh's splat data is read directly and copied into the compound's retained source buffers. * @param other - The other mesh to add. Must be fully loaded before calling this method. * @param disposeOther - Whether to dispose the other mesh after adding it. * @returns a placeholder mesh that can be used to manipulate the part transform @@ -37,7 +38,7 @@ export class GaussianSplattingCompoundMesh extends GaussianSplattingMesh { /** * Add multiple meshes to this compound mesh as new parts in a single operation. - * Splat data is written directly into texture arrays without constructing a merged CPU buffer. + * Splat data is written into texture arrays while the compound refreshes its retained merged source buffers. * @param others - The meshes to add. Each must be fully loaded and must not be a compound. * @param disposeOthers - Whether to dispose the other meshes after adding them. * @returns an array of placeholder meshes that can be used to manipulate the part transforms @@ -52,11 +53,38 @@ export class GaussianSplattingCompoundMesh extends GaussianSplattingMesh { /** * Remove a part from this compound mesh. - * The remaining parts are rebuilt directly from their stored source mesh references — - * no merged CPU splat buffer is read back. + * The remaining parts are rebuilt directly from the compound mesh's retained source buffers. * @param index - The index of the part to remove */ public override removePart(index: number): void { super.removePart(index); } + + /** + * Serialize current GaussianSplattingMesh + * @param serializationObject defines the object which will receive the serialization data + * @param encoding the encoding of binary data, defaults to base64 for json serialize, + * kept for future internal use like cloning where base64 encoding wastes cycles and memory + * @returns the serialized object + */ + public override serialize(serializationObject: any = {}, encoding: string = "base64"): any { + serializationObject = super.serialize(serializationObject, encoding); + // Note here, the getClassName() is not overridden, + // as a lot of code currently depend on `getClassName() === "GaussianSplattingMesh"` check, + // to not break those code, serialization uses `_isCompound` to mark the type + serializationObject._isCompound = true; + return serializationObject; + } + + /** + * Parses a serialized GaussianSplattingCompoundMesh + * @param parsedMesh the serialized mesh + * @param scene the scene to create the GaussianSplattingCompoundMesh in + * @returns the created GaussianSplattingCompoundMesh + */ + public static override Parse(parsedMesh: any, scene: Scene): GaussianSplattingCompoundMesh { + return GaussianSplattingMesh._ParseInternal(parsedMesh, scene, GaussianSplattingCompoundMesh); + } } + +Mesh._GaussianSplattingCompoundMeshParser = GaussianSplattingCompoundMesh.Parse; diff --git a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts index fb739748f49..3bfed6f6de9 100644 --- a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts +++ b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts @@ -8,10 +8,76 @@ import { GaussianSplattingMeshBase } from "./gaussianSplattingMeshBase"; import { RawTexture } from "core/Materials/Textures/rawTexture"; import { Constants } from "core/Engines/constants"; +import { DecodeBase64ToBinary, EncodeArrayBufferToBase64 } from "core/Misc/stringTools"; +import { Mesh } from "core/Meshes/mesh"; import "core/Meshes/thinInstanceMesh"; import { GaussianSplattingPartProxyMesh } from "./gaussianSplattingPartProxyMesh"; +import { type BoundingInfo } from "../../Culling/boundingInfo"; import { type BaseTexture } from "../../Materials/Textures/baseTexture"; +const _GaussianSplattingBytesPerSplat = 32; +const _GaussianSplattingBytesPerShTexel = 16; + +interface IGaussianSplattingPartSource { + name: string; + _vertexCount: number; + _splatsData: Nullable; + _shData: Nullable; + _shDegree: number; + isCompound: boolean; + getWorldMatrix(): Matrix; + getBoundingInfo(): BoundingInfo; + dispose(): void; +} + +/** + * Run-Length Encoding (RLE) compression for serialization + * Compressed Uint32Array can be parsed using {@link ParsePartIndices} + * Some notes for devs: We do not expect Uint8Array larger than 4GB, + * so it should be safe to use Uint32Array. + * @param partIndices A view of partIndices from GaussianSplattingMesh + * @returns A compressed Uint32Array of [count, value, ...] + */ +function CompressPartIndices(partIndices: Uint8Array): Uint32Array { + const runs: number[] = []; + const length = partIndices.length; + let i = 0; + while (i < length) { + const value = partIndices[i]; + let count = 1; + while (i + count < length && partIndices[i + count] === value) { + count++; + } + runs.push(count, value); + i += count; + } + return new Uint32Array(runs); +} + +/** + * Parse partIndices compressed by {@link CompressPartIndices} to runtime array + * @param compressed The compressed partIndices of [count, value, ...] + * @returns runtime Uint8Array for GaussianSplattingMesh + */ +function ParsePartIndices(compressed: Uint32Array | number[]): Uint8Array { + let totalCount = 0; + const length = compressed.length; + for (let i = 0; i < length; i += 2) { + totalCount += compressed[i]; + } + + const partIndices = new Uint8Array(totalCount); + let offset = 0; + for (let i = 0; i < length; i += 2) { + const count = compressed[i]; + const value = compressed[i + 1]; + partIndices.fill(value, offset, offset + count); + offset += count; + } + + return partIndices; +} + /** * Class used to render a Gaussian Splatting mesh. Supports both single-cloud and compound * (multi-part) rendering. In compound mode, multiple Gaussian Splatting source meshes are @@ -344,16 +410,119 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase { } } + private _appendPartSourceToArrays( + source: IGaussianSplattingPartSource, + dstOffset: number, + covA: Uint16Array, + covB: Uint16Array, + colorArray: Uint8Array, + sh: Uint8Array[] | undefined, + minimum: Vector3, + maximum: Vector3 + ): void { + this._appendSourceToArrays(source as unknown as GaussianSplattingMeshBase, dstOffset, covA, covB, colorArray, sh, minimum, maximum); + } + + private _createRetainedPartSource(proxy: GaussianSplattingPartProxyMesh): Nullable { + if (!this._splatsData || (this._shDegree > 0 && !this._shData)) { + return null; + } + + const splatByteOffset = proxy._splatsDataOffset * _GaussianSplattingBytesPerSplat; + const splatByteLength = proxy._vertexCount * _GaussianSplattingBytesPerSplat; + const shByteOffset = proxy._shDataOffset * _GaussianSplattingBytesPerShTexel; + const shByteLength = proxy._vertexCount * _GaussianSplattingBytesPerShTexel; + + return { + name: proxy.name, + _vertexCount: proxy._vertexCount, + _splatsData: this._splatsData.slice(splatByteOffset, splatByteOffset + splatByteLength), + _shData: this._shData?.map((texture) => texture.slice(shByteOffset, shByteOffset + shByteLength)) ?? null, + _shDegree: this._shData?.length ?? 0, + isCompound: false, + getWorldMatrix: () => proxy.getWorldMatrix(), + getBoundingInfo: () => proxy.getBoundingInfo(), + dispose: () => {}, + }; + } + + private _retainMergedPartData(existingVertexCount: number, totalCount: number, others: IGaussianSplattingPartSource[], shDegree: number): void { + if (!this._keepInRam && !this._alwaysRetainSplatsData) { + this._splatsData = null; + this._shData = null; + return; + } + + const getSourceBuffer = (data: ArrayBuffer): ArrayBuffer => { + return data instanceof ArrayBuffer ? data : ((data as unknown as ArrayBufferView).buffer as ArrayBuffer); + }; + + const mergedSplatsData = new Uint8Array(totalCount * _GaussianSplattingBytesPerSplat); + let splatByteOffset = 0; + + if (this._splatsData && existingVertexCount > 0) { + mergedSplatsData.set(new Uint8Array(getSourceBuffer(this._splatsData), 0, existingVertexCount * _GaussianSplattingBytesPerSplat), splatByteOffset); + splatByteOffset += existingVertexCount * _GaussianSplattingBytesPerSplat; + } + + for (const other of others) { + if (!other._splatsData) { + continue; + } + + const splatByteLength = other._vertexCount * _GaussianSplattingBytesPerSplat; + mergedSplatsData.set(new Uint8Array(getSourceBuffer(other._splatsData), 0, splatByteLength), splatByteOffset); + splatByteOffset += splatByteLength; + } + + this._splatsData = mergedSplatsData.buffer; + + if (shDegree <= 0) { + this._shData = null; + return; + } + + const mergedShData: Uint8Array[] = []; + for (let textureIndex = 0; textureIndex < shDegree; textureIndex++) { + mergedShData.push(new Uint8Array(totalCount * _GaussianSplattingBytesPerShTexel)); + } + + let shByteOffset = 0; + if (this._shData && existingVertexCount > 0) { + const existingShByteLength = existingVertexCount * _GaussianSplattingBytesPerShTexel; + for (let textureIndex = 0; textureIndex < mergedShData.length; textureIndex++) { + if (textureIndex < this._shData.length) { + mergedShData[textureIndex].set(this._shData[textureIndex].subarray(0, existingShByteLength), shByteOffset); + } + } + shByteOffset += existingShByteLength; + } + + for (const other of others) { + const otherShByteLength = other._vertexCount * _GaussianSplattingBytesPerShTexel; + if (other._shData) { + for (let textureIndex = 0; textureIndex < mergedShData.length; textureIndex++) { + if (textureIndex < other._shData.length) { + mergedShData[textureIndex].set(other._shData[textureIndex].subarray(0, otherShByteLength), shByteOffset); + } + } + } + shByteOffset += otherShByteLength; + } + + this._shData = mergedShData; + } + /** - * Core implementation for adding one or more external GaussianSplattingMesh objects as new - * parts. Writes directly into texture-sized CPU arrays and uploads in one pass — no merged - * CPU splat buffer is ever constructed. + * Core implementation for adding one or more source parts as new + * parts. Writes directly into texture-sized CPU arrays, updates the retained merged source + * buffers, and uploads in one pass. * * @param others - Source meshes to append (must each be non-compound and fully loaded) * @param disposeOthers - Dispose source meshes after appending * @returns Proxy meshes and their assigned part indices */ - protected _addPartsInternal(others: GaussianSplattingMesh[], disposeOthers: boolean): { proxyMeshes: GaussianSplattingPartProxyMesh[]; assignedPartIndices: number[] } { + protected _addPartsInternal(others: IGaussianSplattingPartSource[], disposeOthers: boolean): { proxyMeshes: GaussianSplattingPartProxyMesh[]; assignedPartIndices: number[] } { if (others.length === 0) { return { proxyMeshes: [], assignedPartIndices: [] }; } @@ -420,6 +589,7 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase { this._partIndices.set(partIndicesA.subarray(0, splatCountA)); const assignedPartIndices: number[] = []; + const assignedSplatsDataOffsets: number[] = []; let dstOffset = splatCountA; const maxPartCount = GetGaussianSplattingMaxPartCount(this._scene.getEngine()); for (const other of others) { @@ -428,6 +598,7 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase { } const newPartIndex = nextPartIndex++; assignedPartIndices.push(newPartIndex); + assignedSplatsDataOffsets.push(dstOffset); this._partIndices.fill(newPartIndex, dstOffset, dstOffset + other._vertexCount); dstOffset += other._vertexCount; } @@ -461,7 +632,7 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase { // Rebuild the compound's legacy "own" data at part 0 (scenario A only). // Skipped in the preferred empty-composer path (scenario B). if (!this._partProxies[0] && this._splatsData) { - const proxyVertexCount = this._partProxies.reduce((sum, proxy) => sum + (proxy ? proxy.proxiedMesh._vertexCount : 0), 0); + const proxyVertexCount = this._partProxies.reduce((sum, proxy) => sum + (proxy ? proxy._vertexCount : 0), 0); const part0Count = splatCountA - proxyVertexCount; if (part0Count > 0) { const uBufA = new Uint8Array(this._splatsData); @@ -485,11 +656,15 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase { // scenario B, part 0 is itself a proxied part with no implicit "own" data. for (let partIndex = 0; partIndex < this._partProxies.length; partIndex++) { const proxy = this._partProxies[partIndex]; - if (!proxy || !proxy.proxiedMesh) { + if (!proxy) { continue; } - this._appendSourceToArrays(proxy.proxiedMesh, rebuildOffset, covA, covB, colorArray, sh, minimum, maximum); - rebuildOffset += proxy.proxiedMesh._vertexCount; + const source = this._createRetainedPartSource(proxy); + if (!source) { + throw new Error(`Cannot rebuild compound part "${proxy.name}": the retained compound source data is not available.`); + } + this._appendPartSourceToArrays(source, rebuildOffset, covA, covB, colorArray, sh, minimum, maximum); + rebuildOffset += source._vertexCount; } } else { // No proxies yet: this is the very first addPart call on a mesh that loaded @@ -541,7 +716,7 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase { // Handles both layouts (see full-rebuild comment above): // A) LEGACY: _partProxies[0] absent → seed lookup[0] with this._splatsData // B) PREFERRED: _partProxies[0] present → all entries filled from proxies - const proxyTotal = this._partProxies.reduce((s, p) => s + (p ? p.proxiedMesh._vertexCount : 0), 0); + const proxyTotal = this._partProxies.reduce((s, p) => s + (p ? p._vertexCount : 0), 0); const part0Count = splatCountA - proxyTotal; // > 0 only in legacy scenario A const srcUBufs: (Uint8Array | null)[] = new Array(this._partProxies.length).fill(null); const srcFBufs: (Float32Array | null)[] = new Array(this._partProxies.length).fill(null); @@ -556,14 +731,17 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase { let cumOffset = part0Count; for (let pi = 0; pi < this._partProxies.length; pi++) { const proxy = this._partProxies[pi]; - if (!proxy?.proxiedMesh) { + if (!proxy) { continue; } - const srcData = proxy.proxiedMesh._splatsData ?? null; - srcUBufs[pi] = srcData ? new Uint8Array(srcData) : null; - srcFBufs[pi] = srcData ? new Float32Array(srcData) : null; + const source = this._createRetainedPartSource(proxy); + if (!source || !source._splatsData) { + throw new Error(`Cannot rebuild compound part "${proxy.name}": the retained compound source data is not available.`); + } + srcUBufs[pi] = new Uint8Array(source._splatsData); + srcFBufs[pi] = new Float32Array(source._splatsData); partStarts[pi] = cumOffset; - cumOffset += proxy.proxiedMesh._vertexCount; + cumOffset += source._vertexCount; } for (let splatIdx = firstNewTexel; splatIdx < splatCountA; splatIdx++) { const partIdx = this._partIndices ? this._partIndices[splatIdx] : 0; @@ -580,7 +758,7 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase { // Append each new source dstOffset = splatCountA; for (const other of others) { - this._appendSourceToArrays(other, dstOffset, covA, covB, colorArray, sh, minimum, maximum); + this._appendPartSourceToArrays(other, dstOffset, covA, covB, colorArray, sh, minimum, maximum); dstOffset += other._vertexCount; } @@ -594,6 +772,7 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase { if (totalCount !== this._vertexCount) { this._updateSplatIndexBuffer(totalCount); } + this._retainMergedPartData(splatCountA, totalCount, others, shDegreeNew); this._vertexCount = totalCount; this._shDegree = shDegreeNew; @@ -636,7 +815,16 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase { const partWorldMatrix = other.getWorldMatrix(); this.setWorldMatrixForPart(newPartIndex, partWorldMatrix); - const proxyMesh = new GaussianSplattingPartProxyMesh(other.name, this.getScene(), this, other, newPartIndex); + const proxyMesh = new GaussianSplattingPartProxyMesh( + other.name, + this.getScene(), + this, + newPartIndex, + other.getBoundingInfo(), + other._vertexCount, + assignedSplatsDataOffsets[i], + assignedSplatsDataOffsets[i] + ); if (disposeOthers) { other.dispose(); @@ -679,7 +867,7 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase { /** * Add another mesh to this mesh, as a new part. This makes the current mesh a compound, if not already. - * The source mesh's splat data is read directly — no merged CPU buffer is constructed. + * The source mesh's splat data is read directly and copied into the compound's retained source buffers. * @param other - The other mesh to add. Must be fully loaded before calling this method. * @param disposeOther - Whether to dispose the other mesh after adding it to the current mesh. * @returns a placeholder mesh that can be used to manipulate the part transform @@ -692,9 +880,9 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase { /** * Remove a part from this compound mesh. - * The remaining parts are rebuilt directly from their stored source mesh references — - * no merged CPU splat buffer is read back. The current mesh is reset to a plain (single-part) - * state and then each remaining source is re-added via addParts. + * The remaining parts are rebuilt directly from the compound mesh's retained source buffers. + * The current mesh is reset to a plain (single-part) state and then each remaining source is + * re-added via addParts. * @param index - The index of the part to remove * @deprecated Use {@link GaussianSplattingCompoundMesh.removePart} instead. */ @@ -704,19 +892,27 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase { } // Collect surviving proxy objects (sorted by current part index so part 0 is added first) - const survivors: Array<{ proxyMesh: GaussianSplattingPartProxyMesh; oldIndex: number; worldMatrix: Matrix; visibility: number }> = []; + const survivors: Array<{ proxyMesh: GaussianSplattingPartProxyMesh; source: IGaussianSplattingPartSource; oldIndex: number; worldMatrix: Matrix; visibility: number }> = []; for (let proxyIndex = 0; proxyIndex < this._partProxies.length; proxyIndex++) { const proxy = this._partProxies[proxyIndex]; - if (proxy && proxyIndex !== index) { - survivors.push({ proxyMesh: proxy, oldIndex: proxyIndex, worldMatrix: proxy.getWorldMatrix().clone(), visibility: this._partVisibility[proxyIndex] ?? 1.0 }); + if (!proxy || proxyIndex === index) { + continue; + } + const source = this._createRetainedPartSource(proxy); + if (!source) { + throw new Error(`Cannot remove part: the retained compound source data is not available for part "${proxy.name}".`); } + survivors.push({ proxyMesh: proxy, source, oldIndex: proxyIndex, worldMatrix: proxy.getWorldMatrix().clone(), visibility: this._partVisibility[proxyIndex] ?? 1.0 }); } survivors.sort((a, b) => a.oldIndex - b.oldIndex); // Validate every survivor still has its source data. If even one is missing we cannot rebuild. - for (const { proxyMesh } of survivors) { - if (!proxyMesh.proxiedMesh._splatsData) { - throw new Error(`Cannot remove part: the source mesh for part "${proxyMesh.name}" no longer has its splat data available.`); + for (const { proxyMesh, source } of survivors) { + if (!source._splatsData) { + throw new Error(`Cannot remove part: the source data for part "${proxyMesh.name}" is not available.`); + } + if (source._shDegree > 0 && !source._shData) { + throw new Error(`Cannot remove part: the SH data for part "${proxyMesh.name}" is not available.`); } } @@ -757,6 +953,9 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase { this._partVisibility = []; this._cachedBoundingMin = null; this._cachedBoundingMax = null; + this._splatsData = null; + this._shData = null; + this._shDegree = 0; // Remove the proxy for the removed part and dispose it const proxyToRemove = this._partProxies[index]; @@ -777,7 +976,7 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase { this._rebuilding = true; this._canPostToWorker = false; try { - const sources = survivors.map((s) => s.proxyMesh.proxiedMesh); + const sources = survivors.map((s) => s.source); const { proxyMeshes: newProxies } = this._addPartsInternal(sources, false); // Restore world matrices and re-map proxies @@ -794,8 +993,9 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase { newProxy.rotationQuaternion = quaternion; newProxy.computeWorldMatrix(true); - // Update the old proxy's index so any existing user references still work + // Update the old proxy's index and metadata so existing user references still work. oldProxy.updatePartIndex(newPartIndex); + oldProxy.updatePartMetadata(newProxy._vertexCount, newProxy._splatsDataOffset, newProxy._shDataOffset); this._partProxies[newPartIndex] = oldProxy; // newProxy is redundant — it was created inside _addPartsInternal; dispose it @@ -817,4 +1017,122 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase { throw e; } } + + /** + * Serialize current GaussianSplattingMesh + * @param serializationObject defines the object which will receive the serialization data + * @param encoding the encoding of binary data, defaults to base64 for json serialize, + * kept for future internal use like cloning where base64 encoding wastes cycles and memory + * @returns the serialized object + */ + public override serialize(serializationObject: any = {}, encoding: string = "base64"): any { + serializationObject = super.serialize(serializationObject); + serializationObject.subMeshes = []; + serializationObject.geometryUniqueId = undefined; + serializationObject.geometryId = undefined; + serializationObject.materialUniqueId = undefined; + serializationObject.materialId = undefined; + serializationObject.instances = []; + serializationObject.actions = undefined; + serializationObject.type = this.getClassName(); + serializationObject.keepInRam = this._keepInRam; + serializationObject.disableDepthSort = this._disableDepthSort; + serializationObject.viewUpdateThreshold = this.viewUpdateThreshold; + serializationObject._flipY = this._flipY; + + if (this._splatsData) { + serializationObject.splatsData = encoding === "base64" ? EncodeArrayBufferToBase64(this._splatsData) : this._splatsData; + } + if (this._shData) { + serializationObject.shData = encoding === "base64" ? this._shData.map(EncodeArrayBufferToBase64) : this._shData; + } + if (this._partIndices) { + const compressedIndices = CompressPartIndices(this._partIndices.subarray(0, this._vertexCount)); + serializationObject.partIndices = encoding === "base64" ? EncodeArrayBufferToBase64(compressedIndices) : compressedIndices; + } + if (this._partProxies.length) { + serializationObject.partProxies = this._partProxies.filter((proxy) => !!proxy).map((proxy) => proxy.serialize()); + } + + return serializationObject; + } + + /** + * Internal helper to parses a serialized GaussianSplattingMesh or GaussianSplattingCompoundMesh + * @param parsedMesh the serialized mesh + * @param scene the scene to create the GaussianSplattingMesh or GaussianSplattingCompoundMesh in + * @param ctor the constructor of the mesh to create + * @returns the created GaussianSplattingMesh + * @internal + */ + public static _ParseInternal( + parsedMesh: any, + scene: Scene, + ctor: new (name: string, url: Nullable, scene: Nullable, keepInRam: boolean) => T + ): T { + const mesh = new ctor(parsedMesh.name, null, scene, parsedMesh.keepInRam); + + mesh.disableDepthSort = parsedMesh.disableDepthSort ?? false; + mesh.viewUpdateThreshold = parsedMesh.viewUpdateThreshold ?? GaussianSplattingMeshBase._DefaultViewUpdateThreshold; + + let splatsData: ArrayBuffer | string | undefined = parsedMesh.splatsData; + if (typeof splatsData === "string") { + splatsData = DecodeBase64ToBinary(splatsData); + } + + const shData: string[] | Uint8Array[] | undefined = parsedMesh.shData; + let parsedShData: Uint8Array[] | undefined; + if (Array.isArray(shData) && shData.length) { + const newData: Uint8Array[] = []; + for (let i = 0, length = shData.length; i < length; i++) { + const data = shData[i]; + if (typeof data === "string") { + newData[i] = new Uint8Array(DecodeBase64ToBinary(data)); + } else { + newData[i] = data; + } + } + parsedShData = newData; + } + + let partIndices: string | Uint32Array | number[] | undefined = parsedMesh.partIndices; + let parsedPartIndices: Uint8Array | undefined; + if (typeof partIndices === "string") { + partIndices = new Uint32Array(DecodeBase64ToBinary(partIndices)); + } + if (partIndices) { + parsedPartIndices = ParsePartIndices(partIndices); + } + + if (splatsData) { + const flipY = parsedMesh._flipY ?? false; + mesh.updateData(splatsData, parsedShData, { flipY }, parsedPartIndices); + } + + if (parsedMesh.partProxies) { + for (const serializedPart of parsedMesh.partProxies) { + const part = Object.assign({}, serializedPart); + part.compoundSplatMesh = mesh; + const proxyMesh = Mesh.Parse(part, scene, "") as GaussianSplattingPartProxyMesh; + const newPartIndex = proxyMesh.partIndex; + mesh._partProxies[newPartIndex] = proxyMesh; + mesh.setWorldMatrixForPart(newPartIndex, proxyMesh.getWorldMatrix()); + mesh.setPartVisibility(newPartIndex, proxyMesh.visibility); + } + } + + return mesh; + } + + /** + * Parses a serialized GaussianSplattingMesh + * @param parsedMesh the serialized mesh + * @param scene the scene to create the GaussianSplattingMesh in + * @returns the created GaussianSplattingMesh + */ + public static override Parse(parsedMesh: any, scene: Scene): GaussianSplattingMesh { + return GaussianSplattingMesh._ParseInternal(parsedMesh, scene, GaussianSplattingMesh); + } } + +Mesh._GaussianSplattingMeshParser = GaussianSplattingMesh.Parse; diff --git a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMeshBase.ts b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMeshBase.ts index ae04ba0d947..a7ade34bfb2 100644 --- a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMeshBase.ts +++ b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMeshBase.ts @@ -323,6 +323,7 @@ export class GaussianSplattingMeshBase extends Mesh { private _textureSize: Vector2 = new Vector2(0, 0); protected readonly _keepInRam: boolean = false; protected _alwaysRetainSplatsData: boolean = false; + protected _flipY = false; private _delayedTextureUpdate: Nullable = null; protected _useRGBACovariants = false; @@ -348,7 +349,7 @@ export class GaussianSplattingMeshBase extends Mesh { private static readonly _BatchSize = 16; // 16 splats per instance private _cameraViewInfos = new Map(); - private static readonly _DefaultViewUpdateThreshold = 1e-4; + protected static readonly _DefaultViewUpdateThreshold = 1e-4; /** * Cosine value of the angle threshold to update view dependent splat sorting. Default is 0.0001. @@ -554,6 +555,7 @@ export class GaussianSplattingMeshBase extends Mesh { const gaussianSplattingMaterial = new GaussianSplattingMaterial(this.name + "_material", this._scene); // Cast is safe: GaussianSplattingMeshBase is @internal; all concrete instances are GaussianSplattingMesh. gaussianSplattingMaterial.setSourceMesh(this as any); + gaussianSplattingMaterial.doNotSerialize = true; this._material = gaussianSplattingMaterial; // delete meshes created for cameras on camera removal @@ -658,6 +660,7 @@ export class GaussianSplattingMeshBase extends Mesh { } else { // mesh doesn't exist yet for this camera const cameraMesh = new Mesh(this.name + "_cameraMesh_" + cameraId, this._scene); + cameraMesh.doNotSerialize = true; // not visible with inspector or the scene graph cameraMesh.reservedDataStore = { hidden: true }; cameraMesh.setEnabled(false); @@ -1509,8 +1512,8 @@ export class GaussianSplattingMeshBase extends Mesh { this._cachedBoundingMin = null; this._cachedBoundingMax = null; // Note: _splatsData and _shData are intentionally kept alive after dispose. - // They serve as the source reference for the compound API (addPart/removePart) - // when this mesh is held by a GaussianSplattingPartProxyMesh.compoundSplatMesh. + // They can still be used as runtime source buffers by a compound mesh that retained + // this mesh's data before disposal. this._worker?.terminate(); this._worker = null; @@ -1925,6 +1928,7 @@ export class GaussianSplattingMeshBase extends Mesh { if (!this._covariancesATexture) { this._readyToDisplay = false; } + this._flipY = flipY; const uBuffer = new Uint8Array(data); const fBuffer = new Float32Array(uBuffer.buffer); diff --git a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingPartProxyMesh.ts b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingPartProxyMesh.ts index fd9119296e9..921f258076e 100644 --- a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingPartProxyMesh.ts +++ b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingPartProxyMesh.ts @@ -12,15 +12,36 @@ import { Vector3 } from "../../Maths/math.vector"; */ export class GaussianSplattingPartProxyMesh extends Mesh { /** - * The Gaussian Splatting mesh that this proxy represents a part of + * Local-space bounds for this part, stored directly on the proxy so it does not + * need to retain a reference to the original source mesh. */ - public readonly proxiedMesh: GaussianSplattingMesh; + private _minimum: Vector3; + private _maximum: Vector3; /** * The index of the part in the compound mesh (internal storage) */ private _partIndex: number; + /** + * Number of splats owned by this part. + * @internal + */ + public _vertexCount: number; + + /** + * Offset of this part in the compound splat ordering. + * @internal + */ + public _splatsDataOffset: number; + + /** + * Texel offset of this part inside the compound SH textures. + * This matches the splat offset because SH data is stored one texel per splat. + * @internal + */ + public _shDataOffset: number; + /** * Gets the index of the part in the compound mesh */ @@ -29,41 +50,73 @@ export class GaussianSplattingPartProxyMesh extends Mesh { } /** - * The original Gaussian Splatting mesh that was merged into the compound + * The compound mesh that owns this part proxy */ public readonly compoundSplatMesh: GaussianSplattingMesh; + /** + * Backward-compatible alias for the owning compound mesh. + * @deprecated Use `compoundSplatMesh` instead. + */ + public get proxiedMesh(): GaussianSplattingMesh { + return this.compoundSplatMesh; + } + /** * Creates a new Gaussian Splatting part proxy mesh * @param name The name of the proxy mesh * @param scene The scene the proxy mesh belongs to - * @param compoundSplatMesh The original Gaussian Splatting mesh that was merged into the compound - * @param proxiedMesh The Gaussian Splatting mesh that this proxy represents a part of + * @param compoundSplatMesh The compound mesh that owns this part proxy * @param partIndex The index of the part in the compound mesh + * @param boundingInfo Local-space bounds of the part inside the compound mesh + * @param vertexCount Number of splats owned by the part + * @param splatsDataOffset Offset of the part in the compound splat ordering + * @param shDataOffset Offset of the part in the compound SH textures */ - constructor(name: string, scene: Nullable, compoundSplatMesh: GaussianSplattingMesh, proxiedMesh: GaussianSplattingMesh, partIndex: number) { + constructor( + name: string, + scene: Nullable, + compoundSplatMesh: GaussianSplattingMesh, + partIndex: number, + boundingInfo: BoundingInfo, + vertexCount: number, + splatsDataOffset: number, + shDataOffset: number = splatsDataOffset + ) { super(name, scene); - this.proxiedMesh = proxiedMesh; this._partIndex = partIndex; + this._vertexCount = vertexCount; + this._splatsDataOffset = splatsDataOffset; + this._shDataOffset = shDataOffset; + this._minimum = boundingInfo.minimum.clone(); + this._maximum = boundingInfo.maximum.clone(); this.compoundSplatMesh = compoundSplatMesh; - // Set bounding info based on the source mesh's bounds - this.updateBoundingInfoFromProxiedMesh(); + this._applyBoundingInfo(); this.compoundSplatMesh.setWorldMatrixForPart(this.partIndex, this.getWorldMatrix()); - // Update the proxied mesh's part matrix when this proxy's world matrix changes + // Update the compound mesh's part matrix when this proxy's world matrix changes. this.onAfterWorldMatrixUpdateObservable.add(() => { this.compoundSplatMesh.setWorldMatrixForPart(this.partIndex, this.getWorldMatrix()); - this.updateBoundingInfoFromProxiedMesh(); }); } /** - * Updates the bounding info of this proxy mesh from the proxied mesh + * Updates the bounding info of this proxy mesh from its stored part metadata. + */ + public updateBoundingInfoFromPartData(): void { + this._applyBoundingInfo(); + } + + /** + * Backward-compatible alias retained while callers move away from source-mesh based semantics. */ public updateBoundingInfoFromProxiedMesh(): void { - const boundingInfo = this.proxiedMesh.getBoundingInfo(); - this.setBoundingInfo(new BoundingInfo(boundingInfo.minimum.clone(), boundingInfo.maximum.clone())); + this.updateBoundingInfoFromPartData(); + } + + private _applyBoundingInfo(): void { + this.setBoundingInfo(new BoundingInfo(this._minimum.clone(), this._maximum.clone())); } /** @@ -84,6 +137,20 @@ export class GaussianSplattingPartProxyMesh extends Mesh { this._partIndex = newPartIndex; } + /** + * Updates the per-part metadata for this proxy mesh. + * This is used internally when compound parts are rebuilt and re-indexed. + * @param vertexCount the number of splats owned by the part + * @param splatsDataOffset the new splat offset in the compound + * @param shDataOffset the new SH texel offset in the compound + * @internal + */ + public updatePartMetadata(vertexCount: number, splatsDataOffset: number, shDataOffset: number = splatsDataOffset): void { + this._vertexCount = vertexCount; + this._splatsDataOffset = splatsDataOffset; + this._shDataOffset = shDataOffset; + } + /** * Gets whether the part is visible */ @@ -138,4 +205,58 @@ export class GaussianSplattingPartProxyMesh extends Mesh { return pickingInfo; } + + /** + * Serialize current GaussianSplattingPartProxyMesh + * @param serializationObject defines the object which will receive the serialization data + * @returns the serialized object + */ + public override serialize(serializationObject: any = {}): any { + serializationObject = super.serialize(serializationObject); + // GaussianSplattingPartProxyMesh needs no SubMesh, Geometry, or Material + serializationObject.subMeshes = []; + serializationObject.geometryUniqueId = undefined; + serializationObject.geometryId = undefined; + serializationObject.materialUniqueId = undefined; + serializationObject.materialId = undefined; + serializationObject.instances = []; + serializationObject.actions = undefined; + serializationObject.type = this.getClassName(); + serializationObject.compoundSplatMeshId = this.compoundSplatMesh.id; + // Part metadata is needed to reconnect the proxy to the correct compound segment. + serializationObject.partIndex = this._partIndex; + serializationObject.vertexCount = this._vertexCount; + serializationObject.splatsDataOffset = this._splatsDataOffset; + serializationObject.shDataOffset = this._shDataOffset; + const boundingInfo = this.getBoundingInfo(); + serializationObject.boundingInfo = { + minimum: boundingInfo.minimum.asArray(), + maximum: boundingInfo.maximum.asArray(), + }; + return serializationObject; + } + + /** + * Parses a serialized GaussianSplattingPartProxyMesh + * @param parsedMesh the serialized mesh + * @param scene the scene to create the GaussianSplattingPartProxyMesh in + * @returns the created GaussianSplattingPartProxyMesh + */ + public static override Parse(parsedMesh: any, scene: Scene): GaussianSplattingPartProxyMesh { + const partIndex = parsedMesh.partIndex; + const compoundSplatMesh = + (parsedMesh.compoundSplatMesh as GaussianSplattingMesh | undefined) ?? (scene.getLastMeshById(parsedMesh.compoundSplatMeshId) as GaussianSplattingMesh | null); + if (!compoundSplatMesh) { + throw new Error(`GaussianSplattingPartProxyMesh: compound mesh not found with ID ${parsedMesh.compoundSplatMeshId}`); + } + const minimum = Vector3.FromArray(parsedMesh.boundingInfo.minimum); + const maximum = Vector3.FromArray(parsedMesh.boundingInfo.maximum); + const boundingInfo = new BoundingInfo(minimum, maximum); + const vertexCount = parsedMesh.vertexCount ?? 0; + const splatsDataOffset = parsedMesh.splatsDataOffset ?? 0; + const shDataOffset = parsedMesh.shDataOffset ?? splatsDataOffset; + return new GaussianSplattingPartProxyMesh(parsedMesh.name, scene, compoundSplatMesh, partIndex, boundingInfo, vertexCount, splatsDataOffset, shDataOffset); + } } + +Mesh._GaussianSplattingPartProxyMeshParser = GaussianSplattingPartProxyMesh.Parse; diff --git a/packages/dev/core/src/Meshes/mesh.ts b/packages/dev/core/src/Meshes/mesh.ts index 5b15d456a59..6e9d489be11 100644 --- a/packages/dev/core/src/Meshes/mesh.ts +++ b/packages/dev/core/src/Meshes/mesh.ts @@ -4445,6 +4445,33 @@ export class Mesh extends AbstractMesh implements IGetSetVerticesData { throw _WarnImport("TrailMesh"); }; + /** + * Holder function for GaussianSplattingMesh Parser, should be GaussianSplattingMesh.Parse after imported + * @internal + */ + // eslint-disable-next-line @typescript-eslint/no-unused-vars + public static _GaussianSplattingMeshParser = (parsedMesh: any, scene: Scene): Mesh => { + throw _WarnImport("GaussianSplattingMesh"); + }; + + /** + * Holder function for GaussianSplattingPartProxyMesh Parser, should be GaussianSplattingPartProxyMesh.Parse after imported + * @internal + */ + // eslint-disable-next-line @typescript-eslint/no-unused-vars + public static _GaussianSplattingPartProxyMeshParser = (parsedMesh: any, scene: Scene): Mesh => { + throw _WarnImport("GaussianSplattingPartProxyMesh"); + }; + + /** + * Holder function for GaussianSplattingCompoundMesh Parser, should be GaussianSplattingCompoundMesh.Parse after imported + * @internal + */ + // eslint-disable-next-line @typescript-eslint/no-unused-vars + public static _GaussianSplattingCompoundMeshParser = (parsedMesh: any, scene: Scene): Mesh => { + throw _WarnImport("GaussianSplattingCompoundMesh"); + }; + /** * Returns a new Mesh object parsed from the source provided. * @param parsedMesh is the source @@ -4454,6 +4481,8 @@ export class Mesh extends AbstractMesh implements IGetSetVerticesData { */ public static override Parse(parsedMesh: any, scene: Scene, rootUrl: string): Mesh { let mesh: Mesh; + // Should not import Geometry for GaussianSplattingMesh and GaussianSplattingPartProxyMesh + let skipImportGeometry = false; if (parsedMesh.type && parsedMesh.type === "LinesMesh") { mesh = Mesh._LinesMeshParser(parsedMesh, scene); @@ -4465,6 +4494,16 @@ export class Mesh extends AbstractMesh implements IGetSetVerticesData { mesh = Mesh._GreasedLineMeshParser(parsedMesh, scene); } else if (parsedMesh.type && parsedMesh.type === "TrailMesh") { mesh = Mesh._TrailMeshParser(parsedMesh, scene); + } else if (parsedMesh.type && parsedMesh.type === "GaussianSplattingMesh") { + if (parsedMesh._isCompound) { + mesh = Mesh._GaussianSplattingCompoundMeshParser(parsedMesh, scene); + } else { + mesh = Mesh._GaussianSplattingMeshParser(parsedMesh, scene); + } + skipImportGeometry = true; + } else if (parsedMesh.type && parsedMesh.type === "GaussianSplattingPartProxyMesh") { + mesh = Mesh._GaussianSplattingPartProxyMeshParser(parsedMesh, scene); + skipImportGeometry = true; } else { mesh = new Mesh(parsedMesh.name, scene); } @@ -4638,7 +4677,7 @@ export class Mesh extends AbstractMesh implements IGetSetVerticesData { if (SceneLoaderFlags.ForceFullSceneLoadingForIncremental) { mesh._checkDelayState(); } - } else { + } else if (!skipImportGeometry) { Geometry._ImportGeometry(parsedMesh, mesh); } diff --git a/packages/dev/core/src/Misc/sceneSerializer.ts b/packages/dev/core/src/Misc/sceneSerializer.ts index 2916042d765..166fbcb4ef7 100644 --- a/packages/dev/core/src/Misc/sceneSerializer.ts +++ b/packages/dev/core/src/Misc/sceneSerializer.ts @@ -325,7 +325,8 @@ export class SceneSerializer { for (index = 0; index < scene.meshes.length; index++) { const abstractMesh = scene.meshes[index]; - if (abstractMesh instanceof Mesh) { + // GaussianSplattingPartProxyMesh would be serialized with the GaussianSplattingMesh holding it + if (abstractMesh instanceof Mesh && abstractMesh.getClassName() !== "GaussianSplattingPartProxyMesh") { const mesh = abstractMesh; if (!mesh.doNotSerialize) { if (mesh.delayLoadState === Constants.DELAYLOADSTATE_LOADED || mesh.delayLoadState === Constants.DELAYLOADSTATE_NONE) { diff --git a/packages/tools/tests/test/visualization/config.json b/packages/tools/tests/test/visualization/config.json index 657a34b1448..209129df0e4 100644 --- a/packages/tools/tests/test/visualization/config.json +++ b/packages/tools/tests/test/visualization/config.json @@ -89,6 +89,13 @@ "errorRatio": 5, "referenceImage": "gsplat-part-test.png" }, + { + "title": "Gaussian Splatting Part Serialize Test", + "playgroundId": "#XQZVOI#0", + "renderCount": 15, + "errorRatio": 5, + "referenceImage": "gsplat-part-test.png" + }, { "title": "RH billboard", "playgroundId": "#PDO1L6#1",