diff --git a/.gitignore b/.gitignore index 6fbae5ebc5..78f816145c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ bin/ +pkg/automerge/internal/reference/wasm/target/ node_modules/ .cache/ .turbo diff --git a/GNUmakefile b/GNUmakefile index 247103007c..88c11229e5 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -2,6 +2,7 @@ NPROC ?= $(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || getconf _ MAKEFLAGS := --jobs=$(NPROC) CAT ?= cat +CARGO ?= cargo CP ?= cp DOCKER ?= docker GO ?= go @@ -13,12 +14,14 @@ NPM ?= npm NPX ?= npx OPENSSL ?= openssl SED ?= sed +SHA256SUM ?= sha256sum SYFT ?= syft TAIL ?= tail ECHO ?= echo GOLINTCMD ?= golangci-lint SWIFTLINTCMD ?= swiftlint SWIFTCMD ?= swift +RUST_TOOLCHAIN ?= 1.89.0 SWIFT_ENROLL_UI ?= cmd/probo-agent/installer/macos/enroll-ui SWIFT_FORMAT_CONFIG ?= .swift-format SWIFTLINT_CONFIG ?= .swiftlint.yml @@ -104,6 +107,9 @@ EMBEDDED= apps/console/dist/index.html \ apps/compliance-portal/dist/index.html \ @probo/emails +AUTOMERGE_REFERENCE_DIR= pkg/automerge/internal/reference +AUTOMERGE_REFERENCE_WASM= $(AUTOMERGE_REFERENCE_DIR)/reference.wasm + PROBOD_BIN_EXTRA_DEPS= PROBOD_BIN= bin/probod PROBOD_SRC= cmd/probod/main.go @@ -208,6 +214,11 @@ test-verbose: test ## Run tests with verbose output test-short: TEST_FLAGS+=-short test-short: test ## Run short tests only +.PHONY: test-automerge-conformance +test-automerge-conformance: ## Test Go binary compatibility with official Automerge JS + AUTOMERGE_JS_ORACLE=$(CURDIR)/packages/automerge-conformance/oracle.mjs \ + $(GO_BASE) test -count=1 ./pkg/automerge + .PHONY: coverage-report coverage-report: test ## Generate HTML coverage report $(GO) tool cover -html=coverage.out -o coverage.html @@ -414,6 +425,18 @@ pkg/server/api/complianceportal/v1/schema.graphql: pkg/server/api/complianceport .PHONY: generate generate: $(GENERATED) +.PHONY: generate-automerge-reference +generate-automerge-reference: ## Rebuild the embedded official Automerge WASM backend + cd $(AUTOMERGE_REFERENCE_DIR)/wasm && \ + $(CARGO) +$(RUST_TOOLCHAIN) build --locked --release --target wasm32-wasip1 + $(CP) $(AUTOMERGE_REFERENCE_DIR)/wasm/target/wasm32-wasip1/release/probo_automerge_reference.wasm $(AUTOMERGE_REFERENCE_WASM) + cd $(AUTOMERGE_REFERENCE_DIR) && $(SHA256SUM) reference.wasm > reference.wasm.sha256 + +.PHONY: audit-automerge-reference +audit-automerge-reference: ## Audit Automerge Rust advisories, licenses, bans, and sources + cd $(AUTOMERGE_REFERENCE_DIR)/wasm && \ + $(CARGO) +$(RUST_TOOLCHAIN) deny check + .PHONY: embed embed: $(EMBEDDED) diff --git a/apps/console/package.json b/apps/console/package.json index a031e0174a..e41f900c58 100644 --- a/apps/console/package.json +++ b/apps/console/package.json @@ -10,6 +10,8 @@ "preview": "vite preview" }, "dependencies": { + "@automerge/automerge": "^3.4.0", + "@automerge/prosemirror": "^0.2.0", "@hookform/resolvers": "^5.7.1", "@phosphor-icons/react": "^2.1.10", "@probo/coredata": "^1.0.0", diff --git a/apps/console/src/pages/organizations/documents/description/DocumentDescriptionPage.tsx b/apps/console/src/pages/organizations/documents/description/DocumentDescriptionPage.tsx index 3648cd6c6f..4841eeffda 100644 --- a/apps/console/src/pages/organizations/documents/description/DocumentDescriptionPage.tsx +++ b/apps/console/src/pages/organizations/documents/description/DocumentDescriptionPage.tsx @@ -19,8 +19,13 @@ // SOFTWARE. import { formatError } from "@probo/helpers"; -import { RichEditor, useToast } from "@probo/ui"; -import { useCallback, useState } from "react"; +import { + createRichEditorAutomergeDocument, + RichEditor, + supportsRichEditorCollaboration, + useToast, +} from "@probo/ui"; +import { useCallback, useEffect, useState } from "react"; import { useTranslation } from "react-i18next"; import { type PreloadedQuery, useMutation, usePreloadedQuery } from "react-relay"; import { useOutletContext } from "react-router"; @@ -30,8 +35,20 @@ import { useDebounceCallback } from "usehooks-ts"; import type { DocumentDescriptionPage_updateContentMutation } from "#/__generated__/core/DocumentDescriptionPage_updateContentMutation.graphql"; import type { DocumentDescriptionPageQuery } from "#/__generated__/core/DocumentDescriptionPageQuery.graphql"; +import { + type AutomergeDocumentHandle, + connectAutomergeDocument, +} from "./_lib/AutomergeDocumentHandle"; + const autoSaveIntervalMs = 1000; +type CollaborationState = { + versionID: string; + handle?: AutomergeDocumentHandle; + failed?: boolean; + reconnecting?: boolean; +}; + export const documentDescriptionPageQuery = graphql` query DocumentDescriptionPageQuery($documentId: ID! $versionId: ID! $versionSpecified: Boolean!) { # We use this on /documents/:documentId/versions/:versionId/description @@ -158,6 +175,86 @@ export function DocumentDescriptionPage(props: { && document.canUpdate && document.status !== "ARCHIVED" && document.writeMode !== "GENERATED"; + const collaborationSupported = canEdit + && supportsRichEditorCollaboration(currentVersion.content); + const [collaboration, setCollaboration] = useState(); + + useEffect(() => { + if (!collaborationSupported) return; + + let cancelled = false; + let activeHandle: AutomergeDocumentHandle | undefined; + + void connectAutomergeDocument( + currentVersion.id, + content => createRichEditorAutomergeDocument(content), + (connected) => { + if (cancelled) return; + + setCollaboration({ + versionID: currentVersion.id, + handle: activeHandle, + reconnecting: !connected, + }); + if (!connected) { + toast({ + title: t("documentDescriptionPage.errors.title"), + description: t("documentDescriptionPage.errors.save"), + variant: "error", + }); + } + }, + ).then((handle) => { + if (cancelled) { + handle.close(); + return; + } + + activeHandle = handle; + setCollaboration({ + versionID: currentVersion.id, + handle, + }); + }).catch((error) => { + if (cancelled) return; + + setCollaboration({ + versionID: currentVersion.id, + failed: true, + }); + toast({ + title: t("documentDescriptionPage.errors.title"), + description: error instanceof Error + ? error.message + : t("documentDescriptionPage.errors.save"), + variant: "error", + }); + }); + + return () => { + cancelled = true; + activeHandle?.close(); + }; + }, [ + collaborationSupported, + currentVersion.id, + t, + toast, + ]); + + const collaborationConnecting = collaborationSupported + && ( + collaboration?.versionID !== currentVersion.id + || (!collaboration?.handle && !collaboration?.failed) + ); + const collaborationHandle = collaborationSupported + && collaboration?.versionID === currentVersion.id + && !collaboration.failed + ? collaboration.handle + : undefined; + const collaborationReconnecting = collaborationSupported + && collaboration?.versionID === currentVersion.id + && collaboration.reconnecting; // The editor key must change on explicit actions (delete draft, edit // title/type) but NOT on auto-save side effects (cursor preservation). @@ -199,7 +296,8 @@ export function DocumentDescriptionPage(props: { className="flex-1" content={currentVersion.content} data-theme="document" - disabled={!canEdit} + disabled={!canEdit || collaborationConnecting || collaborationReconnecting} + collaborationHandle={collaborationHandle} onChangeContent={handleUpdate} /> ); diff --git a/apps/console/src/pages/organizations/documents/description/_lib/AutomergeDocumentHandle.ts b/apps/console/src/pages/organizations/documents/description/_lib/AutomergeDocumentHandle.ts new file mode 100644 index 0000000000..52edf8abd7 --- /dev/null +++ b/apps/console/src/pages/organizations/documents/description/_lib/AutomergeDocumentHandle.ts @@ -0,0 +1,509 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +import * as Automerge from "@automerge/automerge"; +import type { DocHandle } from "@automerge/prosemirror"; +import { + collaborationDebug, + type RichEditorAutomergeDocument, + type RichEditorPresence, + summarizeAutomergeSpans, +} from "@probo/ui"; + +const collaborationProtocol = "automerge-sync-v1"; +const maxGeneratedMessages = 100; +const initializationTimeoutMs = 35_000; +const reconnectMaxDelayMs = 10_000; +const presenceHeartbeatMs = 5_000; +const presenceUpdateThrottleMs = 50; + +type CollaborationHandshake = { + type: "ready"; + version: 1; + revision: number; + needsSeed: boolean; + seedContent?: string; + connectionId: string; +}; + +type ChangeListener = ( + payload: DocumentHandleChangePayload, +) => void; + +type DocumentHandleChangePayload = { + handle: DocHandle; + doc: Automerge.Doc; + patches: Automerge.Patch[]; + patchInfo: Automerge.PatchInfo; +}; + +type PresenceSnapshot = { + type: "presence"; + presences: Array<{ + connectionId: string; + identityId: string; + anchorPosition: number; + headPosition: number; + }>; +}; + +export class AutomergeDocumentHandle implements DocHandle { + readonly #endpoint: URL; + #socket: WebSocket; + #document: Automerge.Doc; + #syncState: Automerge.SyncState; + #listeners = new Set(); + #presenceListeners = new Set<(presences: RichEditorPresence[]) => void>(); + #onConnectionState?: (connected: boolean) => void; + #closed = false; + #ready = false; + #reconnectAttempt = 0; + #reconnectTimer?: number; + #presenceTimer?: number; + #presenceHeartbeat?: number; + #pendingPresence?: { anchorPosition: number; headPosition: number }; + + constructor( + endpoint: URL, + socket: WebSocket, + document: Automerge.Doc, + onConnectionState?: (connected: boolean) => void, + ) { + this.#endpoint = endpoint; + this.#socket = socket; + this.#document = document; + this.#syncState = Automerge.initSyncState(); + this.#onConnectionState = onConnectionState; + + this.#attachSocket(socket); + this.#presenceHeartbeat = window.setInterval( + () => this.#sendPresence(), + presenceHeartbeatMs, + ); + this.#sendAvailableMessages(); + } + + doc(): Automerge.Doc { + return this.#document; + } + + change(fn: (document: RichEditorAutomergeDocument) => void): void { + let patches: Automerge.Patch[] = []; + let patchInfo: Automerge.PatchInfo | undefined; + const headsBefore = Automerge.getHeads(this.#document); + + this.#document = Automerge.change( + this.#document, + { + patchCallback: (nextPatches, nextPatchInfo) => { + patches = nextPatches; + patchInfo = nextPatchInfo; + }, + }, + fn, + ); + collaborationDebug("handle-local-change", { + headsBefore, + headsAfter: Automerge.getHeads(this.#document), + patches: summarizePatches(patches), + spans: summarizeAutomergeSpans(this.#document), + }); + if (patchInfo) { + this.#emit(patches, patchInfo); + } + this.#sendAvailableMessages(); + } + + on(event: "change", callback: ChangeListener): void { + if (event === "change") this.#listeners.add(callback); + } + + off(event: "change", callback: ChangeListener): void { + if (event === "change") this.#listeners.delete(callback); + } + + updatePresence(anchorPosition: number, headPosition: number): void { + this.#pendingPresence = { anchorPosition, headPosition }; + if (this.#presenceTimer !== undefined) return; + + this.#presenceTimer = window.setTimeout(() => { + this.#presenceTimer = undefined; + this.#sendPresence(); + }, presenceUpdateThrottleMs); + } + + onPresence(listener: (presences: RichEditorPresence[]) => void): () => void { + this.#presenceListeners.add(listener); + return () => this.#presenceListeners.delete(listener); + } + + close(): void { + this.#closed = true; + if (this.#reconnectTimer !== undefined) { + window.clearTimeout(this.#reconnectTimer); + } + if (this.#presenceTimer !== undefined) { + window.clearTimeout(this.#presenceTimer); + } + if (this.#presenceHeartbeat !== undefined) { + window.clearInterval(this.#presenceHeartbeat); + } + this.#detachSocket(this.#socket); + this.#socket.close(1000); + this.#listeners.clear(); + this.#presenceListeners.clear(); + } + + waitUntilReady(): Promise { + if (typeof this.#document.body === "string") { + this.#ready = true; + return Promise.resolve(); + } + + return new Promise((resolve, reject) => { + const timeout = window.setTimeout(() => { + cleanup(); + reject(new Error("Document collaboration initialization timed out")); + }, initializationTimeoutMs); + + const handleChange: ChangeListener = ({ doc }) => { + if (typeof doc.body !== "string") return; + cleanup(); + this.#ready = true; + resolve(); + }; + const handleClose = () => { + cleanup(); + reject(new Error("Document collaboration closed during initialization")); + }; + const cleanup = () => { + window.clearTimeout(timeout); + this.off("change", handleChange); + this.#socket.removeEventListener("close", handleClose); + }; + + this.on("change", handleChange); + this.#socket.addEventListener("close", handleClose); + }); + } + + #handleMessage = (event: MessageEvent) => { + if (typeof event.data === "string") { + let value: unknown; + try { + value = JSON.parse(event.data); + } catch { + return; + } + if (!isPresenceSnapshot(value)) return; + const presences = value.presences.map(presence => ({ + connectionID: presence.connectionId, + identityID: presence.identityId, + anchorPosition: presence.anchorPosition, + headPosition: presence.headPosition, + })); + for (const listener of this.#presenceListeners) { + listener(presences); + } + return; + } + if (!(event.data instanceof ArrayBuffer)) return; + + let patches: Automerge.Patch[] = []; + let patchInfo: Automerge.PatchInfo | undefined; + const headsBefore = Automerge.getHeads(this.#document); + [this.#document, this.#syncState] = Automerge.receiveSyncMessage( + this.#document, + this.#syncState, + new Uint8Array(event.data), + { + patchCallback: (nextPatches, nextPatchInfo) => { + patches = nextPatches; + patchInfo = nextPatchInfo; + }, + }, + ); + collaborationDebug("sync-receive", { + bytes: event.data.byteLength, + headsBefore, + headsAfter: Automerge.getHeads(this.#document), + patches: summarizePatches(patches), + spans: summarizeAutomergeSpans(this.#document), + }); + if (patchInfo) { + this.#emit(patches, patchInfo); + } + this.#sendAvailableMessages(); + }; + + #handleDisconnect = () => { + if (this.#closed || !this.#ready || this.#reconnectTimer !== undefined) return; + + collaborationDebug("socket-disconnected", { + ready: this.#ready, + attempt: this.#reconnectAttempt, + heads: Automerge.getHeads(this.#document), + }); + this.#detachSocket(this.#socket); + this.#onConnectionState?.(false); + const delay = Math.min( + 500 * 2 ** this.#reconnectAttempt, + reconnectMaxDelayMs, + ); + this.#reconnectAttempt++; + this.#reconnectTimer = window.setTimeout(() => { + this.#reconnectTimer = undefined; + void this.#reconnect(); + }, delay); + }; + + async #reconnect(): Promise { + if (this.#closed) return; + + const socket = createSocket(this.#endpoint); + try { + await waitForHandshake(socket); + if (this.#closed) { + socket.close(1000); + return; + } + + this.#socket = socket; + this.#syncState = Automerge.initSyncState(); + this.#reconnectAttempt = 0; + this.#attachSocket(socket); + this.#sendAvailableMessages(); + this.#sendPresence(); + this.#onConnectionState?.(true); + collaborationDebug("socket-reconnected", { + heads: Automerge.getHeads(this.#document), + }); + } catch { + socket.close(); + if (this.#closed) return; + this.#reconnectTimer = window.setTimeout(() => { + this.#reconnectTimer = undefined; + void this.#reconnect(); + }, reconnectMaxDelayMs); + } + } + + #attachSocket(socket: WebSocket): void { + socket.addEventListener("message", this.#handleMessage); + socket.addEventListener("close", this.#handleDisconnect); + socket.addEventListener("error", this.#handleDisconnect); + } + + #detachSocket(socket: WebSocket): void { + socket.removeEventListener("message", this.#handleMessage); + socket.removeEventListener("close", this.#handleDisconnect); + socket.removeEventListener("error", this.#handleDisconnect); + } + + #sendAvailableMessages(): void { + if (this.#socket.readyState !== WebSocket.OPEN) return; + + for (let i = 0; i < maxGeneratedMessages; i++) { + let message: Automerge.SyncMessage | null; + [this.#syncState, message] = Automerge.generateSyncMessage( + this.#document, + this.#syncState, + ); + if (!message) return; + const data = new ArrayBuffer(message.byteLength); + new Uint8Array(data).set(message); + collaborationDebug("sync-send", { + sequence: i, + bytes: message.byteLength, + heads: Automerge.getHeads(this.#document), + }); + this.#socket.send(data); + } + + throw new Error("Automerge sync protocol did not quiesce"); + } + + #sendPresence(): void { + if ( + !this.#pendingPresence + || this.#socket.readyState !== WebSocket.OPEN + ) { + return; + } + this.#socket.send( + JSON.stringify({ + type: "presence", + ...this.#pendingPresence, + }), + ); + } + + #emit( + patches: Automerge.Patch[], + patchInfo: Automerge.PatchInfo, + ): void { + const payload: DocumentHandleChangePayload = { + handle: this, + doc: this.#document, + patches, + patchInfo, + }; + for (const listener of this.#listeners) { + listener(payload); + } + } +} + +export async function connectAutomergeDocument( + documentVersionID: string, + createSeed: (content: string) => Automerge.Doc, + onConnectionState?: (connected: boolean) => void, +): Promise { + const endpoint = new URL(window.location.origin); + endpoint.protocol = endpoint.protocol === "https:" ? "wss:" : "ws:"; + endpoint.pathname = [ + "api", + "console", + "v1", + "document-versions", + encodeURIComponent(documentVersionID), + "sync", + ].join("/"); + + const socket = createSocket(endpoint); + + const handshake = await waitForHandshake(socket); + collaborationDebug("handshake", { + revision: handshake.revision, + needsSeed: handshake.needsSeed, + connectionId: handshake.connectionId, + }); + const document = handshake.needsSeed + ? createSeed(handshake.seedContent ?? "") + : Automerge.init(); + + const handle = new AutomergeDocumentHandle( + endpoint, + socket, + document, + onConnectionState, + ); + try { + await handle.waitUntilReady(); + return handle; + } catch (error) { + handle.close(); + throw error; + } +} + +function createSocket(endpoint: URL): WebSocket { + const socket = new WebSocket(endpoint, collaborationProtocol); + socket.binaryType = "arraybuffer"; + return socket; +} + +function summarizePatches( + patches: Automerge.Patch[], +): Array> { + return patches.map(patch => ({ + action: patch.action, + path: patch.path, + })); +} + +function waitForHandshake(socket: WebSocket): Promise { + return new Promise((resolve, reject) => { + function cleanup() { + socket.removeEventListener("message", handleMessage); + socket.removeEventListener("error", handleError); + socket.removeEventListener("close", handleClose); + } + + function handleMessage(event: MessageEvent) { + if (typeof event.data !== "string") { + cleanup(); + reject(new Error("Collaboration server sent binary data before handshake")); + return; + } + + let value: unknown; + try { + value = JSON.parse(event.data); + } catch { + cleanup(); + reject(new Error("Collaboration server returned an invalid handshake")); + return; + } + if (!isCollaborationHandshake(value)) { + cleanup(); + reject(new Error("Collaboration server returned an unsupported handshake")); + return; + } + + cleanup(); + resolve(value); + } + + function handleError() { + cleanup(); + reject(new Error("Cannot connect to document collaboration server")); + } + + function handleClose() { + cleanup(); + reject(new Error("Document collaboration connection closed before handshake")); + } + + socket.addEventListener("message", handleMessage); + socket.addEventListener("error", handleError); + socket.addEventListener("close", handleClose); + }); +} + +function isCollaborationHandshake(value: unknown): value is CollaborationHandshake { + if (!value || typeof value !== "object") return false; + const handshake = value as Record; + return handshake.type === "ready" + && handshake.version === 1 + && typeof handshake.revision === "number" + && typeof handshake.needsSeed === "boolean" + && typeof handshake.connectionId === "string" + && ( + handshake.seedContent === undefined + || typeof handshake.seedContent === "string" + ); +} + +function isPresenceSnapshot(value: unknown): value is PresenceSnapshot { + if (!value || typeof value !== "object") return false; + const snapshot = value as Record; + if (snapshot.type !== "presence" || !Array.isArray(snapshot.presences)) { + return false; + } + return snapshot.presences.every((presence) => { + if (!presence || typeof presence !== "object") return false; + const item = presence as Record; + return typeof item.connectionId === "string" + && typeof item.identityId === "string" + && typeof item.anchorPosition === "number" + && typeof item.headPosition === "number"; + }); +} diff --git a/apps/console/vite.config.ts b/apps/console/vite.config.ts index 43a46e2e8a..3a2ef33ff4 100644 --- a/apps/console/vite.config.ts +++ b/apps/console/vite.config.ts @@ -68,6 +68,7 @@ export default defineConfig({ "/api": { target: "http://localhost:8080", changeOrigin: true, + ws: true, }, }, }, diff --git a/go.mod b/go.mod index 07bd220066..b8e97c94e8 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/charmbracelet/lipgloss v1.1.0 github.com/chromedp/cdproto v0.0.0-20260719223732-95f6af754cfe github.com/chromedp/chromedp v0.16.0 + github.com/coder/websocket v1.8.15 github.com/crewjam/saml v0.5.1 github.com/digitorus/timestamp v0.0.0-20250524132541-c45532741eea github.com/go-chi/chi/v5 v5.3.1 @@ -37,6 +38,7 @@ require ( github.com/scim2/filter-parser/v2 v2.3.1 github.com/sigstore/sigstore-go v1.3.0 github.com/stretchr/testify v1.11.1 + github.com/tetratelabs/wazero v1.12.0 github.com/vektah/gqlparser/v2 v2.5.36 github.com/vikstrous/dataloadgen v0.0.10 github.com/yuin/goldmark v1.8.5 @@ -81,7 +83,6 @@ require ( github.com/charmbracelet/x/term v0.2.2 // indirect github.com/clipperhouse/displaywidth v0.11.0 // indirect github.com/cloudflare/circl v1.6.4 // indirect - github.com/coder/websocket v1.8.15 // indirect github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 // indirect github.com/cyphar/filepath-securejoin v0.7.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/go.sum b/go.sum index 16ff1c5ec9..941d033a49 100644 --- a/go.sum +++ b/go.sum @@ -572,6 +572,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU= +github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0= github.com/theupdateframework/go-tuf v0.7.0 h1:CqbQFrWo1ae3/I0UCblSbczevCCbS31Qvs5LdxRWqRI= github.com/theupdateframework/go-tuf v0.7.0/go.mod h1:uEB7WSY+7ZIugK6R1hiBMBjQftaFzn7ZCDJcp1tCUug= github.com/theupdateframework/go-tuf/v2 v2.4.2 h1:w7976/W8uTwlsegP5nRymlpjPgrwSh+AXUf85is6nJk= diff --git a/package-lock.json b/package-lock.json index 4baff0cd45..c3fc18f127 100644 --- a/package-lock.json +++ b/package-lock.json @@ -339,6 +339,8 @@ "name": "@probo/console", "version": "0.0.0", "dependencies": { + "@automerge/automerge": "^3.4.0", + "@automerge/prosemirror": "^0.2.0", "@hookform/resolvers": "^5.7.1", "@phosphor-icons/react": "^2.1.10", "@probo/coredata": "^1.0.0", @@ -771,6 +773,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } @@ -853,7 +856,6 @@ "integrity": "sha512-IjLt0gd3L4jlOfilxVXTifn42FnVffMgDC04RJK1KDZpmkBWLv0XC92MVVmkxrFZNS/7l3xWgP/I3nqtX1sQHw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", @@ -870,7 +872,6 @@ "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~5.26.4" } @@ -880,8 +881,7 @@ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@apm-js-collab/code-transformer": { "version": "0.18.1", @@ -1017,6 +1017,30 @@ "integrity": "sha512-BQebYH9nV1VZttZwoq/fsxcxIJjc8oW2bNV6yJDHLZ8OF8UtM15drFN0JOL8Jwn7jeeD7Ev+tIC8pUijJEditQ==", "license": "MIT" }, + "node_modules/@automerge/automerge": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/@automerge/automerge/-/automerge-3.4.0.tgz", + "integrity": "sha512-THmghtTNGGt2xsI0pM3o1i3PM8oZKcYFgOj25FOzW7l6e94SQOivNtCwy6xc0I8hVJsQSSotoBNs+yk/9hM2dg==", + "license": "MIT" + }, + "node_modules/@automerge/prosemirror": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@automerge/prosemirror/-/prosemirror-0.2.0.tgz", + "integrity": "sha512-Mixxftvfs12tKi7DsStobrQxFayoLJXhEjbrnmNnaQbMzwHrUzuwac+OUvw6ujbtXaulWX+JE8hAuHFAGRDsvQ==", + "license": "MIT", + "dependencies": { + "@automerge/automerge": "^3.1", + "ordered-map": "^0.1.0", + "prosemirror-changeset": "^2.2.1", + "prosemirror-history": "^1.4.1", + "prosemirror-model": "^1.25.2", + "prosemirror-schema-basic": "^1.2.2", + "prosemirror-schema-list": "^1.3.0", + "prosemirror-state": "^1.4.3", + "prosemirror-transform": "^1.7.3", + "prosemirror-view": "^1.40.1" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -1341,7 +1365,6 @@ "integrity": "sha512-DDaRehssg1aNrH4+2hnj1B7vnUGEjU6OIlyRdkMd0aUdIUvKXrJfXsy8LVtXAy7DRvYVluWbMspsRhz2lcW0mQ==", "dev": true, "license": "MIT", - "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/Borewit" @@ -1359,7 +1382,6 @@ "integrity": "sha512-mPAuLRU9jWR7o0KJi9+gQnOBDUSIkoKbbFv4HjrA+80qWVcFacrNPlZmf4mguQnfZ0oP2t5c3ws6yuFyAX9vpA==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", @@ -1376,7 +1398,6 @@ "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~5.26.4" } @@ -1386,8 +1407,7 @@ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@browserbasehq/stagehand": { "version": "1.14.0", @@ -1395,7 +1415,6 @@ "integrity": "sha512-Hi/EzgMFWz+FKyepxHTrqfTPjpsuBS4zRy3e9sbMpBgLPv+9c0R+YZEvS7Bw4mTS66QtvvURRT6zgDGFotthVQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@anthropic-ai/sdk": "^0.27.3", "@browserbasehq/sdk": "^2.0.0", @@ -1416,7 +1435,6 @@ "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", "dev": true, "license": "ISC", - "peer": true, "peerDependencies": { "zod": "^3.25.28 || ^4" } @@ -1550,29 +1568,6 @@ "kuler": "^2.0.0" } }, - "node_modules/@emnapi/core": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.2.tgz", - "integrity": "sha512-UC+ZhH3XtczQYfOlu3lNEkdW/p4dsJ1r/bP7H8+rhao3TTTMO1ATq/4DdIi23XuGoFY+Cz0JmCbdVl0hz9jZcA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.1", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.2.tgz", - "integrity": "sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@emnapi/wasi-threads": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", @@ -2260,6 +2255,7 @@ "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.8.0.tgz", "integrity": "sha512-yXSrzeHZBTZadLOlfyhCkJHNeLJnHRnRInwdZ40L7ZiaAtrBwoYlsDrX3v5zB1Utk7CLfzcOVnVVWoXEky7Ceg==", "license": "MIT", + "peer": true, "dependencies": { "@floating-ui/core": "^1.8.0", "@floating-ui/utils": "^0.2.12" @@ -2568,6 +2564,7 @@ "integrity": "sha512-nXmyH0FbcsASlRmC9sbqX0gjQdxgB9KcS13vkw9PMaH0zzylwZkGFU9sY0XCPa2/AokmaNTU9DOW3IUDfAtQow==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@cfworker/json-schema": "^4.0.2", "@standard-schema/spec": "^1.1.0", @@ -3486,6 +3483,7 @@ "integrity": "sha512-aYCc0C6L864eR6WSYIwQGyXriw/nIyZx0ObvhzOEVuk0zoBDpynjSbrionWI7q65B5H8jJX0DXR9snEzM6bfPg==", "dev": true, "license": "Apache-2.0", + "peer": true, "peerDependencies": { "@aws-sdk/credential-provider-node": ">=3.972.0 <4", "@smithy/hash-node": ">=4.3.0 <5", @@ -3517,6 +3515,7 @@ "integrity": "sha512-idA2YXwpCdqUSKRCACDE6ItZD9TZzy3OZMtpfLoh6oPR47lipysRrJfjzMqFxQ3uJuUPyUeWe1r9vLH33xO/Qw==", "dev": true, "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } @@ -3648,6 +3647,7 @@ "integrity": "sha512-idA2YXwpCdqUSKRCACDE6ItZD9TZzy3OZMtpfLoh6oPR47lipysRrJfjzMqFxQ3uJuUPyUeWe1r9vLH33xO/Qw==", "dev": true, "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } @@ -3785,6 +3785,7 @@ "integrity": "sha512-idA2YXwpCdqUSKRCACDE6ItZD9TZzy3OZMtpfLoh6oPR47lipysRrJfjzMqFxQ3uJuUPyUeWe1r9vLH33xO/Qw==", "dev": true, "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } @@ -3947,6 +3948,7 @@ "integrity": "sha512-idA2YXwpCdqUSKRCACDE6ItZD9TZzy3OZMtpfLoh6oPR47lipysRrJfjzMqFxQ3uJuUPyUeWe1r9vLH33xO/Qw==", "dev": true, "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } @@ -3966,7 +3968,6 @@ "resolved": "https://registry.npmjs.org/@n8n/errors/-/errors-0.10.0.tgz", "integrity": "sha512-Qle4gCsx7qtDRMdie3YkjuXPuo369PP32cR9Svb+INwcjLMxNzmnbJ66jFnOmLn0ntwBccUw1IecJ6axwyTgBQ==", "license": "SEE LICENSE IN LICENSE.md", - "peer": true, "dependencies": { "callsites": "3.1.0" } @@ -3976,7 +3977,6 @@ "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", "license": "MIT", - "peer": true, "engines": { "node": ">=6" } @@ -3986,7 +3986,6 @@ "resolved": "https://registry.npmjs.org/@n8n/expression-runtime/-/expression-runtime-0.19.0.tgz", "integrity": "sha512-ShRp3TEue2dNaP/PIQzD5DqyYUjkiaIsDVwsyfAvhsdyIUKyishsHsnogv527vjuA/WzH5cYn9X75lFeQA8hZg==", "license": "SEE LICENSE IN LICENSE.md", - "peer": true, "dependencies": { "@n8n/errors": "0.10.0", "@n8n/tournament": "1.4.0", @@ -4007,7 +4006,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.67.tgz", "integrity": "sha512-idA2YXwpCdqUSKRCACDE6ItZD9TZzy3OZMtpfLoh6oPR47lipysRrJfjzMqFxQ3uJuUPyUeWe1r9vLH33xO/Qw==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } @@ -4160,6 +4158,7 @@ "integrity": "sha512-GsGizj2Y1rCWDu6XoEekL3RLilp0voSePurjZIkxL3wlm5o5EC9VpgaP7lrCvjnkuLvzFBQWB3vWB3K5KQTveQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.12.1", @@ -4308,7 +4307,6 @@ "resolved": "https://registry.npmjs.org/@n8n/tournament/-/tournament-1.4.0.tgz", "integrity": "sha512-llDdmIj3Kdfq2vX/NpLBOUU4kLjhITtIMeJmOSTApAOeAArj1D45ey4D7CFaYcYsPBw6DgF0vxifyskPDlgekw==", "license": "SEE LICENSE IN LICENSE.md", - "peer": true, "dependencies": { "ast-types": "^0.16.1", "esprima-next": "^5.8.4", @@ -4635,6 +4633,7 @@ "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", "dev": true, "license": "Apache-2.0", + "peer": true, "engines": { "node": ">=8.0.0" } @@ -4658,6 +4657,7 @@ "integrity": "sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ==", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, @@ -4674,6 +4674,7 @@ "integrity": "sha512-xQx3E2WxP1mDvKzxLxX+CTCtNLa560YJZ3087qYHerl2YmiKpv7AH+dAy7vmx+eVrZ5BwhfWUAVoKOoxCNHcpw==", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "@opentelemetry/api-logs": "0.220.0", "import-in-the-middle": "^3.0.0", @@ -4727,6 +4728,7 @@ "integrity": "sha512-GuYQQT7QD2EeO8lcZLRQzcbOyhqAzL+6WWTKTU9mSUBYBazkEDl+VrQcXQhbB08OWM9anD1aHleVadzulpOaUQ==", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "@opentelemetry/core": "2.10.0", "@opentelemetry/resources": "2.10.0", @@ -5469,6 +5471,10 @@ "integrity": "sha512-4PZ9wMYI8m8AqJuZ9YR1IAHGVtSnYbBVBgTevrKpzZbcSe/OUwhEV2Ks//rJh/L8eMTU8R5DrD4D/hlrOwaAiQ==", "license": "MIT" }, + "node_modules/@probo/automerge-conformance": { + "resolved": "packages/automerge-conformance", + "link": true + }, "node_modules/@probo/compliance-portal": { "resolved": "apps/compliance-portal", "link": true @@ -6829,7 +6835,8 @@ "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", "devOptional": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@standard-schema/utils": { "version": "0.3.0", @@ -7315,27 +7322,6 @@ "node": ">=14.0.0" } }, - "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/core": { - "version": "1.11.1", - "dev": true, - "inBundle": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.2", - "tslib": "^2.4.0" - } - }, - "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/runtime": { - "version": "1.11.1", - "dev": true, - "inBundle": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/wasi-threads": { "version": "1.2.2", "dev": true, @@ -7556,6 +7542,7 @@ "resolved": "https://registry.npmjs.org/@tiptap/core/-/core-3.29.2.tgz", "integrity": "sha512-oKUkiPUB7noilVYxI9lNzUD4rX17sHub+PYjMfHMWHG9A3nvIy+FdePIVIIhThKWF7ijhr3eIqHY51Bn+GAFtw==", "license": "MIT", + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/ueberdosis" @@ -7745,6 +7732,7 @@ "resolved": "https://registry.npmjs.org/@tiptap/extension-list/-/extension-list-3.29.2.tgz", "integrity": "sha512-WPZ9BHAPT6QeIm1vdVkuoOWvy9a8/EZeJwV2VhU8LXyTAttvzyj4rsbbHyJWvYWlUSTt/QF2AZ2zhKo7u1w3/A==", "license": "MIT", + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/ueberdosis" @@ -7850,6 +7838,7 @@ "resolved": "https://registry.npmjs.org/@tiptap/extensions/-/extensions-3.29.2.tgz", "integrity": "sha512-BCz+FCAChSYtUe4BFj97HEO+nSK+J7GxbJgZG4Hg7DT/gI+hRyeNndU8efiQAx3WGdzsFi3UxRpcF1tTQM7iMQ==", "license": "MIT", + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/ueberdosis" @@ -7864,6 +7853,7 @@ "resolved": "https://registry.npmjs.org/@tiptap/pm/-/pm-3.29.2.tgz", "integrity": "sha512-GCOme7xHaS+DSoaA4CDcAD3l6JyBlvZhvCyfsy2Vp6j8tEoBkZWio7soYVosmlyn7zq8/64VeFZP5s47yfG7fQ==", "license": "MIT", + "peer": true, "dependencies": { "prosemirror-changeset": "^2.4.1", "prosemirror-commands": "^1.7.1", @@ -7926,7 +7916,6 @@ "integrity": "sha512-2mAv+8pkG6GIZiF1kNg1jAjh27IDxEPKwdGul3snfztFerfPGI1LjDezZp3i7BElXompqEtPmoPx6c2wgtWsOA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "debug": "^4.4.3", "token-types": "^6.1.1" @@ -7944,8 +7933,7 @@ "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@ts-morph/common": { "version": "0.28.1", @@ -8492,7 +8480,6 @@ "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/node": "*", "form-data": "^4.0.4" @@ -8510,6 +8497,7 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.18.tgz", "integrity": "sha512-AnzbBERsrLKtk2XSfTbYRLjQPdy116Sty4q+T+Bp3IC4l6jNBvreVPAHmpq9qhXQM7CXZPjLVmGMw9sy+hxQ3w==", "license": "MIT", + "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -8519,6 +8507,7 @@ "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.4.tgz", "integrity": "sha512-Bsc+QHgp+P/F02XDzNCY9jnZNCUuLki36KT7VKrTXXLdHf+vHMNZnW1rVu5DNW/rCK+fya3DATySbLM4yhtKUw==", "license": "MIT", + "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -8553,8 +8542,7 @@ "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.0.tgz", "integrity": "sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/triple-beam": { "version": "1.3.5", @@ -8636,6 +8624,7 @@ "integrity": "sha512-CZ4nMxWwgu1HEEFNkeaCptra9QCtkmKdgf3sWh1rl1trIhmxLilgTV4cwcbQ4wemnT4sWQN8CaKOmdYx+g2gMA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.65.0", "@typescript-eslint/types": "8.65.0", @@ -9676,6 +9665,7 @@ "integrity": "sha512-IKI6kpIH+LmpROplyLwBBaCfMgOZOMsygVa6BARD6ahA04VRuJSa6OaVG7kRvSEMD870Vd91rSSw0eegtWyLGg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@vitest/utils": "4.1.10", "pathe": "^2.0.3" @@ -9738,7 +9728,6 @@ "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "event-target-shim": "^5.0.0" }, @@ -9765,6 +9754,7 @@ "integrity": "sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -9798,7 +9788,6 @@ "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "humanize-ms": "^1.2.1" }, @@ -9812,6 +9801,7 @@ "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -9828,6 +9818,7 @@ "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", "license": "MIT", + "peer": true, "dependencies": { "ajv": "^8.0.0" }, @@ -10234,6 +10225,7 @@ "integrity": "sha512-E32NzpYKp++W7XRe52rHiXV2ehxmh3wbdgO7MHeFM+vqxLBYHzt0ElkiImtOBxtOmyp0yoC8C6uESVV84Y2/hw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "follow-redirects": "^1.16.0", "form-data": "^4.0.5", @@ -10515,6 +10507,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.38", "caniuse-lite": "^1.0.30001799", @@ -10534,8 +10527,7 @@ "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", "dev": true, - "license": "BSD-3-Clause", - "peer": true + "license": "BSD-3-Clause" }, "node_modules/buildcheck": { "version": "0.0.7", @@ -10684,7 +10676,6 @@ "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -11384,6 +11375,7 @@ "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.34.0.tgz", "integrity": "sha512-62rNSrioXw93uliKFBwjukeQyeWwH2PqDrTac31r2P6464u3AUvTk0xS4LVvT251g7IgkFunrI48ZEZGjywSOg==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10" } @@ -11577,6 +11569,7 @@ "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz", "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==", "license": "ISC", + "peer": true, "dependencies": { "commander": "7", "iconv-lite": "0.6", @@ -11784,6 +11777,7 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -12017,6 +12011,7 @@ "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -12302,7 +12297,6 @@ "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "safe-buffer": "^5.0.1" } @@ -12712,6 +12706,7 @@ "integrity": "sha512-nuKKvN+oIBO0koN7Tm7dlkmnkc21mtt0QJLwAKzjLq14y6lRTdVG36MZHJ8eQHwdJMwZbQNMlPOYedMq/oVJvQ==", "dev": true, "license": "MIT", + "peer": true, "workspaces": [ "packages/*" ], @@ -13169,7 +13164,6 @@ "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6" } @@ -13372,7 +13366,6 @@ "integrity": "sha512-DLkUvGwep3poOV2wpzbHCOnSKGk1LzyXTv+aHFgN2VFl96wnp8YA9YjO2qPzg5PuL8q/SW9Pdi6WTkYOIh995w==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@tokenizer/inflate": "^0.4.1", "strtok3": "^10.3.4", @@ -13551,8 +13544,7 @@ "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/formdata-node": { "version": "4.4.1", @@ -13560,7 +13552,6 @@ "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "node-domexception": "1.0.0", "web-streams-polyfill": "4.0.0-beta.3" @@ -14210,7 +14201,6 @@ "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ms": "^2.0.0" } @@ -14234,6 +14224,7 @@ } ], "license": "MIT", + "peer": true, "peerDependencies": { "typescript": "^5 || ^6 || ^7" }, @@ -14249,7 +14240,6 @@ "integrity": "sha512-7balLY8WKk+bOhe5Vgg4zG2X6Z0zhpG/3VtYPC69evj+lVJSId8xYP7ISRzRfoeXAlJfzLNMbi2Na/0IdDiIkQ==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "@types/debug": "4.1.12", "@types/node": "18.19.80", @@ -14278,7 +14268,6 @@ "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/ms": "*" } @@ -14289,7 +14278,6 @@ "integrity": "sha512-kEWeMwMeIvxYkeg1gTc01awpwLbfMRZXdIhwRcakd/KlK53jmRC26LqcbIt7fnAQTu5GzlnWmzA3H6+l1u6xxQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~5.26.4" } @@ -14300,7 +14288,6 @@ "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ms": "2.1.2" }, @@ -14319,7 +14306,6 @@ "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", "dev": true, "license": "BSD-2-Clause", - "peer": true, "engines": { "node": ">=12" }, @@ -14332,16 +14318,14 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/ibm-cloud-sdk-core/node_modules/undici-types": { "version": "5.26.5", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/iconv-lite": { "version": "0.6.3", @@ -14374,8 +14358,7 @@ "url": "https://feross.org/support" } ], - "license": "BSD-3-Clause", - "peer": true + "license": "BSD-3-Clause" }, "node_modules/ignore": { "version": "5.3.2", @@ -14383,6 +14366,7 @@ "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">= 4" } @@ -15182,8 +15166,7 @@ "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/istanbul-lib-coverage": { "version": "3.2.2", @@ -15454,7 +15437,6 @@ "integrity": "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "jws": "^4.0.1", "lodash.includes": "^4.3.0", @@ -15478,7 +15460,6 @@ "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", "dev": true, "license": "ISC", - "peer": true, "bin": { "semver": "bin/semver.js" }, @@ -15517,7 +15498,6 @@ "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", @@ -15530,7 +15510,6 @@ "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "jwa": "^2.0.1", "safe-buffer": "^5.0.1" @@ -15978,7 +15957,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=13.2.0" } @@ -16003,7 +15981,8 @@ "version": "4.18.1", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/lodash-es": { "version": "4.18.1", @@ -16029,48 +16008,42 @@ "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/lodash.isboolean": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/lodash.isinteger": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/lodash.isnumber": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/lodash.isstring": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/lodash.merge": { "version": "4.6.2", @@ -16084,8 +16057,7 @@ "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/log-symbols": { "version": "7.0.1", @@ -17377,7 +17349,6 @@ "resolved": "https://registry.npmjs.org/n8n-workflow/-/n8n-workflow-2.28.2.tgz", "integrity": "sha512-fGJ6yxRdgeea5EAywfAplYWDLMRNVMZRFikVTFLCBOInxLRDB89LL66sZHx2SC9dl/FALJAGZ6twY5MyUlc+HA==", "license": "SEE LICENSE IN LICENSE.md", - "peer": true, "dependencies": { "@n8n/errors": "0.10.0", "@n8n/expression-runtime": "0.19.0", @@ -17406,7 +17377,6 @@ "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", "license": "MIT", - "peer": true, "engines": { "node": ">=6" } @@ -17416,7 +17386,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.67.tgz", "integrity": "sha512-idA2YXwpCdqUSKRCACDE6ItZD9TZzy3OZMtpfLoh6oPR47lipysRrJfjzMqFxQ3uJuUPyUeWe1r9vLH33xO/Qw==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } @@ -17515,7 +17484,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=10.5.0" } @@ -17814,7 +17782,6 @@ "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~5.26.4" } @@ -17824,8 +17791,7 @@ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/openapi-types": { "version": "12.1.3", @@ -17852,6 +17818,12 @@ "node": ">= 0.8.0" } }, + "node_modules/ordered-map": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/ordered-map/-/ordered-map-0.1.0.tgz", + "integrity": "sha512-ttpssyEqKXIeTapKeFTFG+oUCnrC+Fmyy1DPjys0vPJtccBE3baKDFTbJPDuKd9/XHbnKKe+KPQKTzBLri6Mtw==", + "license": "MIT" + }, "node_modules/orderedmap": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/orderedmap/-/orderedmap-2.1.1.tgz", @@ -18253,6 +18225,7 @@ "integrity": "sha512-mHU89HGh7v+4u2ubfnevJ03lmPgQ5WU4CxAVmTSh/sxVTEDYd1er/dKS/A6vg77NX47KTEoihq8jZBLr8Cxuwg==", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "@napi-rs/canvas": "0.1.80", "pdfjs-dist": "5.4.296" @@ -18678,6 +18651,15 @@ "orderedmap": "^2.0.0" } }, + "node_modules/prosemirror-schema-basic": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/prosemirror-schema-basic/-/prosemirror-schema-basic-1.2.4.tgz", + "integrity": "sha512-ELxP4TlX3yr2v5rM7Sb70SqStq5NvI15c0j9j/gjsrO5vaw+fnnpovCLEGIcpeGfifkuqJwl4fon6b+KdrODYQ==", + "license": "MIT", + "dependencies": { + "prosemirror-model": "^1.25.0" + } + }, "node_modules/prosemirror-schema-list": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/prosemirror-schema-list/-/prosemirror-schema-list-1.5.1.tgz", @@ -18746,7 +18728,6 @@ "integrity": "sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "punycode": "^2.3.1" }, @@ -18791,8 +18772,7 @@ "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/queue-microtask": { "version": "1.2.3", @@ -18820,6 +18800,7 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.8.tgz", "integrity": "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -18874,6 +18855,7 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.8.tgz", "integrity": "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ==", "license": "MIT", + "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -19033,6 +19015,7 @@ "resolved": "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.84.0.tgz", "integrity": "sha512-+hWvQP6GLco56mDwrbU4XnHix8t1z90ltZsDIrREl+jnQFQxYLX8oAzqe/Xn8nHpmoXTY5M6oEXrAhbP1qevNQ==", "license": "MIT", + "peer": true, "engines": { "node": ">=18.0.0" }, @@ -19154,6 +19137,7 @@ "resolved": "https://registry.npmjs.org/react-relay/-/react-relay-21.0.1.tgz", "integrity": "sha512-08BBmpdolP8vslClnszG+g4vkcbVHvaoR0BDHyhicwpCfbtnVGe73/MfPja/xj30eLWSP5WPYNqZupTdEPZvtw==", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.25.0", "fbjs": "^3.0.2", @@ -19217,6 +19201,7 @@ "resolved": "https://registry.npmjs.org/react-router/-/react-router-8.3.0.tgz", "integrity": "sha512-qyPMvW83jGIct3yiieisxdk9M745anqhpIMKN5m1t6yBMfgVPpt77aHOqs5fUlEJRMCGffg9BaQLH9oPVOL7xQ==", "license": "MIT", + "peer": true, "dependencies": { "cookie-es": "^3.1.1" }, @@ -19419,6 +19404,7 @@ "resolved": "https://registry.npmjs.org/relay-runtime/-/relay-runtime-21.0.1.tgz", "integrity": "sha512-OS+We56sp6gkGVEcce9VCNOeg/sMbxT9unEoerQPgNAICHowSWrXNc6LJxF2OqjypxYF7kaLDSPPiaPii2Pw1g==", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.25.0", "fbjs": "^3.0.2", @@ -19528,8 +19514,7 @@ "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/reselect": { "version": "5.2.0", @@ -19585,7 +19570,6 @@ "integrity": "sha512-pOLi+Gdll3JekwuFjXO3fTq+L9lzMQGcSq7M5gIjExcl3Gu1hd4XXuf5o3+LuSBsaULQH7DiNbsqPd1chVpQGQ==", "dev": true, "license": "Apache-2.0", - "peer": true, "engines": { "node": ">=10.7.0" }, @@ -20203,6 +20187,7 @@ "integrity": "sha512-VhYwqxPySa24CVXKoWD6gCZXx9//DTmo43YpusGuAoHDYj5Osjt8wuBRQVeGoaLUWnHiPWv8S+GYHrJEaBM6Rg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@storybook/global": "^5.0.0", "@storybook/icons": "^2.0.2", @@ -20521,7 +20506,6 @@ "integrity": "sha512-ki4hZQfh5rX0QDLLkOCj+h+CVNkqmp/CMf8v8kZpkNVK6jGQooMytqzLZYUVYIZcFZ6yDB70EfD8POcFXiF5oA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@tokenizer/token": "^0.3.0" }, @@ -20631,6 +20615,7 @@ "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-3.6.0.tgz", "integrity": "sha512-uxL7qAVQriqRQPAyK3pj66VqskWqoZ37PW94jwOTwNfq/z9oyu1V+eqrZqtR2+fCiXdYOZe/Modt8GtvqNzu+w==", "license": "MIT", + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/dcastil" @@ -20662,7 +20647,8 @@ "version": "4.3.3", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.3.3.tgz", "integrity": "sha512-gOhV3P7ufE62QDGg1zVaTgCR+EtPv92k2nIhVcVKcLmxT1sUBsQGhnZj175j+MqRt4zLF7ic+sCYjfhxMxj7YQ==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/tapable": { "version": "2.3.3", @@ -20792,7 +20778,6 @@ "integrity": "sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@borewit/text-codec": "^0.2.1", "@tokenizer/token": "^0.3.0", @@ -20822,7 +20807,6 @@ "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==", "dev": true, "license": "BSD-3-Clause", - "peer": true, "dependencies": { "psl": "^1.1.33", "punycode": "^2.1.1", @@ -20839,7 +20823,6 @@ "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">= 4.0.0" } @@ -21127,7 +21110,6 @@ "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", "devOptional": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -21455,7 +21437,6 @@ "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "querystringify": "^2.1.1", "requires-port": "^1.0.0" @@ -21630,6 +21611,7 @@ "integrity": "sha512-pn+CFpM0lwDeKwmOq1ZaBK/9sjorZcgqxki6MbY/jPEVd9vichIlmlD4HmQ5wdP5EgqQCFRaACBxMC7uEGc6lQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "lightningcss": "^1.33.0", "picomatch": "^4.0.5", @@ -21984,6 +21966,7 @@ "integrity": "sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@vitest/expect": "4.1.10", "@vitest/mocker": "4.1.10", @@ -22127,7 +22110,6 @@ "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">= 14" } @@ -22623,6 +22605,7 @@ "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", "dev": true, "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } @@ -22689,6 +22672,14 @@ "url": "https://github.com/sponsors/wooorm" } }, + "packages/automerge-conformance": { + "name": "@probo/automerge-conformance", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "@automerge/automerge": "^3.4.0" + } + }, "packages/cookie-banner": { "name": "@probo/cookie-banner", "version": "0.13.0", @@ -23417,6 +23408,8 @@ "version": "1.0.0", "dependencies": { "@ariakit/react": "^0.4.35", + "@automerge/automerge": "^3.4.0", + "@automerge/prosemirror": "^0.2.0", "@base-ui/react": "^1.6.0", "@floating-ui/react": "^0.27.20", "@fontsource-variable/inter": "^5.3.0", @@ -23433,6 +23426,7 @@ "@radix-ui/react-scroll-area": "^1.2.18", "@radix-ui/react-select": "^2.3.7", "@radix-ui/react-tabs": "^1.1.21", + "@tiptap/core": "^3.29.2", "@tiptap/extension-table": "^3.29.2", "@tiptap/pm": "^3.29.2", "@tiptap/react": "^3.29.2", @@ -23562,6 +23556,7 @@ "integrity": "sha512-UDwuWGwXj646CBx/bQHOaJSX7np0I8JL/UKQYa1e4QrVHH8VdWtx8eaOuf8sy0ShwDgR6NjJAsp5eF6vjF6qng==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@blazediff/core": "1.9.1", "@vitest/mocker": "4.1.10", diff --git a/packages/automerge-conformance/oracle.mjs b/packages/automerge-conformance/oracle.mjs new file mode 100644 index 0000000000..849a86c1aa --- /dev/null +++ b/packages/automerge-conformance/oracle.mjs @@ -0,0 +1,251 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +import * as Automerge from "@automerge/automerge"; +import { Buffer } from "node:buffer"; +import process from "node:process"; + +const chunks = []; +for await (const chunk of process.stdin) { + chunks.push(chunk); +} + +const request = JSON.parse(Buffer.concat(chunks).toString("utf8")); + +switch (request.action) { + case "create": { + let document = Automerge.init({ actor: request.actor }); + document = Automerge.change( + document, + { + message: request.message, + time: request.timestamp, + }, + draft => { + draft.body = ""; + Automerge.splice(draft, ["body"], 0, 0, request.text); + }, + ); + process.stdout.write( + JSON.stringify({ + document: Buffer.from(Automerge.save(document)).toString("base64"), + heads: Automerge.getHeads(document), + }), + ); + break; + } + case "createRichText": { + let document = Automerge.from({ body: "" }, { actor: request.actor }); + document = Automerge.change(document, draft => { + Automerge.updateSpans(draft, ["body"], [ + { + type: "block", + value: { + type: "heading", + parents: [], + isEmbed: false, + attrs: { level: 2 }, + }, + }, + { + type: "text", + value: "Policy", + marks: { strong: true }, + }, + ]); + }); + process.stdout.write( + JSON.stringify({ + document: Buffer.from(Automerge.save(document)).toString("base64"), + heads: Automerge.getHeads(document), + }), + ); + break; + } + case "createChange": { + let document = Automerge.init({ actor: request.actor }); + document = Automerge.change( + document, + { + message: request.message, + time: request.timestamp, + }, + draft => { + draft.title = "Policy"; + }, + ); + const changes = Automerge.getAllChanges(document); + process.stdout.write( + JSON.stringify({ + change: Buffer.from(changes[0]).toString("base64"), + heads: Automerge.getHeads(document), + }), + ); + break; + } + case "createConcurrentChanges": { + const base = Automerge.from({ body: "" }, { actor: request.actor }); + let left = Automerge.load(Automerge.save(base), { actor: request.actorB }); + let right = Automerge.load(Automerge.save(base), { actor: request.actorC }); + left = Automerge.change(left, draft => { + Automerge.splice(draft, ["body"], 0, 0, "L"); + }); + right = Automerge.change(right, draft => { + Automerge.splice(draft, ["body"], 0, 0, "R"); + }); + const changes = [ + ...Automerge.getAllChanges(base), + ...Automerge.getChanges(base, left), + ...Automerge.getChanges(base, right), + ]; + const merged = Automerge.merge(left, right); + process.stdout.write( + JSON.stringify({ + body: merged.body, + changes: changes.map(change => Buffer.from(change).toString("base64")), + heads: Automerge.getHeads(merged), + }), + ); + break; + } + case "createSyncMessage": { + const document = Automerge.from( + { title: "Policy" }, + { actor: request.actor }, + ); + const [, message] = Automerge.generateSyncMessage( + document, + Automerge.initSyncState(), + ); + if (!message) throw new Error("expected an initial sync message"); + process.stdout.write( + JSON.stringify({ + sync: Buffer.from(message).toString("base64"), + heads: Automerge.getHeads(document), + }), + ); + break; + } + case "inspectChange": { + const change = Buffer.from(request.change, "base64"); + const decoded = Automerge.decodeChange(change); + const [document] = Automerge.applyChanges(Automerge.init(), [change]); + process.stdout.write( + JSON.stringify({ + body: document.title, + message: decoded.message, + heads: [decoded.hash], + }), + ); + break; + } + case "createComplexRichText": { + let document = Automerge.from({ body: "" }, { actor: request.actor }); + document = Automerge.change(document, draft => { + Automerge.updateSpans(draft, ["body"], [ + { + type: "block", + value: { + type: "paragraph", + parents: [], + isEmbed: false, + attrs: {}, + }, + }, + { + type: "text", + value: "A", + marks: { strong: true, em: true }, + }, + { + type: "text", + value: "B", + marks: { em: true }, + }, + ]); + }); + process.stdout.write( + JSON.stringify({ + document: Buffer.from(Automerge.save(document)).toString("base64"), + heads: Automerge.getHeads(document), + }), + ); + break; + } + case "createTableRichText": { + let document = Automerge.from({ body: "" }, { actor: request.actor }); + document = Automerge.change(document, draft => { + Automerge.updateSpans(draft, ["body"], [ + tableBlock("table", []), + tableBlock("table-row", ["table"]), + tableBlock("table-header", ["table", "table-row"], { + colspan: 1, + rowspan: 1, + colwidth: null, + }), + { type: "text", value: "A" }, + tableBlock("table-cell", ["table", "table-row"], { + colspan: 1, + rowspan: 1, + colwidth: null, + }), + { type: "text", value: "B" }, + tableBlock("table-row", ["table"]), + tableBlock("table-cell", ["table", "table-row"], { + colspan: 1, + rowspan: 1, + colwidth: null, + }), + { type: "text", value: "C" }, + ]); + }); + process.stdout.write( + JSON.stringify({ + document: Buffer.from(Automerge.save(document)).toString("base64"), + heads: Automerge.getHeads(document), + }), + ); + break; + } + case "inspect": { + const document = Automerge.load(Buffer.from(request.document, "base64")); + process.stdout.write( + JSON.stringify({ + body: document.body, + heads: Automerge.getHeads(document), + }), + ); + break; + } + default: + throw new Error(`unsupported action: ${request.action}`); +} + +function tableBlock(type, parents, attrs = {}) { + return { + type: "block", + value: { + type, + parents, + attrs, + isEmbed: false, + }, + }; +} diff --git a/packages/automerge-conformance/package.json b/packages/automerge-conformance/package.json new file mode 100644 index 0000000000..910127c77c --- /dev/null +++ b/packages/automerge-conformance/package.json @@ -0,0 +1,13 @@ +{ + "name": "@probo/automerge-conformance", + "version": "1.0.0", + "private": true, + "type": "module", + "license": "MIT", + "scripts": { + "check": "node --check oracle.mjs" + }, + "dependencies": { + "@automerge/automerge": "^3.4.0" + } +} diff --git a/packages/ui/package.json b/packages/ui/package.json index bb51d7491b..74f50474ac 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -14,6 +14,8 @@ }, "dependencies": { "@ariakit/react": "^0.4.35", + "@automerge/automerge": "^3.4.0", + "@automerge/prosemirror": "^0.2.0", "@base-ui/react": "^1.6.0", "@floating-ui/react": "^0.27.20", "@fontsource-variable/inter": "^5.3.0", @@ -30,6 +32,7 @@ "@radix-ui/react-scroll-area": "^1.2.18", "@radix-ui/react-select": "^2.3.7", "@radix-ui/react-tabs": "^1.1.21", + "@tiptap/core": "^3.29.2", "@tiptap/extension-table": "^3.29.2", "@tiptap/pm": "^3.29.2", "@tiptap/react": "^3.29.2", diff --git a/packages/ui/src/RichEditor/AutomergeSyncPlugin.ts b/packages/ui/src/RichEditor/AutomergeSyncPlugin.ts new file mode 100644 index 0000000000..cde21d5e7c --- /dev/null +++ b/packages/ui/src/RichEditor/AutomergeSyncPlugin.ts @@ -0,0 +1,255 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +// Adapted from @automerge/prosemirror's MIT-licensed sync plugin. The +// structural-leaf path deliberately bypasses its text-offset fast path. + +import * as Automerge from "@automerge/automerge"; +import type { + DocHandle, + SchemaAdapter, +} from "@automerge/prosemirror"; +import { patchesToTr } from "@automerge/prosemirror/dist/patchesToTr.js"; +import pmToAm from "@automerge/prosemirror/dist/pmToAm.js"; +import { + pmDocFromSpans, + pmNodeToSpans, +} from "@automerge/prosemirror/dist/traversal.js"; +import type { Fragment } from "@tiptap/pm/model"; +import { + type EditorState, + Plugin, + PluginKey, + type Transaction, +} from "@tiptap/pm/state"; + +import type { RichEditorAutomergeDocument } from "./collaboration"; +import { + collaborationDebug, + summarizeAutomergeSpans, + summarizeProseMirrorDocument, +} from "./collaborationDebug"; + +export const automergeSyncPluginKey = new PluginKey("automerge-sync"); + +export function createAutomergeSyncPlugin( + adapter: SchemaAdapter, + handle: DocHandle, + path: Automerge.Prop[], +): Plugin { + let ignoreTransaction = false; + + return new Plugin({ + key: automergeSyncPluginKey, + view: (view) => { + const onPatch = ({ + doc, + patches, + patchInfo, + }: { + doc: Automerge.Doc; + patches: Automerge.Patch[]; + patchInfo: Automerge.PatchInfo; + }) => { + if (ignoreTransaction) return; + + collaborationDebug("remote-patches", { + patchCount: patches.length, + patches: patches.map(patch => ({ + action: patch.action, + path: patch.path, + })), + heads: Automerge.getHeads(doc), + spans: summarizeAutomergeSpans(doc), + prosemirror: summarizeProseMirrorDocument(view.state.doc), + }); + + const transaction = hasHorizontalRule(view.state.doc) + ? reconcileAutomergeDocument(adapter, path, doc, view.state) + : patchesToTr({ + adapter, + path, + before: patchInfo.before, + after: doc, + patches, + state: view.state, + }); + ignoreTransaction = true; + view.dispatch(transaction); + ignoreTransaction = false; + collaborationDebug("remote-applied", { + stepCount: transaction.steps.length, + prosemirror: summarizeProseMirrorDocument(view.state.doc), + }); + }; + + handle.on("change", onPatch); + + return { + destroy() { + handle.off("change", onPatch); + }, + }; + }, + appendTransaction(transactions, _oldState, state) { + if (ignoreTransaction) return undefined; + + const changedTransactions = transactions.filter( + transaction => transaction.docChanged, + ); + if (changedTransactions.length === 0) return undefined; + + collaborationDebug("local-before", { + transactionCount: changedTransactions.length, + steps: changedTransactions.flatMap(transaction => + transaction.steps.map(step => step.constructor.name), + ), + heads: Automerge.getHeads(handle.doc()), + spans: summarizeAutomergeSpans(handle.doc()), + prosemirror: summarizeProseMirrorDocument(state.doc), + structuralLeaf: hasHorizontalRule(state.doc), + }); + + ignoreTransaction = true; + handle.change((document) => { + if (hasHorizontalRule(state.doc)) { + Automerge.updateSpans( + document, + path, + pmNodeToSpans(adapter, state.doc), + adapter.updateSpansConfig(), + ); + } else { + for (const transaction of changedTransactions) { + const spans = Automerge.spans(document, path); + pmToAm( + adapter, + spans, + transaction.steps, + document, + transaction.docs[0], + path, + ); + } + } + }); + ignoreTransaction = false; + + collaborationDebug("local-after", { + heads: Automerge.getHeads(handle.doc()), + spans: summarizeAutomergeSpans(handle.doc()), + prosemirror: summarizeProseMirrorDocument(state.doc), + }); + + return undefined; + }, + }); +} + +export function reconcileAutomergeDocument( + adapter: SchemaAdapter, + path: Automerge.Prop[], + document: Automerge.Doc, + state: EditorState, +): Transaction { + const nextDocument = pmDocFromSpans( + adapter, + Automerge.spans(document, path), + ); + const transaction = state.tr; + const change = findDocumentDiff(state.doc.content, nextDocument.content); + if (!change) return transaction; + + transaction.replace( + change.start, + change.endBefore, + nextDocument.slice(change.start, change.endAfter), + ); + transaction.setMeta("addToHistory", false); + return transaction; +} + +function hasHorizontalRule(document: { + descendants: ( + callback: (node: { type: { name: string } }) => boolean, + ) => void; +}): boolean { + let found = false; + document.descendants((node) => { + if (node.type.name === "horizontalRule") { + found = true; + return false; + } + + return true; + }); + return found; +} + +function findDocumentDiff( + before: Fragment, + after: Fragment, +): { + start: number; + endBefore: number; + endAfter: number; +} | null { + let start = before.findDiffStart(after); + if (start === null) return null; + + const end = before.findDiffEnd(after); + if (!end) return null; + + let endBefore = end.a; + let endAfter = end.b; + if (endBefore < start && before.size < after.size) { + if ( + start > 0 + && start < after.size + && isSurrogatePair(after.textBetween(start - 1, start + 1)) + ) { + start--; + } + endAfter = start + endAfter - endBefore; + endBefore = start; + } else if (endAfter < start) { + if ( + start > 0 + && start < before.size + && isSurrogatePair(before.textBetween(start - 1, start + 1)) + ) { + start--; + } + endBefore = start + endBefore - endAfter; + endAfter = start; + } + + return { start, endBefore, endAfter }; +} + +function isSurrogatePair(value: string): boolean { + if (value.length !== 2) return false; + const first = value.charCodeAt(0); + const second = value.charCodeAt(1); + return first >= 0xdc00 + && first <= 0xdfff + && second >= 0xd800 + && second <= 0xdbff; +} diff --git a/packages/ui/src/RichEditor/AutomergeTableStructureExtension.ts b/packages/ui/src/RichEditor/AutomergeTableStructureExtension.ts new file mode 100644 index 0000000000..63bedd746b --- /dev/null +++ b/packages/ui/src/RichEditor/AutomergeTableStructureExtension.ts @@ -0,0 +1,67 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +import { Extension } from "@tiptap/core"; + +export const AutomergeTableStructureExtension = Extension.create({ + name: "automergeTableStructure", + + addGlobalAttributes() { + return [ + { + types: [ + "automergeUnknownBlock", + "blockquote", + "codeBlock", + "heading", + "listItem", + "paragraph", + ], + attributes: { + isAmgBlock: { + default: false, + rendered: false, + keepOnSplit: true, + }, + unknownAttrs: { + default: null, + rendered: false, + }, + }, + }, + { + types: [ + "horizontalRule", + "table", + "tableRow", + "tableCell", + "tableHeader", + ], + attributes: { + isAmgBlock: { + default: true, + rendered: false, + keepOnSplit: true, + }, + }, + }, + ]; + }, +}); diff --git a/packages/ui/src/RichEditor/AutomergeUnknownBlockExtension.ts b/packages/ui/src/RichEditor/AutomergeUnknownBlockExtension.ts new file mode 100644 index 0000000000..e713aaea01 --- /dev/null +++ b/packages/ui/src/RichEditor/AutomergeUnknownBlockExtension.ts @@ -0,0 +1,35 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +import { Node } from "@tiptap/core"; + +export const AutomergeUnknownBlockExtension = Node.create({ + name: "automergeUnknownBlock", + group: "block", + content: "block+", + + parseHTML() { + return [{ tag: "div[data-automerge-unknown-block]" }]; + }, + + renderHTML() { + return ["div", { "data-automerge-unknown-block": "" }, 0]; + }, +}); diff --git a/packages/ui/src/RichEditor/BlockMenu/BlockMenuContent.tsx b/packages/ui/src/RichEditor/BlockMenu/BlockMenuContent.tsx index 59a5e3a493..7e972460b5 100644 --- a/packages/ui/src/RichEditor/BlockMenu/BlockMenuContent.tsx +++ b/packages/ui/src/RichEditor/BlockMenu/BlockMenuContent.tsx @@ -27,7 +27,6 @@ import { import { type Editor } from "@tiptap/react"; import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; -import { getSlashStorage } from "../_lib/getSlashStorage"; import { MenuButton } from "../MenuButton"; import { deactivateSlashCommand } from "../SlashCommandExtension"; @@ -63,7 +62,10 @@ type BlockMenuContentProps = { slashState: { active: boolean; query: string; from: number }; }; -export function BlockMenuContent({ editor, slashState }: BlockMenuContentProps) { +export function BlockMenuContent({ + editor, + slashState, +}: BlockMenuContentProps) { const [slashNav, setSlashNav] = useState({ index: 0, query: "" }); const slashDropdownRef = useRef(null); @@ -109,30 +111,20 @@ export function BlockMenuContent({ editor, slashState }: BlockMenuContentProps) }, [slashState.active, slashState.from, editor, slashMenuRefs]); const deactivateSlash = useCallback(() => { - if (!editor) return; - const s = getSlashStorage(editor); - if (s) deactivateSlashCommand(s); + deactivateSlashCommand(editor.view); setSlashNav({ index: 0, query: "" }); }, [editor]); const handleSlashAction = useCallback( (item: BlockItem) => { if (!slashState.active) return; - const { from } = slashState; - const cursorPos = editor.state.selection.from; try { - editor.chain() - .focus() - .deleteRange({ from, to: cursorPos }) - .run(); - + deactivateSlash(); item.action(editor.chain().focus()).run(); } catch { // Block may no longer be in the document } - - deactivateSlash(); }, [editor, slashState, deactivateSlash], ); diff --git a/packages/ui/src/RichEditor/BlockMenu/BlockMenuTrigger.tsx b/packages/ui/src/RichEditor/BlockMenu/BlockMenuTrigger.tsx index e405e8df8e..0cd20b012f 100644 --- a/packages/ui/src/RichEditor/BlockMenu/BlockMenuTrigger.tsx +++ b/packages/ui/src/RichEditor/BlockMenu/BlockMenuTrigger.tsx @@ -5,7 +5,6 @@ import { PlusIcon } from "@phosphor-icons/react"; import { type Editor } from "@tiptap/react"; -import { getSlashStorage } from "../_lib/getSlashStorage"; import { useBlockTrigger } from "../_lib/useBlockTrigger"; import { activateSlashCommand } from "../SlashCommandExtension"; @@ -35,11 +34,9 @@ export function BlockMenuTrigger({ editor, hoveredBlock }: BlockMenuTriggerProps editor.chain() .focus() .setTextSelection(textPos) - .insertContent("/") .run(); - const s = getSlashStorage(editor); - if (s) activateSlashCommand(s, textPos); + activateSlashCommand(editor.view, textPos); return; } @@ -57,11 +54,9 @@ export function BlockMenuTrigger({ editor, hoveredBlock }: BlockMenuTriggerProps .focus() .insertContentAt(insertPos, { type: "paragraph" }) .setTextSelection(textPos) - .insertContent("/") .run(); - const s = getSlashStorage(editor); - if (s) activateSlashCommand(s, textPos); + activateSlashCommand(editor.view, textPos); } catch { // Block may no longer be in the document } diff --git a/packages/ui/src/RichEditor/RichEditor.tsx b/packages/ui/src/RichEditor/RichEditor.tsx index 899b5e3cee..b2bd20eba0 100644 --- a/packages/ui/src/RichEditor/RichEditor.tsx +++ b/packages/ui/src/RichEditor/RichEditor.tsx @@ -18,22 +18,38 @@ import { Text } from "@tiptap/extension-text"; import { Underline } from "@tiptap/extension-underline"; import { Dropcursor, UndoRedo } from "@tiptap/extensions"; import { type Content, Editor, EditorContent, useEditor } from "@tiptap/react"; -import { type ComponentProps, useCallback, useEffect } from "react"; +import { type ComponentProps, useCallback, useEffect, useMemo } from "react"; import { tv } from "tailwind-variants"; +import { AutomergeTableStructureExtension } from "./AutomergeTableStructureExtension"; +import { AutomergeUnknownBlockExtension } from "./AutomergeUnknownBlockExtension"; import { BlockMenu } from "./BlockMenu/BlockMenu"; import { BubbleMenu } from "./BubbleMenu"; import { CodeBlockExtension } from "./CodeBlockExtension"; +import { + createRichEditorAutomergeDocument as createAutomergeDocument, + createRichEditorCollaborationExtension, + richEditorAutomergeContent, + supportsRichEditorCollaboration as supportsCollaboration, +} from "./collaboration"; import { LinkExtension } from "./LinkExtension"; import { MarkdownPasteExtension } from "./MarkdownPasteExtension"; import { OptionsMenu } from "./OptionsMenu/OptionsMenu"; import { PlaceholderExtension } from "./PlaceholderExtension"; +import { + createRichEditorPresenceExtension, + type RichEditorCollaborationHandle, +} from "./presence"; import { SlashCommandExtension } from "./SlashCommandExtension"; import { TableCellMenu } from "./TableCellMenu/TableCellMenu"; import { TableColumnMenu } from "./TableColumnMenu/TableColumnMenu"; import { TableRowMenu } from "./TableRowMenu/TableRowMenu"; import { TableSelectionOverlay } from "./TableSelectionOverlay"; +const tableExtension = TableKit.configure({ + table: { resizable: true }, +}); + const extensions = [ Document, Paragraph, @@ -60,12 +76,16 @@ const extensions = [ width: 2, }), UndoRedo, - TableKit.configure({ - table: { resizable: true }, - }), + tableExtension, + AutomergeUnknownBlockExtension, MarkdownPasteExtension, ]; +export const richEditorCollaborationExtensions = [ + ...extensions, + AutomergeTableStructureExtension, +]; + const richEditorVariants = tv({ base: ["relative flex-1 min-w-0 overflow-auto py-14 pr-8 bg-level-1 shadow-base"], variants: { @@ -80,18 +100,48 @@ type RichEditorProps = ComponentProps<"div"> & { content: string; disabled?: boolean; onChangeContent: (content: string) => void; + collaborationHandle?: RichEditorCollaborationHandle; }; export function RichEditor(props: RichEditorProps) { - const { className, content, disabled = false, onChangeContent, ...divProps } = props; + const { + className, + content, + disabled = false, + onChangeContent, + collaborationHandle, + ...divProps + } = props; const handleUpdate = useCallback( ({ editor }: { editor: Editor }) => { + if (collaborationHandle) return; + onChangeContent(JSON.stringify(editor.getJSON())); }, - [onChangeContent], + [collaborationHandle, onChangeContent], ); + const editorExtensions = useMemo( + () => collaborationHandle + ? [ + ...richEditorCollaborationExtensions, + createRichEditorCollaborationExtension( + collaborationHandle, + richEditorCollaborationExtensions, + ), + createRichEditorPresenceExtension(collaborationHandle), + ] + : extensions, + [collaborationHandle], + ); + const initialContent = collaborationHandle + ? richEditorAutomergeContent( + collaborationHandle, + richEditorCollaborationExtensions, + ) + : (content ? JSON.parse(content) : "") as Content; + const editor = useEditor({ editorProps: { attributes: { @@ -99,10 +149,16 @@ export function RichEditor(props: RichEditorProps) { }, }, editable: !disabled, - extensions, - content: (content ? JSON.parse(content) : "") as Content, + extensions: editorExtensions, + content: initialContent, onUpdate: handleUpdate, - }); + onSelectionUpdate: ({ editor }) => { + collaborationHandle?.updatePresence?.( + editor.state.selection.anchor, + editor.state.selection.head, + ); + }, + }, [collaborationHandle]); useEffect(() => { if (!editor) return; @@ -130,3 +186,13 @@ export function RichEditor(props: RichEditorProps) { ); } + +export function createRichEditorAutomergeDocument( + content: string, +) { + return createAutomergeDocument(content, richEditorCollaborationExtensions); +} + +export function supportsRichEditorCollaboration(content: string): boolean { + return supportsCollaboration(content); +} diff --git a/packages/ui/src/RichEditor/SlashCommandExtension.test.ts b/packages/ui/src/RichEditor/SlashCommandExtension.test.ts new file mode 100644 index 0000000000..1703ba2822 --- /dev/null +++ b/packages/ui/src/RichEditor/SlashCommandExtension.test.ts @@ -0,0 +1,127 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +import { Schema } from "@tiptap/pm/model"; +import { EditorState, type Transaction } from "@tiptap/pm/state"; +import type { EditorView } from "@tiptap/pm/view"; +import { describe, expect, it } from "vitest"; + +import { + createSlashCommandPlugin, + type SlashCommandStorage, +} from "./SlashCommandExtension"; + +const schema = new Schema({ + nodes: { + doc: { content: "block+" }, + paragraph: { content: "inline*", group: "block" }, + text: { group: "inline" }, + }, +}); + +describe("SlashCommandExtension", () => { + it("keeps slash queries out of the document", () => { + const storage = slashCommandStorage(); + const plugin = createSlashCommandPlugin(storage); + let state = EditorState.create({ + schema, + doc: schema.node("doc", null, [schema.node("paragraph")]), + plugins: [plugin], + }); + const transactions: Transaction[] = []; + const view = { + get state() { + return state; + }, + dispatch(transaction: Transaction) { + transactions.push(transaction); + state = state.apply(transaction); + }, + } as EditorView; + const handleTextInput = plugin.props.handleTextInput; + if (!handleTextInput) throw new Error("expected text input handler"); + + expect( + handleTextInput.call(plugin, view, 1, 1, "/", () => state.tr), + ).toBe(true); + expect( + handleTextInput.call(plugin, view, 1, 1, "head", () => state.tr), + ).toBe(true); + + expect(storage).toEqual({ + active: true, + query: "head", + from: 1, + }); + expect(state.doc.textContent).toBe(""); + expect(transactions.every(transaction => !transaction.docChanged)).toBe(true); + }); + + it("edits and closes a local query with keyboard commands", () => { + const storage = slashCommandStorage(); + const plugin = createSlashCommandPlugin(storage); + let state = EditorState.create({ + schema, + doc: schema.node("doc", null, [schema.node("paragraph")]), + plugins: [plugin], + }); + const view = { + get state() { + return state; + }, + dispatch(transaction: Transaction) { + state = state.apply(transaction); + }, + } as EditorView; + const handleTextInput = plugin.props.handleTextInput; + const handleKeyDown = plugin.props.handleKeyDown; + if (!handleTextInput || !handleKeyDown) { + throw new Error("expected slash command keyboard handlers"); + } + + handleTextInput.call(plugin, view, 1, 1, "/", () => state.tr); + handleTextInput.call(plugin, view, 1, 1, "ab", () => state.tr); + expect( + handleKeyDown.call( + plugin, + view, + { key: "Backspace" } as KeyboardEvent, + ), + ).toBe(true); + expect(storage.query).toBe("a"); + expect( + handleKeyDown.call( + plugin, + view, + { key: "Escape" } as KeyboardEvent, + ), + ).toBe(true); + expect(storage.active).toBe(false); + expect(state.doc.textContent).toBe(""); + }); +}); + +function slashCommandStorage(): SlashCommandStorage { + return { + active: false, + query: "", + from: 0, + }; +} diff --git a/packages/ui/src/RichEditor/SlashCommandExtension.ts b/packages/ui/src/RichEditor/SlashCommandExtension.ts index 981b83ca82..4ef82c5838 100644 --- a/packages/ui/src/RichEditor/SlashCommandExtension.ts +++ b/packages/ui/src/RichEditor/SlashCommandExtension.ts @@ -3,10 +3,32 @@ // that can be found in the LICENSE file. import { Extension } from "@tiptap/core"; -import { Plugin, PluginKey } from "@tiptap/pm/state"; +import { + Plugin, + PluginKey, + type Transaction, +} from "@tiptap/pm/state"; +import type { EditorView } from "@tiptap/pm/view"; import { Decoration, DecorationSet } from "@tiptap/pm/view"; -const slashCommandKey = new PluginKey("slashCommand"); +type SlashCommandPluginState = { + active: boolean; + query: string; + from: number; +}; + +type SlashCommandAction + = { type: "open"; from: number } + | { type: "append"; text: string } + | { type: "backspace" } + | { type: "close" }; + +const slashCommandKey = new PluginKey("slashCommand"); +const inactiveSlashCommand: SlashCommandPluginState = { + active: false, + query: "", + from: 0, +}; export type SlashCommandStorage = { active: boolean; @@ -14,16 +36,12 @@ export type SlashCommandStorage = { from: number; }; -export function activateSlashCommand(storage: SlashCommandStorage, from: number) { - storage.active = true; - storage.query = ""; - storage.from = from; +export function activateSlashCommand(view: EditorView, from: number): void { + dispatchSlashCommand(view, { type: "open", from }); } -export function deactivateSlashCommand(storage: SlashCommandStorage) { - storage.active = false; - storage.query = ""; - storage.from = 0; +export function deactivateSlashCommand(view: EditorView): void { + dispatchSlashCommand(view, { type: "close" }); } export const SlashCommandExtension = Extension.create({ @@ -38,108 +56,164 @@ export const SlashCommandExtension = Extension.create m.type.name === "code")) return false; - - const blockStart = $from.start($from.depth); - if (from !== blockStart) return false; - if ($from.parent.textContent.length !== 0) return false; - - storage.active = true; - storage.from = from; - storage.query = ""; - - return false; - }, - - handleKeyDown(view, event) { - if (!storage.active) return false; - - if (event.key === "Escape") { - const { state } = view; - const cursorPos = state.selection.from; - const from = storage.from; - - deactivateSlashCommand(storage); - - if (cursorPos > from) { - const tr = state.tr.delete(from, cursorPos); - view.dispatch(tr); - } - - return true; - } - - if (event.key === "Backspace") { - const { state } = view; - const cursorPos = state.selection.from; - - if (cursorPos <= storage.from + 1) { - deactivateSlashCommand(storage); - } - } - - return false; - }, - - decorations(state) { - if (!storage.active) return DecorationSet.empty; - - const { from } = storage; - const cursorPos = state.selection.from; - - try { - const $from = state.doc.resolve(from); - const blockStart = $from.start($from.depth); - const blockEnd = $from.end($from.depth); - - if (cursorPos < blockStart || cursorPos > blockEnd) { - deactivateSlashCommand(storage); - return DecorationSet.empty; - } - - const text = state.doc.textBetween(from, cursorPos); - if (!text.startsWith("/")) { - deactivateSlashCommand(storage); - return DecorationSet.empty; - } - - storage.query = text.slice(1); - - const decoEnd = Math.max(cursorPos, from + 1); - const isEmpty = storage.query.length === 0; - - return DecorationSet.create(state.doc, [ - Decoration.inline(from, decoEnd, { - "nodeName": "span", - "class": "slash-search", - "data-placeholder": "Search", - "data-empty": isEmpty - ? "true" - : "false", - }), - ]); - } catch { - deactivateSlashCommand(storage); - return DecorationSet.empty; - } - }, - }, - }), - ]; + return [createSlashCommandPlugin(this.storage)]; }, }); + +export function createSlashCommandPlugin( + storage: SlashCommandStorage, +): Plugin { + return new Plugin({ + key: slashCommandKey, + + state: { + init() { + updateSlashCommandStorage(storage, inactiveSlashCommand); + return inactiveSlashCommand; + }, + + apply(transaction, current) { + const action = slashCommandAction(transaction); + let next = applySlashCommandAction(current, action); + if (next.active && transaction.docChanged) { + const mapped = transaction.mapping.mapResult(next.from, 1); + next = mapped.deleted + ? inactiveSlashCommand + : { ...next, from: mapped.pos }; + } + if ( + next.active + && transaction.selectionSet + && transaction.selection.from !== next.from + ) { + next = inactiveSlashCommand; + } + + updateSlashCommandStorage(storage, next); + return next; + }, + }, + + props: { + handleTextInput(view, from, _to, text) { + const current = slashCommandKey.getState(view.state); + if (current?.active) { + dispatchSlashCommand(view, { type: "append", text }); + return true; + } + if (text !== "/" || !canOpenSlashCommand(view, from)) { + return false; + } + + dispatchSlashCommand(view, { type: "open", from }); + return true; + }, + + handleKeyDown(view, event) { + const current = slashCommandKey.getState(view.state); + if (!current?.active) return false; + + if (event.key === "Escape") { + dispatchSlashCommand(view, { type: "close" }); + return true; + } + + if (event.key === "Backspace") { + dispatchSlashCommand(view, { type: "backspace" }); + return true; + } + + return false; + }, + + decorations(state) { + const current = slashCommandKey.getState(state); + if (!current?.active) return DecorationSet.empty; + + try { + const $from = state.doc.resolve(current.from); + const decorations = [ + Decoration.widget( + current.from, + () => createSlashCommandWidget(current.query), + { key: `slash-command:${current.query}`, side: -1 }, + ), + ]; + if ($from.depth > 0 && $from.parent.isTextblock) { + decorations.push( + Decoration.node( + $from.before($from.depth), + $from.after($from.depth), + { class: "slash-command-active" }, + ), + ); + } + return DecorationSet.create(state.doc, decorations); + } catch { + return DecorationSet.empty; + } + }, + }, + }); +} + +function applySlashCommandAction( + current: SlashCommandPluginState, + action: SlashCommandAction | undefined, +): SlashCommandPluginState { + switch (action?.type) { + case "open": + return { active: true, query: "", from: action.from }; + case "append": + return { ...current, query: current.query + action.text }; + case "backspace": + return current.query.length === 0 + ? inactiveSlashCommand + : { ...current, query: current.query.slice(0, -1) }; + case "close": + return inactiveSlashCommand; + default: + return current; + } +} + +function canOpenSlashCommand(view: EditorView, from: number): boolean { + const $from = view.state.doc.resolve(from); + if ($from.parent.type.name === "codeBlock") return false; + if ($from.marks().some(mark => mark.type.name === "code")) return false; + + const blockStart = $from.start($from.depth); + return from === blockStart && $from.parent.textContent.length === 0; +} + +function createSlashCommandWidget(query: string): HTMLElement { + const widget = document.createElement("span"); + widget.className = "slash-search"; + widget.dataset.placeholder = "Search"; + widget.dataset.empty = query.length === 0 ? "true" : "false"; + widget.contentEditable = "false"; + widget.textContent = `/${query}`; + return widget; +} + +function dispatchSlashCommand( + view: EditorView, + action: SlashCommandAction, +): void { + view.dispatch(view.state.tr.setMeta(slashCommandKey, action)); +} + +function slashCommandAction( + transaction: Transaction, +): SlashCommandAction | undefined { + return transaction.getMeta(slashCommandKey) as SlashCommandAction | undefined; +} + +function updateSlashCommandStorage( + storage: SlashCommandStorage, + state: SlashCommandPluginState, +): void { + storage.active = state.active; + storage.query = state.query; + storage.from = state.from; +} diff --git a/packages/ui/src/RichEditor/collaboration.test.ts b/packages/ui/src/RichEditor/collaboration.test.ts new file mode 100644 index 0000000000..6fb5e7f4a7 --- /dev/null +++ b/packages/ui/src/RichEditor/collaboration.test.ts @@ -0,0 +1,746 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +import * as Automerge from "@automerge/automerge"; +import { + type DocHandle, + pmDocFromSpans, + pmNodeToSpans, + type SchemaAdapter, +} from "@automerge/prosemirror"; +import { getSchema } from "@tiptap/core"; +import { history, undo } from "@tiptap/pm/history"; +import { Fragment } from "@tiptap/pm/model"; +import { EditorState, type Transaction } from "@tiptap/pm/state"; +import { describe, expect, it } from "vitest"; + +import { + createAutomergeSyncPlugin, + reconcileAutomergeDocument, +} from "./AutomergeSyncPlugin"; +import { + createSchemaAdapter, + explicitBlockIdentityPlugin, + richEditorAutomergeContent, + type RichEditorAutomergeDocument, +} from "./collaboration"; +import { + createRichEditorAutomergeDocument, + richEditorCollaborationExtensions, + supportsRichEditorCollaboration, +} from "./RichEditor"; + +function syncPlugin({ + adapter, + handle, + path, +}: { + adapter: SchemaAdapter; + handle: DocHandle; + path: Automerge.Prop[]; +}) { + return createAutomergeSyncPlugin(adapter, handle, path); +} + +describe("RichEditor collaboration", () => { + it("imports supported ProseMirror content into Automerge rich text", () => { + const document = createRichEditorAutomergeDocument( + JSON.stringify({ + type: "doc", + content: [ + { + type: "heading", + attrs: { level: 2 }, + content: [{ type: "text", text: "Policy" }], + }, + { + type: "paragraph", + content: [ + { type: "text", text: "Hello ", marks: [{ type: "bold" }] }, + { type: "text", text: "world" }, + ], + }, + ], + }), + ); + + expect(Automerge.spans(document, ["body"])).not.toHaveLength(0); + expect(document.body).toContain("Hello world"); + }); + + it("preserves table and row boundaries", () => { + const content = tableDocumentJSON(); + + expect(supportsRichEditorCollaboration(content)).toBe(true); + const document = createRichEditorAutomergeDocument(content); + const spans = Automerge.spans(document, ["body"]); + expect( + spans + .filter(span => span.type === "block") + .map(span => Automerge.isImmutableString(span.value.type) + ? span.value.type.val + : span.value.type), + ).toEqual([ + "table", + "table-row", + "table-cell", + "table-cell", + "table-row", + "table-cell", + "table-cell", + ]); + + const handle: DocHandle = { + doc: () => document, + change: () => {}, + on: () => {}, + off: () => {}, + }; + const roundTrip = richEditorAutomergeContent( + handle, + richEditorCollaborationExtensions, + ) as { + content: Array<{ + type: string; + content: Array<{ + type: string; + content: unknown[]; + }>; + }>; + }; + expect(roundTrip.content[0].type).toBe("table"); + expect(roundTrip.content[0].content).toHaveLength(2); + expect(roundTrip.content[0].content[0].type).toBe("tableRow"); + expect(roundTrip.content[0].content[0].content).toHaveLength(2); + }); + + it("applies table cell edits through the Automerge sync plugin", () => { + let document = createRichEditorAutomergeDocument(tableDocumentJSON()); + const handle: DocHandle = { + doc: () => document, + change: (change) => { + document = Automerge.change(document, change); + }, + on: () => {}, + off: () => {}, + }; + const adapter = createSchemaAdapter(richEditorCollaborationExtensions); + const pmDocument = pmDocFromSpans( + adapter, + Automerge.spans(document, ["body"]), + ); + const state = EditorState.create({ + schema: adapter.schema, + doc: pmDocument, + plugins: [ + syncPlugin({ + adapter, + handle, + path: ["body"], + }), + ], + }); + let textPosition: number | undefined; + state.doc.descendants((node, position) => { + if (node.isText && node.text === "A") { + textPosition = position; + return false; + } + + return true; + }); + expect(textPosition).toBeDefined(); + + state.applyTransaction(state.tr.insertText("X", textPosition! + 1)); + + expect( + Automerge.spans(document, ["body"]) + .filter(span => span.type === "text") + .map(span => span.value) + .join(""), + ).toContain("AX"); + }); + + it("preserves rows inserted through ProseMirror transactions", () => { + let document = createRichEditorAutomergeDocument(tableDocumentJSON()); + const handle: DocHandle = { + doc: () => document, + change: (change) => { + document = Automerge.change(document, change); + }, + on: () => {}, + off: () => {}, + }; + const adapter = createSchemaAdapter(richEditorCollaborationExtensions); + const pmDocument = pmDocFromSpans( + adapter, + Automerge.spans(document, ["body"]), + ); + const state = EditorState.create({ + schema: adapter.schema, + doc: pmDocument, + plugins: [ + syncPlugin({ + adapter, + handle, + path: ["body"], + }), + ], + }); + const paragraph = adapter.schema.nodes.paragraph.create( + null, + adapter.schema.text("E"), + ); + const cell = adapter.schema.nodes.tableCell.create( + { + isAmgBlock: true, + colspan: 1, + rowspan: 1, + colwidth: null, + }, + paragraph, + ); + const row = adapter.schema.nodes.tableRow.create( + { isAmgBlock: true }, + [cell, cell], + ); + const table = state.doc.firstChild; + if (!table) throw new Error("expected table node"); + + state.applyTransaction( + state.tr.insert(table.nodeSize - 1, row), + ); + + const spans = Automerge.spans(document, ["body"]); + expect( + spans.filter(span => + span.type === "block" + && Automerge.isImmutableString(span.value.type) + && span.value.type.val === "table-row", + ), + ).toHaveLength(3); + + const roundTrip = pmDocFromSpans(adapter, spans); + expect(roundTrip.firstChild?.childCount).toBe(3); + }); + + it("converges concurrent row insertions", () => { + const base = createRichEditorAutomergeDocument(tableDocumentJSON()); + const left = insertTableRow( + Automerge.clone(base, { actor: "01000000000000000000000000000000" }), + "L", + ); + const right = insertTableRow( + Automerge.clone(base, { actor: "02000000000000000000000000000000" }), + "R", + ); + const merged = Automerge.merge(left, right); + const adapter = createSchemaAdapter(richEditorCollaborationExtensions); + const spans = Automerge.spans(merged, ["body"]); + const pmDocument = pmDocFromSpans(adapter, spans); + + expect(pmDocument.firstChild?.childCount).toBe(4); + expect(pmDocument.textContent).toContain("L"); + expect(pmDocument.textContent).toContain("R"); + }); + + it("records local table edits in collaborative undo history", () => { + let document = createRichEditorAutomergeDocument(tableDocumentJSON()); + const handle: DocHandle = { + doc: () => document, + change: (change) => { + document = Automerge.change(document, change); + }, + on: () => {}, + off: () => {}, + }; + const adapter = createSchemaAdapter(richEditorCollaborationExtensions); + let state = EditorState.create({ + schema: adapter.schema, + doc: pmDocFromSpans(adapter, Automerge.spans(document, ["body"])), + plugins: [ + history(), + syncPlugin({ + adapter, + handle, + path: ["body"], + }), + ], + }); + let textPosition: number | undefined; + state.doc.descendants((node, position) => { + if (node.isText && node.text === "A") { + textPosition = position; + return false; + } + + return true; + }); + if (textPosition === undefined) throw new Error("expected cell text"); + + state = state.applyTransaction( + state.tr.insertText("X", textPosition + 1), + ).state; + expect(document.body).toContain("AX"); + + let undoTransaction: Transaction | undefined; + expect( + undo(state, (transaction) => { + undoTransaction = transaction; + }), + ).toBe(true); + if (!undoTransaction) throw new Error("expected undo transaction"); + + state.applyTransaction(undoTransaction); + expect(document.body).not.toContain("AX"); + }); + + it("inserts a divider as a structural block", () => { + let document = createRichEditorAutomergeDocument( + JSON.stringify({ + type: "doc", + content: [ + { + type: "paragraph", + content: [{ type: "text", text: "A" }], + }, + { + type: "paragraph", + content: [{ type: "text", text: "B" }], + }, + ], + }), + ); + const handle: DocHandle = { + doc: () => document, + change: (change) => { + document = Automerge.change(document, change); + }, + on: () => {}, + off: () => {}, + }; + const adapter = createSchemaAdapter(richEditorCollaborationExtensions); + const pmDocument = pmDocFromSpans( + adapter, + Automerge.spans(document, ["body"]), + ); + const state = EditorState.create({ + schema: adapter.schema, + doc: pmDocument, + plugins: [ + syncPlugin({ + adapter, + handle, + path: ["body"], + }), + ], + }); + const firstParagraph = state.doc.firstChild; + if (!firstParagraph) throw new Error("expected first paragraph"); + + state.applyTransaction( + state.tr.insert( + firstParagraph.nodeSize, + adapter.schema.nodes.horizontalRule.create({ isAmgBlock: true }), + ), + ); + + const spans = Automerge.spans(document, ["body"]); + expect( + spans.some(span => + span.type === "block" + && Automerge.isImmutableString(span.value.type) + && span.value.type.val === "horizontal-rule", + ), + ).toBe(true); + expect( + pmDocFromSpans(adapter, spans).toJSON(), + ).toMatchObject({ + content: [ + { type: "paragraph" }, + { type: "horizontalRule" }, + { type: "paragraph" }, + ], + }); + }); + + it("synchronizes text typed after a newly inserted divider", () => { + let document = createRichEditorAutomergeDocument( + JSON.stringify({ + type: "doc", + content: [ + { + type: "paragraph", + content: [{ type: "text", text: "Before" }], + }, + ], + }), + ); + const targetSchema = getSchema(richEditorCollaborationExtensions); + const adapter = createSchemaAdapter( + richEditorCollaborationExtensions, + targetSchema, + ); + const handle: DocHandle = { + doc: () => document, + change: (change) => { + document = Automerge.change(document, change); + }, + on: () => {}, + off: () => {}, + }; + const initialAdapter = createSchemaAdapter( + richEditorCollaborationExtensions, + ); + const pmDocument = targetSchema.nodeFromJSON( + pmDocFromSpans( + initialAdapter, + Automerge.spans(document, ["body"]), + ).toJSON(), + ); + let state = EditorState.create({ + schema: targetSchema, + doc: pmDocument, + plugins: [ + explicitBlockIdentityPlugin(), + createAutomergeSyncPlugin(adapter, handle, ["body"]), + ], + }); + const divider = targetSchema.nodes.horizontalRule.create(); + const paragraph = targetSchema.nodes.paragraph.create(); + state = state.applyTransaction( + state.tr.insert( + state.doc.content.size, + Fragment.fromArray([divider, paragraph]), + ), + ).state; + expect( + Automerge.spans(document, ["body"]) + .filter(span => span.type === "block") + .map(span => Automerge.isImmutableString(span.value.type) + ? span.value.type.val + : span.value.type), + ).toEqual([ + "paragraph", + "horizontal-rule", + "paragraph", + ]); + expect(state.doc.lastChild?.attrs.isAmgBlock).toBe(true); + + const lastParagraphPosition = state.doc.content.size - paragraph.nodeSize; + let insertionError: unknown; + try { + state = state.applyTransaction( + state.tr.insertText("After", lastParagraphPosition + 1), + ).state; + } catch (error) { + insertionError = error; + } + + const spans = Automerge.spans(document, ["body"]); + expect( + spans + .filter(span => span.type === "block") + .map(span => Automerge.isImmutableString(span.value.type) + ? span.value.type.val + : span.value.type), + ).toEqual([ + "paragraph", + "horizontal-rule", + "paragraph", + ]); + expect( + spans + .filter(span => span.type === "block") + .map(span => ({ + type: automergeString(span.value.type), + isEmbed: span.value.isEmbed, + parents: automergeStringArray(span.value.parents), + })), + ).toEqual([ + { type: "paragraph", isEmbed: false, parents: [] }, + { type: "horizontal-rule", isEmbed: false, parents: [] }, + { type: "paragraph", isEmbed: false, parents: [] }, + ]); + expect( + spans + .filter(span => span.type === "text") + .map(span => span.value) + .join(""), + ).toBe("BeforeAfter"); + expect( + spans.map(span => span.type === "text" + ? `text:${span.value}` + : `block:${automergeString(span.value.type)}`), + ).toEqual([ + "block:paragraph", + "text:Before", + "block:horizontal-rule", + "block:paragraph", + "text:After", + ]); + expect(insertionError).toBeUndefined(); + + const remoteDocument = pmDocFromSpans(initialAdapter, spans); + expect(remoteDocument.textContent).toBe("BeforeAfter"); + expect(remoteDocument.child(1).type.name).toBe("horizontalRule"); + expect(remoteDocument.child(2).textContent).toBe("After"); + + const nextParagraph = targetSchema.nodes.paragraph.create(); + state = state.applyTransaction( + state.tr.insert(state.doc.content.size, nextParagraph), + ).state; + const nextPosition = state.doc.content.size - nextParagraph.nodeSize + 1; + state.applyTransaction(state.tr.insertText("Later", nextPosition)); + + const laterSpans = Automerge.spans(document, ["body"]); + expect( + laterSpans.map(span => span.type === "text" + ? `text:${span.value}` + : `block:${automergeString(span.value.type)}`), + ).toEqual([ + "block:paragraph", + "text:Before", + "block:horizontal-rule", + "block:paragraph", + "text:After", + "block:paragraph", + "text:Later", + ]); + const laterRemote = pmDocFromSpans(initialAdapter, laterSpans); + expect(laterRemote.textContent).toBe("BeforeAfterLater"); + expect(laterRemote.child(3).textContent).toBe("Later"); + }); + + it("reconciles remote text between empty structural blocks", () => { + const adapter = createSchemaAdapter(richEditorCollaborationExtensions); + let document = createRichEditorAutomergeDocument( + JSON.stringify({ + type: "doc", + content: [ + { + type: "paragraph", + content: [{ type: "text", text: "Before" }], + }, + { type: "horizontalRule" }, + { type: "paragraph" }, + { type: "paragraph" }, + { + type: "paragraph", + content: [{ type: "text", text: "After" }], + }, + ], + }), + ); + const initialDocument = pmDocFromSpans( + adapter, + Automerge.spans(document, ["body"]), + ); + const state = EditorState.create({ + schema: adapter.schema, + doc: initialDocument, + }); + const emptyParagraphPositions: number[] = []; + state.doc.descendants((node, position) => { + if (node.type.name === "paragraph" && node.content.size === 0) { + emptyParagraphPositions.push(position); + } + }); + expect(emptyParagraphPositions).toHaveLength(2); + + const changedDocument = state.tr.insertText( + "X", + emptyParagraphPositions[0] + 1, + ).doc; + document = Automerge.change(document, (draft) => { + Automerge.updateSpans( + draft, + ["body"], + pmNodeToSpans(adapter, changedDocument), + adapter.updateSpansConfig(), + ); + }); + + const transaction = reconcileAutomergeDocument( + adapter, + ["body"], + document, + state, + ); + const reconciled = state.apply(transaction); + expect(reconciled.doc.eq(changedDocument)).toBe(true); + }); + + it("preserves Mermaid code-block language", () => { + const document = createRichEditorAutomergeDocument( + JSON.stringify({ + type: "doc", + content: [ + { + type: "codeBlock", + attrs: { language: "mermaid" }, + content: [{ type: "text", text: "graph TD; A-->B" }], + }, + ], + }), + ); + const spans = Automerge.spans(document, ["body"]); + const block = spans.find(span => span.type === "block"); + if (block?.type !== "block") throw new Error("expected block span"); + const attrs = block.value.attrs; + if ( + attrs === null + || typeof attrs !== "object" + || !("language" in attrs) + ) { + throw new Error("expected code-block language"); + } + expect(attrs.language).toBe("mermaid"); + + const handle: DocHandle = { + doc: () => document, + change: () => {}, + on: () => {}, + off: () => {}, + }; + const roundTrip = richEditorAutomergeContent( + handle, + richEditorCollaborationExtensions, + ) as { + content: Array<{ + type: string; + attrs: { language: string | null }; + }>; + }; + expect(roundTrip.content[0]).toMatchObject({ + type: "codeBlock", + attrs: { language: "mermaid" }, + }); + }); +}); + +function tableDocumentJSON(): string { + return JSON.stringify({ + type: "doc", + content: [ + { + type: "table", + content: [ + { + type: "tableRow", + content: [ + tableCell("A"), + tableCell("B"), + ], + }, + { + type: "tableRow", + content: [ + tableCell("C"), + tableCell("D"), + ], + }, + ], + }, + ], + }); +} + +function automergeString(value: unknown): string { + if (typeof value === "string") return value; + if (Automerge.isImmutableString(value)) return value.val; + throw new Error("expected Automerge string"); +} + +function automergeStringArray(value: unknown): string[] { + if (!Array.isArray(value)) throw new Error("expected Automerge string array"); + return value.map(automergeString); +} + +function tableCell(text: string) { + return { + type: "tableCell", + attrs: { + colspan: 1, + rowspan: 1, + colwidth: null, + }, + content: [ + { + type: "paragraph", + content: [{ type: "text", text }], + }, + ], + }; +} + +function insertTableRow( + initial: Automerge.Doc, + text: string, +): Automerge.Doc { + let document = initial; + const handle: DocHandle = { + doc: () => document, + change: (change) => { + document = Automerge.change(document, change); + }, + on: () => {}, + off: () => {}, + }; + const adapter = createSchemaAdapter(richEditorCollaborationExtensions); + const pmDocument = pmDocFromSpans( + adapter, + Automerge.spans(document, ["body"]), + ); + const state = EditorState.create({ + schema: adapter.schema, + doc: pmDocument, + plugins: [ + syncPlugin({ + adapter, + handle, + path: ["body"], + }), + ], + }); + const paragraph = adapter.schema.nodes.paragraph.create( + null, + adapter.schema.text(text), + ); + const cell = adapter.schema.nodes.tableCell.create( + { + isAmgBlock: true, + colspan: 1, + rowspan: 1, + colwidth: null, + }, + paragraph, + ); + const row = adapter.schema.nodes.tableRow.create( + { isAmgBlock: true }, + [cell], + ); + const table = state.doc.firstChild; + if (!table) throw new Error("expected table node"); + + state.applyTransaction(state.tr.insert(table.nodeSize - 1, row)); + + return document; +} diff --git a/packages/ui/src/RichEditor/collaboration.ts b/packages/ui/src/RichEditor/collaboration.ts new file mode 100644 index 0000000000..2be59e5193 --- /dev/null +++ b/packages/ui/src/RichEditor/collaboration.ts @@ -0,0 +1,437 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +import * as Automerge from "@automerge/automerge"; +import { + type DocHandle, + type MappedMarkSpec, + type MappedNodeSpec, + type MappedSchemaSpec, + pmDocFromSpans, + pmNodeToSpans, + SchemaAdapter, +} from "@automerge/prosemirror"; +import { Extension, type Extensions, getSchema } from "@tiptap/core"; +import type { + Mark, + Node as ProseMirrorNode, + Schema, +} from "@tiptap/pm/model"; +import { Plugin } from "@tiptap/pm/state"; + +import { createAutomergeSyncPlugin } from "./AutomergeSyncPlugin"; + +export type RichEditorAutomergeDocument = { + body: string; +}; + +const textPath: Automerge.Prop[] = ["body"]; + +const supportedNodeNames = new Set([ + "doc", + "paragraph", + "text", + "heading", + "blockquote", + "codeBlock", + "horizontalRule", + "hardBreak", + "bulletList", + "orderedList", + "listItem", + "table", + "tableRow", + "tableCell", + "tableHeader", +]); + +const supportedMarkNames = new Set([ + "bold", + "italic", + "strike", + "underline", + "code", + "link", +]); + +export function supportsRichEditorCollaboration(content: string): boolean { + if (!content) return true; + + const document = JSON.parse(content) as { + type?: string; + marks?: Array<{ type?: string }>; + content?: unknown[]; + }; + + function supportsNode(value: unknown): boolean { + if (!value || typeof value !== "object") return false; + + const node = value as { + type?: string; + marks?: Array<{ type?: string }>; + content?: unknown[]; + }; + if (!node.type || !supportedNodeNames.has(node.type)) return false; + if (node.marks?.some(mark => !mark.type || !supportedMarkNames.has(mark.type))) { + return false; + } + return node.content?.every(supportsNode) ?? true; + } + + return supportsNode(document); +} + +export function createRichEditorAutomergeDocument( + content: string, + extensions: Extensions, +): Automerge.Doc { + const adapter = createSchemaAdapter(extensions); + const documentJSON: Record = content + ? parseJSONObject(content) + : { type: "doc", content: [{ type: "paragraph" }] }; + markTableStructure(documentJSON); + const pmDocument = adapter.schema.nodeFromJSON(documentJSON); + const spans = pmNodeToSpans(adapter, pmDocument); + const document = Automerge.from({ body: "" }); + + return Automerge.change(document, (draft) => { + Automerge.updateSpans(draft, textPath, spans, adapter.updateSpansConfig()); + }); +} + +export function richEditorAutomergeContent( + handle: DocHandle, + extensions: Extensions, +): object { + const adapter = createSchemaAdapter(extensions); + const content: unknown = pmDocFromSpans( + adapter, + Automerge.spans(handle.doc(), textPath), + ).toJSON(); + if (!isJSONObject(content)) { + throw new Error("Automerge produced invalid ProseMirror content"); + } + return content; +} + +export function reconcileRichEditorAutomergeDocument( + handle: DocHandle, + document: ProseMirrorNode, + extensions: Extensions, +): void { + let hasStructuralLeaf = false; + document.descendants((node) => { + if (node.type.name === "horizontalRule") { + hasStructuralLeaf = true; + return false; + } + + return true; + }); + if (!hasStructuralLeaf) return; + + const adapter = createSchemaAdapter(extensions, document.type.schema); + const spans = pmNodeToSpans(adapter, document); + handle.change((draft) => { + Automerge.updateSpans( + draft, + textPath, + spans, + adapter.updateSpansConfig(), + ); + }); +} + +export function createRichEditorCollaborationExtension( + handle: DocHandle, + extensions: Extensions, +): Extension { + return Extension.create({ + name: "automergeCollaboration", + priority: 1_000, + + addProseMirrorPlugins() { + const adapter = createSchemaAdapter(extensions, this.editor.schema); + return [ + explicitBlockIdentityPlugin(), + createAutomergeSyncPlugin(adapter, handle, textPath), + ]; + }, + }); +} + +export function explicitBlockIdentityPlugin(): Plugin { + return new Plugin({ + appendTransaction(transactions, _oldState, state) { + if (!transactions.some(transaction => transaction.docChanged)) { + return null; + } + + const transaction = state.tr; + let changed = false; + state.doc.descendants((node, position, parent) => { + if (node.attrs.isAmgBlock !== false) return true; + + const isTopLevelTextBlock = parent === state.doc && ( + node.type.name === "paragraph" + || node.type.name === "heading" + || node.type.name === "blockquote" + || node.type.name === "codeBlock" + ); + if (!isTopLevelTextBlock && node.type.name !== "listItem") { + return true; + } + + transaction.setNodeMarkup( + position, + undefined, + { + ...node.attrs, + isAmgBlock: true, + }, + ); + changed = true; + + return true; + }); + if (!changed) return null; + + transaction.setMeta("addToHistory", false); + + return transaction; + }, + }); +} + +export function createSchemaAdapter( + extensions: Extensions, + targetSchema?: Schema, +): SchemaAdapter { + const sourceSchema = getSchema(extensions); + const nodes: Record = {}; + const marks: Record = {}; + + sourceSchema.spec.nodes.forEach((name, spec) => { + nodes[name] = { + ...spec, + automerge: automergeNodeMapping(name), + }; + }); + sourceSchema.spec.marks.forEach((name, spec) => { + marks[name] = { + ...spec, + automerge: automergeMarkMapping(name), + }; + }); + + const adapter = new SchemaAdapter({ nodes, marks } satisfies MappedSchemaSpec); + if (!targetSchema) return adapter; + + adapter.schema = targetSchema; + adapter.unknownBlock = targetSchema.nodes.automergeUnknownBlock; + adapter.nodeMappings = adapter.nodeMappings.map(mapping => ({ + ...mapping, + outer: mapping.outer ? targetSchema.nodes[mapping.outer.name] : null, + content: targetSchema.nodes[mapping.content.name], + })); + adapter.markMappings = adapter.markMappings.map(mapping => ({ + ...mapping, + prosemirrorMark: targetSchema.marks[mapping.prosemirrorMark.name], + })); + + return adapter; +} + +function automergeNodeMapping(name: string): MappedNodeSpec["automerge"] { + switch (name) { + case "paragraph": + return { block: "paragraph" }; + case "heading": + return { + block: "heading", + attrParsers: { + fromAutomerge: block => ({ + level: readNumberAttribute(block.attrs, "level", 1), + }), + fromProsemirror: node => ({ + level: readNumberAttribute(node.attrs, "level", 1), + }), + }, + }; + case "blockquote": + return { block: "blockquote" }; + case "codeBlock": + return { + block: "code-block", + attrParsers: { + fromAutomerge: block => ({ + language: readNullableStringAttribute(block.attrs, "language"), + }), + fromProsemirror: node => ({ + language: readNullableStringAttribute(node.attrs, "language"), + }), + }, + }; + case "horizontalRule": + return { block: "horizontal-rule" }; + case "hardBreak": + return { block: "hard-break", isEmbed: true }; + case "listItem": + return { + block: { + within: { + orderedList: "ordered-list-item", + bulletList: "unordered-list-item", + }, + }, + }; + case "table": + return { block: "table" }; + case "tableRow": + return { block: "table-row" }; + case "tableCell": + case "tableHeader": + return { + block: name === "tableCell" ? "table-cell" : "table-header", + attrParsers: { + fromAutomerge: block => ({ + colspan: readNumberAttribute(block.attrs, "colspan", 1), + rowspan: readNumberAttribute(block.attrs, "rowspan", 1), + colwidth: readNumberArrayAttribute(block.attrs, "colwidth"), + }), + fromProsemirror: node => ({ + colspan: readNumberAttribute(node.attrs, "colspan", 1), + rowspan: readNumberAttribute(node.attrs, "rowspan", 1), + colwidth: readNumberArrayAttribute(node.attrs, "colwidth"), + }), + }, + }; + case "automergeUnknownBlock": + return { unknownBlock: true }; + default: + return undefined; + } +} + +function automergeMarkMapping(name: string): MappedMarkSpec["automerge"] { + switch (name) { + case "bold": + return { markName: "strong" }; + case "italic": + return { markName: "em" }; + case "strike": + case "underline": + case "code": + return { markName: name }; + case "link": + return { + markName: "link", + parsers: { + fromAutomerge: (value) => { + if (typeof value !== "string") return { href: "", title: null }; + const parsed = parseJSONObject(value); + return { + href: readStringAttribute(parsed, "href", ""), + title: readNullableStringAttribute(parsed, "title"), + }; + }, + fromProsemirror: (mark: Mark) => JSON.stringify({ + href: readStringAttribute(mark.attrs, "href", ""), + title: readNullableStringAttribute(mark.attrs, "title"), + }), + }, + }; + default: + return undefined; + } +} + +function parseJSONObject(value: string): Record { + const parsed: unknown = JSON.parse(value); + if (!isJSONObject(parsed)) { + throw new Error("expected a JSON object"); + } + return parsed; +} + +function isJSONObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function markTableStructure(node: Record): void { + const type = node.type; + if ( + type === "horizontalRule" + || type === "table" + || type === "tableRow" + || type === "tableCell" + || type === "tableHeader" + ) { + const attrs = isJSONObject(node.attrs) ? node.attrs : {}; + attrs.isAmgBlock = true; + node.attrs = attrs; + } + + if (!Array.isArray(node.content)) return; + for (const child of node.content) { + if (isJSONObject(child)) markTableStructure(child); + } +} + +function readNumberAttribute( + attributes: unknown, + name: string, + fallback: number, +): number { + if (!isJSONObject(attributes)) return fallback; + const value = attributes[name]; + return typeof value === "number" ? value : fallback; +} + +function readStringAttribute( + attributes: unknown, + name: string, + fallback: string, +): string { + if (!isJSONObject(attributes)) return fallback; + const value = attributes[name]; + return typeof value === "string" ? value : fallback; +} + +function readNullableStringAttribute( + attributes: unknown, + name: string, +): string | null { + if (!isJSONObject(attributes)) return null; + const value = attributes[name]; + return typeof value === "string" ? value : null; +} + +function readNumberArrayAttribute( + attributes: unknown, + name: string, +): number[] | null { + if (!isJSONObject(attributes)) return null; + const value = attributes[name]; + if (!Array.isArray(value)) return null; + + const numbers = value.filter(item => typeof item === "number"); + return numbers.length === value.length ? numbers : null; +} diff --git a/packages/ui/src/RichEditor/collaborationDebug.ts b/packages/ui/src/RichEditor/collaborationDebug.ts new file mode 100644 index 0000000000..7a7fd10c93 --- /dev/null +++ b/packages/ui/src/RichEditor/collaborationDebug.ts @@ -0,0 +1,97 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +import * as Automerge from "@automerge/automerge"; +import type { Node as ProseMirrorNode } from "@tiptap/pm/model"; + +const collaborationDebugStorageKey = "probo:collaboration-debug"; +const collaborationDebugPrefix = "[probo:collaboration]"; + +export function collaborationDebug( + event: string, + details: Record, +): void { + if (!collaborationDebugEnabled()) return; + console.info(collaborationDebugPrefix, event, JSON.stringify(details)); +} + +export function summarizeAutomergeSpans( + document: Automerge.Doc<{ body: string }>, +): Array> { + return Automerge.spans(document, ["body"]).map((span, index) => { + if (span.type === "text") { + return { + index, + kind: "text", + length: span.value.length, + marks: Object.keys(span.marks ?? {}).sort(), + }; + } + + return { + index, + kind: "block", + type: automergeString(span.value.type), + parents: automergeStringArray(span.value.parents), + isEmbed: span.value.isEmbed, + attrs: Object.keys( + isRecord(span.value.attrs) ? span.value.attrs : {}, + ).sort(), + }; + }); +} + +export function summarizeProseMirrorDocument( + document: ProseMirrorNode, +): Array> { + const nodes: Array> = []; + document.descendants((node, position, parent) => { + nodes.push({ + position, + type: node.type.name, + parent: parent?.type.name ?? null, + isText: node.isText, + textLength: node.isText ? node.text?.length ?? 0 : 0, + isAmgBlock: node.attrs.isAmgBlock ?? null, + }); + return true; + }); + return nodes; +} + +function collaborationDebugEnabled(): boolean { + return typeof window !== "undefined" + && window.localStorage.getItem(collaborationDebugStorageKey) === "1"; +} + +function automergeString(value: unknown): string { + if (typeof value === "string") return value; + if (Automerge.isImmutableString(value)) return value.val; + return ""; +} + +function automergeStringArray(value: unknown): string[] { + if (!Array.isArray(value)) return []; + return value.map(automergeString); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/ui/src/RichEditor/presence.ts b/packages/ui/src/RichEditor/presence.ts new file mode 100644 index 0000000000..9e3055426f --- /dev/null +++ b/packages/ui/src/RichEditor/presence.ts @@ -0,0 +1,143 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +import type { DocHandle } from "@automerge/prosemirror"; +import { Extension } from "@tiptap/core"; +import type { Node as ProseMirrorNode } from "@tiptap/pm/model"; +import { Plugin, PluginKey } from "@tiptap/pm/state"; +import { Decoration, DecorationSet } from "@tiptap/pm/view"; + +import type { RichEditorAutomergeDocument } from "./collaboration"; + +export type RichEditorPresence = { + connectionID: string; + identityID: string; + anchorPosition: number; + headPosition: number; +}; + +export type RichEditorCollaborationHandle = DocHandle & { + updatePresence?: (anchorPosition: number, headPosition: number) => void; + onPresence?: (listener: (presences: RichEditorPresence[]) => void) => () => void; +}; + +const presencePluginKey = new PluginKey("automerge-presence"); + +export function createRichEditorPresenceExtension( + handle: RichEditorCollaborationHandle, +): Extension { + return Extension.create({ + name: "automergePresence", + + addProseMirrorPlugins() { + return [ + new Plugin({ + key: presencePluginKey, + state: { + init: () => DecorationSet.empty, + apply(transaction, decorations) { + const presences = transaction.getMeta(presencePluginKey) as + | RichEditorPresence[] + | undefined; + if (!presences) return decorations.map(transaction.mapping, transaction.doc); + return decorationsForPresences(transaction.doc, presences); + }, + }, + props: { + decorations(state) { + return presencePluginKey.getState(state); + }, + }, + view: (view) => { + const unsubscribe = handle.onPresence?.((presences) => { + view.dispatch(view.state.tr.setMeta(presencePluginKey, presences)); + }); + return { + destroy() { + unsubscribe?.(); + }, + }; + }, + }), + ]; + }, + }); +} + +function decorationsForPresences( + pmDocument: ProseMirrorNode, + presences: RichEditorPresence[], +): DecorationSet { + const decorations: Decoration[] = []; + for (const presence of presences) { + const documentSize = pmDocument.content.size; + const anchor = clamp(presence.anchorPosition, 0, documentSize); + const head = clamp(presence.headPosition, 0, documentSize); + const from = Math.min(anchor, head); + const to = Math.max(anchor, head); + const color = presenceColor(presence.identityID); + + if (from !== to) { + decorations.push( + Decoration.inline( + from, + to, + { + style: `background-color: ${color}33`, + }, + ), + ); + } + decorations.push( + Decoration.widget( + head, + () => { + const cursor = document.createElement("span"); + cursor.setAttribute("aria-label", "Collaborator cursor"); + cursor.setAttribute("title", "Collaborator"); + cursor.style.borderLeft = `2px solid ${color}`; + cursor.style.height = "1.2em"; + cursor.style.marginLeft = "-1px"; + cursor.style.pointerEvents = "none"; + return cursor; + }, + { + key: presence.connectionID, + side: 1, + }, + ), + ); + } + + return DecorationSet.create(pmDocument, decorations); +} + +function presenceColor(identityID: string): string { + let hash = 0; + for (const character of identityID) { + hash = ((hash << 5) - hash + character.charCodeAt(0)) | 0; + } + const hue = Math.abs(hash) % 360; + return `hsl(${hue} 70% 45%)`; +} + +function clamp(value: number, minimum: number, maximum: number): number { + return Math.min(Math.max(value, minimum), maximum); +} diff --git a/packages/ui/src/index.ts b/packages/ui/src/index.ts index 2f067a528e..56796ea4e2 100644 --- a/packages/ui/src/index.ts +++ b/packages/ui/src/index.ts @@ -119,4 +119,17 @@ export { EditableRow } from "./Molecules/Table/EditableRow"; export { Toasts, useToast } from "./Atoms/Toasts/Toasts"; // Rich editor -export { RichEditor } from "./RichEditor/RichEditor"; +export { + createRichEditorAutomergeDocument, + RichEditor, + supportsRichEditorCollaboration, +} from "./RichEditor/RichEditor"; +export type { RichEditorAutomergeDocument } from "./RichEditor/collaboration"; +export type { + RichEditorCollaborationHandle, + RichEditorPresence, +} from "./RichEditor/presence"; +export { + collaborationDebug, + summarizeAutomergeSpans, +} from "./RichEditor/collaborationDebug"; diff --git a/packages/ui/src/rich-editor.css b/packages/ui/src/rich-editor.css index 2631eafb57..0d445e1d4f 100644 --- a/packages/ui/src/rich-editor.css +++ b/packages/ui/src/rich-editor.css @@ -49,6 +49,10 @@ @apply text-txt-tertiary pointer-events-none float-left h-0; } + p.is-empty-focused.slash-command-active::before { + content: none; + } + .slash-search { @apply bg-subtle rounded px-1.5 py-0.5; } diff --git a/pkg/automerge/README.md b/pkg/automerge/README.md new file mode 100644 index 0000000000..03f61dbfcd --- /dev/null +++ b/pkg/automerge/README.md @@ -0,0 +1,54 @@ +# Automerge + +This package is Probo's owned, no-CGO boundary for Automerge documents. + +## Engines + +The default backend is a clean-room, pure-Go Automerge 0.10 engine. It: + +- decodes document, change, compressed-change, and v1/v2 sync formats; +- validates checksums, actor ownership, causal frontiers, sequences, and limits; +- preserves unknown columns and scalars for forward compatibility; +- supports maps, text, UTF-16 splices, rich-text materialization, cursors, + changes, heads, merges, and synchronization; and +- emits changes accepted by official Rust and JavaScript implementations. + +The package also retains a first-party WASI adapter around the official +[`automerge`](https://crates.io/crates/automerge) Rust crate as an independent +differential oracle. The adapter: + +- pins `automerge` 0.10.0 and every transitive crate in `Cargo.lock`; +- compiles with UTF-16 indexing to match the JavaScript editor; +- runs in-process through wazero without CGO or native shared libraries; and +- gives every open document an isolated WASM instance. + +Use `NewReference` and `LoadReference` only in conformance tests or when +diagnosing native parity. Production `New` and `Load` use the Go engine. + +The committed `reference.wasm` is reproducible from reviewed Rust source: + +```sh +rustup toolchain install 1.89.0 --profile minimal --target wasm32-wasip1 +make generate-automerge-reference +``` + +## Compatibility checks + +Ordinary Go tests cover binary round trips, UTF-16 text offsets, concurrent +changes, randomized text histories, merge convergence, native/reference sync, +rich-text spans, cursor movement, and lifecycle behavior: + +```sh +go test -race ./pkg/automerge/... +``` + +The cross-language suite additionally loads Go documents in the official +JavaScript implementation and JavaScript documents in Go: + +```sh +npm install +make test-automerge-conformance +``` + +The conformance oracle is deliberately separate from the Go implementation so +the two paths do not share adapter code. diff --git a/pkg/automerge/automerge.go b/pkg/automerge/automerge.go new file mode 100644 index 0000000000..ad414c5b53 --- /dev/null +++ b/pkg/automerge/automerge.go @@ -0,0 +1,630 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +// Package automerge provides a no-CGO Go API for Automerge documents. +// +// The initial backend embeds the official Rust Automerge engine as a WASI +// module. A native Go engine can implement the private backend contract and be +// checked against this reference implementation without changing callers. +package automerge + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "sync" + "time" + + "go.probo.inc/probo/pkg/automerge/internal/native" + "go.probo.inc/probo/pkg/automerge/internal/reference" +) + +type ( + // ActorID identifies one writer in an Automerge history. + ActorID [16]byte + + // Hash identifies an Automerge change. + Hash [32]byte + + // Cursor is a stable position in an Automerge sequence. + Cursor []byte + + // Change is one immutable encoded Automerge change. + Change struct { + Hash Hash + Bytes []byte + } + + // Document is a concurrency-safe Automerge document. + Document struct { + mu sync.Mutex + backend backend + closed bool + } + + // Text is a collaborative UTF-16-indexed text object. + Text struct { + document *Document + handle uint32 + } + + // SyncState tracks one document's synchronization with one remote peer. + SyncState struct { + document *Document + handle uint32 + closed bool + } + + backend interface { + Close(context.Context) error + Save(context.Context) ([]byte, error) + SetActor(context.Context, []byte) error + PutString(context.Context, uint32, string, string) error + PutText(context.Context, uint32, string) (uint32, error) + GetText(context.Context, uint32, string) (uint32, error) + SpliceText(context.Context, uint32, uint32, int32, string) error + Text(context.Context, uint32) (string, error) + TextSpans(context.Context, uint32) ([]byte, error) + TextCursor(context.Context, uint32, uint32) ([]byte, error) + TextCursorPosition(context.Context, uint32, []byte) (uint32, error) + Commit(context.Context, string, time.Time) ([32]byte, error) + Heads(context.Context) ([][32]byte, error) + Merge(context.Context, []byte) ([][32]byte, error) + NewSyncState(context.Context) (uint32, error) + CloseSyncState(context.Context, uint32) error + GenerateSyncMessage(context.Context, uint32) ([]byte, bool, error) + ReceiveSyncMessage(context.Context, uint32, []byte) error + SaveSyncState(context.Context, uint32) ([]byte, error) + LoadSyncState(context.Context, []byte) (uint32, error) + } + + changeBackend interface { + ChangesSince(context.Context, [][32]byte) ([][]byte, [][32]byte, error) + } + + changeApplier interface { + ApplyChanges(context.Context, [][]byte) error + } +) + +var ( + ErrClosed = errors.New("automerge document is closed") + ErrSameDocument = errors.New("cannot merge an Automerge document into itself") + ErrSyncStateClosed = errors.New("automerge sync state is closed") + + _ backend = (*reference.Backend)(nil) + _ backend = (*native.Backend)(nil) +) + +const rootObject uint32 = 0 + +// NewActorID returns a cryptographically random actor ID. +func NewActorID() (ActorID, error) { + var actorID ActorID + if _, err := rand.Read(actorID[:]); err != nil { + return ActorID{}, fmt.Errorf("cannot generate Automerge actor ID: %w", err) + } + + return actorID, nil +} + +// New creates an empty document using the native Go engine. +func New(ctx context.Context, actorID ActorID) (*Document, error) { + return NewPureGo(ctx, actorID) +} + +// NewReference creates an empty document using the official WASM reference +// engine. It is retained as a differential oracle for the native backend. +func NewReference(ctx context.Context, actorID ActorID) (*Document, error) { + b, err := reference.New(ctx) + if err != nil { + return nil, fmt.Errorf("cannot create Automerge backend: %w", err) + } + + if err := b.SetActor(ctx, actorID[:]); err != nil { + _ = b.Close(ctx) + return nil, fmt.Errorf("cannot initialize Automerge actor: %w", err) + } + + return &Document{backend: b}, nil +} + +// NewPureGo creates an empty document using the experimental native Go engine. +// +// The native engine is intended for differential testing until its complete +// feature surface reaches parity with the reference backend. +func NewPureGo(ctx context.Context, actorID ActorID) (*Document, error) { + b, err := native.NewBackend(ctx) + if err != nil { + return nil, fmt.Errorf("cannot create native Automerge backend: %w", err) + } + + if err := b.SetActor(ctx, actorID[:]); err != nil { + _ = b.Close(ctx) + return nil, fmt.Errorf("cannot initialize native Automerge actor: %w", err) + } + + return &Document{backend: b}, nil +} + +// Load creates a document using the native Go engine and assigns a new writer. +func Load(ctx context.Context, data []byte, actorID ActorID) (*Document, error) { + return LoadPureGo(ctx, data, actorID) +} + +// LoadReference loads a document using the official WASM reference engine. +func LoadReference( + ctx context.Context, + data []byte, + actorID ActorID, +) (*Document, error) { + b, err := reference.Load(ctx, data) + if err != nil { + return nil, fmt.Errorf("cannot load Automerge backend: %w", err) + } + + if err := b.SetActor(ctx, actorID[:]); err != nil { + _ = b.Close(ctx) + return nil, fmt.Errorf("cannot assign loaded Automerge actor: %w", err) + } + + return &Document{backend: b}, nil +} + +// LoadPureGo loads Automerge data using the experimental native Go engine. +func LoadPureGo( + ctx context.Context, + data []byte, + actorID ActorID, +) (*Document, error) { + b, err := native.LoadBackend(ctx, data) + if err != nil { + return nil, fmt.Errorf("cannot load native Automerge backend: %w", err) + } + + if err := b.SetActor(ctx, actorID[:]); err != nil { + _ = b.Close(ctx) + return nil, fmt.Errorf("cannot assign native Automerge actor: %w", err) + } + + return &Document{backend: b}, nil +} + +// Close releases the document's WASM module instance. +func (d *Document) Close(ctx context.Context) error { + d.mu.Lock() + defer d.mu.Unlock() + + if d.closed { + return nil + } + + d.closed = true + + if err := d.backend.Close(ctx); err != nil { + return fmt.Errorf("cannot close Automerge document: %w", err) + } + + return nil +} + +// Save serializes the complete Automerge history. +func (d *Document) Save(ctx context.Context) ([]byte, error) { + d.mu.Lock() + defer d.mu.Unlock() + + if d.closed { + return nil, ErrClosed + } + + data, err := d.backend.Save(ctx) + if err != nil { + return nil, fmt.Errorf("cannot save Automerge document: %w", err) + } + + return data, nil +} + +// PutString assigns a string at a key in the root map. +func (d *Document) PutString(ctx context.Context, key, value string) error { + d.mu.Lock() + defer d.mu.Unlock() + + if d.closed { + return ErrClosed + } + + if err := d.backend.PutString(ctx, rootObject, key, value); err != nil { + return fmt.Errorf("cannot put Automerge string: %w", err) + } + + return nil +} + +// CreateText creates collaborative text at a key in the root map. +func (d *Document) CreateText(ctx context.Context, key string) (*Text, error) { + d.mu.Lock() + defer d.mu.Unlock() + + if d.closed { + return nil, ErrClosed + } + + handle, err := d.backend.PutText(ctx, rootObject, key) + if err != nil { + return nil, fmt.Errorf("cannot create Automerge text: %w", err) + } + + return &Text{document: d, handle: handle}, nil +} + +// Text returns existing collaborative text from the root map. +func (d *Document) Text(ctx context.Context, key string) (*Text, error) { + d.mu.Lock() + defer d.mu.Unlock() + + if d.closed { + return nil, ErrClosed + } + + handle, err := d.backend.GetText(ctx, rootObject, key) + if err != nil { + return nil, fmt.Errorf("cannot get Automerge text: %w", err) + } + + return &Text{document: d, handle: handle}, nil +} + +// Commit records pending operations as one Automerge change. +func (d *Document) Commit( + ctx context.Context, + message string, + timestamp time.Time, +) (Hash, error) { + d.mu.Lock() + defer d.mu.Unlock() + + if d.closed { + return Hash{}, ErrClosed + } + + hash, err := d.backend.Commit(ctx, message, timestamp) + if err != nil { + return Hash{}, fmt.Errorf("cannot commit Automerge document: %w", err) + } + + return Hash(hash), nil +} + +// Heads returns the hashes at the document's current frontier. +func (d *Document) Heads(ctx context.Context) ([]Hash, error) { + d.mu.Lock() + defer d.mu.Unlock() + + if d.closed { + return nil, ErrClosed + } + + backendHeads, err := d.backend.Heads(ctx) + if err != nil { + return nil, fmt.Errorf("cannot get Automerge heads: %w", err) + } + + heads := make([]Hash, len(backendHeads)) + for i := range backendHeads { + heads[i] = Hash(backendHeads[i]) + } + + return heads, nil +} + +// ChangesSince returns encoded changes not covered by heads. +func (d *Document) ChangesSince( + ctx context.Context, + heads []Hash, +) ([]Change, error) { + d.mu.Lock() + defer d.mu.Unlock() + + if d.closed { + return nil, ErrClosed + } + + changeSource, ok := d.backend.(changeBackend) + if !ok { + return nil, fmt.Errorf("automerge backend does not expose incremental changes") + } + + backendHeads := make([][32]byte, len(heads)) + for i, head := range heads { + backendHeads[i] = [32]byte(head) + } + + raw, hashes, err := changeSource.ChangesSince(ctx, backendHeads) + if err != nil { + return nil, fmt.Errorf("cannot get incremental Automerge changes: %w", err) + } + + if len(raw) != len(hashes) { + return nil, fmt.Errorf( + "cannot get incremental Automerge changes: %d changes for %d hashes", + len(raw), + len(hashes), + ) + } + + changes := make([]Change, len(raw)) + for i := range raw { + changes[i] = Change{ + Hash: Hash(hashes[i]), + Bytes: append([]byte(nil), raw[i]...), + } + } + + return changes, nil +} + +// ApplyChanges applies encoded changes whose dependencies may already exist in +// the document. +func (d *Document) ApplyChanges( + ctx context.Context, + changes [][]byte, +) error { + d.mu.Lock() + defer d.mu.Unlock() + + if d.closed { + return ErrClosed + } + + applier, ok := d.backend.(changeApplier) + if !ok { + return fmt.Errorf("automerge backend does not accept incremental changes") + } + + if err := applier.ApplyChanges(ctx, changes); err != nil { + return fmt.Errorf("cannot apply incremental Automerge changes: %w", err) + } + + return nil +} + +// Merge applies all changes from another document. +func (d *Document) Merge(ctx context.Context, other *Document) ([]Hash, error) { + if d == other { + return nil, ErrSameDocument + } + + otherData, err := other.Save(ctx) + if err != nil { + return nil, fmt.Errorf("cannot save Automerge merge source: %w", err) + } + + d.mu.Lock() + defer d.mu.Unlock() + + if d.closed { + return nil, ErrClosed + } + + backendHeads, err := d.backend.Merge(ctx, otherData) + if err != nil { + return nil, fmt.Errorf("cannot merge Automerge document: %w", err) + } + + heads := make([]Hash, len(backendHeads)) + for i := range backendHeads { + heads[i] = Hash(backendHeads[i]) + } + + return heads, nil +} + +// NewSyncState starts synchronization with a remote peer. +func (d *Document) NewSyncState(ctx context.Context) (*SyncState, error) { + d.mu.Lock() + defer d.mu.Unlock() + + if d.closed { + return nil, ErrClosed + } + + handle, err := d.backend.NewSyncState(ctx) + if err != nil { + return nil, fmt.Errorf("cannot create Automerge sync state: %w", err) + } + + return &SyncState{document: d, handle: handle}, nil +} + +// LoadSyncState resumes a previously serialized remote-peer session. +func (d *Document) LoadSyncState(ctx context.Context, data []byte) (*SyncState, error) { + d.mu.Lock() + defer d.mu.Unlock() + + if d.closed { + return nil, ErrClosed + } + + handle, err := d.backend.LoadSyncState(ctx, data) + if err != nil { + return nil, fmt.Errorf("cannot load Automerge sync state: %w", err) + } + + return &SyncState{document: d, handle: handle}, nil +} + +// Splice replaces deleteCount UTF-16 code units at index with value. +func (t *Text) Splice(ctx context.Context, index uint32, deleteCount int32, value string) error { + t.document.mu.Lock() + defer t.document.mu.Unlock() + + if t.document.closed { + return ErrClosed + } + + if err := t.document.backend.SpliceText(ctx, t.handle, index, deleteCount, value); err != nil { + return fmt.Errorf("cannot splice Automerge text: %w", err) + } + + return nil +} + +// String returns the current materialized text. +func (t *Text) String(ctx context.Context) (string, error) { + t.document.mu.Lock() + defer t.document.mu.Unlock() + + if t.document.closed { + return "", ErrClosed + } + + value, err := t.document.backend.Text(ctx, t.handle) + if err != nil { + return "", fmt.Errorf("cannot read Automerge text: %w", err) + } + + return value, nil +} + +// Cursor returns a stable address for the UTF-16 position at index. +func (t *Text) Cursor(ctx context.Context, index uint32) (Cursor, error) { + t.document.mu.Lock() + defer t.document.mu.Unlock() + + if t.document.closed { + return nil, ErrClosed + } + + cursor, err := t.document.backend.TextCursor(ctx, t.handle, index) + if err != nil { + return nil, fmt.Errorf("cannot create Automerge text cursor: %w", err) + } + + return Cursor(cursor), nil +} + +// CursorPosition resolves a stable cursor in the current document. +func (t *Text) CursorPosition(ctx context.Context, cursor Cursor) (uint32, error) { + t.document.mu.Lock() + defer t.document.mu.Unlock() + + if t.document.closed { + return 0, ErrClosed + } + + position, err := t.document.backend.TextCursorPosition(ctx, t.handle, cursor) + if err != nil { + return 0, fmt.Errorf("cannot resolve Automerge text cursor: %w", err) + } + + return position, nil +} + +// Close releases the peer-specific synchronization state. +func (s *SyncState) Close(ctx context.Context) error { + s.document.mu.Lock() + defer s.document.mu.Unlock() + + if s.closed { + return nil + } + + s.closed = true + + if s.document.closed { + return nil + } + + if err := s.document.backend.CloseSyncState(ctx, s.handle); err != nil { + return fmt.Errorf("cannot close Automerge sync state: %w", err) + } + + return nil +} + +// GenerateMessage returns the next message for the remote peer. +func (s *SyncState) GenerateMessage(ctx context.Context) ([]byte, bool, error) { + s.document.mu.Lock() + defer s.document.mu.Unlock() + + if s.document.closed { + return nil, false, ErrClosed + } + + if s.closed { + return nil, false, ErrSyncStateClosed + } + + message, ok, err := s.document.backend.GenerateSyncMessage(ctx, s.handle) + if err != nil { + return nil, false, fmt.Errorf("cannot generate Automerge sync message: %w", err) + } + + return message, ok, nil +} + +// ReceiveMessage applies a message received from the remote peer. +func (s *SyncState) ReceiveMessage(ctx context.Context, message []byte) error { + s.document.mu.Lock() + defer s.document.mu.Unlock() + + if s.document.closed { + return ErrClosed + } + + if s.closed { + return ErrSyncStateClosed + } + + if err := s.document.backend.ReceiveSyncMessage(ctx, s.handle, message); err != nil { + return fmt.Errorf("cannot receive Automerge sync message: %w", err) + } + + return nil +} + +// Save serializes the peer-specific synchronization state. +func (s *SyncState) Save(ctx context.Context) ([]byte, error) { + s.document.mu.Lock() + defer s.document.mu.Unlock() + + if s.document.closed { + return nil, ErrClosed + } + + if s.closed { + return nil, ErrSyncStateClosed + } + + data, err := s.document.backend.SaveSyncState(ctx, s.handle) + if err != nil { + return nil, fmt.Errorf("cannot save Automerge sync state: %w", err) + } + + return data, nil +} + +// String returns the lowercase hexadecimal change hash. +func (h Hash) String() string { + return hex.EncodeToString(h[:]) +} diff --git a/pkg/automerge/automerge_test.go b/pkg/automerge/automerge_test.go new file mode 100644 index 0000000000..42daaed144 --- /dev/null +++ b/pkg/automerge/automerge_test.go @@ -0,0 +1,339 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package automerge_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/automerge" +) + +var commitTime = time.Date(2026, time.August, 7, 12, 0, 0, 0, time.UTC) + +func actor(value byte) automerge.ActorID { + var actorID automerge.ActorID + + actorID[0] = value + + return actorID +} + +func closeDocument(t *testing.T, document *automerge.Document) { + t.Helper() + t.Cleanup( + func() { + require.NoError(t, document.Close(context.Background())) + }, + ) +} + +func closeSyncState(t *testing.T, state *automerge.SyncState) { + t.Helper() + t.Cleanup( + func() { + require.NoError(t, state.Close(context.Background())) + }, + ) +} + +func synchronize(t *testing.T, left, right *automerge.SyncState) { + t.Helper() + + ctx := context.Background() + + for range 100 { + progressed := false + + message, ok, err := left.GenerateMessage(ctx) + require.NoError(t, err) + + if ok { + require.NoError(t, right.ReceiveMessage(ctx, message)) + + progressed = true + } + + message, ok, err = right.GenerateMessage(ctx) + require.NoError(t, err) + + if ok { + require.NoError(t, left.ReceiveMessage(ctx, message)) + + progressed = true + } + + if !progressed { + return + } + } + + require.Fail(t, "sync did not quiesce") +} + +func newBaseDocument(t *testing.T) []byte { + t.Helper() + + ctx := context.Background() + document, err := automerge.NewReference(ctx, actor(1)) + require.NoError(t, err) + closeDocument(t, document) + + text, err := document.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 0, 0, "Hello")) + _, err = document.Commit(ctx, "Create document", commitTime) + require.NoError(t, err) + + data, err := document.Save(ctx) + require.NoError(t, err) + + return data +} + +func TestDocument_SaveAndLoad(t *testing.T) { + t.Parallel() + + ctx := context.Background() + document, err := automerge.New(ctx, actor(1)) + require.NoError(t, err) + closeDocument(t, document) + + require.NoError(t, document.PutString(ctx, "title", "Policy")) + text, err := document.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 0, 0, "Hello")) + + hash, err := document.Commit(ctx, "Create policy", commitTime) + require.NoError(t, err) + assert.Len(t, hash.String(), 64) + + data, err := document.Save(ctx) + require.NoError(t, err) + + loaded, err := automerge.Load(ctx, data, actor(2)) + require.NoError(t, err) + closeDocument(t, loaded) + + loadedText, err := loaded.Text(ctx, "body") + require.NoError(t, err) + value, err := loadedText.String(ctx) + require.NoError(t, err) + assert.Equal(t, "Hello", value) + + heads, err := loaded.Heads(ctx) + require.NoError(t, err) + assert.Equal(t, []automerge.Hash{hash}, heads) +} + +func TestDocument_ConcurrentChangesConverge(t *testing.T) { + t.Parallel() + + ctx := context.Background() + base := newBaseDocument(t) + + left, err := automerge.Load(ctx, base, actor(2)) + require.NoError(t, err) + closeDocument(t, left) + leftText, err := left.Text(ctx, "body") + require.NoError(t, err) + require.NoError(t, leftText.Splice(ctx, 5, 0, " left")) + _, err = left.Commit(ctx, "Edit left", commitTime.Add(time.Second)) + require.NoError(t, err) + + right, err := automerge.Load(ctx, base, actor(3)) + require.NoError(t, err) + closeDocument(t, right) + rightText, err := right.Text(ctx, "body") + require.NoError(t, err) + require.NoError(t, rightText.Splice(ctx, 5, 0, " right")) + _, err = right.Commit(ctx, "Edit right", commitTime.Add(2*time.Second)) + require.NoError(t, err) + + leftData, err := left.Save(ctx) + require.NoError(t, err) + rightData, err := right.Save(ctx) + require.NoError(t, err) + + leftFirst, err := automerge.Load(ctx, leftData, actor(4)) + require.NoError(t, err) + closeDocument(t, leftFirst) + + rightFirst, err := automerge.Load(ctx, rightData, actor(5)) + require.NoError(t, err) + closeDocument(t, rightFirst) + + _, err = leftFirst.Merge(ctx, right) + require.NoError(t, err) + _, err = rightFirst.Merge(ctx, left) + require.NoError(t, err) + + leftMergedText, err := leftFirst.Text(ctx, "body") + require.NoError(t, err) + rightMergedText, err := rightFirst.Text(ctx, "body") + require.NoError(t, err) + leftValue, err := leftMergedText.String(ctx) + require.NoError(t, err) + rightValue, err := rightMergedText.String(ctx) + require.NoError(t, err) + assert.Equal(t, leftValue, rightValue) + + leftHeads, err := leftFirst.Heads(ctx) + require.NoError(t, err) + rightHeads, err := rightFirst.Heads(ctx) + require.NoError(t, err) + assert.ElementsMatch(t, leftHeads, rightHeads) +} + +func TestText_SpliceUsesUTF16Offsets(t *testing.T) { + t.Parallel() + + ctx := context.Background() + document, err := automerge.New(ctx, actor(1)) + require.NoError(t, err) + closeDocument(t, document) + + text, err := document.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 0, 0, "A😀B")) + require.NoError(t, text.Splice(ctx, 1, 2, "")) + + value, err := text.String(ctx) + require.NoError(t, err) + assert.Equal(t, "AB", value) +} + +func TestText_CursorTracksConcurrentEdits(t *testing.T) { + t.Parallel() + + ctx := context.Background() + document, err := automerge.Load(ctx, newBaseDocument(t), actor(2)) + require.NoError(t, err) + closeDocument(t, document) + text, err := document.Text(ctx, "body") + require.NoError(t, err) + + cursor, err := text.Cursor(ctx, 4) + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 0, 0, "A")) + _, err = document.Commit(ctx, "Insert prefix", commitTime.Add(time.Second)) + require.NoError(t, err) + + position, err := text.CursorPosition(ctx, cursor) + require.NoError(t, err) + assert.Equal(t, uint32(5), position) +} + +func TestSyncState_ExchangesConcurrentChanges(t *testing.T) { + t.Parallel() + + ctx := context.Background() + left, err := automerge.Load(ctx, newBaseDocument(t), actor(2)) + require.NoError(t, err) + closeDocument(t, left) + + right, err := automerge.New(ctx, actor(3)) + require.NoError(t, err) + closeDocument(t, right) + + leftSync, err := left.NewSyncState(ctx) + require.NoError(t, err) + closeSyncState(t, leftSync) + + rightSync, err := right.NewSyncState(ctx) + require.NoError(t, err) + closeSyncState(t, rightSync) + + synchronize(t, leftSync, rightSync) + + leftText, err := left.Text(ctx, "body") + require.NoError(t, err) + rightText, err := right.Text(ctx, "body") + require.NoError(t, err) + require.NoError(t, leftText.Splice(ctx, 5, 0, " left")) + _, err = left.Commit(ctx, "Edit left", commitTime.Add(time.Second)) + require.NoError(t, err) + require.NoError(t, rightText.Splice(ctx, 5, 0, " right")) + _, err = right.Commit(ctx, "Edit right", commitTime.Add(2*time.Second)) + require.NoError(t, err) + + synchronize(t, leftSync, rightSync) + + leftValue, err := leftText.String(ctx) + require.NoError(t, err) + rightValue, err := rightText.String(ctx) + require.NoError(t, err) + assert.Equal(t, leftValue, rightValue) + + leftHeads, err := left.Heads(ctx) + require.NoError(t, err) + rightHeads, err := right.Heads(ctx) + require.NoError(t, err) + assert.ElementsMatch(t, leftHeads, rightHeads) +} + +func TestSyncState_SaveAndLoad(t *testing.T) { + t.Parallel() + + ctx := context.Background() + document, err := automerge.New(ctx, actor(1)) + require.NoError(t, err) + closeDocument(t, document) + + state, err := document.NewSyncState(ctx) + require.NoError(t, err) + data, err := state.Save(ctx) + require.NoError(t, err) + require.NoError(t, state.Close(ctx)) + + loaded, err := document.LoadSyncState(ctx, data) + require.NoError(t, err) + closeSyncState(t, loaded) + _, _, err = loaded.GenerateMessage(ctx) + require.NoError(t, err) +} + +func TestDocument_CloseIsIdempotent(t *testing.T) { + t.Parallel() + + ctx := context.Background() + document, err := automerge.New(ctx, actor(1)) + require.NoError(t, err) + + require.NoError(t, document.Close(ctx)) + require.NoError(t, document.Close(ctx)) + + _, err = document.Save(ctx) + assert.ErrorIs(t, err, automerge.ErrClosed) +} + +func TestLoad_InvalidDocument(t *testing.T) { + t.Parallel() + + document, err := automerge.Load(context.Background(), []byte("invalid"), actor(1)) + assert.Nil(t, document) + assert.Error(t, err) + assert.False(t, errors.Is(err, automerge.ErrClosed)) +} diff --git a/pkg/automerge/conformance_test.go b/pkg/automerge/conformance_test.go new file mode 100644 index 0000000000..b9a9e1c240 --- /dev/null +++ b/pkg/automerge/conformance_test.go @@ -0,0 +1,467 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package automerge_test + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/hex" + "encoding/json" + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/automerge" + "go.probo.inc/probo/pkg/automerge/internal/native" + automergeprosemirror "go.probo.inc/probo/pkg/automerge/prosemirror" +) + +type ( + oracleRequest struct { + Action string `json:"action"` + Actor string `json:"actor,omitempty"` + ActorB string `json:"actorB,omitempty"` + ActorC string `json:"actorC,omitempty"` + Change string `json:"change,omitempty"` + Document string `json:"document,omitempty"` + Message string `json:"message,omitempty"` + Text string `json:"text,omitempty"` + Timestamp int64 `json:"timestamp,omitempty"` + } + + oracleResponse struct { + Body string `json:"body"` + Change string `json:"change"` + Changes []string `json:"changes"` + Document string `json:"document"` + Heads []string `json:"heads"` + Message string `json:"message"` + Sync string `json:"sync"` + } +) + +func runOracle(t *testing.T, request oracleRequest) oracleResponse { + t.Helper() + + oracle := os.Getenv("AUTOMERGE_JS_ORACLE") + if oracle == "" { + t.Skip("AUTOMERGE_JS_ORACLE is not configured") + } + + input, err := json.Marshal(request) + require.NoError(t, err) + + command := exec.CommandContext(context.Background(), "node", oracle) + command.Stdin = bytes.NewReader(input) + output, err := command.CombinedOutput() + require.NoError(t, err, string(output)) + + var response oracleResponse + require.NoError(t, json.Unmarshal(output, &response)) + + return response +} + +func TestConformance_JavaScriptLoadsGoDocument(t *testing.T) { + t.Parallel() + + ctx := context.Background() + document, err := automerge.New(ctx, actor(1)) + require.NoError(t, err) + closeDocument(t, document) + + text, err := document.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 0, 0, "Hello 😀")) + hash, err := document.Commit(ctx, "Create in Go", commitTime) + require.NoError(t, err) + data, err := document.Save(ctx) + require.NoError(t, err) + + response := runOracle( + t, + oracleRequest{ + Action: "inspect", + Document: base64.StdEncoding.EncodeToString(data), + }, + ) + + assert.Equal(t, "Hello 😀", response.Body) + assert.Equal(t, []string{hash.String()}, response.Heads) +} + +func TestConformance_GoLoadsJavaScriptDocument(t *testing.T) { + t.Parallel() + + actorID := actor(9) + response := runOracle( + t, + oracleRequest{ + Action: "create", + Actor: hex.EncodeToString(actorID[:]), + Message: "Create in JavaScript", + Text: "Hello from JavaScript 😀", + Timestamp: commitTime.Unix(), + }, + ) + + data, err := base64.StdEncoding.DecodeString(response.Document) + require.NoError(t, err) + document, err := automerge.Load(context.Background(), data, actor(10)) + require.NoError(t, err) + closeDocument(t, document) + + text, err := document.Text(context.Background(), "body") + require.NoError(t, err) + value, err := text.String(context.Background()) + require.NoError(t, err) + assert.Equal(t, "Hello from JavaScript 😀", value) + + heads, err := document.Heads(context.Background()) + require.NoError(t, err) + require.Len(t, heads, 1) + assert.Equal(t, response.Heads[0], heads[0].String()) + + nativeDocument, err := native.Decode(data) + require.NoError(t, err) + nativeState, err := native.NewStateFromDocument(nativeDocument) + require.NoError(t, err) + nativeText, err := nativeState.Text("body") + require.NoError(t, err) + assert.Equal(t, "Hello from JavaScript 😀", nativeText) +} + +func TestConformance_GoReadsJavaScriptRichTextSpans(t *testing.T) { + t.Parallel() + + actorID := actor(11) + response := runOracle( + t, + oracleRequest{ + Action: "createRichText", + Actor: hex.EncodeToString(actorID[:]), + }, + ) + + data, err := base64.StdEncoding.DecodeString(response.Document) + require.NoError(t, err) + document, err := automerge.LoadReference(context.Background(), data, actor(12)) + require.NoError(t, err) + closeDocument(t, document) + + text, err := document.Text(context.Background(), "body") + require.NoError(t, err) + spans, err := text.Spans(context.Background()) + require.NoError(t, err) + require.Len(t, spans, 2) + assert.Equal(t, automerge.SpanTypeBlock, spans[0].Type) + assert.Equal(t, "heading", spans[0].Block["type"]) + attrs, ok := spans[0].Block["attrs"].(map[string]any) + require.True(t, ok) + assert.Equal(t, float64(2), attrs["level"]) + assert.Equal(t, automerge.SpanTypeText, spans[1].Type) + assert.Equal(t, "Policy", spans[1].Text) + assert.Equal(t, true, spans[1].Marks["strong"]) + + nativeDocument, err := automerge.LoadPureGo(context.Background(), data, actor(14)) + require.NoError(t, err) + closeDocument(t, nativeDocument) + nativeText, err := nativeDocument.Text(context.Background(), "body") + require.NoError(t, err) + nativeSpans, err := nativeText.Spans(context.Background()) + require.NoError(t, err) + assert.Equal(t, spans, nativeSpans) +} + +func TestConformance_NativeParsesJavaScriptChange(t *testing.T) { + t.Parallel() + + actorID := actor(13) + response := runOracle( + t, + oracleRequest{ + Action: "createChange", + Actor: hex.EncodeToString(actorID[:]), + Message: "Create policy", + Timestamp: commitTime.Unix(), + }, + ) + + data, err := base64.StdEncoding.DecodeString(response.Change) + require.NoError(t, err) + decoded, err := native.Decode(data) + require.NoError(t, err) + require.Len(t, decoded.Changes, 1) + change := &decoded.Changes[0] + assert.Equal(t, actorID[:], change.Actor.Bytes()) + assert.Equal(t, uint64(1), change.Sequence) + assert.Equal(t, uint64(1), change.StartOp) + assert.Equal(t, commitTime.Unix(), change.Time) + assert.Equal(t, "Create policy", change.Message) + assert.Empty(t, change.Dependencies) + require.NotNil(t, change.Hash) + assert.Equal(t, response.Heads[0], change.Hash.String()) + + operations := change.Operations + require.Len(t, operations, 7) + assert.Equal(t, native.ActionMakeText, operations[0].Action) + assert.True(t, operations[0].Object.IsRoot) + require.NotNil(t, operations[0].Key.Property) + assert.Equal(t, "title", *operations[0].Key.Property) + assert.Equal(t, uint64(1), operations[0].ID.Counter) + assert.Equal(t, native.ActorID(string(actorID[:])), operations[0].ID.Actor) + assert.Equal(t, native.ActionSet, operations[1].Action) + assert.Equal(t, uint64(1), operations[1].Object.OpID.Counter) + assert.True(t, operations[1].Key.IsHead) + require.NotNil(t, operations[1].Value) + assert.Equal(t, "P", operations[1].Value.String) + require.NotNil(t, operations[6].Key.Element) + assert.Equal(t, uint64(6), operations[6].Key.Element.Counter) + require.NotNil(t, operations[6].Value) + assert.Equal(t, "y", operations[6].Value.String) + + state := native.NewState() + require.NoError(t, state.ApplyChange(change)) + title, err := state.Text("title") + require.NoError(t, err) + assert.Equal(t, "Policy", title) + + nativeEncoded, err := native.EncodeChange(change) + require.NoError(t, err) + inspection := runOracle( + t, + oracleRequest{ + Action: "inspectChange", + Change: base64.StdEncoding.EncodeToString(nativeEncoded), + }, + ) + assert.Equal(t, "Policy", inspection.Body) + assert.Equal(t, "Create policy", inspection.Message) + require.NotNil(t, change.Hash) + assert.Equal(t, inspection.Heads[0], change.Hash.String()) +} + +func TestConformance_NativeConcurrentChangesConverge(t *testing.T) { + t.Parallel() + + actorA := actor(20) + actorB := actor(21) + actorC := actor(22) + response := runOracle( + t, + oracleRequest{ + Action: "createConcurrentChanges", + Actor: hex.EncodeToString(actorA[:]), + ActorB: hex.EncodeToString(actorB[:]), + ActorC: hex.EncodeToString(actorC[:]), + }, + ) + require.Len(t, response.Changes, 3) + + var ( + combined []byte + rawChanges [][]byte + ) + + for _, encoded := range response.Changes { + data, err := base64.StdEncoding.DecodeString(encoded) + require.NoError(t, err) + + combined = append(combined, data...) + rawChanges = append(rawChanges, data) + } + + decoded, err := native.Decode(combined) + require.NoError(t, err) + require.Len(t, decoded.Changes, 3) + changes := []*native.Change{ + &decoded.Changes[0], + &decoded.Changes[1], + &decoded.Changes[2], + } + + leftFirst := native.NewState() + require.NoError(t, leftFirst.ApplyChange(changes[0])) + require.NoError(t, leftFirst.ApplyChange(changes[1])) + require.NoError(t, leftFirst.ApplyChange(changes[2])) + + rightFirst := native.NewState() + require.NoError(t, rightFirst.ApplyChange(changes[0])) + require.NoError(t, rightFirst.ApplyChange(changes[2])) + require.NoError(t, rightFirst.ApplyChange(changes[1])) + + leftText, err := leftFirst.Text("body") + require.NoError(t, err) + rightText, err := rightFirst.Text("body") + require.NoError(t, err) + assert.Equal(t, response.Body, leftText) + assert.Equal(t, response.Body, rightText) + + leftHeads := leftFirst.Heads() + rightHeads := rightFirst.Heads() + + require.Len(t, leftHeads, 2) + require.Len(t, rightHeads, 2) + assert.ElementsMatch( + t, + response.Heads, + []string{ + hex.EncodeToString(leftHeads[0][:]), + hex.EncodeToString(leftHeads[1][:]), + }, + ) + assert.Equal(t, leftHeads, rightHeads) + + backend, err := native.LoadBackend(context.Background(), rawChanges[0]) + require.NoError(t, err) + _, err = backend.Merge(context.Background(), rawChanges[1]) + require.NoError(t, err) + saved, err := backend.Save(context.Background()) + require.NoError(t, err) + assert.True(t, bytes.Contains(saved, rawChanges[1])) +} + +func TestConformance_NativeSyncMessageRoundTrip(t *testing.T) { + t.Parallel() + + actorID := actor(30) + response := runOracle( + t, + oracleRequest{ + Action: "createSyncMessage", + Actor: hex.EncodeToString(actorID[:]), + }, + ) + data, err := base64.StdEncoding.DecodeString(response.Sync) + require.NoError(t, err) + + message, err := native.ParseSyncMessage(data) + require.NoError(t, err) + assert.Contains( + t, + []native.SyncMessageVersion{ + native.SyncMessageVersion1, + native.SyncMessageVersion2, + }, + message.Version, + ) + require.Len(t, message.Heads, 1) + assert.Equal(t, response.Heads[0], hex.EncodeToString(message.Heads[0][:])) + + encoded, err := message.Encode() + require.NoError(t, err) + assert.Equal(t, data, encoded) +} + +func TestConformance_NativeComplexRichTextSpans(t *testing.T) { + t.Parallel() + + actorID := actor(31) + response := runOracle( + t, + oracleRequest{ + Action: "createComplexRichText", + Actor: hex.EncodeToString(actorID[:]), + }, + ) + data, err := base64.StdEncoding.DecodeString(response.Document) + require.NoError(t, err) + + referenceDocument, err := automerge.LoadReference( + context.Background(), + data, + actor(32), + ) + require.NoError(t, err) + closeDocument(t, referenceDocument) + referenceText, err := referenceDocument.Text(context.Background(), "body") + require.NoError(t, err) + referenceSpans, err := referenceText.Spans(context.Background()) + require.NoError(t, err) + + nativeDocument, err := automerge.Load(context.Background(), data, actor(33)) + require.NoError(t, err) + closeDocument(t, nativeDocument) + nativeText, err := nativeDocument.Text(context.Background(), "body") + require.NoError(t, err) + nativeSpans, err := nativeText.Spans(context.Background()) + require.NoError(t, err) + + assert.Equal(t, referenceSpans, nativeSpans) +} + +func TestConformance_NativeTableRichTextSpans(t *testing.T) { + t.Parallel() + + actorID := actor(34) + response := runOracle( + t, + oracleRequest{ + Action: "createTableRichText", + Actor: hex.EncodeToString(actorID[:]), + }, + ) + data, err := base64.StdEncoding.DecodeString(response.Document) + require.NoError(t, err) + + referenceDocument, err := automerge.LoadReference( + context.Background(), + data, + actor(35), + ) + require.NoError(t, err) + closeDocument(t, referenceDocument) + referenceText, err := referenceDocument.Text(context.Background(), "body") + require.NoError(t, err) + referenceSpans, err := referenceText.Spans(context.Background()) + require.NoError(t, err) + + nativeDocument, err := automerge.Load(context.Background(), data, actor(36)) + require.NoError(t, err) + closeDocument(t, nativeDocument) + nativeText, err := nativeDocument.Text(context.Background(), "body") + require.NoError(t, err) + nativeSpans, err := nativeText.Spans(context.Background()) + require.NoError(t, err) + assert.Equal(t, referenceSpans, nativeSpans) + + content, err := automergeprosemirror.Render(nativeSpans) + require.NoError(t, err) + + var document struct { + Content []struct { + Type string `json:"type"` + Content []struct { + Type string `json:"type"` + Content []json.RawMessage `json:"content"` + } `json:"content"` + } `json:"content"` + } + require.NoError(t, json.Unmarshal([]byte(content), &document)) + require.Len(t, document.Content, 1) + assert.Equal(t, "table", document.Content[0].Type) + require.Len(t, document.Content[0].Content, 2) + assert.Equal(t, "tableRow", document.Content[0].Content[0].Type) + assert.Len(t, document.Content[0].Content[0].Content, 2) +} diff --git a/pkg/automerge/fuzz_test.go b/pkg/automerge/fuzz_test.go new file mode 100644 index 0000000000..9082cbe369 --- /dev/null +++ b/pkg/automerge/fuzz_test.go @@ -0,0 +1,47 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package automerge_test + +import ( + "context" + "testing" + + "go.probo.inc/probo/pkg/automerge" +) + +func FuzzLoad(f *testing.F) { + f.Add([]byte{}) + f.Add([]byte("invalid")) + f.Add([]byte{0x85, 0x6f, 0x4a, 0x83}) + + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > 1024*1024 { + t.Skip() + } + + document, err := automerge.Load(context.Background(), data, actor(255)) + if err != nil { + return + } + + _ = document.Close(context.Background()) + }) +} diff --git a/pkg/automerge/internal/native/backend.go b/pkg/automerge/internal/native/backend.go new file mode 100644 index 0000000000..409fb1a700 --- /dev/null +++ b/pkg/automerge/internal/native/backend.go @@ -0,0 +1,1035 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package native + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/json" + "fmt" + "sort" + "time" + "unicode/utf16" +) + +type Backend struct { + state *State + actor ActorID + base []byte + appended [][]byte + pending []Operation + objects map[uint32]ObjectID + nextHandle uint32 + syncStates map[uint32]*nativeSyncState + nextSyncState uint32 + queuedChanges map[ChangeHash]*Change + queuedBytes int +} + +type nativeSyncState struct { + RemoteHeads [][32]byte `json:"remoteHeads"` + Need [][32]byte `json:"need"` + Requested [][32]byte `json:"requested"` + NeedsAck bool `json:"needsAck"` + InFlight bool `json:"inFlight"` +} + +const ( + maxQueuedChangeBytes = 64 * 1024 * 1024 + maxQueuedChanges = 100_000 +) + +func NewBackend(ctx context.Context) (*Backend, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + actor, err := randomActorID() + if err != nil { + return nil, err + } + + base := encodeEmptyDocument() + + document, err := Decode(base) + if err != nil { + return nil, fmt.Errorf("cannot decode native empty document: %w", err) + } + + state, err := NewStateFromDocument(document) + if err != nil { + return nil, fmt.Errorf("cannot initialize native empty state: %w", err) + } + + return &Backend{ + state: state, + actor: actor, + base: base, + objects: map[uint32]ObjectID{0: RootObject()}, + nextHandle: 1, + syncStates: make(map[uint32]*nativeSyncState), + nextSyncState: 1, + queuedChanges: make(map[ChangeHash]*Change), + }, nil +} + +func LoadBackend(ctx context.Context, data []byte) (*Backend, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + document, err := Decode(data) + if err != nil { + return nil, fmt.Errorf("cannot decode native document: %w", err) + } + + state, err := NewStateFromDocument(document) + if err != nil { + return nil, fmt.Errorf("cannot initialize native document state: %w", err) + } + + actor, err := randomActorID() + if err != nil { + return nil, err + } + + return &Backend{ + state: state, + actor: actor, + base: append([]byte(nil), data...), + objects: map[uint32]ObjectID{0: RootObject()}, + nextHandle: 1, + syncStates: make(map[uint32]*nativeSyncState), + nextSyncState: 1, + queuedChanges: make(map[ChangeHash]*Change), + }, nil +} + +func (b *Backend) Close(context.Context) error { + return nil +} + +func (b *Backend) Save(ctx context.Context) ([]byte, error) { + if len(b.pending) > 0 { + if _, err := b.Commit(ctx, "", time.Time{}); err != nil { + return nil, err + } + } + + total := len(b.base) + for _, change := range b.appended { + total += len(change) + } + + data := make([]byte, 0, total) + + data = append(data, b.base...) + for _, change := range b.appended { + data = append(data, change...) + } + + return data, nil +} + +func (b *Backend) SetActor(ctx context.Context, value []byte) error { + if err := ctx.Err(); err != nil { + return err + } + + actor, err := NewActorID(value) + if err != nil { + return err + } + + if len(b.pending) > 0 { + return fmt.Errorf("cannot change actor with pending operations") + } + + b.actor = actor + + return nil +} + +func (b *Backend) PutString( + ctx context.Context, + object uint32, + key string, + value string, +) error { + if err := b.requireRoot(ctx, object); err != nil { + return err + } + + property := key + + operation := Operation{ + ID: b.nextOperationID(), + Object: RootObject(), + Key: Key{Property: &property}, + Action: ActionSet, + Value: &Scalar{Type: ScalarString, String: value}, + } + for _, predecessor := range b.state.visibleMapOperations(key) { + operation.Predecessors = append(operation.Predecessors, predecessor.ID) + } + + return b.addPending(operation) +} + +func (b *Backend) PutText( + ctx context.Context, + object uint32, + key string, +) (uint32, error) { + if err := b.requireRoot(ctx, object); err != nil { + return 0, err + } + + property := key + + operation := Operation{ + ID: b.nextOperationID(), + Object: RootObject(), + Key: Key{Property: &property}, + Action: ActionMakeText, + } + for _, predecessor := range b.state.visibleMapOperations(key) { + operation.Predecessors = append(operation.Predecessors, predecessor.ID) + } + + if err := b.addPending(operation); err != nil { + return 0, err + } + + return b.pushObject(ObjectID{OpID: operation.ID}), nil +} + +func (b *Backend) GetText( + ctx context.Context, + object uint32, + key string, +) (uint32, error) { + if err := b.requireRoot(ctx, object); err != nil { + return 0, err + } + + operation, ok := b.state.visibleMapOperation(key, ActionMakeText) + if !ok { + return 0, fmt.Errorf("text property %q does not exist", key) + } + + return b.pushObject(ObjectID{OpID: operation.ID}), nil +} + +func (b *Backend) SpliceText( + ctx context.Context, + handle uint32, + index uint32, + deleteCount int32, + value string, +) error { + if err := ctx.Err(); err != nil { + return err + } + + if deleteCount < 0 { + return fmt.Errorf("negative text deletion is unsupported") + } + + object, err := b.object(handle) + if err != nil { + return err + } + + sequence := b.state.sequence(object.OpID) + + start, end, previous, err := sequenceRange(sequence, index, uint32(deleteCount)) + if err != nil { + return err + } + + for _, target := range sequence[start:end] { + operation := Operation{ + ID: b.nextOperationID(), + Object: object, + Key: Key{Element: new(target.ID)}, + Action: ActionDelete, + Predecessors: []OpID{target.ID}, + } + if err := b.addPending(operation); err != nil { + return err + } + } + + for _, character := range value { + key := Key{IsHead: previous == nil} + if previous != nil { + key.Element = new(*previous) + } + + operation := Operation{ + ID: b.nextOperationID(), + Object: object, + Key: key, + Insert: true, + Action: ActionSet, + Value: &Scalar{Type: ScalarString, String: string(character)}, + } + if err := b.addPending(operation); err != nil { + return err + } + + previous = new(operation.ID) + } + + return nil +} + +func (b *Backend) Text(ctx context.Context, handle uint32) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + + object, err := b.object(handle) + if err != nil { + return "", err + } + + for property, operation := range b.rootTextObjects() { + if operation.ID == object.OpID { + return b.state.Text(property) + } + } + + return "", fmt.Errorf("text object does not exist") +} + +func (b *Backend) TextSpans( + ctx context.Context, + handle uint32, +) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + object, err := b.object(handle) + if err != nil { + return nil, err + } + + spans, err := b.state.RichTextSpans(object.OpID) + if err != nil { + return nil, err + } + + data, err := json.Marshal(spans) + if err != nil { + return nil, fmt.Errorf("cannot encode native rich-text spans: %w", err) + } + + return data, nil +} + +func (b *Backend) TextCursor( + ctx context.Context, + handle uint32, + index uint32, +) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + object, err := b.object(handle) + if err != nil { + return nil, err + } + + sequence := b.state.sequence(object.OpID) + + position := uint32(0) + for _, operation := range sequence { + if position == index { + data := []byte{1, 3} + data = appendLengthPrefixedNative(data, operation.ID.Actor.Bytes()) + data = appendULEB(data, operation.ID.Counter) + data = append(data, 2) + + return data, nil + } + + position += uint32(utf16Length(operation)) + } + + return nil, fmt.Errorf("text cursor index %d is out of bounds", index) +} + +func (b *Backend) TextCursorPosition( + ctx context.Context, + handle uint32, + cursor []byte, +) (uint32, error) { + if err := ctx.Err(); err != nil { + return 0, err + } + + object, err := b.object(handle) + if err != nil { + return 0, err + } + + target, _, err := decodeCursor(cursor) + if err != nil { + return 0, err + } + + position := uint32(0) + + for _, operation := range b.state.sequenceAll(object.OpID) { + if operation.ID == target { + return position, nil + } + + if !b.state.isSuperseded(operation.ID) { + position += uint32(utf16Length(operation)) + } + } + + return 0, fmt.Errorf("text cursor target does not exist") +} + +func (b *Backend) Commit( + ctx context.Context, + message string, + timestamp time.Time, +) ([32]byte, error) { + if err := ctx.Err(); err != nil { + return [32]byte{}, err + } + + if len(b.pending) == 0 { + return [32]byte{}, fmt.Errorf("change contains no operations") + } + + dependencies := b.state.Heads() + + change := &Change{ + Actor: b.actor, + Sequence: b.state.sequenceForActor(b.actor) + 1, + StartOp: b.pending[0].ID.Counter, + MaxOp: b.pending[len(b.pending)-1].ID.Counter, + Time: timestamp.Unix(), + Message: message, + Dependencies: dependencies, + Operations: append([]Operation(nil), b.pending...), + } + if timestamp.IsZero() { + change.Time = 0 + } + + raw, err := EncodeChange(change) + if err != nil { + return [32]byte{}, fmt.Errorf("cannot encode native change: %w", err) + } + + if err := b.state.recordAppliedChange(change); err != nil { + return [32]byte{}, err + } + + b.appended = append(b.appended, raw) + b.pending = nil + + return [32]byte(*change.Hash), nil +} + +func (b *Backend) Heads(ctx context.Context) ([][32]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + heads := b.state.Heads() + + result := make([][32]byte, len(heads)) + for i := range heads { + result[i] = [32]byte(heads[i]) + } + + return result, nil +} + +func (b *Backend) ChangesSince( + ctx context.Context, + heads [][32]byte, +) ([][]byte, [][32]byte, error) { + if err := ctx.Err(); err != nil { + return nil, nil, err + } + + knownHeads := make([]ChangeHash, len(heads)) + for i, head := range heads { + knownHeads[i] = ChangeHash(head) + } + + changes, ok := b.state.changesSince(knownHeads) + if !ok { + return nil, nil, fmt.Errorf("cannot compute changes from unknown heads") + } + + raw := make([][]byte, len(changes)) + + hashes := make([][32]byte, len(changes)) + for i, change := range changes { + if change.Hash == nil { + return nil, nil, fmt.Errorf("change %d has no hash", i) + } + + raw[i] = append([]byte(nil), change.Raw...) + hashes[i] = [32]byte(*change.Hash) + } + + return raw, hashes, nil +} + +func (b *Backend) ApplyChanges( + ctx context.Context, + changes [][]byte, +) error { + for i, change := range changes { + if _, err := b.Merge(ctx, change); err != nil { + return fmt.Errorf("cannot apply native change %d: %w", i, err) + } + } + + return nil +} + +func (b *Backend) Merge(ctx context.Context, data []byte) ([][32]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + document, err := Decode(data) + if err != nil { + document, err = DecodePartial(data) + } + + if err != nil { + return nil, err + } + + if len(b.state.Heads()) == 0 && len(b.pending) == 0 { + state, err := NewStateFromDocument(document) + if err != nil { + return nil, fmt.Errorf("cannot initialize merged native state: %w", err) + } + + b.state = state + + b.base = append([]byte(nil), data...) + b.appended = nil + + return b.Heads(ctx) + } + + if err := b.applyMergedChanges(document.Changes); err != nil { + return nil, err + } + + return b.Heads(ctx) +} + +func (b *Backend) applyMergedChanges(changes []Change) error { + for i := range changes { + change := &changes[i] + if change.Hash == nil || b.state.hasChange(*change.Hash) { + continue + } + + if _, queued := b.queuedChanges[*change.Hash]; queued { + continue + } + + if len(change.Raw) == 0 { + return fmt.Errorf( + "cannot preserve merged change %s: original bytes are unavailable", + change.Hash, + ) + } + + if len(b.queuedChanges) >= maxQueuedChanges || + b.queuedBytes+len(change.Raw) > maxQueuedChangeBytes { + return fmt.Errorf("merged change queue exceeds its resource limit") + } + + clone := *change + clone.Raw = append([]byte(nil), change.Raw...) + b.queuedChanges[*change.Hash] = &clone + b.queuedBytes += len(clone.Raw) + } + + for len(b.queuedChanges) > 0 { + progressed := false + + for hash, change := range b.queuedChanges { + if !b.state.hasDependencies(change) { + continue + } + + if err := b.state.ApplyChange(change); err != nil { + return fmt.Errorf("cannot apply merged native change: %w", err) + } + + b.appended = append( + b.appended, + append([]byte(nil), change.Raw...), + ) + b.queuedBytes -= len(change.Raw) + delete(b.queuedChanges, hash) + + progressed = true + } + + if !progressed { + break + } + } + + return nil +} + +func (b *Backend) NewSyncState(ctx context.Context) (uint32, error) { + if err := ctx.Err(); err != nil { + return 0, err + } + + handle := b.nextSyncState + b.nextSyncState++ + b.syncStates[handle] = &nativeSyncState{} + + return handle, nil +} + +func (b *Backend) CloseSyncState(ctx context.Context, handle uint32) error { + if err := ctx.Err(); err != nil { + return err + } + + if _, ok := b.syncStates[handle]; !ok { + return fmt.Errorf("invalid sync state %d", handle) + } + + delete(b.syncStates, handle) + + return nil +} + +func (b *Backend) GenerateSyncMessage( + ctx context.Context, + handle uint32, +) ([]byte, bool, error) { + if err := ctx.Err(); err != nil { + return nil, false, err + } + + state, err := b.syncState(handle) + if err != nil { + return nil, false, err + } + + if state.InFlight { + return nil, false, nil + } + + heads, err := b.Heads(ctx) + if err != nil { + return nil, false, err + } + + if !state.NeedsAck && equalHashes(heads, state.RemoteHeads) { + return nil, false, nil + } + + message := SyncMessage{ + Version: SyncMessageVersion2, + Heads: heads, + Need: append([][32]byte(nil), state.Need...), + } + if !state.NeedsAck { + switch { + case len(state.Requested) > 0: + for _, requested := range state.Requested { + change, ok := b.state.changes[ChangeHash(requested)] + if !ok || len(change.Raw) == 0 { + continue + } + + message.Changes = append( + message.Changes, + append([]byte(nil), change.Raw...), + ) + } + + state.Requested = nil + case len(state.Need) == 0: + remoteHeads := make([]ChangeHash, len(state.RemoteHeads)) + for i, head := range state.RemoteHeads { + remoteHeads[i] = ChangeHash(head) + } + + changes, incremental := b.state.changesSince(remoteHeads) + if incremental { + for _, change := range changes { + message.Changes = append( + message.Changes, + append([]byte(nil), change.Raw...), + ) + } + } else { + document, err := b.Save(ctx) + if err != nil { + return nil, false, err + } + + message.Changes = [][]byte{document} + } + } + } + + if !state.NeedsAck { + state.InFlight = true + } + + state.NeedsAck = false + + data, err := message.Encode() + if err != nil { + return nil, false, err + } + + return data, true, nil +} + +func (b *Backend) ReceiveSyncMessage( + ctx context.Context, + handle uint32, + data []byte, +) error { + state, err := b.syncState(handle) + if err != nil { + return err + } + + message, err := ParseSyncMessage(data) + if err != nil { + return err + } + + state.InFlight = false + + for _, change := range message.Changes { + if _, err := b.Merge(ctx, change); err != nil { + return fmt.Errorf("cannot merge native sync payload: %w", err) + } + } + + state.RemoteHeads = append([][32]byte(nil), message.Heads...) + state.Requested = append(state.Requested[:0], message.Need...) + + needed := make(map[[32]byte]struct{}) + + for _, head := range message.Heads { + if _, ok := b.state.changes[ChangeHash(head)]; !ok { + needed[head] = struct{}{} + } + } + + for _, change := range b.queuedChanges { + for _, dependency := range change.Dependencies { + if !b.state.hasChange(dependency) { + needed[[32]byte(dependency)] = struct{}{} + } + } + } + + state.Need = state.Need[:0] + for dependency := range needed { + state.Need = append(state.Need, dependency) + } + + sort.Slice(state.Need, func(i, j int) bool { + return bytes.Compare(state.Need[i][:], state.Need[j][:]) < 0 + }) + + state.NeedsAck = len(message.Changes) > 0 + + return nil +} + +func (b *Backend) SaveSyncState( + ctx context.Context, + handle uint32, +) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + state, err := b.syncState(handle) + if err != nil { + return nil, err + } + + data, err := json.Marshal(state) + if err != nil { + return nil, fmt.Errorf("cannot encode native sync state: %w", err) + } + + return data, nil +} + +func (b *Backend) LoadSyncState( + ctx context.Context, + data []byte, +) (uint32, error) { + if err := ctx.Err(); err != nil { + return 0, err + } + + var state nativeSyncState + if err := json.Unmarshal(data, &state); err != nil { + return 0, fmt.Errorf("cannot decode native sync state: %w", err) + } + + handle := b.nextSyncState + b.nextSyncState++ + b.syncStates[handle] = &state + + return handle, nil +} + +func (b *Backend) addPending(operation Operation) error { + if err := b.state.applyPending([]Operation{operation}); err != nil { + return err + } + + b.pending = append(b.pending, operation) + + return nil +} + +func (b *Backend) nextOperationID() OpID { + return OpID{ + Actor: b.actor, + Counter: b.state.maxOpGlobal() + 1, + } +} + +func (b *Backend) requireRoot(ctx context.Context, handle uint32) error { + if err := ctx.Err(); err != nil { + return err + } + + object, err := b.object(handle) + if err != nil { + return err + } + + if !object.IsRoot { + return fmt.Errorf("object is not the root map") + } + + return nil +} + +func (b *Backend) object(handle uint32) (ObjectID, error) { + object, ok := b.objects[handle] + if !ok { + return ObjectID{}, fmt.Errorf("invalid object handle %d", handle) + } + + return object, nil +} + +func (b *Backend) pushObject(object ObjectID) uint32 { + handle := b.nextHandle + b.nextHandle++ + b.objects[handle] = object + + return handle +} + +func (b *Backend) syncState(handle uint32) (*nativeSyncState, error) { + state, ok := b.syncStates[handle] + if !ok { + return nil, fmt.Errorf("invalid sync state %d", handle) + } + + return state, nil +} + +func (b *Backend) rootTextObjects() map[string]Operation { + objects := make(map[string]Operation) + + for _, operation := range b.state.operations { + if operation.Object.IsRoot && + operation.Key.Property != nil && + operation.Action == ActionMakeText && + !b.state.isSuperseded(operation.ID) { + objects[*operation.Key.Property] = operation + } + } + + return objects +} + +func sequenceRange( + sequence []Operation, + index uint32, + deleteCount uint32, +) (int, int, *OpID, error) { + position := uint32(0) + start := -1 + + var previous *OpID + + for i, operation := range sequence { + if position == index { + start = i + break + } + + length := uint32(utf16Length(operation)) + if position+length > index { + return 0, 0, nil, fmt.Errorf("text index splits a Unicode character") + } + + position += length + previous = new(operation.ID) + } + + if start == -1 { + if position != index { + return 0, 0, nil, fmt.Errorf("text index %d is out of bounds", index) + } + + start = len(sequence) + } + + target := index + deleteCount + + end := start + for end < len(sequence) && position < target { + position += uint32(utf16Length(sequence[end])) + if position > target { + return 0, 0, nil, fmt.Errorf("text deletion splits a Unicode character") + } + + end++ + } + + if position != target { + return 0, 0, nil, fmt.Errorf("text deletion extends beyond the document") + } + + return start, end, previous, nil +} + +func utf16Length(operation Operation) int { + if operation.Value == nil || operation.Value.Type != ScalarString { + return 0 + } + + return len(utf16.Encode([]rune(operation.Value.String))) +} + +func randomActorID() (ActorID, error) { + var value [16]byte + if _, err := rand.Read(value[:]); err != nil { + return "", fmt.Errorf("cannot generate native actor ID: %w", err) + } + + return NewActorID(value[:]) +} + +func encodeEmptyDocument() []byte { + body := []byte{0, 0, 0, 0} + hashInput := []byte{byte(ChunkDocument)} + hashInput = appendULEB(hashInput, uint64(len(body))) + hashInput = append(hashInput, body...) + hash := sha256.Sum256(hashInput) + + raw := []byte{0x85, 0x6f, 0x4a, 0x83} + raw = append(raw, hash[:4]...) + raw = append(raw, byte(ChunkDocument)) + raw = appendULEB(raw, uint64(len(body))) + + return append(raw, body...) +} + +func decodeCursor(data []byte) (OpID, byte, error) { + r := &reader{data: data} + + version, err := r.byte() + if err != nil || version != 1 { + return OpID{}, 0, fmt.Errorf("invalid cursor version") + } + + cursorType, err := r.byte() + if err != nil || cursorType != 3 { + return OpID{}, 0, fmt.Errorf("unsupported cursor type") + } + + actorBytes, err := decodeLengthPrefixed(r) + if err != nil { + return OpID{}, 0, fmt.Errorf("cannot decode cursor actor: %w", err) + } + + actor, err := NewActorID(actorBytes) + if err != nil { + return OpID{}, 0, err + } + + counter, err := r.uleb() + if err != nil { + return OpID{}, 0, fmt.Errorf("cannot decode cursor counter: %w", err) + } + + move, err := r.byte() + if err != nil || (move != 1 && move != 2) || r.remaining() != 0 { + return OpID{}, 0, fmt.Errorf("invalid cursor movement") + } + + return OpID{Actor: actor, Counter: counter}, move, nil +} + +func equalHashes(left, right [][32]byte) bool { + if len(left) != len(right) { + return false + } + + for i := range left { + if left[i] != right[i] { + return false + } + } + + return true +} diff --git a/pkg/automerge/internal/native/columns.go b/pkg/automerge/internal/native/columns.go new file mode 100644 index 0000000000..fc64c6a72f --- /dev/null +++ b/pkg/automerge/internal/native/columns.go @@ -0,0 +1,637 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package native + +import ( + "bytes" + "compress/flate" + "encoding/binary" + "fmt" + "io" + "math" + "unicode/utf8" +) + +type ( + reader struct { + data []byte + offset int + } + + columnMeta struct { + specification uint32 + normalized uint32 + length uint64 + compressed bool + } + + column struct { + specification uint32 + data []byte + } + + optional[T any] struct { + value T + valid bool + } +) + +const ( + maxDecodedItems = 100_000_000 + maxInflatedBytes = 512 << 20 +) + +func (r *reader) remaining() int { + return len(r.data) - r.offset +} + +func (r *reader) bytes(length uint64) ([]byte, error) { + if length > uint64(r.remaining()) { + return nil, fmt.Errorf( + "need %d bytes at offset %d, only %d remain", + length, + r.offset, + r.remaining(), + ) + } + + if length > uint64(math.MaxInt) { + return nil, fmt.Errorf("byte length %d exceeds platform capacity", length) + } + + start := r.offset + r.offset += int(length) + + return r.data[start:r.offset], nil +} + +func (r *reader) byte() (byte, error) { + value, err := r.bytes(1) + if err != nil { + return 0, err + } + + return value[0], nil +} + +func (r *reader) uleb() (uint64, error) { + start := r.offset + + var value uint64 + + for i := 0; i < 10; i++ { + current, err := r.byte() + if err != nil { + return 0, fmt.Errorf("cannot decode uLEB at offset %d: %w", start, err) + } + + payload := uint64(current & 0x7f) + if i == 9 && payload > 1 { + return 0, fmt.Errorf("uLEB at offset %d overflows uint64", start) + } + + value |= payload << (7 * i) + if current&0x80 == 0 { + if i > 0 && payload == 0 { + return 0, fmt.Errorf("uLEB at offset %d is not minimally encoded", start) + } + + return value, nil + } + } + + return 0, fmt.Errorf("uLEB at offset %d exceeds 10 bytes", start) +} + +func (r *reader) leb() (int64, error) { + start := r.offset + + var ( + value uint64 + current byte + shift uint + ) + + for i := 0; i < 10; i++ { + var err error + + current, err = r.byte() + if err != nil { + return 0, fmt.Errorf("cannot decode LEB at offset %d: %w", start, err) + } + + payload := uint64(current & 0x7f) + if i == 9 { + if payload != 0 && payload != 0x7f { + return 0, fmt.Errorf("LEB at offset %d overflows int64", start) + } + + if payload == 0 && current&0x40 != 0 { + return 0, fmt.Errorf("LEB at offset %d overflows int64", start) + } + + if payload == 0x7f && current&0x40 == 0 { + return 0, fmt.Errorf("LEB at offset %d overflows int64", start) + } + } + + value |= payload << shift + shift += 7 + + if current&0x80 == 0 { + if i > 0 { + previous := r.data[r.offset-2] + if current == 0 && previous&0x40 == 0 { + return 0, fmt.Errorf("LEB at offset %d is not minimally encoded", start) + } + + if current == 0x7f && previous&0x40 != 0 { + return 0, fmt.Errorf("LEB at offset %d is not minimally encoded", start) + } + } + + if shift < 64 && current&0x40 != 0 { + value |= ^uint64(0) << shift + } + + return int64(value), nil + } + } + + return 0, fmt.Errorf("LEB at offset %d exceeds 10 bytes", start) +} + +func decodeRLE[T any](data []byte, decodeValue func(*reader) (T, error)) ([]optional[T], error) { + r := &reader{data: data} + values := make([]optional[T], 0) + + for r.remaining() > 0 { + run, err := r.leb() + if err != nil { + return nil, fmt.Errorf("cannot decode run length: %w", err) + } + + switch { + case run > 0: + value, err := decodeValue(r) + if err != nil { + return nil, fmt.Errorf("cannot decode repeated value: %w", err) + } + + if err := appendRepeated(&values, optional[T]{value: value, valid: true}, uint64(run)); err != nil { + return nil, err + } + case run == 0: + count, err := r.uleb() + if err != nil { + return nil, fmt.Errorf("cannot decode null run length: %w", err) + } + + if count == 0 { + return nil, fmt.Errorf("null run cannot be empty") + } + + if err := appendRepeated(&values, optional[T]{}, count); err != nil { + return nil, err + } + default: + if run == math.MinInt64 { + return nil, fmt.Errorf("literal run length overflows") + } + + count := uint64(-run) + if err := reserveItems(len(values), count); err != nil { + return nil, err + } + + for range count { + value, err := decodeValue(r) + if err != nil { + return nil, fmt.Errorf("cannot decode literal value: %w", err) + } + + values = append(values, optional[T]{value: value, valid: true}) + } + } + } + + return values, nil +} + +func appendRepeated[T any](values *[]optional[T], value optional[T], count uint64) error { + if err := reserveItems(len(*values), count); err != nil { + return err + } + + for range count { + *values = append(*values, value) + } + + return nil +} + +func reserveItems(existing int, additional uint64) error { + if additional > maxDecodedItems || uint64(existing)+additional > maxDecodedItems { + return fmt.Errorf("decoded column exceeds %d items", maxDecodedItems) + } + + return nil +} + +func decodeULEBColumn(data []byte) ([]optional[uint64], error) { + return decodeRLE(data, func(r *reader) (uint64, error) { + return r.uleb() + }) +} + +func decodeDeltaColumn(data []byte) ([]optional[uint64], error) { + deltas, err := decodeRLE(data, func(r *reader) (int64, error) { + return r.leb() + }) + if err != nil { + return nil, err + } + + values := make([]optional[uint64], len(deltas)) + + var previous uint64 + + for i, delta := range deltas { + if !delta.valid { + continue + } + + next, err := addSigned(previous, delta.value) + if err != nil { + return nil, fmt.Errorf("delta item %d: %w", i, err) + } + + values[i] = optional[uint64]{value: next, valid: true} + previous = next + } + + return values, nil +} + +func decodeSignedDeltaColumn(data []byte) ([]optional[int64], error) { + deltas, err := decodeRLE(data, func(r *reader) (int64, error) { + return r.leb() + }) + if err != nil { + return nil, err + } + + values := make([]optional[int64], len(deltas)) + + var previous int64 + + for i, delta := range deltas { + if !delta.valid { + continue + } + + if delta.value > 0 && previous > math.MaxInt64-delta.value { + return nil, fmt.Errorf("delta item %d overflows int64", i) + } + + if delta.value < 0 && previous < math.MinInt64-delta.value { + return nil, fmt.Errorf("delta item %d underflows int64", i) + } + + previous += delta.value + values[i] = optional[int64]{value: previous, valid: true} + } + + return values, nil +} + +func addSigned(value uint64, delta int64) (uint64, error) { + if delta >= 0 { + addition := uint64(delta) + if value > math.MaxUint64-addition { + return 0, fmt.Errorf("positive delta overflows uint64") + } + + return value + addition, nil + } + + if delta == math.MinInt64 { + subtraction := uint64(math.MaxInt64) + 1 + if subtraction > value { + return 0, fmt.Errorf("negative delta underflows uint64") + } + + return value - subtraction, nil + } + + subtraction := uint64(-delta) + if subtraction > value { + return 0, fmt.Errorf("negative delta underflows uint64") + } + + return value - subtraction, nil +} + +func decodeStringColumn(data []byte) ([]optional[string], error) { + return decodeRLE(data, func(r *reader) (string, error) { + length, err := r.uleb() + if err != nil { + return "", err + } + + value, err := r.bytes(length) + if err != nil { + return "", err + } + + if !utf8.Valid(value) { + return "", fmt.Errorf("string is not valid UTF-8") + } + + return string(value), nil + }) +} + +func decodeBooleanColumn(data []byte, expected int) ([]bool, error) { + r := &reader{data: data} + values := make([]bool, 0, expected) + current := false + + for r.remaining() > 0 { + count, err := r.uleb() + if err != nil { + return nil, fmt.Errorf("cannot decode boolean run: %w", err) + } + + if err := reserveItems(len(values), count); err != nil { + return nil, err + } + + for range count { + values = append(values, current) + } + + current = !current + } + + if len(values) != expected { + return nil, fmt.Errorf("boolean column has %d items, expected %d", len(values), expected) + } + + return values, nil +} + +func parseColumnMetadata(r *reader, allowCompressed bool) ([]columnMeta, error) { + count, err := r.uleb() + if err != nil { + return nil, fmt.Errorf("cannot decode column count: %w", err) + } + + if count > maxDecodedItems { + return nil, fmt.Errorf("column count %d exceeds limit", count) + } + + metadata := make([]columnMeta, 0, count) + + var previous uint32 + + for i := uint64(0); i < count; i++ { + rawSpec, err := r.uleb() + if err != nil { + return nil, fmt.Errorf("cannot decode column %d specification: %w", i, err) + } + + if rawSpec > math.MaxUint32 { + return nil, fmt.Errorf("column %d specification %d exceeds uint32", i, rawSpec) + } + + specification := uint32(rawSpec) + compressed := specification&8 != 0 + normalized := specification &^ 8 + + if compressed && !allowCompressed { + return nil, fmt.Errorf("column %d is compressed in a change chunk", i) + } + + if i > 0 && normalized <= previous { + return nil, fmt.Errorf("column %d specification %d is not strictly sorted", i, normalized) + } + + previous = normalized + + length, err := r.uleb() + if err != nil { + return nil, fmt.Errorf("cannot decode column %d length: %w", i, err) + } + + metadata = append( + metadata, + columnMeta{ + specification: specification, + normalized: normalized, + length: length, + compressed: compressed, + }, + ) + } + + return metadata, nil +} + +func readColumns(r *reader, metadata []columnMeta) (map[uint32]column, error) { + columns := make(map[uint32]column, len(metadata)) + for i, meta := range metadata { + data, err := r.bytes(meta.length) + if err != nil { + return nil, fmt.Errorf("cannot read column %d: %w", i, err) + } + + if meta.compressed { + data, err = inflate(data) + if err != nil { + return nil, fmt.Errorf("cannot inflate column %d: %w", i, err) + } + } + + columns[meta.normalized] = column{ + specification: meta.specification, + data: append([]byte(nil), data...), + } + } + + return columns, nil +} + +func inflate(data []byte) ([]byte, error) { + compressed := flate.NewReader(bytes.NewReader(data)) + defer func() { _ = compressed.Close() }() + + output, err := io.ReadAll(io.LimitReader(compressed, maxInflatedBytes+1)) + if err != nil { + return nil, fmt.Errorf("cannot read DEFLATE stream: %w", err) + } + + if len(output) > maxInflatedBytes { + return nil, fmt.Errorf("inflated data exceeds %d bytes", maxInflatedBytes) + } + + return output, nil +} + +func decodeScalars(metaData, rawData []byte, expected int) ([]optional[Scalar], error) { + metadata, err := decodeULEBColumn(metaData) + if err != nil { + return nil, fmt.Errorf("cannot decode value metadata: %w", err) + } + + if len(metadata) != expected { + return nil, fmt.Errorf("value metadata has %d items, expected %d", len(metadata), expected) + } + + raw := &reader{data: rawData} + values := make([]optional[Scalar], expected) + + for i, item := range metadata { + if !item.valid { + continue + } + + scalarType := ScalarType(item.value & 0x0f) + length := item.value >> 4 + + valueBytes, err := raw.bytes(length) + if err != nil { + return nil, fmt.Errorf("cannot read scalar %d: %w", i, err) + } + + scalar, err := decodeScalar(scalarType, valueBytes) + if err != nil { + return nil, fmt.Errorf("cannot decode scalar %d: %w", i, err) + } + + values[i] = optional[Scalar]{value: scalar, valid: true} + } + + if raw.remaining() != 0 { + return nil, fmt.Errorf("value column has %d trailing bytes", raw.remaining()) + } + + return values, nil +} + +func decodeScalar(scalarType ScalarType, data []byte) (Scalar, error) { + scalar := Scalar{Type: scalarType} + switch scalarType { + case ScalarNull: + if len(data) != 0 { + return Scalar{}, fmt.Errorf("null has length %d, expected 0", len(data)) + } + case ScalarFalse, ScalarTrue: + if len(data) != 0 { + return Scalar{}, fmt.Errorf("boolean has length %d, expected 0", len(data)) + } + + scalar.Bool = scalarType == ScalarTrue + case ScalarUint: + r := &reader{data: data} + + value, err := r.uleb() + if err != nil || r.remaining() != 0 { + return Scalar{}, fmt.Errorf("invalid unsigned integer scalar") + } + + scalar.Uint = value + case ScalarInt, ScalarCounter, ScalarTimestamp: + r := &reader{data: data} + + value, err := r.leb() + if err != nil || r.remaining() != 0 { + return Scalar{}, fmt.Errorf("invalid signed integer scalar") + } + + scalar.Int = value + case ScalarFloat64: + if len(data) != 8 { + return Scalar{}, fmt.Errorf("float has length %d, expected 8", len(data)) + } + + scalar.Float = math.Float64frombits(binary.LittleEndian.Uint64(data)) + case ScalarString: + if !utf8.Valid(data) { + return Scalar{}, fmt.Errorf("string scalar is not valid UTF-8") + } + + scalar.String = string(data) + case ScalarBytes: + scalar.Bytes = append([]byte(nil), data...) + default: + scalar.Raw = append([]byte(nil), data...) + } + + return scalar, nil +} + +func requireColumn(columns map[uint32]column, specification uint32) ([]byte, error) { + value, ok := columns[specification] + if !ok { + return nil, fmt.Errorf("required column %d is missing", specification) + } + + delete(columns, specification) + + return value.data, nil +} + +func optionalColumn(columns map[uint32]column, specification uint32) []byte { + value, ok := columns[specification] + if !ok { + return nil + } + + delete(columns, specification) + + return value.data +} + +func requireItems[T any](name string, values []optional[T], expected int, nullable bool) error { + if len(values) != expected { + return fmt.Errorf("%s column has %d items, expected %d", name, len(values), expected) + } + + if nullable { + return nil + } + + for i, value := range values { + if !value.valid { + return fmt.Errorf("%s column item %d is null", name, i) + } + } + + return nil +} + +func copyHash(data []byte) ChangeHash { + var hash ChangeHash + copy(hash[:], data) + + return hash +} diff --git a/pkg/automerge/internal/native/decode.go b/pkg/automerge/internal/native/decode.go new file mode 100644 index 0000000000..b79202217c --- /dev/null +++ b/pkg/automerge/internal/native/decode.go @@ -0,0 +1,734 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package native + +import ( + "bytes" + "crypto/sha256" + "fmt" + "math" + "unicode/utf8" +) + +var magic = [4]byte{0x85, 0x6f, 0x4a, 0x83} + +type decodedChunk struct { + kind ChunkType + content []byte + hash *ChangeHash + raw []byte +} + +type implicitOperationIDs struct { + actorIndex uint64 + startOp uint64 +} + +// Decode parses all chunks in data and validates the resulting dependency +// graph. It accepts document, change, and compressed change chunks. +func Decode(data []byte) (*Document, error) { + return decode(data, true) +} + +// DecodePartial parses chunks whose causal dependencies may already exist in +// another document. Column and operation ownership validation still happens +// while whole-history frontier validation is deferred to the caller. +func DecodePartial(data []byte) (*Document, error) { + return decode(data, false) +} + +func decode(data []byte, validateHistory bool) (*Document, error) { + if len(data) == 0 { + return nil, fmt.Errorf("automerge file is empty") + } + + r := &reader{data: data} + document := &Document{} + + for r.remaining() > 0 { + chunk, err := decodeChunk(r) + if err != nil { + return nil, fmt.Errorf("cannot decode chunk %d: %w", len(document.ChunkTypes), err) + } + + document.ChunkTypes = append(document.ChunkTypes, chunk.kind) + + switch chunk.kind { + case ChunkDocument: + if len(document.ChunkTypes) != 1 { + return nil, fmt.Errorf("only the first chunk may be a document chunk") + } + + if err := decodeDocumentChunk(document, chunk.content); err != nil { + return nil, fmt.Errorf("cannot decode document chunk: %w", err) + } + case ChunkChange, ChunkCompressedChange: + change, actors, unknown, err := decodeChangeChunk(chunk.content, *chunk.hash) + if err != nil { + return nil, fmt.Errorf("cannot decode change chunk: %w", err) + } + + change.Raw = append([]byte(nil), chunk.raw...) + + document.Changes = append(document.Changes, change) + document.Actors = mergeActors(document.Actors, actors) + document.UnknownColumns = append(document.UnknownColumns, unknown...) + default: + return nil, fmt.Errorf("unsupported chunk type %d", chunk.kind) + } + } + + if validateHistory { + if err := validateDocument(document); err != nil { + return nil, fmt.Errorf("invalid dependency graph: %w", err) + } + } + + return document, nil +} + +func decodeChunk(r *reader) (decodedChunk, error) { + start := r.offset + + header, err := r.bytes(4) + if err != nil { + return decodedChunk{}, fmt.Errorf("cannot read magic bytes: %w", err) + } + + if !bytes.Equal(header, magic[:]) { + return decodedChunk{}, fmt.Errorf("invalid magic bytes %x", header) + } + + checksum, err := r.bytes(4) + if err != nil { + return decodedChunk{}, fmt.Errorf("cannot read checksum: %w", err) + } + + rawType, err := r.byte() + if err != nil { + return decodedChunk{}, fmt.Errorf("cannot read chunk type: %w", err) + } + + kind := ChunkType(rawType) + if kind > ChunkCompressedChange { + return decodedChunk{}, fmt.Errorf("unknown chunk type %d", kind) + } + + lengthStart := r.offset + + length, err := r.uleb() + if err != nil { + return decodedChunk{}, fmt.Errorf("cannot read chunk length: %w", err) + } + + lengthBytes := append([]byte(nil), r.data[lengthStart:r.offset]...) + + content, err := r.bytes(length) + if err != nil { + return decodedChunk{}, fmt.Errorf("cannot read chunk content: %w", err) + } + + hashInput := make([]byte, 0, 1+len(lengthBytes)+len(content)) + hashKind := kind + hashContent := content + + if kind == ChunkCompressedChange { + hashKind = ChunkChange + + hashContent, err = inflate(content) + if err != nil { + return decodedChunk{}, fmt.Errorf("cannot inflate compressed change: %w", err) + } + + lengthBytes = appendULEB(nil, uint64(len(hashContent))) + } + + hashInput = append(hashInput, byte(hashKind)) + hashInput = append(hashInput, lengthBytes...) + hashInput = append(hashInput, hashContent...) + + digest := sha256.Sum256(hashInput) + if !bytes.Equal(checksum, digest[:4]) { + return decodedChunk{}, fmt.Errorf( + "checksum mismatch: encoded %x, calculated %x", + checksum, + digest[:4], + ) + } + + chunk := decodedChunk{ + kind: kind, + content: hashContent, + raw: append([]byte(nil), r.data[start:r.offset]...), + } + if kind == ChunkChange || kind == ChunkCompressedChange { + hash := ChangeHash(digest) + chunk.hash = &hash + } + + return chunk, nil +} + +func appendULEB(destination []byte, value uint64) []byte { + for { + current := byte(value & 0x7f) + + value >>= 7 + if value != 0 { + current |= 0x80 + } + + destination = append(destination, current) + if value == 0 { + return destination + } + } +} + +func decodeDocumentChunk(document *Document, data []byte) error { + r := &reader{data: data} + + actors, err := decodeActorArray(r, true) + if err != nil { + return fmt.Errorf("cannot decode actors: %w", err) + } + + heads, err := decodeHashArray(r, true) + if err != nil { + return fmt.Errorf("cannot decode heads: %w", err) + } + + changeMetadata, err := parseColumnMetadata(r, true) + if err != nil { + return fmt.Errorf("cannot decode change column metadata: %w", err) + } + + operationMetadata, err := parseColumnMetadata(r, true) + if err != nil { + return fmt.Errorf("cannot decode operation column metadata: %w", err) + } + + changeColumns, err := readColumns(r, changeMetadata) + if err != nil { + return fmt.Errorf("cannot decode change columns: %w", err) + } + + operationColumns, err := readColumns(r, operationMetadata) + if err != nil { + return fmt.Errorf("cannot decode operation columns: %w", err) + } + + changes, unknownChanges, err := decodeDocumentChanges(changeColumns, actors) + if err != nil { + return fmt.Errorf("cannot decode changes: %w", err) + } + + operations, unknownOperations, err := decodeOperations(operationColumns, actors, false, nil) + if err != nil { + return fmt.Errorf("cannot decode operations: %w", err) + } + + if err := assignOperations(changes, operations); err != nil { + return fmt.Errorf("cannot assign operations: %w", err) + } + + headIndexes := make([]uint64, len(heads)) + for i := range heads { + index, err := r.uleb() + if err != nil { + return fmt.Errorf("cannot decode head index %d: %w", i, err) + } + + if index >= uint64(len(changes)) { + return fmt.Errorf("head index %d is out of bounds", index) + } + + headIndexes[i] = index + + hash := heads[i] + if changes[index].Hash != nil && *changes[index].Hash != hash { + return fmt.Errorf("head index %d is assigned conflicting hashes", index) + } + + changes[index].Hash = &hash + } + + if r.remaining() != 0 { + return fmt.Errorf("document chunk has %d trailing bytes", r.remaining()) + } + + if err := validateSnapshotGraph(changes, headIndexes); err != nil { + return err + } + + document.Actors = actors + document.Heads = heads + document.Changes = changes + + document.UnknownColumns = append(unknownChanges, unknownOperations...) + + return nil +} + +func decodeDocumentChanges( + columns map[uint32]column, + actors []ActorID, +) ([]Change, []RawColumn, error) { + if len(columns) == 0 { + return nil, nil, nil + } + + actorData, err := requireColumn(columns, 1) + if err != nil { + return nil, nil, err + } + + actorIndexes, err := decodeULEBColumn(actorData) + if err != nil { + return nil, nil, fmt.Errorf("cannot decode actor column: %w", err) + } + + count := len(actorIndexes) + if err := requireItems("actor", actorIndexes, count, false); err != nil { + return nil, nil, err + } + + sequence, err := decodeRequiredDelta(columns, 3, "sequence", count) + if err != nil { + return nil, nil, err + } + + maxOps, err := decodeRequiredDelta(columns, 19, "maxOp", count) + if err != nil { + return nil, nil, err + } + + times, err := decodeOptionalSignedDelta(columns, 35, "time", count) + if err != nil { + return nil, nil, err + } + + messages, err := decodeOptionalStrings(columns, 53, "message", count) + if err != nil { + return nil, nil, err + } + + groupData, err := requireColumn(columns, 64) + if err != nil { + return nil, nil, err + } + + groups, err := decodeULEBColumn(groupData) + if err != nil { + return nil, nil, fmt.Errorf("cannot decode dependency groups: %w", err) + } + + if err := requireItems("dependency group", groups, count, false); err != nil { + return nil, nil, err + } + + dependencyCount, err := sumGroups(groups) + if err != nil { + return nil, nil, err + } + + dependencies := make([]optional[uint64], 0) + + if dependencyCount > 0 { + dependencyData, err := requireColumn(columns, 67) + if err != nil { + return nil, nil, err + } + + dependencies, err = decodeDeltaColumn(dependencyData) + if err != nil { + return nil, nil, fmt.Errorf("cannot decode dependencies: %w", err) + } + + if err := requireItems("dependency", dependencies, dependencyCount, false); err != nil { + return nil, nil, err + } + } + + extras, err := decodeOptionalScalars(columns, 86, 87, count) + if err != nil { + return nil, nil, fmt.Errorf("cannot decode change extras: %w", err) + } + + changes := make([]Change, count) + dependencyOffset := 0 + + for i := range changes { + actorIndex := actorIndexes[i].value + if actorIndex >= uint64(len(actors)) { + return nil, nil, fmt.Errorf("change %d actor index %d is out of bounds", i, actorIndex) + } + + changes[i] = Change{ + Actor: actors[actorIndex], + Sequence: sequence[i].value, + MaxOp: maxOps[i].value, + } + if times[i].valid { + changes[i].Time = times[i].value + } + + if messages[i].valid { + changes[i].Message = messages[i].value + } + + if extras[i].valid { + extra := extras[i].value + changes[i].Extra = &extra + } + + groupLength := int(groups[i].value) + + changes[i].DependencyIndexes = make([]uint64, groupLength) + for j := range groupLength { + changes[i].DependencyIndexes[j] = dependencies[dependencyOffset+j].value + } + + dependencyOffset += groupLength + } + + unknown := collectUnknown(columns) + + return changes, unknown, nil +} + +func decodeChangeChunk( + data []byte, + hash ChangeHash, +) (Change, []ActorID, []RawColumn, error) { + r := &reader{data: data} + + dependencies, err := decodeHashArray(r, false) + if err != nil { + return Change{}, nil, nil, fmt.Errorf("cannot decode dependencies: %w", err) + } + + actorBytes, err := decodeLengthPrefixed(r) + if err != nil { + return Change{}, nil, nil, fmt.Errorf("cannot decode actor: %w", err) + } + + actor, err := NewActorID(actorBytes) + if err != nil { + return Change{}, nil, nil, err + } + + sequence, err := r.uleb() + if err != nil { + return Change{}, nil, nil, fmt.Errorf("cannot decode sequence: %w", err) + } + + if sequence == 0 { + return Change{}, nil, nil, fmt.Errorf("sequence is zero") + } + + startOp, err := r.uleb() + if err != nil { + return Change{}, nil, nil, fmt.Errorf("cannot decode start op: %w", err) + } + + if startOp == 0 { + return Change{}, nil, nil, fmt.Errorf("start op is zero") + } + + timestamp, err := r.leb() + if err != nil { + return Change{}, nil, nil, fmt.Errorf("cannot decode time: %w", err) + } + + messageBytes, err := decodeLengthPrefixed(r) + if err != nil { + return Change{}, nil, nil, fmt.Errorf("cannot decode message: %w", err) + } + + if !utf8.Valid(messageBytes) { + return Change{}, nil, nil, fmt.Errorf("message is not valid UTF-8") + } + + otherActors, err := decodeActorArray(r, true) + if err != nil { + return Change{}, nil, nil, fmt.Errorf("cannot decode other actors: %w", err) + } + + for _, other := range otherActors { + if other == actor { + return Change{}, nil, nil, fmt.Errorf("other actors contains the change actor") + } + } + + metadata, err := parseColumnMetadata(r, false) + if err != nil { + return Change{}, nil, nil, fmt.Errorf("cannot decode operation metadata: %w", err) + } + + columns, err := readColumns(r, metadata) + if err != nil { + return Change{}, nil, nil, fmt.Errorf("cannot decode operation columns: %w", err) + } + + actors := append([]ActorID{actor}, otherActors...) + + operations, unknown, err := decodeOperations( + columns, + actors, + true, + &implicitOperationIDs{actorIndex: 0, startOp: startOp}, + ) + if err != nil { + return Change{}, nil, nil, err + } + + if len(operations) == 0 { + return Change{}, nil, nil, fmt.Errorf("change contains no operations") + } + + if startOp > math.MaxUint64-uint64(len(operations))+1 { + return Change{}, nil, nil, fmt.Errorf("operation range overflows uint64") + } + + maxOp := startOp + uint64(len(operations)) - 1 + for i, operation := range operations { + expected := startOp + uint64(i) + if operation.ID.Actor != actor || operation.ID.Counter != expected { + return Change{}, nil, nil, fmt.Errorf( + "operation %d has ID %s@%d, expected %s@%d", + i, + operation.ID.Actor, + operation.ID.Counter, + actor, + expected, + ) + } + } + + change := Change{ + Hash: new(hash), + Actor: actor, + Sequence: sequence, + StartOp: startOp, + MaxOp: maxOp, + Time: timestamp, + Message: string(messageBytes), + Dependencies: dependencies, + Operations: operations, + } + if r.remaining() > 0 { + change.ExtraBytes = append([]byte(nil), r.data[r.offset:]...) + } + + return change, actors, unknown, nil +} + +func decodeOperations( + columns map[uint32]column, + actors []ActorID, + changeChunk bool, + implicitIDs *implicitOperationIDs, +) ([]Operation, []RawColumn, error) { + if len(columns) == 0 { + return nil, nil, nil + } + + actionData, err := requireColumn(columns, 66) + if err != nil { + return nil, nil, err + } + + actions, err := decodeULEBColumn(actionData) + if err != nil { + return nil, nil, fmt.Errorf("cannot decode actions: %w", err) + } + + count := len(actions) + if err := requireItems("action", actions, count, false); err != nil { + return nil, nil, err + } + + idActors := make([]optional[uint64], count) + if idActorData := optionalColumn(columns, 33); idActorData != nil { + idActors, err = decodeULEBColumn(idActorData) + if err != nil { + return nil, nil, fmt.Errorf("cannot decode operation actors: %w", err) + } + + if err := requireItems("operation actor", idActors, count, false); err != nil { + return nil, nil, err + } + } else if implicitIDs != nil { + for i := range idActors { + idActors[i] = optional[uint64]{value: implicitIDs.actorIndex, valid: true} + } + } else { + return nil, nil, fmt.Errorf("required column 33 is missing") + } + + idCounters := make([]optional[uint64], count) + if idCounterData := optionalColumn(columns, 35); idCounterData != nil { + idCounters, err = decodeDeltaColumn(idCounterData) + if err != nil { + return nil, nil, fmt.Errorf("cannot decode operation counters: %w", err) + } + + if err := requireItems("operation counter", idCounters, count, false); err != nil { + return nil, nil, err + } + } else if implicitIDs != nil { + for i := range idCounters { + if implicitIDs.startOp > math.MaxUint64-uint64(i) { + return nil, nil, fmt.Errorf("implicit operation counter overflows uint64") + } + + idCounters[i] = optional[uint64]{ + value: implicitIDs.startOp + uint64(i), + valid: true, + } + } + } else { + return nil, nil, fmt.Errorf("required column 35 is missing") + } + + objectActors, err := decodeOptionalULEB(columns, 1, "object actor", count) + if err != nil { + return nil, nil, err + } + + objectCounters, err := decodeOptionalULEB(columns, 2, "object counter", count) + if err != nil { + return nil, nil, err + } + + keyActors, err := decodeOptionalULEB(columns, 17, "key actor", count) + if err != nil { + return nil, nil, err + } + + keyCounters, err := decodeOptionalDelta(columns, 19, "key counter", count) + if err != nil { + return nil, nil, err + } + + keyStrings, err := decodeOptionalStrings(columns, 21, "key string", count) + if err != nil { + return nil, nil, err + } + + inserts := make([]bool, count) + if insertData := optionalColumn(columns, 52); insertData != nil { + inserts, err = decodeBooleanColumn(insertData, count) + if err != nil { + return nil, nil, fmt.Errorf("cannot decode insert column: %w", err) + } + } + + values, err := decodeOptionalScalars(columns, 86, 87, count) + if err != nil { + return nil, nil, fmt.Errorf("cannot decode values: %w", err) + } + + var related [][]OpID + if changeChunk { + related, err = decodeGroupedOpIDs(columns, actors, 112, 113, 115, count, "predecessor") + } else { + related, err = decodeGroupedOpIDs(columns, actors, 128, 129, 131, count, "successor") + } + + if err != nil { + return nil, nil, err + } + + markExpand, err := decodeOptionalBooleans(columns, 148, count) + if err != nil { + return nil, nil, fmt.Errorf("cannot decode mark expand: %w", err) + } + + markNames, err := decodeOptionalStrings(columns, 165, "mark name", count) + if err != nil { + return nil, nil, fmt.Errorf("cannot decode mark name: %w", err) + } + + operations := make([]Operation, count) + for i := range operations { + id, err := opIDFromIndexes(idActors[i], idCounters[i], actors) + if err != nil { + return nil, nil, fmt.Errorf("operation %d ID: %w", i, err) + } + + object, err := objectIDFromIndexes(objectActors[i], objectCounters[i], actors) + if err != nil { + return nil, nil, fmt.Errorf("operation %d object: %w", i, err) + } + + key, err := keyFromColumns( + keyActors[i], + keyCounters[i], + keyStrings[i], + actors, + inserts[i], + ) + if err != nil { + return nil, nil, fmt.Errorf( + "operation %d key (insert=%t, action=%d): %w", + i, + inserts[i], + actions[i].value, + err, + ) + } + + operations[i] = Operation{ + ID: id, + Object: object, + Key: key, + Insert: inserts[i], + Action: Action(actions[i].value), + } + if values[i].valid { + value := values[i].value + operations[i].Value = &value + } + + if changeChunk { + operations[i].Predecessors = related[i] + } else { + operations[i].Successors = related[i] + if operations[i].Action == ActionDelete { + return nil, nil, fmt.Errorf("document operation %d explicitly encodes a delete", i) + } + } + + if markExpand[i].valid { + value := markExpand[i].value + operations[i].MarkExpand = &value + } + + if markNames[i].valid { + value := markNames[i].value + operations[i].MarkName = &value + } + } + + return operations, collectUnknown(columns), nil +} diff --git a/pkg/automerge/internal/native/decode_test.go b/pkg/automerge/internal/native/decode_test.go new file mode 100644 index 0000000000..ec9767a40a --- /dev/null +++ b/pkg/automerge/internal/native/decode_test.go @@ -0,0 +1,289 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package native + +import ( + "bytes" + "compress/flate" + "context" + "encoding/base64" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/automerge/internal/reference" +) + +const ( + officialChangeFixture = "hW9Kg5nDjoUBzgEAEAECAwQFBgcICQoLDA0ODxABAYDiz6oGF29mZmljaWFsIHNjYWxhciBmaXh0dXJlAAoBBAIHEQYTBxU4NAJCCFYQVxtwAgALBAAACwILfgwNAAx/AAACAAt8AAx0AHUDbmlsAm5vA3llcwR1aW50A2ludAVmbG9hdAR0ZXh0BWJ5dGVzBHdoZW4FY291bnQEbGlzdAAECwQKAX8CAgQCAXYAAQITFIUBVjdpGAMAAhYqeQAAAAAAAPg/aGVsbG8A/wf70JX/vDEJYWIPAA==" + officialDocumentFixture = "hW9Kg3tNcOoAogIBEAECAwQFBgcICQoLDA0ODxABmcOOhfOq6K9fyRtQMpEkw5nRGiPrg0/hSLI3KA5LqKcHAQIDAhMCIwY1GUACVgIMAQQCBxEGEwcVOCECIw80AkIKVhJXG4ABAn8AfwF/D3+A4s+qBn8Xb2ZmaWNpYWwgc2NhbGFyIGZpeHR1cmV/AH8HAAsEAAALAgt+DA0ADH8AAAIAC3wADHQAdQVieXRlcwVjb3VudAVmbG9hdANpbnQEbGlzdANuaWwCbm8EdGV4dAR1aW50BHdoZW4DeWVzAAQPAHQIAnx/BnYBBX0FegkDAQsEBAF/AgYBAgQCAXw3GIUBFAIAewFWE2kCAgACFgD/BwkAAAAAAAD4P3loZWxsbyr70JX/vDFhYg8AAA==" +) + +func fixture(t *testing.T, encoded string) []byte { + t.Helper() + + data, err := base64.StdEncoding.DecodeString(encoded) + require.NoError(t, err) + + return data +} + +func scalarOperations(document *Document) map[string]Scalar { + result := make(map[string]Scalar) + + for _, operation := range document.Changes[0].Operations { + if operation.Action == ActionSet && operation.Key.Property != nil && operation.Value != nil { + result[*operation.Key.Property] = *operation.Value + } + } + + return result +} + +func TestDecode_OfficialDocumentFixture(t *testing.T) { + t.Parallel() + + document, err := Decode(fixture(t, officialDocumentFixture)) + require.NoError(t, err) + require.Len(t, document.Changes, 1) + require.Len(t, document.Heads, 1) + + change := document.Changes[0] + assert.Equal(t, uint64(1), change.Sequence) + assert.Equal(t, "official scalar fixture", change.Message) + assert.Equal(t, document.Heads[0], *change.Hash) + assert.Equal(t, "99c38e85f3aae8af5fc91b50329124c399d11a23eb834fe148b237280e4ba8a7", change.Hash.String()) + + values := scalarOperations(document) + require.Len(t, values, 10) + assert.Equal(t, ScalarNull, values["nil"].Type) + assert.Equal(t, ScalarFalse, values["no"].Type) + assert.Equal(t, ScalarTrue, values["yes"].Type) + assert.Equal(t, uint64(42), values["uint"].Uint) + assert.Equal(t, int64(-7), values["int"].Int) + assert.Equal(t, 1.5, values["float"].Float) + assert.Equal(t, "hello", values["text"].String) + assert.Equal(t, []byte{0, 255, 7}, values["bytes"].Bytes) + assert.Equal(t, int64(1_700_000_000_123), values["when"].Int) + assert.Equal(t, int64(9), values["count"].Int) +} + +func TestDecode_OfficialChangeFixture(t *testing.T) { + t.Parallel() + + document, err := Decode(fixture(t, officialChangeFixture)) + require.NoError(t, err) + require.Len(t, document.Changes, 1) + require.Len(t, document.Heads, 1) + + change := document.Changes[0] + assert.Equal(t, uint64(1), change.Sequence) + assert.Equal(t, uint64(1), change.StartOp) + assert.Equal(t, uint64(15), change.MaxOp) + assert.Equal(t, document.Heads[0], *change.Hash) + assert.Equal(t, "99c38e85f3aae8af5fc91b50329124c399d11a23eb834fe148b237280e4ba8a7", change.Hash.String()) +} + +func TestDecode_OfficialEmptyDocumentFixture(t *testing.T) { + t.Parallel() + + data := []byte{ + 0x85, 0x6f, 0x4a, 0x83, + 0xb8, 0x1a, 0x95, 0x44, + 0x00, 0x04, + 0x00, 0x00, 0x00, 0x00, + } + document, err := Decode(data) + require.NoError(t, err) + assert.Empty(t, document.Actors) + assert.Empty(t, document.Heads) + assert.Empty(t, document.Changes) +} + +func TestDecode_CompressedOfficialChangeFixture(t *testing.T) { + t.Parallel() + + uncompressed := fixture(t, officialChangeFixture) + r := &reader{data: uncompressed, offset: 9} + length, err := r.uleb() + require.NoError(t, err) + content, err := r.bytes(length) + require.NoError(t, err) + + var compressed bytes.Buffer + + writer, err := flate.NewWriter(&compressed, flate.BestCompression) + require.NoError(t, err) + _, err = writer.Write(content) + require.NoError(t, err) + require.NoError(t, writer.Close()) + + data := append([]byte(nil), uncompressed[:8]...) + data = append(data, byte(ChunkCompressedChange)) + data = appendULEB(data, uint64(compressed.Len())) + data = append(data, compressed.Bytes()...) + + document, err := Decode(data) + require.NoError(t, err) + require.Len(t, document.Changes, 1) + assert.Equal(t, ChunkCompressedChange, document.ChunkTypes[0]) + assert.Equal(t, "99c38e85f3aae8af5fc91b50329124c399d11a23eb834fe148b237280e4ba8a7", document.Heads[0].String()) +} + +func TestDecode_ReferenceBackendDocument(t *testing.T) { + t.Parallel() + + ctx := context.Background() + backend, err := reference.New(ctx) + require.NoError(t, err) + t.Cleanup(func() { + assert.NoError(t, backend.Close(ctx)) + }) + + actor := []byte{1, 3, 3, 7} + require.NoError(t, backend.SetActor(ctx, actor)) + require.NoError(t, backend.PutString(ctx, 0, "policy", "approved")) + _, err = backend.Commit(ctx, "reference fixture", time.Unix(1_700_000_000, 0)) + require.NoError(t, err) + data, err := backend.Save(ctx) + require.NoError(t, err) + + document, err := Decode(data) + require.NoError(t, err) + require.Len(t, document.Changes, 1) + assert.Equal(t, "01030307", document.Changes[0].Actor.String()) + assert.Equal(t, "reference fixture", document.Changes[0].Message) +} + +func TestDecode_ReferenceBackendConcurrentGraph(t *testing.T) { + t.Parallel() + + ctx := context.Background() + base, err := reference.New(ctx) + require.NoError(t, err) + t.Cleanup(func() { + assert.NoError(t, base.Close(ctx)) + }) + require.NoError(t, base.SetActor(ctx, []byte{1})) + require.NoError(t, base.PutString(ctx, 0, "base", "value")) + _, err = base.Commit(ctx, "base", time.Unix(1, 0)) + require.NoError(t, err) + baseData, err := base.Save(ctx) + require.NoError(t, err) + + left, err := reference.Load(ctx, baseData) + require.NoError(t, err) + t.Cleanup(func() { + assert.NoError(t, left.Close(ctx)) + }) + require.NoError(t, left.SetActor(ctx, []byte{2})) + require.NoError(t, left.PutString(ctx, 0, "left", "value")) + _, err = left.Commit(ctx, "left", time.Unix(2, 0)) + require.NoError(t, err) + + right, err := reference.Load(ctx, baseData) + require.NoError(t, err) + t.Cleanup(func() { + assert.NoError(t, right.Close(ctx)) + }) + require.NoError(t, right.SetActor(ctx, []byte{3})) + require.NoError(t, right.PutString(ctx, 0, "right", "value")) + _, err = right.Commit(ctx, "right", time.Unix(3, 0)) + require.NoError(t, err) + rightData, err := right.Save(ctx) + require.NoError(t, err) + + _, err = left.Merge(ctx, rightData) + require.NoError(t, err) + mergedData, err := left.Save(ctx) + require.NoError(t, err) + + document, err := Decode(mergedData) + require.NoError(t, err) + require.Len(t, document.Changes, 3) + assert.Len(t, document.Heads, 2) + assert.Empty(t, document.Changes[0].DependencyIndexes) + assert.Equal(t, []uint64{0}, document.Changes[1].DependencyIndexes) + assert.Equal(t, []uint64{0}, document.Changes[2].DependencyIndexes) +} + +func TestDecode_RejectsCorruptChecksum(t *testing.T) { + t.Parallel() + + data := fixture(t, officialDocumentFixture) + data[len(data)-1] ^= 0xff + + _, err := Decode(data) + require.Error(t, err) + assert.ErrorContains(t, err, "checksum mismatch") +} + +func TestReaderULEB_RejectsNonCanonicalValue(t *testing.T) { + t.Parallel() + + r := &reader{data: []byte{0x80, 0x00}} + _, err := r.uleb() + require.Error(t, err) + assert.ErrorContains(t, err, "not minimally encoded") +} + +func TestValidateSnapshotGraph_RejectsCycle(t *testing.T) { + t.Parallel() + + actor, err := NewActorID([]byte{1}) + require.NoError(t, err) + + changes := []Change{ + { + Actor: actor, + Sequence: 1, + MaxOp: 1, + DependencyIndexes: []uint64{1}, + }, + { + Actor: actor, + Sequence: 2, + MaxOp: 2, + DependencyIndexes: []uint64{0}, + }, + } + + err = validateSnapshotGraph(changes, nil) + require.Error(t, err) + assert.ErrorContains(t, err, "dependency cycle") +} + +func TestValidateSnapshotGraph_RejectsSequenceGap(t *testing.T) { + t.Parallel() + + actor, err := NewActorID([]byte{1}) + require.NoError(t, err) + + changes := []Change{ + {Actor: actor, Sequence: 1, MaxOp: 1}, + {Actor: actor, Sequence: 3, MaxOp: 2, DependencyIndexes: []uint64{0}}, + } + + err = validateSnapshotGraph(changes, []uint64{1}) + require.Error(t, err) + assert.ErrorContains(t, err, "sequence 3, expected 2") +} diff --git a/pkg/automerge/internal/native/encode.go b/pkg/automerge/internal/native/encode.go new file mode 100644 index 0000000000..704976a519 --- /dev/null +++ b/pkg/automerge/internal/native/encode.go @@ -0,0 +1,423 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package native + +import ( + "crypto/sha256" + "encoding/binary" + "fmt" + "math" + "sort" +) + +type encodedColumn struct { + specification uint32 + data []byte +} + +func EncodeChange(change *Change) ([]byte, error) { + if len(change.Actor) == 0 { + return nil, fmt.Errorf("change actor cannot be empty") + } + + if change.Sequence == 0 || change.StartOp == 0 { + return nil, fmt.Errorf("change sequence and start operation must be positive") + } + + actors, actorIndexes, err := changeActorTable(change) + if err != nil { + return nil, err + } + + columns, err := encodeOperationColumns(change, actorIndexes) + if err != nil { + return nil, err + } + + var body []byte + + body = appendHashesNative(body, change.Dependencies) + body = appendLengthPrefixedNative(body, change.Actor.Bytes()) + body = appendULEB(body, change.Sequence) + body = appendULEB(body, change.StartOp) + body = appendLEB(body, change.Time) + body = appendLengthPrefixedNative(body, []byte(change.Message)) + + body = appendULEB(body, uint64(len(actors))) + for _, actor := range actors { + body = appendLengthPrefixedNative(body, actor.Bytes()) + } + + body = appendColumns(body, columns) + body = append(body, change.ExtraBytes...) + + hashInput := []byte{byte(ChunkChange)} + hashInput = appendULEB(hashInput, uint64(len(body))) + hashInput = append(hashInput, body...) + hash := ChangeHash(sha256.Sum256(hashInput)) + change.Hash = new(hash) + + raw := []byte{0x85, 0x6f, 0x4a, 0x83} + raw = append(raw, hash[:4]...) + raw = append(raw, byte(ChunkChange)) + raw = appendULEB(raw, uint64(len(body))) + raw = append(raw, body...) + change.Raw = append([]byte(nil), raw...) + + return raw, nil +} + +func changeActorTable( + change *Change, +) ([]ActorID, map[ActorID]uint64, error) { + actorSet := make(map[ActorID]struct{}) + add := func(actor ActorID) { + if actor != "" && actor != change.Actor { + actorSet[actor] = struct{}{} + } + } + + for i, operation := range change.Operations { + expectedID := OpID{ + Actor: change.Actor, + Counter: change.StartOp + uint64(i), + } + if operation.ID != expectedID { + return nil, nil, fmt.Errorf( + "operation %d ID %v does not match implicit ID %v", + i, + operation.ID, + expectedID, + ) + } + + if !operation.Object.IsRoot { + add(operation.Object.OpID.Actor) + } + + if operation.Key.Element != nil { + add(operation.Key.Element.Actor) + } + + for _, predecessor := range operation.Predecessors { + add(predecessor.Actor) + } + } + + actors := make([]ActorID, 0, len(actorSet)) + for actor := range actorSet { + actors = append(actors, actor) + } + + sort.Slice(actors, func(i, j int) bool { + return actors[i].Compare(actors[j]) < 0 + }) + + indexes := map[ActorID]uint64{change.Actor: 0} + for i, actor := range actors { + indexes[actor] = uint64(i + 1) + } + + return actors, indexes, nil +} + +func encodeOperationColumns( + change *Change, + actorIndexes map[ActorID]uint64, +) ([]encodedColumn, error) { + count := len(change.Operations) + objActors := make([]optional[uint64], count) + objCounters := make([]optional[uint64], count) + keyActors := make([]optional[uint64], count) + keyCounters := make([]optional[int64], count) + keyStrings := make([]optional[string], count) + inserts := make([]bool, count) + actions := make([]optional[uint64], count) + valueMetadata := make([]optional[uint64], count) + predGroups := make([]optional[uint64], count) + + var ( + valueData []byte + predActors []optional[uint64] + predCounters []optional[int64] + ) + + for i, operation := range change.Operations { + if !operation.Object.IsRoot { + actorIndex, ok := actorIndexes[operation.Object.OpID.Actor] + if !ok { + return nil, fmt.Errorf("operation %d object actor is unknown", i) + } + + objActors[i] = some(actorIndex) + objCounters[i] = some(operation.Object.OpID.Counter) + } + + switch { + case operation.Key.Property != nil: + keyStrings[i] = some(*operation.Key.Property) + case operation.Key.IsHead: + keyCounters[i] = some(int64(0)) + case operation.Key.Element != nil: + actorIndex, ok := actorIndexes[operation.Key.Element.Actor] + if !ok { + return nil, fmt.Errorf("operation %d key actor is unknown", i) + } + + keyActors[i] = some(actorIndex) + keyCounters[i] = some(int64(operation.Key.Element.Counter)) + default: + return nil, fmt.Errorf("operation %d has no key", i) + } + + inserts[i] = operation.Insert + actions[i] = some(uint64(operation.Action)) + + meta, data, err := encodeScalar(operation.Value) + if err != nil { + return nil, fmt.Errorf("cannot encode operation %d value: %w", i, err) + } + + valueMetadata[i] = meta + + valueData = append(valueData, data...) + + predGroups[i] = some(uint64(len(operation.Predecessors))) + for _, predecessor := range operation.Predecessors { + actorIndex, ok := actorIndexes[predecessor.Actor] + if !ok { + return nil, fmt.Errorf("operation %d predecessor actor is unknown", i) + } + + predActors = append(predActors, some(actorIndex)) + predCounters = append(predCounters, some(int64(predecessor.Counter))) + } + } + + columns := []encodedColumn{ + {specification: 1, data: encodeRLE(objActors, appendULEB)}, + {specification: 2, data: encodeRLE(objCounters, appendULEB)}, + {specification: 17, data: encodeRLE(keyActors, appendULEB)}, + {specification: 19, data: encodeDelta(keyCounters)}, + {specification: 21, data: encodeStrings(keyStrings)}, + {specification: 52, data: encodeBooleans(inserts)}, + {specification: 66, data: encodeRLE(actions, appendULEB)}, + {specification: 86, data: encodeRLE(valueMetadata, appendULEB)}, + {specification: 87, data: valueData}, + {specification: 112, data: encodeRLE(predGroups, appendULEB)}, + {specification: 113, data: encodeRLE(predActors, appendULEB)}, + {specification: 115, data: encodeDelta(predCounters)}, + } + + filtered := columns[:0] + for _, column := range columns { + if len(column.data) > 0 { + filtered = append(filtered, column) + } + } + + return filtered, nil +} + +func encodeScalar(value *Scalar) (optional[uint64], []byte, error) { + if value == nil { + return some(uint64(ScalarNull)), nil, nil + } + + var data []byte + + switch value.Type { + case ScalarNull: + case ScalarFalse, ScalarTrue: + case ScalarUint: + data = appendULEB(data, value.Uint) + case ScalarInt, ScalarCounter, ScalarTimestamp: + data = appendLEB(data, value.Int) + case ScalarFloat64: + var encoded [8]byte + binary.LittleEndian.PutUint64(encoded[:], math.Float64bits(value.Float)) + data = encoded[:] + case ScalarString: + data = []byte(value.String) + case ScalarBytes: + data = append([]byte(nil), value.Bytes...) + default: + data = append([]byte(nil), value.Raw...) + } + + meta := uint64(len(data))<<4 | uint64(value.Type) + + return some(meta), data, nil +} + +func appendColumns(data []byte, columns []encodedColumn) []byte { + sort.Slice(columns, func(i, j int) bool { + return columns[i].specification < columns[j].specification + }) + + data = appendULEB(data, uint64(len(columns))) + for _, column := range columns { + data = appendULEB(data, uint64(column.specification)) + data = appendULEB(data, uint64(len(column.data))) + } + + for _, column := range columns { + data = append(data, column.data...) + } + + return data +} + +func encodeRLE[T comparable]( + values []optional[T], + appendValue func([]byte, T) []byte, +) []byte { + var data []byte + + for index := 0; index < len(values); { + if !values[index].valid { + end := index + 1 + for end < len(values) && !values[end].valid { + end++ + } + + data = appendLEB(data, 0) + data = appendULEB(data, uint64(end-index)) + index = end + + continue + } + + end := index + 1 + for end < len(values) && values[end].valid { + end++ + } + + data = appendLEB(data, -int64(end-index)) + for _, value := range values[index:end] { + data = appendValue(data, value.value) + } + + index = end + } + + if allNull(values) { + return nil + } + + return data +} + +func encodeDelta(values []optional[int64]) []byte { + var ( + previous int64 + deltas = make([]optional[int64], len(values)) + ) + for i, value := range values { + if !value.valid { + continue + } + + deltas[i] = some(value.value - previous) + previous = value.value + } + + return encodeRLE(deltas, appendLEB) +} + +func encodeStrings(values []optional[string]) []byte { + return encodeRLE( + values, + func(data []byte, value string) []byte { + return appendLengthPrefixedNative(data, []byte(value)) + }, + ) +} + +func encodeBooleans(values []bool) []byte { + if len(values) == 0 { + return nil + } + + var ( + data []byte + current bool + count uint64 + ) + for _, value := range values { + if value == current { + count++ + continue + } + + data = appendULEB(data, count) + current = value + count = 1 + } + + return appendULEB(data, count) +} + +func appendLEB(data []byte, value int64) []byte { + for { + current := byte(value & 0x7f) + value >>= 7 + + done := (value == 0 && current&0x40 == 0) || + (value == -1 && current&0x40 != 0) + if !done { + current |= 0x80 + } + + data = append(data, current) + if done { + return data + } + } +} + +func appendHashesNative(data []byte, hashes []ChangeHash) []byte { + data = appendULEB(data, uint64(len(hashes))) + for _, hash := range hashes { + data = append(data, hash[:]...) + } + + return data +} + +func appendLengthPrefixedNative(data, value []byte) []byte { + data = appendULEB(data, uint64(len(value))) + return append(data, value...) +} + +func some[T any](value T) optional[T] { + return optional[T]{value: value, valid: true} +} + +func allNull[T any](values []optional[T]) bool { + for _, value := range values { + if value.valid { + return false + } + } + + return true +} diff --git a/pkg/automerge/internal/native/fuzz_test.go b/pkg/automerge/internal/native/fuzz_test.go new file mode 100644 index 0000000000..de9fe001b1 --- /dev/null +++ b/pkg/automerge/internal/native/fuzz_test.go @@ -0,0 +1,47 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package native + +import ( + "encoding/base64" + "testing" + + "github.com/stretchr/testify/require" +) + +func FuzzDecode(f *testing.F) { + change, err := base64.StdEncoding.DecodeString(officialChangeFixture) + require.NoError(f, err) + document, err := base64.StdEncoding.DecodeString(officialDocumentFixture) + require.NoError(f, err) + + f.Add(change) + f.Add(document) + f.Add([]byte{0x85, 0x6f, 0x4a, 0x83}) + + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > 1024*1024 { + t.Skip() + } + + _, _ = Decode(data) + }) +} diff --git a/pkg/automerge/internal/native/incremental_sync_test.go b/pkg/automerge/internal/native/incremental_sync_test.go new file mode 100644 index 0000000000..a7821c9d18 --- /dev/null +++ b/pkg/automerge/internal/native/incremental_sync_test.go @@ -0,0 +1,85 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package native + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBackendSync_SendsOnlyChangesSinceRemoteHeads(t *testing.T) { + t.Parallel() + + ctx := context.Background() + backend, err := NewBackend(ctx) + require.NoError(t, err) + require.NoError(t, backend.SetActor(ctx, []byte{1})) + text, err := backend.PutText(ctx, 0, "body") + require.NoError(t, err) + require.NoError(t, backend.SpliceText(ctx, text, 0, 0, "A")) + _, err = backend.Commit(ctx, "first", time.Unix(1, 0)) + require.NoError(t, err) + + syncHandle, err := backend.NewSyncState(ctx) + require.NoError(t, err) + firstMessage, ok, err := backend.GenerateSyncMessage(ctx, syncHandle) + require.NoError(t, err) + assert.True(t, ok) + + first, err := ParseSyncMessage(firstMessage) + require.NoError(t, err) + require.Len(t, first.Changes, 1) + firstDocument, err := Decode(first.Changes[0]) + require.NoError(t, err) + assert.Equal(t, []ChunkType{ChunkChange}, firstDocument.ChunkTypes) + + ackMessage := SyncMessage{ + Version: SyncMessageVersion2, + Heads: first.Heads, + } + ack, err := ackMessage.Encode() + require.NoError(t, err) + require.NoError(t, backend.ReceiveSyncMessage(ctx, syncHandle, ack)) + + require.NoError(t, backend.SpliceText(ctx, text, 1, 0, "B")) + _, err = backend.Commit(ctx, "second", time.Unix(2, 0)) + require.NoError(t, err) + secondMessage, ok, err := backend.GenerateSyncMessage(ctx, syncHandle) + require.NoError(t, err) + assert.True(t, ok) + + second, err := ParseSyncMessage(secondMessage) + require.NoError(t, err) + require.Len(t, second.Changes, 1) + + secondDocument, err := DecodePartial(second.Changes[0]) + require.NoError(t, err) + require.Len(t, secondDocument.Changes, 1) + assert.Equal(t, "second", secondDocument.Changes[0].Message) + + fullDocument, err := backend.Save(ctx) + require.NoError(t, err) + assert.Less(t, len(secondMessage), len(fullDocument)) +} diff --git a/pkg/automerge/internal/native/merge_test.go b/pkg/automerge/internal/native/merge_test.go new file mode 100644 index 0000000000..22462b5816 --- /dev/null +++ b/pkg/automerge/internal/native/merge_test.go @@ -0,0 +1,89 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package native + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBackendMerge_AppliesReversedDependentChanges(t *testing.T) { + t.Parallel() + + ctx := context.Background() + base, err := NewBackend(ctx) + require.NoError(t, err) + require.NoError(t, base.SetActor(ctx, []byte{1})) + text, err := base.PutText(ctx, 0, "body") + require.NoError(t, err) + require.NoError(t, base.SpliceText(ctx, text, 0, 0, "A")) + _, err = base.Commit(ctx, "base", time.Unix(1, 0)) + require.NoError(t, err) + baseData, err := base.Save(ctx) + require.NoError(t, err) + + source, err := LoadBackend(ctx, baseData) + require.NoError(t, err) + require.NoError(t, source.SetActor(ctx, []byte{2})) + sourceText, err := source.GetText(ctx, 0, "body") + require.NoError(t, err) + require.NoError(t, source.SpliceText(ctx, sourceText, 1, 0, "B")) + _, err = source.Commit(ctx, "parent", time.Unix(2, 0)) + require.NoError(t, err) + + parent := append([]byte(nil), source.appended[len(source.appended)-1]...) + + require.NoError(t, source.SpliceText(ctx, sourceText, 2, 0, "C")) + _, err = source.Commit(ctx, "child", time.Unix(3, 0)) + require.NoError(t, err) + + child := append([]byte(nil), source.appended[len(source.appended)-1]...) + + target, err := LoadBackend(ctx, baseData) + require.NoError(t, err) + _, err = target.Merge(ctx, append(child, parent...)) + require.NoError(t, err) + targetText, err := target.GetText(ctx, 0, "body") + require.NoError(t, err) + value, err := target.Text(ctx, targetText) + require.NoError(t, err) + assert.Equal(t, "ABC", value) + + separate, err := LoadBackend(ctx, baseData) + require.NoError(t, err) + _, err = separate.Merge(ctx, child) + require.NoError(t, err) + assert.Len(t, separate.queuedChanges, 1) + + _, err = separate.Merge(ctx, parent) + require.NoError(t, err) + assert.Empty(t, separate.queuedChanges) + + separateText, err := separate.GetText(ctx, 0, "body") + require.NoError(t, err) + separateValue, err := separate.Text(ctx, separateText) + require.NoError(t, err) + assert.Equal(t, "ABC", separateValue) +} diff --git a/pkg/automerge/internal/native/state.go b/pkg/automerge/internal/native/state.go new file mode 100644 index 0000000000..2493f9967f --- /dev/null +++ b/pkg/automerge/internal/native/state.go @@ -0,0 +1,731 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package native + +import ( + "bytes" + "fmt" + "reflect" + "sort" + "strings" +) + +type ( + State struct { + changes map[ChangeHash]*Change + actorSequence map[ActorID]uint64 + operations map[OpID]Operation + superseded map[OpID]struct{} + heads map[ChangeHash]struct{} + } + + RichSpan struct { + Type string `json:"type"` + Value any `json:"value"` + Marks map[string]any `json:"marks,omitempty"` + } +) + +func NewState() *State { + return &State{ + changes: make(map[ChangeHash]*Change), + actorSequence: make(map[ActorID]uint64), + operations: make(map[OpID]Operation), + superseded: make(map[OpID]struct{}), + heads: make(map[ChangeHash]struct{}), + } +} + +func NewStateFromDocument(document *Document) (*State, error) { + state := NewState() + + for i := range document.Changes { + change := &document.Changes[i] + if change.Sequence > state.actorSequence[change.Actor] { + state.actorSequence[change.Actor] = change.Sequence + } + + if change.Hash != nil { + state.changes[*change.Hash] = change + } + + for _, operation := range change.Operations { + if _, exists := state.operations[operation.ID]; exists { + return nil, fmt.Errorf("duplicate snapshot operation ID %v", operation.ID) + } + + state.operations[operation.ID] = operation + for _, successor := range operation.Successors { + state.superseded[operation.ID] = struct{}{} + if successor.Counter == 0 { + return nil, fmt.Errorf("invalid zero successor for operation %v", operation.ID) + } + } + } + } + + for _, head := range document.Heads { + state.heads[head] = struct{}{} + } + + return state, nil +} + +func (s *State) ApplyChange(change *Change) error { + if change.Hash == nil { + return fmt.Errorf("change hash is required") + } + + if _, ok := s.changes[*change.Hash]; ok { + return nil + } + + for _, dependency := range change.Dependencies { + if _, ok := s.changes[dependency]; !ok { + return fmt.Errorf("missing change dependency %s", dependency) + } + } + + expectedSequence := s.actorSequence[change.Actor] + 1 + if change.Sequence != expectedSequence { + return fmt.Errorf( + "actor sequence is %d, expected %d", + change.Sequence, + expectedSequence, + ) + } + + for _, operation := range change.Operations { + if _, exists := s.operations[operation.ID]; exists { + return fmt.Errorf("duplicate operation ID %v", operation.ID) + } + } + + for _, operation := range change.Operations { + s.operations[operation.ID] = operation + for _, predecessor := range operation.Predecessors { + s.superseded[predecessor] = struct{}{} + } + } + + s.changes[*change.Hash] = change + + s.actorSequence[change.Actor] = change.Sequence + for _, dependency := range change.Dependencies { + delete(s.heads, dependency) + } + + s.heads[*change.Hash] = struct{}{} + + return nil +} + +func (s *State) Heads() []ChangeHash { + heads := make([]ChangeHash, 0, len(s.heads)) + for head := range s.heads { + heads = append(heads, head) + } + + sort.Slice( + heads, + func(i, j int) bool { + return bytes.Compare(heads[i][:], heads[j][:]) < 0 + }, + ) + + return heads +} + +func (s *State) Text(property string) (string, error) { + objectOperation, ok := s.visibleMapOperation(property, ActionMakeText) + if !ok { + return "", fmt.Errorf("text property %q does not exist", property) + } + + sequence := s.sequence(objectOperation.ID) + + var output strings.Builder + + for _, operation := range sequence { + if operation.Value != nil && operation.Value.Type == ScalarString { + output.WriteString(operation.Value.String) + } + } + + return output.String(), nil +} + +func (s *State) visibleMapOperation(property string, action Action) (Operation, bool) { + var ( + result Operation + found bool + ) + + for _, operation := range s.operations { + if !operation.Object.IsRoot || + operation.Key.Property == nil || + *operation.Key.Property != property || + operation.Action != action || + s.isSuperseded(operation.ID) { + continue + } + + if !found || operation.ID.Compare(result.ID) > 0 { + result = operation + found = true + } + } + + return result, found +} + +func (s *State) visibleMapOperations(property string) []Operation { + operations := make([]Operation, 0) + for _, operation := range s.operations { + if !operation.Object.IsRoot || + operation.Key.Property == nil || + *operation.Key.Property != property || + s.isSuperseded(operation.ID) { + continue + } + + operations = append(operations, operation) + } + + sort.Slice(operations, func(i, j int) bool { + return operations[i].ID.Compare(operations[j].ID) < 0 + }) + + return operations +} + +func (s *State) sequence(object OpID) []Operation { + operations := s.sequenceElements(object) + + result := operations[:0] + for _, operation := range operations { + if operation.Action == ActionSet { + result = append(result, operation) + } + } + + return result +} + +func (s *State) sequenceElements(object OpID) []Operation { + children := make(map[OpID][]Operation) + + var head []Operation + + for _, operation := range s.operations { + if operation.Object.IsRoot || + operation.Object.OpID != object || + !operation.Insert || + operation.Action == ActionMark { + continue + } + + switch { + case operation.Key.IsHead: + head = append(head, operation) + case operation.Key.Element != nil: + children[*operation.Key.Element] = append( + children[*operation.Key.Element], + operation, + ) + } + } + + operations := make([]Operation, 0) + s.appendSequence( + &operations, + head, + children, + make(map[OpID]struct{}), + false, + ) + + return operations +} + +func (s *State) sequenceAll(object OpID) []Operation { + children := make(map[OpID][]Operation) + + var head []Operation + + for _, operation := range s.operations { + if operation.Object.IsRoot || + operation.Object.OpID != object || + !operation.Insert || + operation.Action == ActionMark { + continue + } + + if operation.Key.IsHead { + head = append(head, operation) + } else if operation.Key.Element != nil { + children[*operation.Key.Element] = append( + children[*operation.Key.Element], + operation, + ) + } + } + + operations := make([]Operation, 0) + s.appendSequence( + &operations, + head, + children, + make(map[OpID]struct{}), + true, + ) + + return operations +} + +func (s *State) RichTextSpans(object OpID) ([]RichSpan, error) { + elements := s.sequenceElements(object) + marks := s.richTextMarks(object, elements) + spans := make([]RichSpan, 0) + + for i, operation := range elements { + switch operation.Action { + case ActionMakeMap: + value, err := s.mapValue(operation.ID, make(map[OpID]struct{})) + if err != nil { + return nil, fmt.Errorf("cannot hydrate block %v: %w", operation.ID, err) + } + + spans = append(spans, RichSpan{Type: "block", Value: value}) + case ActionSet: + if operation.Value == nil || operation.Value.Type != ScalarString { + continue + } + + activeMarks := make(map[string]any) + + for _, mark := range marks { + if i >= mark.start && i < mark.end { + activeMarks[mark.name] = mark.value + } + } + + if len(activeMarks) == 0 { + activeMarks = nil + } + + if len(spans) > 0 && + spans[len(spans)-1].Type == "text" && + reflect.DeepEqual(spans[len(spans)-1].Marks, activeMarks) { + spans[len(spans)-1].Value = spans[len(spans)-1].Value.(string) + + operation.Value.String + } else { + spans = append( + spans, + RichSpan{ + Type: "text", + Value: operation.Value.String, + Marks: activeMarks, + }, + ) + } + } + } + + return spans, nil +} + +type richTextMark struct { + start int + end int + name string + value any +} + +func (s *State) richTextMarks(object OpID, elements []Operation) []richTextMark { + positions := make(map[OpID]int, len(elements)) + for i, element := range elements { + positions[element.ID] = i + } + + markOperations := make([]Operation, 0) + + for _, operation := range s.operations { + if !operation.Object.IsRoot && + operation.Object.OpID == object && + operation.Action == ActionMark && + !s.isSuperseded(operation.ID) { + markOperations = append(markOperations, operation) + } + } + + sort.Slice(markOperations, func(i, j int) bool { + return markOperations[i].ID.Compare(markOperations[j].ID) < 0 + }) + + marks := make([]richTextMark, 0, len(markOperations)/2) + for i := 0; i+1 < len(markOperations); i += 2 { + begin := markOperations[i] + + end := markOperations[i+1] + if begin.MarkName == nil || begin.Key.Element == nil || end.Key.Element == nil { + continue + } + + startPosition, startOK := positions[*begin.Key.Element] + + endPosition, endOK := positions[*end.Key.Element] + if !startOK || !endOK || startPosition >= endPosition { + continue + } + + marks = append( + marks, + richTextMark{ + start: startPosition + 1, + end: endPosition + 1, + name: *begin.MarkName, + value: scalarMaterializedValue(begin.Value), + }, + ) + } + + return marks +} + +func (s *State) mapValue( + object OpID, + visited map[OpID]struct{}, +) (map[string]any, error) { + if _, ok := visited[object]; ok { + return nil, fmt.Errorf("object cycle detected") + } + + visited[object] = struct{}{} + defer delete(visited, object) + + properties := make(map[string][]Operation) + + for _, operation := range s.operations { + if operation.Object.IsRoot || + operation.Object.OpID != object || + operation.Key.Property == nil || + s.isSuperseded(operation.ID) { + continue + } + + property := *operation.Key.Property + properties[property] = append(properties[property], operation) + } + + result := make(map[string]any, len(properties)) + for property, operations := range properties { + sort.Slice(operations, func(i, j int) bool { + return operations[i].ID.Compare(operations[j].ID) > 0 + }) + + operation := operations[0] + switch operation.Action { + case ActionMakeMap: + value, err := s.mapValue(operation.ID, visited) + if err != nil { + return nil, err + } + + result[property] = value + case ActionMakeList: + value, err := s.listValue(operation.ID, visited) + if err != nil { + return nil, err + } + + result[property] = value + case ActionMakeText: + var value strings.Builder + + for _, element := range s.sequence(operation.ID) { + if element.Value != nil && element.Value.Type == ScalarString { + value.WriteString(element.Value.String) + } + } + + result[property] = value.String() + case ActionSet: + result[property] = scalarMaterializedValue(operation.Value) + } + } + + return result, nil +} + +func (s *State) listValue( + object OpID, + visited map[OpID]struct{}, +) ([]any, error) { + if _, ok := visited[object]; ok { + return nil, fmt.Errorf("object cycle detected") + } + + visited[object] = struct{}{} + defer delete(visited, object) + + elements := s.sequenceElements(object) + result := make([]any, 0, len(elements)) + + for _, element := range elements { + switch element.Action { + case ActionMakeMap: + value, err := s.mapValue(element.ID, visited) + if err != nil { + return nil, err + } + + result = append(result, value) + case ActionMakeList: + value, err := s.listValue(element.ID, visited) + if err != nil { + return nil, err + } + + result = append(result, value) + case ActionMakeText: + var value strings.Builder + + for _, textElement := range s.sequence(element.ID) { + if textElement.Value != nil && + textElement.Value.Type == ScalarString { + value.WriteString(textElement.Value.String) + } + } + + result = append(result, value.String()) + case ActionSet: + result = append(result, scalarMaterializedValue(element.Value)) + } + } + + return result, nil +} + +func scalarMaterializedValue(value *Scalar) any { + if value == nil { + return nil + } + + switch value.Type { + case ScalarNull: + return nil + case ScalarFalse: + return false + case ScalarTrue: + return true + case ScalarUint: + return value.Uint + case ScalarInt, ScalarCounter, ScalarTimestamp: + return value.Int + case ScalarFloat64: + return value.Float + case ScalarString: + return value.String + case ScalarBytes: + return append([]byte(nil), value.Bytes...) + default: + return append([]byte(nil), value.Raw...) + } +} + +func (s *State) appendSequence( + output *[]Operation, + operations []Operation, + children map[OpID][]Operation, + visited map[OpID]struct{}, + includeSuperseded bool, +) { + sort.Slice( + operations, + func(i, j int) bool { + return operations[i].ID.Compare(operations[j].ID) > 0 + }, + ) + + for _, operation := range operations { + if _, ok := visited[operation.ID]; ok { + continue + } + + visited[operation.ID] = struct{}{} + if includeSuperseded || !s.isSuperseded(operation.ID) { + *output = append(*output, operation) + } + + s.appendSequence( + output, + children[operation.ID], + children, + visited, + includeSuperseded, + ) + } +} + +func (s *State) isSuperseded(id OpID) bool { + _, ok := s.superseded[id] + return ok +} + +func (s *State) maxOpGlobal() uint64 { + var maximum uint64 + for id := range s.operations { + if id.Counter > maximum { + maximum = id.Counter + } + } + + return maximum +} + +func (s *State) sequenceForActor(actor ActorID) uint64 { + return s.actorSequence[actor] +} + +func (s *State) applyPending(operations []Operation) error { + for _, operation := range operations { + if _, exists := s.operations[operation.ID]; exists { + return fmt.Errorf("duplicate pending operation ID %v", operation.ID) + } + + s.operations[operation.ID] = operation + for _, predecessor := range operation.Predecessors { + s.superseded[predecessor] = struct{}{} + } + } + + return nil +} + +func (s *State) recordAppliedChange(change *Change) error { + if change.Hash == nil { + return fmt.Errorf("change hash is required") + } + + for _, dependency := range change.Dependencies { + delete(s.heads, dependency) + } + + s.heads[*change.Hash] = struct{}{} + s.changes[*change.Hash] = change + s.actorSequence[change.Actor] = change.Sequence + + return nil +} + +func (s *State) hasChange(hash ChangeHash) bool { + _, ok := s.changes[hash] + return ok +} + +func (s *State) hasDependencies(change *Change) bool { + for _, dependency := range change.Dependencies { + if !s.hasChange(dependency) { + return false + } + } + + return true +} + +func (s *State) changesSince(heads []ChangeHash) ([]*Change, bool) { + known, ok := s.changeClosure(heads) + if !ok { + return nil, false + } + + ordered := make([]*Change, 0) + visited := make(map[ChangeHash]struct{}) + + var visit func(ChangeHash) bool + + visit = func(hash ChangeHash) bool { + if _, ok := visited[hash]; ok { + return true + } + + visited[hash] = struct{}{} + + change, ok := s.changes[hash] + if !ok { + return false + } + + for _, dependency := range change.Dependencies { + if !visit(dependency) { + return false + } + } + + if _, ok := known[hash]; ok { + return true + } + + if len(change.Raw) == 0 { + return false + } + + ordered = append(ordered, change) + + return true + } + + for _, head := range s.Heads() { + if !visit(head) { + return nil, false + } + } + + return ordered, true +} + +func (s *State) changeClosure(heads []ChangeHash) (map[ChangeHash]struct{}, bool) { + closure := make(map[ChangeHash]struct{}) + + pending := append([]ChangeHash(nil), heads...) + + for len(pending) > 0 { + index := len(pending) - 1 + hash := pending[index] + pending = pending[:index] + + if _, ok := closure[hash]; ok { + continue + } + + change, ok := s.changes[hash] + if !ok { + return nil, false + } + + closure[hash] = struct{}{} + + pending = append(pending, change.Dependencies...) + } + + return closure, true +} diff --git a/pkg/automerge/internal/native/sync.go b/pkg/automerge/internal/native/sync.go new file mode 100644 index 0000000000..b4ebe582b2 --- /dev/null +++ b/pkg/automerge/internal/native/sync.go @@ -0,0 +1,223 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package native + +import "fmt" + +type ( + SyncMessageVersion byte + + SyncHave struct { + LastSync [][32]byte + Bloom []byte + } + + SyncMessage struct { + Version SyncMessageVersion + Heads [][32]byte + Need [][32]byte + Have []SyncHave + Changes [][]byte + Flags []byte + } +) + +const ( + SyncMessageVersion1 SyncMessageVersion = 0x42 + SyncMessageVersion2 SyncMessageVersion = 0x43 + + maxSyncHaveEntries = 1024 + maxSyncChanges = 1024 * 1024 + maxSyncBloomBytes = 1024 * 1024 + maxSyncFlagsBytes = 1024 + maxSyncChunkBytes = 64 * 1024 * 1024 + maxSyncHashes = 1024 * 1024 +) + +func ParseSyncMessage(data []byte) (*SyncMessage, error) { + r := &reader{data: data} + + versionByte, err := r.byte() + if err != nil { + return nil, fmt.Errorf("cannot read sync message version: %w", err) + } + + version := SyncMessageVersion(versionByte) + if version != SyncMessageVersion1 && version != SyncMessageVersion2 { + return nil, fmt.Errorf("unsupported sync message version 0x%02x", version) + } + + heads, err := readSyncHashes(r) + if err != nil { + return nil, fmt.Errorf("cannot read sync heads: %w", err) + } + + need, err := readSyncHashes(r) + if err != nil { + return nil, fmt.Errorf("cannot read sync needs: %w", err) + } + + haveCount, err := r.uleb() + if err != nil { + return nil, fmt.Errorf("cannot read sync have count: %w", err) + } + + if haveCount > maxSyncHaveEntries { + return nil, fmt.Errorf("sync have count %d exceeds limit", haveCount) + } + + have := make([]SyncHave, int(haveCount)) + for i := range have { + have[i].LastSync, err = readSyncHashes(r) + if err != nil { + return nil, fmt.Errorf("cannot read sync have %d heads: %w", i, err) + } + + have[i].Bloom, err = readSyncBytes(r, maxSyncBloomBytes) + if err != nil { + return nil, fmt.Errorf("cannot read sync have %d bloom: %w", i, err) + } + } + + changeCount, err := r.uleb() + if err != nil { + return nil, fmt.Errorf("cannot read sync change count: %w", err) + } + + if changeCount > maxSyncChanges { + return nil, fmt.Errorf("sync change count %d exceeds limit", changeCount) + } + + changes := make([][]byte, int(changeCount)) + for i := range changes { + changes[i], err = readSyncBytes(r, maxSyncChunkBytes) + if err != nil { + return nil, fmt.Errorf("cannot read sync change %d: %w", i, err) + } + } + + var flags []byte + if r.remaining() > 0 { + flags, err = readSyncBytes(r, maxSyncFlagsBytes) + if err != nil { + return nil, fmt.Errorf("cannot read sync flags: %w", err) + } + } + + if r.remaining() > 0 { + return nil, fmt.Errorf("sync message contains trailing bytes") + } + + return &SyncMessage{ + Version: version, + Heads: heads, + Need: need, + Have: have, + Changes: changes, + Flags: flags, + }, nil +} + +func (m SyncMessage) Encode() ([]byte, error) { + if m.Version != SyncMessageVersion1 && m.Version != SyncMessageVersion2 { + return nil, fmt.Errorf("unsupported sync message version 0x%02x", m.Version) + } + + if len(m.Have) > maxSyncHaveEntries || len(m.Changes) > maxSyncChanges { + return nil, fmt.Errorf("sync message exceeds collection limits") + } + + data := []byte{byte(m.Version)} + data = appendHashes(data, m.Heads) + data = appendHashes(data, m.Need) + + data = appendULEB(data, uint64(len(m.Have))) + for _, have := range m.Have { + data = appendHashes(data, have.LastSync) + data = appendLengthPrefixedBytes(data, have.Bloom) + } + + data = appendULEB(data, uint64(len(m.Changes))) + for _, change := range m.Changes { + data = appendLengthPrefixedBytes(data, change) + } + + if m.Flags != nil { + data = appendLengthPrefixedBytes(data, m.Flags) + } + + return data, nil +} + +func appendHashes(data []byte, hashes [][32]byte) []byte { + data = appendULEB(data, uint64(len(hashes))) + for _, hash := range hashes { + data = append(data, hash[:]...) + } + + return data +} + +func appendLengthPrefixedBytes(data, value []byte) []byte { + data = appendULEB(data, uint64(len(value))) + return append(data, value...) +} + +func readSyncHashes(r *reader) ([][32]byte, error) { + count, err := r.uleb() + if err != nil { + return nil, err + } + + if count > maxSyncHashes { + return nil, fmt.Errorf("sync hash count %d exceeds limit", count) + } + + hashes := make([][32]byte, int(count)) + for i := range hashes { + value, err := r.bytes(32) + if err != nil { + return nil, fmt.Errorf("cannot read sync hash %d: %w", i, err) + } + + copy(hashes[i][:], value) + } + + return hashes, nil +} + +func readSyncBytes(r *reader, limit uint64) ([]byte, error) { + length, err := r.uleb() + if err != nil { + return nil, err + } + + if length > limit { + return nil, fmt.Errorf("sync byte length %d exceeds limit %d", length, limit) + } + + value, err := r.bytes(length) + if err != nil { + return nil, err + } + + return append([]byte(nil), value...), nil +} diff --git a/pkg/automerge/internal/native/types.go b/pkg/automerge/internal/native/types.go new file mode 100644 index 0000000000..79a091bff2 --- /dev/null +++ b/pkg/automerge/internal/native/types.go @@ -0,0 +1,212 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +// Package native decodes and validates the Automerge 0.10 binary format. +// +// This package is deliberately independent from the reference WASM backend. +// It is not yet a mutation or materialization engine. +package native + +import ( + "bytes" + "encoding/hex" + "fmt" + "math" +) + +type ( + // ActorID is an immutable, arbitrary-length Automerge actor identifier. + ActorID string + + // ChangeHash identifies a change by the SHA-256 digest of its change chunk. + ChangeHash [32]byte + + // OpID is an Automerge Lamport timestamp. + OpID struct { + Actor ActorID + Counter uint64 + } + + // ObjectID identifies either the root map or the operation that made an + // object. + ObjectID struct { + OpID OpID + IsRoot bool + } + + // Key identifies either a map property or a sequence element. + Key struct { + Property *string + Element *OpID + IsHead bool + } + + // Action is the numeric action stored in an operation. + Action uint64 + + // ScalarType is the four-bit scalar tag from a value metadata column. + ScalarType uint8 + + // Scalar preserves the exact Automerge scalar type and its typed value. + // Raw is populated for unknown, forward-compatible scalar types. + Scalar struct { + Type ScalarType + Bool bool + Uint uint64 + Int int64 + Float float64 + String string + Bytes []byte + Raw []byte + } + + // Operation is one decoded Automerge operation. + Operation struct { + ID OpID + Object ObjectID + Key Key + Insert bool + Action Action + Value *Scalar + Predecessors []OpID + Successors []OpID + MarkExpand *bool + MarkName *string + } + + // Change is one decoded change. Hash is nil for non-head changes in a + // document chunk because snapshots store only head hashes. + Change struct { + Hash *ChangeHash + Actor ActorID + Sequence uint64 + StartOp uint64 + MaxOp uint64 + Time int64 + Message string + Dependencies []ChangeHash + DependencyIndexes []uint64 + Operations []Operation + Extra *Scalar + ExtraBytes []byte + Raw []byte + } + + // ChunkType identifies an Automerge storage chunk. + ChunkType uint8 + + // RawColumn retains a column not interpreted by this milestone. + RawColumn struct { + Specification uint32 + Data []byte + } + + // Document is a validated Automerge history. + Document struct { + Actors []ActorID + Heads []ChangeHash + Changes []Change + UnknownColumns []RawColumn + ChunkTypes []ChunkType + } +) + +const ( + ActionMakeMap Action = 0 + ActionSet Action = 1 + ActionMakeList Action = 2 + ActionDelete Action = 3 + ActionMakeText Action = 4 + ActionIncrement Action = 5 + ActionMakeTable Action = 6 + ActionMark Action = 7 + + ScalarNull ScalarType = 0 + ScalarFalse ScalarType = 1 + ScalarTrue ScalarType = 2 + ScalarUint ScalarType = 3 + ScalarInt ScalarType = 4 + ScalarFloat64 ScalarType = 5 + ScalarString ScalarType = 6 + ScalarBytes ScalarType = 7 + ScalarCounter ScalarType = 8 + ScalarTimestamp ScalarType = 9 + + ChunkDocument ChunkType = 0 + ChunkChange ChunkType = 1 + ChunkCompressedChange ChunkType = 2 +) + +// NewActorID validates and copies an actor ID. +func NewActorID(value []byte) (ActorID, error) { + if len(value) == 0 { + return "", fmt.Errorf("actor ID cannot be empty") + } + + return ActorID(string(value)), nil +} + +// Bytes returns a copy of the actor ID bytes. +func (a ActorID) Bytes() []byte { + return []byte(string(a)) +} + +// String returns the lowercase hexadecimal actor ID. +func (a ActorID) String() string { + return hex.EncodeToString(a.Bytes()) +} + +// Compare returns -1, 0, or 1 using Automerge's bytewise actor ordering. +func (a ActorID) Compare(other ActorID) int { + return bytes.Compare(a.Bytes(), other.Bytes()) +} + +// String returns the lowercase hexadecimal change hash. +func (h ChangeHash) String() string { + return hex.EncodeToString(h[:]) +} + +// Compare orders operation IDs by their Automerge Lamport timestamp. +func (o OpID) Compare(other OpID) int { + switch { + case o.Counter < other.Counter: + return -1 + case o.Counter > other.Counter: + return 1 + default: + return o.Actor.Compare(other.Actor) + } +} + +// RootObject returns the distinguished root map identifier. +func RootObject() ObjectID { + return ObjectID{IsRoot: true} +} + +// IsKnown reports whether the scalar type is defined by Automerge 0.10. +func (s Scalar) IsKnown() bool { + return s.Type <= ScalarTimestamp +} + +// IsFinite reports whether a float scalar is finite. Non-float scalars are +// always finite. +func (s Scalar) IsFinite() bool { + return s.Type != ScalarFloat64 || (!math.IsInf(s.Float, 0) && !math.IsNaN(s.Float)) +} diff --git a/pkg/automerge/internal/native/validate.go b/pkg/automerge/internal/native/validate.go new file mode 100644 index 0000000000..9160150d35 --- /dev/null +++ b/pkg/automerge/internal/native/validate.go @@ -0,0 +1,804 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package native + +import ( + "bytes" + "fmt" + "math" + "slices" +) + +func decodeActorArray(r *reader, sorted bool) ([]ActorID, error) { + count, err := r.uleb() + if err != nil { + return nil, err + } + + if count > maxDecodedItems { + return nil, fmt.Errorf("actor count %d exceeds limit", count) + } + + actors := make([]ActorID, 0, count) + for i := uint64(0); i < count; i++ { + value, err := decodeLengthPrefixed(r) + if err != nil { + return nil, fmt.Errorf("cannot decode actor %d: %w", i, err) + } + + actor, err := NewActorID(value) + if err != nil { + return nil, fmt.Errorf("actor %d: %w", i, err) + } + + if sorted && len(actors) > 0 && actors[len(actors)-1].Compare(actor) >= 0 { + return nil, fmt.Errorf("actor IDs are not strictly sorted at index %d", i) + } + + actors = append(actors, actor) + } + + return actors, nil +} + +func decodeLengthPrefixed(r *reader) ([]byte, error) { + length, err := r.uleb() + if err != nil { + return nil, err + } + + return r.bytes(length) +} + +func decodeHashArray(r *reader, sorted bool) ([]ChangeHash, error) { + count, err := r.uleb() + if err != nil { + return nil, err + } + + if count > maxDecodedItems { + return nil, fmt.Errorf("hash count %d exceeds limit", count) + } + + hashes := make([]ChangeHash, 0, count) + for i := uint64(0); i < count; i++ { + value, err := r.bytes(32) + if err != nil { + return nil, fmt.Errorf("cannot decode hash %d: %w", i, err) + } + + hash := copyHash(value) + if sorted && len(hashes) > 0 && bytes.Compare(hashes[len(hashes)-1][:], hash[:]) >= 0 { + return nil, fmt.Errorf("hashes are not strictly sorted at index %d", i) + } + + hashes = append(hashes, hash) + } + + return hashes, nil +} + +func decodeRequiredDelta( + columns map[uint32]column, + specification uint32, + name string, + expected int, +) ([]optional[uint64], error) { + data, err := requireColumn(columns, specification) + if err != nil { + return nil, err + } + + values, err := decodeDeltaColumn(data) + if err != nil { + return nil, fmt.Errorf("cannot decode %s: %w", name, err) + } + + if err := requireItems(name, values, expected, false); err != nil { + return nil, err + } + + return values, nil +} + +func decodeOptionalDelta( + columns map[uint32]column, + specification uint32, + name string, + expected int, +) ([]optional[uint64], error) { + data := optionalColumn(columns, specification) + if data == nil { + return make([]optional[uint64], expected), nil + } + + values, err := decodeDeltaColumn(data) + if err != nil { + return nil, fmt.Errorf("cannot decode %s: %w", name, err) + } + + if err := requireItems(name, values, expected, true); err != nil { + return nil, err + } + + return values, nil +} + +func decodeOptionalSignedDelta( + columns map[uint32]column, + specification uint32, + name string, + expected int, +) ([]optional[int64], error) { + data := optionalColumn(columns, specification) + if data == nil { + return make([]optional[int64], expected), nil + } + + values, err := decodeSignedDeltaColumn(data) + if err != nil { + return nil, fmt.Errorf("cannot decode %s: %w", name, err) + } + + if err := requireItems(name, values, expected, true); err != nil { + return nil, err + } + + return values, nil +} + +func decodeRequiredULEB( + columns map[uint32]column, + specification uint32, + name string, + expected int, +) ([]optional[uint64], error) { + data, err := requireColumn(columns, specification) + if err != nil { + return nil, err + } + + values, err := decodeULEBColumn(data) + if err != nil { + return nil, fmt.Errorf("cannot decode %s: %w", name, err) + } + + if err := requireItems(name, values, expected, false); err != nil { + return nil, err + } + + return values, nil +} + +func decodeOptionalULEB( + columns map[uint32]column, + specification uint32, + name string, + expected int, +) ([]optional[uint64], error) { + data := optionalColumn(columns, specification) + if data == nil { + return make([]optional[uint64], expected), nil + } + + values, err := decodeULEBColumn(data) + if err != nil { + return nil, fmt.Errorf("cannot decode %s: %w", name, err) + } + + if err := requireItems(name, values, expected, true); err != nil { + return nil, err + } + + return values, nil +} + +func decodeOptionalStrings( + columns map[uint32]column, + specification uint32, + name string, + expected int, +) ([]optional[string], error) { + data := optionalColumn(columns, specification) + if data == nil { + return make([]optional[string], expected), nil + } + + values, err := decodeStringColumn(data) + if err != nil { + return nil, fmt.Errorf("cannot decode %s: %w", name, err) + } + + if err := requireItems(name, values, expected, true); err != nil { + return nil, err + } + + return values, nil +} + +func decodeOptionalBooleans( + columns map[uint32]column, + specification uint32, + expected int, +) ([]optional[bool], error) { + data := optionalColumn(columns, specification) + + values := make([]optional[bool], expected) + if data == nil { + return values, nil + } + + decoded, err := decodeBooleanColumn(data, expected) + if err != nil { + return nil, err + } + + for i, value := range decoded { + values[i] = optional[bool]{value: value, valid: true} + } + + return values, nil +} + +func decodeOptionalScalars( + columns map[uint32]column, + metaSpecification uint32, + rawSpecification uint32, + expected int, +) ([]optional[Scalar], error) { + meta := optionalColumn(columns, metaSpecification) + + raw := optionalColumn(columns, rawSpecification) + if meta == nil { + if raw != nil { + return nil, fmt.Errorf("raw value column is missing metadata") + } + + return make([]optional[Scalar], expected), nil + } + + return decodeScalars(meta, raw, expected) +} + +func sumGroups(groups []optional[uint64]) (int, error) { + var total uint64 + + for i, group := range groups { + if !group.valid { + return 0, fmt.Errorf("group %d is null", i) + } + + if total > math.MaxUint64-group.value { + return 0, fmt.Errorf("group sizes overflow uint64") + } + + total += group.value + if total > maxDecodedItems { + return 0, fmt.Errorf("grouped column exceeds %d items", maxDecodedItems) + } + } + + return int(total), nil +} + +func decodeGroupedOpIDs( + columns map[uint32]column, + actors []ActorID, + groupSpec uint32, + actorSpec uint32, + counterSpec uint32, + expected int, + name string, +) ([][]OpID, error) { + result := make([][]OpID, expected) + + groupData := optionalColumn(columns, groupSpec) + if groupData == nil { + return result, nil + } + + groups, err := decodeULEBColumn(groupData) + if err != nil { + return nil, fmt.Errorf("cannot decode %s groups: %w", name, err) + } + + if err := requireItems(name+" group", groups, expected, false); err != nil { + return nil, err + } + + count, err := sumGroups(groups) + if err != nil { + return nil, err + } + + if count == 0 { + return result, nil + } + + actorData, err := requireColumn(columns, actorSpec) + if err != nil { + return nil, err + } + + actorIndexes, err := decodeULEBColumn(actorData) + if err != nil { + return nil, fmt.Errorf("cannot decode %s actors: %w", name, err) + } + + if err := requireItems(name+" actor", actorIndexes, count, false); err != nil { + return nil, err + } + + counters, err := decodeRequiredDelta(columns, counterSpec, name+" counter", count) + if err != nil { + return nil, err + } + + offset := 0 + + for i, group := range groups { + result[i] = make([]OpID, int(group.value)) + for j := range result[i] { + id, err := opIDFromIndexes(actorIndexes[offset+j], counters[offset+j], actors) + if err != nil { + return nil, fmt.Errorf("%s %d: %w", name, offset+j, err) + } + + result[i][j] = id + } + + offset += int(group.value) + } + + return result, nil +} + +func opIDFromIndexes( + actorIndex optional[uint64], + counter optional[uint64], + actors []ActorID, +) (OpID, error) { + if !actorIndex.valid || !counter.valid { + return OpID{}, fmt.Errorf("actor or counter is null") + } + + if actorIndex.value >= uint64(len(actors)) { + return OpID{}, fmt.Errorf("actor index %d is out of bounds", actorIndex.value) + } + + if counter.value == 0 { + return OpID{}, fmt.Errorf("counter is zero") + } + + return OpID{Actor: actors[actorIndex.value], Counter: counter.value}, nil +} + +func objectIDFromIndexes( + actorIndex optional[uint64], + counter optional[uint64], + actors []ActorID, +) (ObjectID, error) { + if !actorIndex.valid && !counter.valid { + return RootObject(), nil + } + + id, err := opIDFromIndexes(actorIndex, counter, actors) + if err != nil { + return ObjectID{}, err + } + + return ObjectID{OpID: id}, nil +} + +func keyFromColumns( + actorIndex optional[uint64], + counter optional[uint64], + property optional[string], + actors []ActorID, + insert bool, +) (Key, error) { + if property.valid { + if actorIndex.valid || counter.valid { + return Key{}, fmt.Errorf("property key also has an element ID") + } + + value := property.value + + return Key{Property: &value}, nil + } + + if !actorIndex.valid && insert && (!counter.valid || counter.value == 0) { + return Key{IsHead: true}, nil + } + + id, err := opIDFromIndexes(actorIndex, counter, actors) + if err != nil { + return Key{}, err + } + + return Key{Element: &id}, nil +} + +func collectUnknown(columns map[uint32]column) []RawColumn { + specifications := make([]uint32, 0, len(columns)) + for specification := range columns { + specifications = append(specifications, specification) + } + + slices.Sort(specifications) + + result := make([]RawColumn, 0, len(columns)) + for _, specification := range specifications { + value := columns[specification] + result = append( + result, + RawColumn{ + Specification: value.specification, + Data: append([]byte(nil), value.data...), + }, + ) + } + + return result +} + +func assignOperations(changes []Change, operations []Operation) error { + byActor := make(map[ActorID][]int) + for i := range changes { + byActor[changes[i].Actor] = append(byActor[changes[i].Actor], i) + } + + for actor, indexes := range byActor { + slices.SortFunc( + indexes, + func(left, right int) int { + switch { + case changes[left].MaxOp < changes[right].MaxOp: + return -1 + case changes[left].MaxOp > changes[right].MaxOp: + return 1 + default: + return 0 + } + }, + ) + + var previous uint64 + for _, index := range indexes { + changes[index].StartOp = previous + 1 + previous = changes[index].MaxOp + } + + byActor[actor] = indexes + } + + for _, operation := range operations { + indexes := byActor[operation.ID.Actor] + + index, found := slices.BinarySearchFunc( + indexes, + operation.ID.Counter, + func(index int, counter uint64) int { + switch { + case changes[index].MaxOp < counter: + return -1 + case changes[index].MaxOp > counter: + return 1 + default: + return 0 + } + }, + ) + if !found { + if index >= len(indexes) { + return fmt.Errorf( + "operation %s@%d has no containing change", + operation.ID.Actor, + operation.ID.Counter, + ) + } + + // BinarySearchFunc returns the insertion point, which is the first + // change whose maxOp is greater than this counter. + } + + changeIndex := indexes[index] + if operation.ID.Counter < changes[changeIndex].StartOp { + return fmt.Errorf("operation counter precedes containing change start") + } + + changes[changeIndex].Operations = append(changes[changeIndex].Operations, operation) + } + + for i := range changes { + slices.SortFunc(changes[i].Operations, func(left, right Operation) int { + return left.ID.Compare(right.ID) + }) + } + + return nil +} + +func validateSnapshotGraph(changes []Change, heads []uint64) error { + dependedOn := make([]bool, len(changes)) + for i, change := range changes { + seen := make(map[uint64]struct{}, len(change.DependencyIndexes)) + for _, dependency := range change.DependencyIndexes { + if dependency >= uint64(len(changes)) { + return fmt.Errorf("change %d dependency %d is out of bounds", i, dependency) + } + + if dependency == uint64(i) { + return fmt.Errorf("change %d depends on itself", i) + } + + if _, ok := seen[dependency]; ok { + return fmt.Errorf("change %d repeats dependency %d", i, dependency) + } + + seen[dependency] = struct{}{} + dependedOn[dependency] = true + } + } + + if err := detectIndexCycle(changes); err != nil { + return err + } + + expectedHeads := make(map[uint64]struct{}) + + for i, isDependedOn := range dependedOn { + if !isDependedOn { + expectedHeads[uint64(i)] = struct{}{} + } + } + + actualHeads := make(map[uint64]struct{}, len(heads)) + for _, head := range heads { + if _, exists := actualHeads[head]; exists { + return fmt.Errorf("head index %d is repeated", head) + } + + actualHeads[head] = struct{}{} + } + + if !mapsEqual(expectedHeads, actualHeads) { + return fmt.Errorf("head indexes do not match graph frontier") + } + + return validateActorSequences(changes) +} + +func detectIndexCycle(changes []Change) error { + state := make([]uint8, len(changes)) + + var visit func(int) error + + visit = func(index int) error { + switch state[index] { + case 1: + return fmt.Errorf("dependency cycle includes change %d", index) + case 2: + return nil + } + + state[index] = 1 + for _, dependency := range changes[index].DependencyIndexes { + if err := visit(int(dependency)); err != nil { + return err + } + } + + state[index] = 2 + + return nil + } + for i := range changes { + if err := visit(i); err != nil { + return err + } + } + + return nil +} + +func mapsEqual[K comparable](left, right map[K]struct{}) bool { + if len(left) != len(right) { + return false + } + + for key := range left { + if _, ok := right[key]; !ok { + return false + } + } + + return true +} + +func validateActorSequences(changes []Change) error { + byActor := make(map[ActorID][]Change) + for _, change := range changes { + byActor[change.Actor] = append(byActor[change.Actor], change) + } + + for actor, actorChanges := range byActor { + slices.SortFunc(actorChanges, func(left, right Change) int { + switch { + case left.Sequence < right.Sequence: + return -1 + case left.Sequence > right.Sequence: + return 1 + default: + return 0 + } + }) + + var previousMax uint64 + + for i, change := range actorChanges { + expectedSequence := uint64(i + 1) + if change.Sequence != expectedSequence { + return fmt.Errorf( + "actor %s has sequence %d, expected %d", + actor, + change.Sequence, + expectedSequence, + ) + } + + if change.MaxOp <= previousMax { + return fmt.Errorf( + "actor %s sequence %d has non-increasing maxOp %d", + actor, + change.Sequence, + change.MaxOp, + ) + } + + previousMax = change.MaxOp + } + } + + return nil +} + +func mergeActors(existing, additions []ActorID) []ActorID { + set := make(map[ActorID]struct{}, len(existing)+len(additions)) + for _, actor := range existing { + set[actor] = struct{}{} + } + + for _, actor := range additions { + set[actor] = struct{}{} + } + + result := make([]ActorID, 0, len(set)) + for actor := range set { + result = append(result, actor) + } + + slices.SortFunc(result, func(left, right ActorID) int { + return left.Compare(right) + }) + + return result +} + +func validateDocument(document *Document) error { + if len(document.Changes) == 0 { + if len(document.Heads) != 0 { + return fmt.Errorf("empty history has heads") + } + + return nil + } + + if len(document.ChunkTypes) > 0 && document.ChunkTypes[0] == ChunkDocument { + return validateChangeChunksAfterSnapshot(document) + } + + return validateChangeChunkGraph(document) +} + +func validateChangeChunksAfterSnapshot(document *Document) error { + known := make(map[ChangeHash]struct{}) + dependedOn := make(map[ChangeHash]struct{}) + + for _, change := range document.Changes { + if change.Hash != nil { + if _, exists := known[*change.Hash]; exists { + return fmt.Errorf("duplicate known change hash %s", change.Hash) + } + + known[*change.Hash] = struct{}{} + } + } + + for _, change := range document.Changes { + if len(change.DependencyIndexes) > 0 { + continue + } + + for _, dependency := range change.Dependencies { + if _, ok := known[dependency]; !ok { + return fmt.Errorf("change %s has missing dependency %s", change.Hash, dependency) + } + + dependedOn[dependency] = struct{}{} + } + } + + document.Heads = document.Heads[:0] + + for hash := range known { + if _, ok := dependedOn[hash]; !ok { + document.Heads = append(document.Heads, hash) + } + } + + slices.SortFunc(document.Heads, func(left, right ChangeHash) int { + return bytes.Compare(left[:], right[:]) + }) + + return validateActorSequences(document.Changes) +} + +func validateChangeChunkGraph(document *Document) error { + changes := make(map[ChangeHash]Change, len(document.Changes)) + dependedOn := make(map[ChangeHash]struct{}) + + for _, change := range document.Changes { + if change.Hash == nil { + return fmt.Errorf("change chunk has no hash") + } + + if _, exists := changes[*change.Hash]; exists { + return fmt.Errorf("duplicate change %s", change.Hash) + } + + changes[*change.Hash] = change + } + + for _, change := range document.Changes { + for _, dependency := range change.Dependencies { + if _, ok := changes[dependency]; !ok { + return fmt.Errorf("change %s has missing dependency %s", change.Hash, dependency) + } + + dependedOn[dependency] = struct{}{} + } + } + + document.Heads = document.Heads[:0] + + for hash := range changes { + if _, ok := dependedOn[hash]; !ok { + document.Heads = append(document.Heads, hash) + } + } + + slices.SortFunc(document.Heads, func(left, right ChangeHash) int { + return bytes.Compare(left[:], right[:]) + }) + + return validateActorSequences(document.Changes) +} diff --git a/pkg/automerge/internal/reference/reference.go b/pkg/automerge/internal/reference/reference.go new file mode 100644 index 0000000000..4657ea64b7 --- /dev/null +++ b/pkg/automerge/internal/reference/reference.go @@ -0,0 +1,705 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package reference + +import ( + "context" + "crypto/rand" + "crypto/sha256" + _ "embed" + "encoding/hex" + "fmt" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" +) + +//go:embed reference.wasm +var wasm []byte + +//go:embed reference.wasm.sha256 +var wasmChecksum []byte + +const ( + referenceABIVersion uint64 = 1 + referenceMemoryLimitPages uint32 = 1_024 +) + +type ( + Object = uint32 + + Backend struct { + module api.Module + } +) + +var ( + runtimeOnce sync.Once + runtimeInstance wazero.Runtime + compiledModule wazero.CompiledModule + runtimeErr error + moduleSequence atomic.Uint64 +) + +func New(ctx context.Context) (*Backend, error) { + backend, err := instantiate(ctx) + if err != nil { + return nil, fmt.Errorf("cannot instantiate Automerge reference backend: %w", err) + } + + if err := backend.run(ctx, "am_create"); err != nil { + _ = backend.Close(ctx) + return nil, fmt.Errorf("cannot create Automerge document: %w", err) + } + + return backend, nil +} + +func Load(ctx context.Context, document []byte) (*Backend, error) { + backend, err := instantiate(ctx) + if err != nil { + return nil, fmt.Errorf("cannot instantiate Automerge reference backend: %w", err) + } + + if err := backend.runBytes(ctx, "am_load", document); err != nil { + _ = backend.Close(ctx) + return nil, fmt.Errorf("cannot load Automerge document: %w", err) + } + + return backend, nil +} + +func instantiate(ctx context.Context) (*Backend, error) { + runtimeOnce.Do( + func() { + fields := strings.Fields(string(wasmChecksum)) + if len(fields) == 0 { + runtimeErr = fmt.Errorf("automerge reference checksum is empty") + return + } + + checksum := sha256.Sum256(wasm) + + actualChecksum := hex.EncodeToString(checksum[:]) + if !strings.EqualFold(fields[0], actualChecksum) { + runtimeErr = fmt.Errorf( + "automerge reference checksum mismatch: expected %s, got %s", + fields[0], + actualChecksum, + ) + + return + } + + runtimeContext := context.Background() + + runtimeInstance = wazero.NewRuntimeWithConfig( + runtimeContext, + wazero.NewRuntimeConfig().WithMemoryLimitPages(referenceMemoryLimitPages), + ) + if _, err := wasi_snapshot_preview1.Instantiate(runtimeContext, runtimeInstance); err != nil { + runtimeErr = fmt.Errorf("cannot instantiate WASI: %w", err) + return + } + + compiledModule, runtimeErr = runtimeInstance.CompileModule(runtimeContext, wasm) + if runtimeErr != nil { + runtimeErr = fmt.Errorf("cannot compile reference module: %w", runtimeErr) + } + }, + ) + + if runtimeErr != nil { + return nil, runtimeErr + } + + name := fmt.Sprintf("automerge-reference-%d", moduleSequence.Add(1)) + + module, err := runtimeInstance.InstantiateModule( + ctx, + compiledModule, + wazero.NewModuleConfig().WithName(name).WithRandSource(rand.Reader), + ) + if err != nil { + return nil, fmt.Errorf("cannot instantiate reference module: %w", err) + } + + backend := &Backend{module: module} + + version, err := backend.call(ctx, "am_abi_version") + if err != nil { + _ = module.Close(ctx) + return nil, fmt.Errorf("cannot read reference ABI version: %w", err) + } + + if version[0] != referenceABIVersion { + _ = module.Close(ctx) + + return nil, fmt.Errorf( + "unsupported reference ABI version %d, expected %d", + version[0], + referenceABIVersion, + ) + } + + return backend, nil +} + +func (b *Backend) Close(ctx context.Context) error { + if err := b.module.Close(ctx); err != nil { + return fmt.Errorf("cannot close reference module: %w", err) + } + + return nil +} + +func (b *Backend) Save(ctx context.Context) ([]byte, error) { + if err := b.run(ctx, "am_save"); err != nil { + return nil, fmt.Errorf("cannot save reference document: %w", err) + } + + output, err := b.output(ctx) + if err != nil { + return nil, fmt.Errorf("cannot read saved reference document: %w", err) + } + + return output, nil +} + +func (b *Backend) SetActor(ctx context.Context, actor []byte) error { + if err := b.runBytes(ctx, "am_set_actor", actor); err != nil { + return fmt.Errorf("cannot set reference actor: %w", err) + } + + return nil +} + +func (b *Backend) PutString(ctx context.Context, object Object, key, value string) error { + keyPointer, keyLength, err := b.write(ctx, []byte(key)) + if err != nil { + return fmt.Errorf("cannot write map key: %w", err) + } + defer b.free(ctx, keyPointer, keyLength) + + valuePointer, valueLength, err := b.write(ctx, []byte(value)) + if err != nil { + return fmt.Errorf("cannot write map value: %w", err) + } + defer b.free(ctx, valuePointer, valueLength) + + if err := b.run( + ctx, + "am_put_string", + uint64(object), + uint64(keyPointer), + uint64(keyLength), + uint64(valuePointer), + uint64(valueLength), + ); err != nil { + return fmt.Errorf("cannot put reference map value: %w", err) + } + + return nil +} + +func (b *Backend) PutText(ctx context.Context, object Object, key string) (Object, error) { + keyPointer, keyLength, err := b.write(ctx, []byte(key)) + if err != nil { + return 0, fmt.Errorf("cannot write text key: %w", err) + } + defer b.free(ctx, keyPointer, keyLength) + + result, err := b.call( + ctx, + "am_put_text", + uint64(object), + uint64(keyPointer), + uint64(keyLength), + ) + if err != nil { + return 0, fmt.Errorf("cannot create reference text: %w", err) + } + + handle := int64(result[0]) + if handle < 0 { + return 0, b.operationError(ctx, "cannot create reference text") + } + + return Object(handle), nil +} + +func (b *Backend) GetText(ctx context.Context, object Object, key string) (Object, error) { + keyPointer, keyLength, err := b.write(ctx, []byte(key)) + if err != nil { + return 0, fmt.Errorf("cannot write text key: %w", err) + } + defer b.free(ctx, keyPointer, keyLength) + + result, err := b.call( + ctx, + "am_get_text", + uint64(object), + uint64(keyPointer), + uint64(keyLength), + ) + if err != nil { + return 0, fmt.Errorf("cannot get reference text: %w", err) + } + + handle := int64(result[0]) + if handle < 0 { + return 0, b.operationError(ctx, "cannot get reference text") + } + + return Object(handle), nil +} + +func (b *Backend) SpliceText( + ctx context.Context, + object Object, + index uint32, + deleteCount int32, + value string, +) error { + valuePointer, valueLength, err := b.write(ctx, []byte(value)) + if err != nil { + return fmt.Errorf("cannot write splice value: %w", err) + } + defer b.free(ctx, valuePointer, valueLength) + + if err := b.run( + ctx, + "am_text_splice", + uint64(object), + uint64(index), + uint64(uint32(deleteCount)), + uint64(valuePointer), + uint64(valueLength), + ); err != nil { + return fmt.Errorf("cannot splice reference text: %w", err) + } + + return nil +} + +func (b *Backend) Text(ctx context.Context, object Object) (string, error) { + if err := b.run(ctx, "am_text", uint64(object)); err != nil { + return "", fmt.Errorf("cannot read reference text: %w", err) + } + + output, err := b.output(ctx) + if err != nil { + return "", fmt.Errorf("cannot copy reference text: %w", err) + } + + return string(output), nil +} + +func (b *Backend) TextSpans(ctx context.Context, object Object) ([]byte, error) { + if err := b.run(ctx, "am_text_spans", uint64(object)); err != nil { + return nil, fmt.Errorf("cannot read reference text spans: %w", err) + } + + output, err := b.output(ctx) + if err != nil { + return nil, fmt.Errorf("cannot copy reference text spans: %w", err) + } + + return output, nil +} + +func (b *Backend) TextCursor(ctx context.Context, object Object, index uint32) ([]byte, error) { + if err := b.run( + ctx, + "am_text_cursor", + uint64(object), + uint64(index), + ); err != nil { + return nil, fmt.Errorf("cannot create reference text cursor: %w", err) + } + + cursor, err := b.output(ctx) + if err != nil { + return nil, fmt.Errorf("cannot copy reference text cursor: %w", err) + } + + return cursor, nil +} + +func (b *Backend) TextCursorPosition( + ctx context.Context, + object Object, + cursor []byte, +) (uint32, error) { + pointer, length, err := b.write(ctx, cursor) + if err != nil { + return 0, fmt.Errorf("cannot write reference text cursor: %w", err) + } + defer b.free(ctx, pointer, length) + + result, err := b.call( + ctx, + "am_text_cursor_position", + uint64(object), + uint64(pointer), + uint64(length), + ) + if err != nil { + return 0, fmt.Errorf("cannot resolve reference text cursor: %w", err) + } + + position := int64(result[0]) + if position < 0 { + return 0, b.operationError(ctx, "cannot resolve reference text cursor") + } + + if position > int64(^uint32(0)) { + return 0, fmt.Errorf("cannot resolve reference text cursor: position %d exceeds uint32", position) + } + + return uint32(position), nil +} + +func (b *Backend) Commit( + ctx context.Context, + message string, + timestamp time.Time, +) ([32]byte, error) { + var hash [32]byte + + messagePointer, messageLength, err := b.write(ctx, []byte(message)) + if err != nil { + return hash, fmt.Errorf("cannot write commit message: %w", err) + } + defer b.free(ctx, messagePointer, messageLength) + + if err := b.run( + ctx, + "am_commit", + uint64(messagePointer), + uint64(messageLength), + uint64(timestamp.Unix()), + ); err != nil { + return hash, fmt.Errorf("cannot commit reference document: %w", err) + } + + output, err := b.output(ctx) + if err != nil { + return hash, fmt.Errorf("cannot copy reference commit hash: %w", err) + } + + if len(output) != len(hash) { + return hash, fmt.Errorf("cannot decode reference commit hash: expected 32 bytes, got %d", len(output)) + } + + copy(hash[:], output) + + return hash, nil +} + +func (b *Backend) Heads(ctx context.Context) ([][32]byte, error) { + if err := b.run(ctx, "am_heads"); err != nil { + return nil, fmt.Errorf("cannot read reference heads: %w", err) + } + + output, err := b.output(ctx) + if err != nil { + return nil, fmt.Errorf("cannot copy reference heads: %w", err) + } + + if len(output)%32 != 0 { + return nil, fmt.Errorf("cannot decode reference heads: length %d is not divisible by 32", len(output)) + } + + heads := make([][32]byte, len(output)/32) + for i := range heads { + copy(heads[i][:], output[i*32:(i+1)*32]) + } + + return heads, nil +} + +func (b *Backend) Merge(ctx context.Context, other []byte) ([][32]byte, error) { + if err := b.runBytes(ctx, "am_merge", other); err != nil { + return nil, fmt.Errorf("cannot merge reference document: %w", err) + } + + output, err := b.output(ctx) + if err != nil { + return nil, fmt.Errorf("cannot copy merged reference heads: %w", err) + } + + if len(output)%32 != 0 { + return nil, fmt.Errorf("cannot decode merged reference heads: length %d is not divisible by 32", len(output)) + } + + heads := make([][32]byte, len(output)/32) + for i := range heads { + copy(heads[i][:], output[i*32:(i+1)*32]) + } + + return heads, nil +} + +func (b *Backend) NewSyncState(ctx context.Context) (uint32, error) { + result, err := b.call(ctx, "am_sync_new") + if err != nil { + return 0, fmt.Errorf("cannot create reference sync state: %w", err) + } + + handle := int64(result[0]) + if handle < 0 { + return 0, b.operationError(ctx, "cannot create reference sync state") + } + + return uint32(handle), nil +} + +func (b *Backend) CloseSyncState(ctx context.Context, handle uint32) error { + if err := b.run(ctx, "am_sync_free", uint64(handle)); err != nil { + return fmt.Errorf("cannot close reference sync state: %w", err) + } + + return nil +} + +func (b *Backend) GenerateSyncMessage( + ctx context.Context, + handle uint32, +) ([]byte, bool, error) { + if err := b.run(ctx, "am_sync_generate", uint64(handle)); err != nil { + return nil, false, fmt.Errorf("cannot generate reference sync message: %w", err) + } + + message, err := b.output(ctx) + if err != nil { + return nil, false, fmt.Errorf("cannot copy reference sync message: %w", err) + } + + return message, len(message) > 0, nil +} + +func (b *Backend) ReceiveSyncMessage(ctx context.Context, handle uint32, message []byte) error { + pointer, length, err := b.write(ctx, message) + if err != nil { + return fmt.Errorf("cannot write reference sync message: %w", err) + } + defer b.free(ctx, pointer, length) + + if err := b.run( + ctx, + "am_sync_receive", + uint64(handle), + uint64(pointer), + uint64(length), + ); err != nil { + return fmt.Errorf("cannot receive reference sync message: %w", err) + } + + return nil +} + +func (b *Backend) SaveSyncState(ctx context.Context, handle uint32) ([]byte, error) { + if err := b.run(ctx, "am_sync_save", uint64(handle)); err != nil { + return nil, fmt.Errorf("cannot save reference sync state: %w", err) + } + + data, err := b.output(ctx) + if err != nil { + return nil, fmt.Errorf("cannot copy reference sync state: %w", err) + } + + return data, nil +} + +func (b *Backend) LoadSyncState(ctx context.Context, data []byte) (uint32, error) { + pointer, length, err := b.write(ctx, data) + if err != nil { + return 0, fmt.Errorf("cannot write reference sync state: %w", err) + } + defer b.free(ctx, pointer, length) + + result, err := b.call(ctx, "am_sync_load", uint64(pointer), uint64(length)) + if err != nil { + return 0, fmt.Errorf("cannot load reference sync state: %w", err) + } + + handle := int64(result[0]) + if handle < 0 { + return 0, b.operationError(ctx, "cannot load reference sync state") + } + + return uint32(handle), nil +} + +func (b *Backend) runBytes(ctx context.Context, function string, value []byte) error { + pointer, length, err := b.write(ctx, value) + if err != nil { + return fmt.Errorf("cannot write operation input: %w", err) + } + defer b.free(ctx, pointer, length) + + if err := b.run(ctx, function, uint64(pointer), uint64(length)); err != nil { + return fmt.Errorf("cannot run operation with input: %w", err) + } + + return nil +} + +func (b *Backend) run(ctx context.Context, function string, parameters ...uint64) error { + result, err := b.call(ctx, function, parameters...) + if err != nil { + return fmt.Errorf("cannot call %s: %w", function, err) + } + + if int32(result[0]) != 0 { + return b.operationError(ctx, fmt.Sprintf("%s failed", function)) + } + + return nil +} + +func (b *Backend) operationError(ctx context.Context, fallback string) error { + result, err := b.call(ctx, "am_error_len") + if err != nil { + return fmt.Errorf("%s: cannot read error length: %w", fallback, err) + } + + length := uint32(result[0]) + if length == 0 { + return fmt.Errorf("%s", fallback) + } + + pointer, err := b.alloc(ctx, length) + if err != nil { + return fmt.Errorf("%s: cannot allocate error buffer: %w", fallback, err) + } + defer b.free(ctx, pointer, length) + + if _, err := b.call(ctx, "am_error_copy", uint64(pointer)); err != nil { + return fmt.Errorf("%s: cannot copy error: %w", fallback, err) + } + + message, err := b.read(pointer, length) + if err != nil { + return fmt.Errorf("%s: cannot read error: %w", fallback, err) + } + + return fmt.Errorf("%s: %s", fallback, message) +} + +func (b *Backend) output(ctx context.Context) ([]byte, error) { + result, err := b.call(ctx, "am_output_len") + if err != nil { + return nil, fmt.Errorf("cannot read output length: %w", err) + } + + length := uint32(result[0]) + if length == 0 { + return nil, nil + } + + pointer, err := b.alloc(ctx, length) + if err != nil { + return nil, fmt.Errorf("cannot allocate output buffer: %w", err) + } + defer b.free(ctx, pointer, length) + + if _, err := b.call(ctx, "am_output_copy", uint64(pointer)); err != nil { + return nil, fmt.Errorf("cannot copy output: %w", err) + } + + output, err := b.read(pointer, length) + if err != nil { + return nil, fmt.Errorf("cannot read output: %w", err) + } + + return output, nil +} + +func (b *Backend) call(ctx context.Context, function string, parameters ...uint64) ([]uint64, error) { + exported := b.module.ExportedFunction(function) + if exported == nil { + return nil, fmt.Errorf("reference module does not export %q", function) + } + + result, err := exported.Call(ctx, parameters...) + if err != nil { + return nil, fmt.Errorf("cannot execute reference export %q: %w", function, err) + } + + return result, nil +} + +func (b *Backend) alloc(ctx context.Context, length uint32) (uint32, error) { + result, err := b.call(ctx, "am_alloc", uint64(length)) + if err != nil { + return 0, fmt.Errorf("cannot allocate reference memory: %w", err) + } + + pointer := uint32(result[0]) + if pointer == 0 && length > 0 { + return 0, fmt.Errorf("cannot allocate %d bytes of reference memory", length) + } + + return pointer, nil +} + +func (b *Backend) free(ctx context.Context, pointer, length uint32) { + if pointer == 0 || length == 0 { + return + } + + _, _ = b.call(ctx, "am_free", uint64(pointer), uint64(length)) +} + +func (b *Backend) write(ctx context.Context, value []byte) (uint32, uint32, error) { + length := uint32(len(value)) + if length == 0 { + return 0, 0, nil + } + + pointer, err := b.alloc(ctx, length) + if err != nil { + return 0, 0, fmt.Errorf("cannot allocate input: %w", err) + } + + if !b.module.Memory().Write(pointer, value) { + b.free(ctx, pointer, length) + return 0, 0, fmt.Errorf("cannot write %d bytes at reference memory offset %d", length, pointer) + } + + return pointer, length, nil +} + +func (b *Backend) read(pointer, length uint32) ([]byte, error) { + value, ok := b.module.Memory().Read(pointer, length) + if !ok { + return nil, fmt.Errorf("cannot read %d bytes at reference memory offset %d", length, pointer) + } + + return append([]byte(nil), value...), nil +} diff --git a/pkg/automerge/internal/reference/reference.wasm b/pkg/automerge/internal/reference/reference.wasm new file mode 100755 index 0000000000..8103a952a2 Binary files /dev/null and b/pkg/automerge/internal/reference/reference.wasm differ diff --git a/pkg/automerge/internal/reference/reference.wasm.sha256 b/pkg/automerge/internal/reference/reference.wasm.sha256 new file mode 100644 index 0000000000..162f05e65b --- /dev/null +++ b/pkg/automerge/internal/reference/reference.wasm.sha256 @@ -0,0 +1 @@ +45f1556553f5a265141adc4565079b68fa1b6aa75c27b29bcc0d309feba1fa96 reference.wasm diff --git a/pkg/automerge/internal/reference/wasm/Cargo.lock b/pkg/automerge/internal/reference/wasm/Cargo.lock new file mode 100644 index 0000000000..a3f3349362 --- /dev/null +++ b/pkg/automerge/internal/reference/wasm/Cargo.lock @@ -0,0 +1,470 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "automerge" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b78abcbba93428b9465b26cb2816a5b4654cce507f099a84a8c1b311cb3633" +dependencies = [ + "cfg-if", + "flate2", + "getrandom", + "hex", + "hexane", + "itertools", + "leb128", + "rand", + "rustc-hash", + "serde", + "sha2", + "smol_str", + "thiserror", + "tinyvec", + "tracing", + "unicode-segmentation", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "borsh" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a88b7ea17d208c4193f2c1e6de3c35fe71f98c96982d5ced308bdcc749ff6e1f" +dependencies = [ + "bytes", + "cfg_aliases", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hexane" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f4ecba0bb4e14df997df7cab6d1b584c6432d8865cf2adfced814706d715a7b" +dependencies = [ + "leb128", + "thiserror", +] + +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "leb128" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c83bff1d572d6b9aeef67ddfc8448e4a3737909cb28e81f97c791b9018703e52" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "probo-automerge-reference" +version = "0.1.0" +dependencies = [ + "automerge", + "serde_json", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" +dependencies = [ + "borsh", + "serde_core", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/pkg/automerge/internal/reference/wasm/Cargo.toml b/pkg/automerge/internal/reference/wasm/Cargo.toml new file mode 100644 index 0000000000..14bce320d2 --- /dev/null +++ b/pkg/automerge/internal/reference/wasm/Cargo.toml @@ -0,0 +1,40 @@ +# Copyright (c) 2026 Probo Inc . +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +[package] +name = "probo-automerge-reference" +version = "0.1.0" +edition = "2021" +license = "MIT" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +automerge = { version = "=0.10.0", features = ["utf16-indexing"] } +serde_json = "1" + +[profile.release] +codegen-units = 1 +lto = true +opt-level = "z" +panic = "abort" +strip = true diff --git a/pkg/automerge/internal/reference/wasm/deny.toml b/pkg/automerge/internal/reference/wasm/deny.toml new file mode 100644 index 0000000000..58b6d073f0 --- /dev/null +++ b/pkg/automerge/internal/reference/wasm/deny.toml @@ -0,0 +1,257 @@ +# Copyright (c) 2026 Probo Inc . +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# This template contains all of the possible sections and their default values + +# Note that all fields that take a lint level have these possible values: +# * deny - An error will be produced and the check will fail +# * warn - A warning will be produced, but the check will not fail +# * allow - No warning or error will be produced, though in some cases a note +# will be + +# The values provided in this template are the default values that will be used +# when any section or field is not specified in your own configuration + +# Root options + +# The graph table configures how the dependency graph is constructed and thus +# which crates the checks are performed against +[graph] +# If 1 or more target triples (and optionally, target_features) are specified, +# only the specified targets will be checked when running `cargo deny check`. +# This means, if a particular package is only ever used as a target specific +# dependency, such as, for example, the `nix` crate only being used via the +# `target_family = "unix"` configuration, that only having windows targets in +# this list would mean the nix crate, as well as any of its exclusive +# dependencies not shared by any other crates, would be ignored, as the target +# list here is effectively saying which targets you are building for. +targets = [ + "wasm32-wasip1", + # You can also specify which target_features you promise are enabled for a + # particular target. target_features are currently not validated against + # the actual valid features supported by the target architecture. + #{ triple = "wasm32-unknown-unknown", features = ["atomics"] }, +] +# When creating the dependency graph used as the source of truth when checks are +# executed, this field can be used to prune crates from the graph, removing them +# from the view of cargo-deny. This is an extremely heavy hammer, as if a crate +# is pruned from the graph, all of its dependencies will also be pruned unless +# they are connected to another crate in the graph that hasn't been pruned, +# so it should be used with care. The identifiers are [Package ID Specifications] +# (https://doc.rust-lang.org/cargo/reference/pkgid-spec.html) +#exclude = [] +# If true, metadata will be collected with `--all-features`. Note that this can't +# be toggled off if true, if you want to conditionally enable `--all-features` it +# is recommended to pass `--all-features` on the cmd line instead +all-features = false +# If true, metadata will be collected with `--no-default-features`. The same +# caveat with `all-features` applies +no-default-features = false +# If set, these feature will be enabled when collecting metadata. If `--features` +# is specified on the cmd line they will take precedence over this option. +#features = [] + +# The output table provides options for how/if diagnostics are outputted +[output] +# When outputting inclusion graphs in diagnostics that include features, this +# option can be used to specify the depth at which feature edges will be added. +# This option is included since the graphs can be quite large and the addition +# of features from the crate(s) to all of the graph roots can be far too verbose. +# This option can be overridden via `--feature-depth` on the cmd line +feature-depth = 1 + +# This section is considered when running `cargo deny check advisories` +# More documentation for the advisories section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html +[advisories] +# The path where the advisory databases are cloned/fetched into +#db-path = "$CARGO_HOME/advisory-dbs" +# The url(s) of the advisory databases to use +#db-urls = ["https://github.com/rustsec/advisory-db"] +# A list of advisory IDs to ignore. Note that ignored advisories will still +# output a note when they are encountered. +ignore = [ + #"RUSTSEC-0000-0000", + #{ id = "RUSTSEC-0000-0000", reason = "you can specify a reason the advisory is ignored" }, + #"a-crate-that-is-yanked@0.1.1", # you can also ignore yanked crate versions if you wish + #{ crate = "a-crate-that-is-yanked@0.1.1", reason = "you can specify why you are ignoring the yanked crate" }, +] +# If this is true, then cargo deny will use the git executable to fetch advisory database. +# If this is false, then it uses a built-in git library. +# Setting this to true can be helpful if you have special authentication requirements that cargo-deny does not support. +# See Git Authentication for more information about setting up git authentication. +#git-fetch-with-cli = true + +# This section is considered when running `cargo deny check licenses` +# More documentation for the licenses section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html +[licenses] +# List of explicitly allowed licenses +# See https://spdx.org/licenses/ for list of possible licenses +# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. +allow = [ + "MIT", + "Apache-2.0", + "Unicode-3.0", + "Zlib", +] +# The confidence threshold for detecting a license from license text. +# The higher the value, the more closely the license text must be to the +# canonical license text of a valid SPDX license file. +# [possible values: any between 0.0 and 1.0]. +confidence-threshold = 0.8 +# Allow 1 or more licenses on a per-crate basis, so that particular licenses +# aren't accepted for every possible crate as with the normal allow list +exceptions = [ + # Each entry is the crate and version constraint, and its specific allow + # list + #{ allow = ["Zlib"], crate = "adler32" }, +] + +# Some crates don't have (easily) machine readable licensing information, +# adding a clarification entry for it allows you to manually specify the +# licensing information +#[[licenses.clarify]] +# The package spec the clarification applies to +#crate = "ring" +# The SPDX expression for the license requirements of the crate +#expression = "MIT AND ISC AND OpenSSL" +# One or more files in the crate's source used as the "source of truth" for +# the license expression. If the contents match, the clarification will be used +# when running the license check, otherwise the clarification will be ignored +# and the crate will be checked normally, which may produce warnings or errors +# depending on the rest of your configuration +#license-files = [ +# Each entry is a crate relative path, and the (opaque) hash of its contents +#{ path = "LICENSE", hash = 0xbd0eed23 } +#] + +[licenses.private] +# If true, ignores workspace crates that aren't published, or are only +# published to private registries. +# To see how to mark a crate as unpublished (to the official registry), +# visit https://doc.rust-lang.org/cargo/reference/manifest.html#the-publish-field. +ignore = true +# One or more private registries that you might publish crates to, if a crate +# is only published to private registries, and ignore is true, the crate will +# not have its license(s) checked +registries = [ + #"https://sekretz.com/registry +] + +# This section is considered when running `cargo deny check bans`. +# More documentation about the 'bans' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html +[bans] +# Lint level for when multiple versions of the same crate are detected +multiple-versions = "warn" +# Lint level for when a crate version requirement is `*` +wildcards = "deny" +# The graph highlighting used when creating dotgraphs for crates +# with multiple versions +# * lowest-version - The path to the lowest versioned duplicate is highlighted +# * simplest-path - The path to the version with the fewest edges is highlighted +# * all - Both lowest-version and simplest-path are used +highlight = "all" +# The default lint level for `default` features for crates that are members of +# the workspace that is being checked. This can be overridden by allowing/denying +# `default` on a crate-by-crate basis if desired. +workspace-default-features = "allow" +# The default lint level for `default` features for external crates that are not +# members of the workspace. This can be overridden by allowing/denying `default` +# on a crate-by-crate basis if desired. +external-default-features = "allow" +# List of crates that are allowed. Use with care! +allow = [ + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason it is allowed" }, +] +# If true, workspace members are automatically allowed even when using deny-by-default +# This is useful for organizations that want to deny all external dependencies by default +# but allow their own workspace crates without having to explicitly list them +allow-workspace = false +# List of crates to deny +deny = [ + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason it is banned" }, + # Wrapper crates can optionally be specified to allow the crate when it + # is a direct dependency of the otherwise banned crate + #{ crate = "ansi_term@0.11.0", wrappers = ["this-crate-directly-depends-on-ansi_term"] }, +] + +# List of features to allow/deny +# Each entry the name of a crate and a version range. If version is +# not specified, all versions will be matched. +#[[bans.features]] +#crate = "reqwest" +# Features to not allow +#deny = ["json"] +# Features to allow +#allow = [ +# "rustls", +# "__rustls", +# "__tls", +# "hyper-rustls", +# "rustls", +# "rustls-pemfile", +# "rustls-tls-webpki-roots", +# "tokio-rustls", +# "webpki-roots", +#] +# If true, the allowed features must exactly match the enabled feature set. If +# this is set there is no point setting `deny` +#exact = true + +# Certain crates/versions that will be skipped when doing duplicate detection. +skip = [ + { crate = "syn@2.0.119", reason = "Automerge tracing dependencies have not migrated to syn 3 yet" }, +] +# Similarly to `skip` allows you to skip certain crates during duplicate +# detection. Unlike skip, it also includes the entire tree of transitive +# dependencies starting at the specified crate, up to a certain depth, which is +# by default infinite. +skip-tree = [ + #"ansi_term@0.11.0", # will be skipped along with _all_ of its direct and transitive dependencies + #{ crate = "ansi_term@0.11.0", depth = 20 }, +] + +# This section is considered when running `cargo deny check sources`. +# More documentation about the 'sources' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html +[sources] +# Lint level for what to happen when a crate from a crate registry that is not +# in the allow list is encountered +unknown-registry = "deny" +# Lint level for what to happen when a crate from a git repository that is not +# in the allow list is encountered +unknown-git = "deny" +# List of URLs for allowed crate registries. Defaults to the crates.io index +# if not specified. If it is specified but empty, no registries are allowed. +allow-registry = ["https://github.com/rust-lang/crates.io-index"] +# List of URLs for allowed Git repositories +allow-git = [] + +[sources.allow-org] +# github.com organizations to allow git sources for +github = [] +# gitlab.com organizations to allow git sources for +gitlab = [] +# bitbucket.org organizations to allow git sources for +bitbucket = [] diff --git a/pkg/automerge/internal/reference/wasm/src/lib.rs b/pkg/automerge/internal/reference/wasm/src/lib.rs new file mode 100644 index 0000000000..ea67238ed4 --- /dev/null +++ b/pkg/automerge/internal/reference/wasm/src/lib.rs @@ -0,0 +1,710 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +use std::cell::RefCell; +use std::mem; +use std::ptr; +use std::slice; + +use automerge::sync::{Message, State as SyncState, SyncDoc}; +use automerge::transaction::{CommitOptions, Transactable}; +use automerge::{ActorId, AutoCommit, Cursor, ObjId, ObjType, ReadDoc, Value, ROOT}; + +struct State { + doc: AutoCommit, + objects: Vec, + sync_states: Vec>, + output: Vec, + error: String, +} + +impl State { + fn new() -> Self { + Self { + doc: AutoCommit::new(), + objects: vec![ROOT], + sync_states: Vec::new(), + output: Vec::new(), + error: String::new(), + } + } + + fn reset(&mut self, doc: AutoCommit) { + self.doc = doc; + self.objects.clear(); + self.objects.push(ROOT); + self.sync_states.clear(); + self.output.clear(); + self.error.clear(); + } + + fn object(&self, handle: u32) -> Result { + self.objects + .get(handle as usize) + .cloned() + .ok_or_else(|| format!("invalid object handle {handle}")) + } + + fn push_object(&mut self, object: ObjId) -> Result { + let handle = + u32::try_from(self.objects.len()).map_err(|_| "too many object handles".to_owned())?; + self.objects.push(object); + Ok(handle) + } + + fn fail(&mut self, error: impl ToString) -> i32 { + self.error = error.to_string(); + -1 + } +} + +thread_local! { + static STATE: RefCell = RefCell::new(State::new()); +} + +#[no_mangle] +pub extern "C" fn am_abi_version() -> u32 { + 1 +} + +fn input_bytes(pointer: u32, length: u32) -> Vec { + if length == 0 { + return Vec::new(); + } + + // The caller allocates this memory through am_alloc and keeps it alive for + // the duration of the call. + unsafe { slice::from_raw_parts(pointer as *const u8, length as usize).to_vec() } +} + +fn input_string(pointer: u32, length: u32) -> Result { + String::from_utf8(input_bytes(pointer, length)).map_err(|error| error.to_string()) +} + +#[no_mangle] +pub extern "C" fn am_alloc(length: u32) -> u32 { + if length == 0 { + return 0; + } + + let mut bytes = Vec::::with_capacity(length as usize); + let pointer = bytes.as_mut_ptr(); + mem::forget(bytes); + pointer as u32 +} + +#[no_mangle] +pub extern "C" fn am_free(pointer: u32, length: u32) { + if pointer == 0 || length == 0 { + return; + } + + unsafe { + drop(Vec::from_raw_parts(pointer as *mut u8, 0, length as usize)); + } +} + +#[no_mangle] +pub extern "C" fn am_output_len() -> u32 { + STATE.with(|state| state.borrow().output.len() as u32) +} + +#[no_mangle] +pub extern "C" fn am_output_copy(pointer: u32) -> i32 { + STATE.with(|state| { + let state = state.borrow(); + if !state.output.is_empty() && pointer == 0 { + return -1; + } + + unsafe { + ptr::copy_nonoverlapping( + state.output.as_ptr(), + pointer as *mut u8, + state.output.len(), + ); + } + 0 + }) +} + +#[no_mangle] +pub extern "C" fn am_error_len() -> u32 { + STATE.with(|state| state.borrow().error.len() as u32) +} + +#[no_mangle] +pub extern "C" fn am_error_copy(pointer: u32) -> i32 { + STATE.with(|state| { + let state = state.borrow(); + if !state.error.is_empty() && pointer == 0 { + return -1; + } + + unsafe { + ptr::copy_nonoverlapping(state.error.as_ptr(), pointer as *mut u8, state.error.len()); + } + 0 + }) +} + +#[no_mangle] +pub extern "C" fn am_create() -> i32 { + STATE.with(|state| state.borrow_mut().reset(AutoCommit::new())); + 0 +} + +#[no_mangle] +pub extern "C" fn am_load(pointer: u32, length: u32) -> i32 { + let bytes = input_bytes(pointer, length); + STATE.with(|state| { + let mut state = state.borrow_mut(); + match AutoCommit::load(&bytes) { + Ok(doc) => { + state.reset(doc); + 0 + } + Err(error) => state.fail(error), + } + }) +} + +#[no_mangle] +pub extern "C" fn am_save() -> i32 { + STATE.with(|state| { + let mut state = state.borrow_mut(); + state.output = state.doc.save(); + state.error.clear(); + }); + 0 +} + +#[no_mangle] +pub extern "C" fn am_set_actor(pointer: u32, length: u32) -> i32 { + let actor = input_bytes(pointer, length); + if actor.is_empty() { + return STATE.with(|state| state.borrow_mut().fail("actor ID cannot be empty")); + } + + STATE.with(|state| { + let mut state = state.borrow_mut(); + state.doc.set_actor(ActorId::from(actor)); + state.error.clear(); + }); + 0 +} + +#[no_mangle] +pub extern "C" fn am_put_string( + object_handle: u32, + key_pointer: u32, + key_length: u32, + value_pointer: u32, + value_length: u32, +) -> i32 { + let key = match input_string(key_pointer, key_length) { + Ok(key) => key, + Err(error) => return STATE.with(|state| state.borrow_mut().fail(error)), + }; + let value = match input_string(value_pointer, value_length) { + Ok(value) => value, + Err(error) => return STATE.with(|state| state.borrow_mut().fail(error)), + }; + + STATE.with(|state| { + let mut state = state.borrow_mut(); + let object = match state.object(object_handle) { + Ok(object) => object, + Err(error) => return state.fail(error), + }; + match state.doc.put(&object, key, value) { + Ok(()) => { + state.error.clear(); + 0 + } + Err(error) => state.fail(error), + } + }) +} + +#[no_mangle] +pub extern "C" fn am_put_text(object_handle: u32, key_pointer: u32, key_length: u32) -> i64 { + let key = match input_string(key_pointer, key_length) { + Ok(key) => key, + Err(error) => { + STATE.with(|state| state.borrow_mut().fail(error)); + return -1; + } + }; + + STATE.with(|state| { + let mut state = state.borrow_mut(); + let object = match state.object(object_handle) { + Ok(object) => object, + Err(error) => { + state.fail(error); + return -1; + } + }; + match state.doc.put_object(&object, key, ObjType::Text) { + Ok(text) => match state.push_object(text) { + Ok(handle) => { + state.error.clear(); + i64::from(handle) + } + Err(error) => { + state.fail(error); + -1 + } + }, + Err(error) => { + state.fail(error); + -1 + } + } + }) +} + +#[no_mangle] +pub extern "C" fn am_get_text(object_handle: u32, key_pointer: u32, key_length: u32) -> i64 { + let key = match input_string(key_pointer, key_length) { + Ok(key) => key, + Err(error) => { + STATE.with(|state| state.borrow_mut().fail(error)); + return -1; + } + }; + + STATE.with(|state| { + let mut state = state.borrow_mut(); + let object = match state.object(object_handle) { + Ok(object) => object, + Err(error) => { + state.fail(error); + return -1; + } + }; + match state.doc.get(&object, key) { + Ok(Some((Value::Object(ObjType::Text), text))) => match state.push_object(text) { + Ok(handle) => { + state.error.clear(); + i64::from(handle) + } + Err(error) => { + state.fail(error); + -1 + } + }, + Ok(Some(_)) => { + state.fail("value is not text"); + -1 + } + Ok(None) => { + state.fail("text does not exist"); + -1 + } + Err(error) => { + state.fail(error); + -1 + } + } + }) +} + +#[no_mangle] +pub extern "C" fn am_text_splice( + object_handle: u32, + index: u32, + delete_count: i32, + value_pointer: u32, + value_length: u32, +) -> i32 { + let value = match input_string(value_pointer, value_length) { + Ok(value) => value, + Err(error) => return STATE.with(|state| state.borrow_mut().fail(error)), + }; + + STATE.with(|state| { + let mut state = state.borrow_mut(); + let object = match state.object(object_handle) { + Ok(object) => object, + Err(error) => return state.fail(error), + }; + match state + .doc + .splice_text(&object, index as usize, delete_count as isize, &value) + { + Ok(()) => { + state.error.clear(); + 0 + } + Err(error) => state.fail(error), + } + }) +} + +#[no_mangle] +pub extern "C" fn am_text(object_handle: u32) -> i32 { + STATE.with(|state| { + let mut state = state.borrow_mut(); + let object = match state.object(object_handle) { + Ok(object) => object, + Err(error) => return state.fail(error), + }; + match state.doc.text(&object) { + Ok(text) => { + state.output = text.into_bytes(); + state.error.clear(); + 0 + } + Err(error) => state.fail(error), + } + }) +} + +#[no_mangle] +pub extern "C" fn am_text_spans(object_handle: u32) -> i32 { + STATE.with(|state| { + let mut state = state.borrow_mut(); + let object = match state.object(object_handle) { + Ok(object) => object, + Err(error) => return state.fail(error), + }; + let spans = match state.doc.spans(&object) { + Ok(spans) => spans, + Err(error) => return state.fail(error), + }; + + let values = spans + .map(|span| match span { + automerge::Span::Text { text, marks } => { + let mut value = serde_json::Map::new(); + value.insert("type".to_owned(), serde_json::json!("text")); + value.insert("value".to_owned(), serde_json::json!(text.to_string())); + if let Some(marks) = marks { + let marks = marks + .iter() + .map(|(name, value)| (name.to_string(), scalar_to_json(value))) + .collect(); + value.insert("marks".to_owned(), serde_json::Value::Object(marks)); + } + serde_json::Value::Object(value) + } + automerge::Span::Block(block) => serde_json::json!({ + "type": "block", + "value": hydrate_map_to_json(&block), + }), + }) + .collect::>(); + + match serde_json::to_vec(&values) { + Ok(output) => { + state.output = output; + state.error.clear(); + 0 + } + Err(error) => state.fail(error), + } + }) +} + +#[no_mangle] +pub extern "C" fn am_text_cursor(object_handle: u32, index: u32) -> i32 { + STATE.with(|state| { + let mut state = state.borrow_mut(); + let object = match state.object(object_handle) { + Ok(object) => object, + Err(error) => return state.fail(error), + }; + match state.doc.get_cursor(&object, index as usize, None) { + Ok(cursor) => { + state.output = cursor.to_bytes(); + state.error.clear(); + 0 + } + Err(error) => state.fail(error), + } + }) +} + +fn hydrate_map_to_json(map: &automerge::hydrate::Map) -> serde_json::Value { + serde_json::Value::Object( + map.iter() + .map(|(key, value)| (key.to_owned(), hydrate_value_to_json(&value.value))) + .collect(), + ) +} + +fn hydrate_value_to_json(value: &automerge::hydrate::Value) -> serde_json::Value { + match value { + automerge::hydrate::Value::Scalar(value) => scalar_to_json(value), + automerge::hydrate::Value::Map(value) => hydrate_map_to_json(value), + automerge::hydrate::Value::List(value) => serde_json::Value::Array( + value + .iter() + .map(|value| hydrate_value_to_json(&value.value)) + .collect(), + ), + automerge::hydrate::Value::Text(value) => serde_json::json!(value.to_string()), + } +} + +fn scalar_to_json(value: &automerge::ScalarValue) -> serde_json::Value { + match value { + automerge::ScalarValue::Bytes(value) => serde_json::json!(value), + automerge::ScalarValue::Str(value) => serde_json::json!(value), + automerge::ScalarValue::Int(value) => serde_json::json!(value), + automerge::ScalarValue::Uint(value) => serde_json::json!(value), + automerge::ScalarValue::F64(value) => serde_json::json!(value), + automerge::ScalarValue::Counter(value) => serde_json::json!(i64::from(value)), + automerge::ScalarValue::Timestamp(value) => serde_json::json!(value), + automerge::ScalarValue::Boolean(value) => serde_json::json!(value), + automerge::ScalarValue::Null => serde_json::Value::Null, + automerge::ScalarValue::Unknown { bytes, .. } => serde_json::json!(bytes), + } +} + +#[no_mangle] +pub extern "C" fn am_text_cursor_position(object_handle: u32, pointer: u32, length: u32) -> i64 { + let bytes = input_bytes(pointer, length); + let cursor = match Cursor::try_from(bytes) { + Ok(cursor) => cursor, + Err(error) => { + STATE.with(|state| state.borrow_mut().fail(error)); + return -1; + } + }; + + STATE.with(|state| { + let mut state = state.borrow_mut(); + let object = match state.object(object_handle) { + Ok(object) => object, + Err(error) => { + state.fail(error); + return -1; + } + }; + match state.doc.get_cursor_position(&object, &cursor, None) { + Ok(position) => match i64::try_from(position) { + Ok(position) => { + state.error.clear(); + position + } + Err(error) => { + state.fail(error); + -1 + } + }, + Err(error) => { + state.fail(error); + -1 + } + } + }) +} + +#[no_mangle] +pub extern "C" fn am_commit( + message_pointer: u32, + message_length: u32, + timestamp_seconds: i64, +) -> i32 { + let message = match input_string(message_pointer, message_length) { + Ok(message) => message, + Err(error) => return STATE.with(|state| state.borrow_mut().fail(error)), + }; + + STATE.with(|state| { + let mut state = state.borrow_mut(); + let options = CommitOptions::default() + .with_message(message) + .with_time(timestamp_seconds); + match state.doc.commit_with(options) { + Some(hash) => { + state.output = hash.as_ref().to_vec(); + state.error.clear(); + 0 + } + None => state.fail("change contains no operations"), + } + }) +} + +#[no_mangle] +pub extern "C" fn am_heads() -> i32 { + STATE.with(|state| { + let mut state = state.borrow_mut(); + state.output = state + .doc + .get_heads() + .into_iter() + .flat_map(|hash| hash.0) + .collect(); + state.error.clear(); + }); + 0 +} + +#[no_mangle] +pub extern "C" fn am_merge(pointer: u32, length: u32) -> i32 { + let bytes = input_bytes(pointer, length); + STATE.with(|state| { + let mut state = state.borrow_mut(); + let mut other = match AutoCommit::load(&bytes) { + Ok(other) => other, + Err(error) => return state.fail(error), + }; + match state.doc.merge(&mut other) { + Ok(heads) => { + state.output = heads.into_iter().flat_map(|hash| hash.0).collect(); + state.error.clear(); + 0 + } + Err(error) => state.fail(error), + } + }) +} + +#[no_mangle] +pub extern "C" fn am_sync_new() -> i64 { + STATE.with(|state| { + let mut state = state.borrow_mut(); + let handle = match u32::try_from(state.sync_states.len()) { + Ok(handle) => handle, + Err(_) => { + state.fail("too many sync states"); + return -1; + } + }; + state.sync_states.push(Some(SyncState::new())); + state.error.clear(); + i64::from(handle) + }) +} + +#[no_mangle] +pub extern "C" fn am_sync_free(handle: u32) -> i32 { + STATE.with(|state| { + let mut state = state.borrow_mut(); + let Some(sync_state) = state.sync_states.get_mut(handle as usize) else { + return state.fail(format!("invalid sync state handle {handle}")); + }; + if sync_state.take().is_none() { + return state.fail(format!("sync state handle {handle} is closed")); + } + state.error.clear(); + 0 + }) +} + +#[no_mangle] +pub extern "C" fn am_sync_generate(handle: u32) -> i32 { + STATE.with(|state| { + let mut state = state.borrow_mut(); + let State { + doc, + sync_states, + output, + error, + .. + } = &mut *state; + let Some(Some(sync_state)) = sync_states.get_mut(handle as usize) else { + return state.fail(format!("invalid sync state handle {handle}")); + }; + + *output = doc + .sync() + .generate_sync_message(sync_state) + .map(Message::encode) + .unwrap_or_default(); + error.clear(); + 0 + }) +} + +#[no_mangle] +pub extern "C" fn am_sync_receive(handle: u32, pointer: u32, length: u32) -> i32 { + let bytes = input_bytes(pointer, length); + let message = match Message::decode(&bytes) { + Ok(message) => message, + Err(error) => return STATE.with(|state| state.borrow_mut().fail(error)), + }; + + STATE.with(|state| { + let mut state = state.borrow_mut(); + if !matches!(state.sync_states.get(handle as usize), Some(Some(_))) { + return state.fail(format!("invalid sync state handle {handle}")); + } + + let result = { + let State { + doc, sync_states, .. + } = &mut *state; + let sync_state = sync_states[handle as usize].as_mut().unwrap(); + doc.sync().receive_sync_message(sync_state, message) + }; + match result { + Ok(()) => { + state.error.clear(); + 0 + } + Err(error) => state.fail(error), + } + }) +} + +#[no_mangle] +pub extern "C" fn am_sync_save(handle: u32) -> i32 { + STATE.with(|state| { + let mut state = state.borrow_mut(); + let Some(Some(sync_state)) = state.sync_states.get(handle as usize) else { + return state.fail(format!("invalid sync state handle {handle}")); + }; + state.output = sync_state.encode(); + state.error.clear(); + 0 + }) +} + +#[no_mangle] +pub extern "C" fn am_sync_load(pointer: u32, length: u32) -> i64 { + let bytes = input_bytes(pointer, length); + let sync_state = match SyncState::decode(&bytes) { + Ok(sync_state) => sync_state, + Err(error) => { + STATE.with(|state| state.borrow_mut().fail(error)); + return -1; + } + }; + + STATE.with(|state| { + let mut state = state.borrow_mut(); + let handle = match u32::try_from(state.sync_states.len()) { + Ok(handle) => handle, + Err(_) => { + state.fail("too many sync states"); + return -1; + } + }; + state.sync_states.push(Some(sync_state)); + state.error.clear(); + i64::from(handle) + }) +} diff --git a/pkg/automerge/native_backend_test.go b/pkg/automerge/native_backend_test.go new file mode 100644 index 0000000000..95484a4997 --- /dev/null +++ b/pkg/automerge/native_backend_test.go @@ -0,0 +1,550 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package automerge_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/automerge" +) + +func TestPureGoDocument_ReferenceLoadsNativeHistory(t *testing.T) { + t.Parallel() + + ctx := context.Background() + nativeDocument, err := automerge.NewPureGo(ctx, actor(40)) + require.NoError(t, err) + closeDocument(t, nativeDocument) + text, err := nativeDocument.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 0, 0, "Hello 😀")) + _, err = nativeDocument.Commit(ctx, "Create native document", commitTime) + require.NoError(t, err) + data, err := nativeDocument.Save(ctx) + require.NoError(t, err) + + referenceDocument, err := automerge.LoadReference(ctx, data, actor(41)) + require.NoError(t, err) + closeDocument(t, referenceDocument) + referenceText, err := referenceDocument.Text(ctx, "body") + require.NoError(t, err) + value, err := referenceText.String(ctx) + require.NoError(t, err) + assert.Equal(t, "Hello 😀", value) +} + +func TestPureGoDocument_ExtendsReferenceSnapshot(t *testing.T) { + t.Parallel() + + ctx := context.Background() + nativeDocument, err := automerge.LoadPureGo(ctx, newBaseDocument(t), actor(42)) + require.NoError(t, err) + closeDocument(t, nativeDocument) + text, err := nativeDocument.Text(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 5, 0, " native")) + _, err = nativeDocument.Commit(ctx, "Extend in Go", commitTime) + require.NoError(t, err) + data, err := nativeDocument.Save(ctx) + require.NoError(t, err) + + referenceDocument, err := automerge.LoadReference(ctx, data, actor(43)) + require.NoError(t, err) + closeDocument(t, referenceDocument) + referenceText, err := referenceDocument.Text(ctx, "body") + require.NoError(t, err) + value, err := referenceText.String(ctx) + require.NoError(t, err) + assert.Equal(t, "Hello native", value) +} + +func TestPureGoDocument_InsertsInsideReferenceText(t *testing.T) { + t.Parallel() + + ctx := context.Background() + nativeDocument, err := automerge.LoadPureGo(ctx, newBaseDocument(t), actor(46)) + require.NoError(t, err) + closeDocument(t, nativeDocument) + text, err := nativeDocument.Text(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 2, 0, "X")) + _, err = nativeDocument.Commit(ctx, "Insert in Go", commitTime) + require.NoError(t, err) + data, err := nativeDocument.Save(ctx) + require.NoError(t, err) + + referenceDocument, err := automerge.LoadReference(ctx, data, actor(47)) + require.NoError(t, err) + closeDocument(t, referenceDocument) + referenceText, err := referenceDocument.Text(ctx, "body") + require.NoError(t, err) + value, err := referenceText.String(ctx) + require.NoError(t, err) + assert.Equal(t, "HeXllo", value) +} + +func TestPureGoDocument_DeletesReferenceText(t *testing.T) { + t.Parallel() + + ctx := context.Background() + nativeDocument, err := automerge.LoadPureGo(ctx, newBaseDocument(t), actor(48)) + require.NoError(t, err) + closeDocument(t, nativeDocument) + text, err := nativeDocument.Text(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 2, 2, "")) + _, err = nativeDocument.Commit(ctx, "Delete in Go", commitTime) + require.NoError(t, err) + data, err := nativeDocument.Save(ctx) + require.NoError(t, err) + + referenceDocument, err := automerge.LoadReference(ctx, data, actor(49)) + require.NoError(t, err) + closeDocument(t, referenceDocument) + referenceText, err := referenceDocument.Text(ctx, "body") + require.NoError(t, err) + value, err := referenceText.String(ctx) + require.NoError(t, err) + assert.Equal(t, "Heo", value) +} + +func TestPureGoDocument_EmptySnapshotLoadsInReference(t *testing.T) { + t.Parallel() + + ctx := context.Background() + nativeDocument, err := automerge.NewPureGo(ctx, actor(44)) + require.NoError(t, err) + closeDocument(t, nativeDocument) + data, err := nativeDocument.Save(ctx) + require.NoError(t, err) + + referenceDocument, err := automerge.LoadReference(ctx, data, actor(45)) + require.NoError(t, err) + closeDocument(t, referenceDocument) + heads, err := referenceDocument.Heads(ctx) + require.NoError(t, err) + assert.Empty(t, heads) +} + +func TestPureGoDocument_ReusesActorAfterLoad(t *testing.T) { + t.Parallel() + + ctx := context.Background() + actorID := actor(55) + document, err := automerge.NewPureGo(ctx, actorID) + require.NoError(t, err) + closeDocument(t, document) + text, err := document.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 0, 0, "A")) + _, err = document.Commit(ctx, "First change", commitTime) + require.NoError(t, err) + data, err := document.Save(ctx) + require.NoError(t, err) + + loaded, err := automerge.LoadPureGo(ctx, data, actorID) + require.NoError(t, err) + closeDocument(t, loaded) + loadedText, err := loaded.Text(ctx, "body") + require.NoError(t, err) + require.NoError(t, loadedText.Splice(ctx, 1, 0, "B")) + _, err = loaded.Commit(ctx, "Second change", commitTime) + require.NoError(t, err) + data, err = loaded.Save(ctx) + require.NoError(t, err) + + reference, err := automerge.LoadReference(ctx, data, actor(56)) + require.NoError(t, err) + closeDocument(t, reference) + referenceText, err := reference.Text(ctx, "body") + require.NoError(t, err) + value, err := referenceText.String(ctx) + require.NoError(t, err) + assert.Equal(t, "AB", value) +} + +func TestPureGoDocument_ConcurrentChangesConverge(t *testing.T) { + t.Parallel() + + ctx := context.Background() + base, err := automerge.NewPureGo(ctx, actor(50)) + require.NoError(t, err) + closeDocument(t, base) + baseText, err := base.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, baseText.Splice(ctx, 0, 0, "A")) + _, err = base.Commit(ctx, "Create body", commitTime) + require.NoError(t, err) + baseData, err := base.Save(ctx) + require.NoError(t, err) + + left, err := automerge.LoadPureGo(ctx, baseData, actor(51)) + require.NoError(t, err) + closeDocument(t, left) + leftText, err := left.Text(ctx, "body") + require.NoError(t, err) + require.NoError(t, leftText.Splice(ctx, 1, 0, "L")) + _, err = left.Commit(ctx, "Edit left", commitTime) + require.NoError(t, err) + + right, err := automerge.LoadPureGo(ctx, baseData, actor(52)) + require.NoError(t, err) + closeDocument(t, right) + rightText, err := right.Text(ctx, "body") + require.NoError(t, err) + require.NoError(t, rightText.Splice(ctx, 1, 0, "R")) + _, err = right.Commit(ctx, "Edit right", commitTime) + require.NoError(t, err) + + _, err = left.Merge(ctx, right) + require.NoError(t, err) + _, err = right.Merge(ctx, left) + require.NoError(t, err) + + leftValue, err := leftText.String(ctx) + require.NoError(t, err) + rightValue, err := rightText.String(ctx) + require.NoError(t, err) + assert.Equal(t, leftValue, rightValue) + + leftHeads, err := left.Heads(ctx) + require.NoError(t, err) + rightHeads, err := right.Heads(ctx) + require.NoError(t, err) + assert.ElementsMatch(t, leftHeads, rightHeads) +} + +func TestPureGoDocument_CursorMatchesReference(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseData := newBaseDocument(t) + nativeDocument, err := automerge.LoadPureGo(ctx, baseData, actor(60)) + require.NoError(t, err) + closeDocument(t, nativeDocument) + nativeText, err := nativeDocument.Text(ctx, "body") + require.NoError(t, err) + nativeCursor, err := nativeText.Cursor(ctx, 2) + require.NoError(t, err) + + referenceDocument, err := automerge.LoadReference(ctx, baseData, actor(61)) + require.NoError(t, err) + closeDocument(t, referenceDocument) + referenceText, err := referenceDocument.Text(ctx, "body") + require.NoError(t, err) + referenceCursor, err := referenceText.Cursor(ctx, 2) + require.NoError(t, err) + assert.Equal(t, referenceCursor, nativeCursor) + + require.NoError(t, nativeText.Splice(ctx, 0, 0, "X")) + position, err := nativeText.CursorPosition(ctx, nativeCursor) + require.NoError(t, err) + assert.Equal(t, uint32(3), position) +} + +func TestPureGoDocument_DeletedCursorMatchesReference(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseData := newBaseDocument(t) + nativeDocument, err := automerge.LoadPureGo(ctx, baseData, actor(62)) + require.NoError(t, err) + closeDocument(t, nativeDocument) + nativeText, err := nativeDocument.Text(ctx, "body") + require.NoError(t, err) + cursor, err := nativeText.Cursor(ctx, 2) + require.NoError(t, err) + require.NoError(t, nativeText.Splice(ctx, 2, 1, "")) + _, err = nativeDocument.Commit(ctx, "Delete cursor target", commitTime) + require.NoError(t, err) + nativePosition, err := nativeText.CursorPosition(ctx, cursor) + require.NoError(t, err) + + data, err := nativeDocument.Save(ctx) + require.NoError(t, err) + referenceDocument, err := automerge.LoadReference(ctx, data, actor(63)) + require.NoError(t, err) + closeDocument(t, referenceDocument) + referenceText, err := referenceDocument.Text(ctx, "body") + require.NoError(t, err) + referencePosition, err := referenceText.CursorPosition(ctx, cursor) + require.NoError(t, err) + assert.Equal(t, referencePosition, nativePosition) +} + +func TestPureGoDocument_SynchronizesWithNativePeer(t *testing.T) { + t.Parallel() + + ctx := context.Background() + left, err := automerge.NewPureGo(ctx, actor(70)) + require.NoError(t, err) + closeDocument(t, left) + text, err := left.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 0, 0, "Native sync")) + _, err = left.Commit(ctx, "Create sync body", commitTime) + require.NoError(t, err) + + right, err := automerge.NewPureGo(ctx, actor(71)) + require.NoError(t, err) + closeDocument(t, right) + + leftSync, err := left.NewSyncState(ctx) + require.NoError(t, err) + closeSyncState(t, leftSync) + + rightSync, err := right.NewSyncState(ctx) + require.NoError(t, err) + closeSyncState(t, rightSync) + + synchronize(t, leftSync, rightSync) + + rightText, err := right.Text(ctx, "body") + require.NoError(t, err) + value, err := rightText.String(ctx) + require.NoError(t, err) + assert.Equal(t, "Native sync", value) +} + +func TestPureGoDocument_SyncWaitsForPeerResponse(t *testing.T) { + t.Parallel() + + ctx := context.Background() + document, err := automerge.NewPureGo(ctx, actor(78)) + require.NoError(t, err) + closeDocument(t, document) + text, err := document.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 0, 0, "Wait")) + _, err = document.Commit(ctx, "Create body", commitTime) + require.NoError(t, err) + + state, err := document.NewSyncState(ctx) + require.NoError(t, err) + closeSyncState(t, state) + first, ok, err := state.GenerateMessage(ctx) + require.NoError(t, err) + assert.True(t, ok) + assert.NotEmpty(t, first) + + second, ok, err := state.GenerateMessage(ctx) + require.NoError(t, err) + assert.False(t, ok) + assert.Empty(t, second) +} + +func TestPureGoDocument_SynchronizesWithReferencePeer(t *testing.T) { + t.Parallel() + + ctx := context.Background() + left, err := automerge.NewPureGo(ctx, actor(72)) + require.NoError(t, err) + closeDocument(t, left) + text, err := left.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 0, 0, "Reference sync")) + _, err = left.Commit(ctx, "Create sync body", commitTime) + require.NoError(t, err) + + right, err := automerge.NewReference(ctx, actor(73)) + require.NoError(t, err) + closeDocument(t, right) + + leftSync, err := left.NewSyncState(ctx) + require.NoError(t, err) + closeSyncState(t, leftSync) + + rightSync, err := right.NewSyncState(ctx) + require.NoError(t, err) + closeSyncState(t, rightSync) + + synchronize(t, leftSync, rightSync) + + rightText, err := right.Text(ctx, "body") + require.NoError(t, err) + value, err := rightText.String(ctx) + require.NoError(t, err) + assert.Equal(t, "Reference sync", value) +} + +func TestPureGoDocument_ReceivesReferenceSync(t *testing.T) { + t.Parallel() + + ctx := context.Background() + left, err := automerge.NewReference(ctx, actor(74)) + require.NoError(t, err) + closeDocument(t, left) + text, err := left.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 0, 0, "From reference")) + _, err = left.Commit(ctx, "Create reference body", commitTime) + require.NoError(t, err) + + right, err := automerge.NewPureGo(ctx, actor(75)) + require.NoError(t, err) + closeDocument(t, right) + + leftSync, err := left.NewSyncState(ctx) + require.NoError(t, err) + closeSyncState(t, leftSync) + + rightSync, err := right.NewSyncState(ctx) + require.NoError(t, err) + closeSyncState(t, rightSync) + + synchronize(t, leftSync, rightSync) + + rightText, err := right.Text(ctx, "body") + require.NoError(t, err) + value, err := rightText.String(ctx) + require.NoError(t, err) + assert.Equal(t, "From reference", value) +} + +func TestPureGoDocument_RepeatedMixedPeerSync(t *testing.T) { + t.Parallel() + + ctx := context.Background() + nativeDocument, err := automerge.NewPureGo(ctx, actor(76)) + require.NoError(t, err) + closeDocument(t, nativeDocument) + nativeText, err := nativeDocument.CreateText(ctx, "body") + require.NoError(t, err) + _, err = nativeDocument.Commit(ctx, "Create native body", commitTime) + require.NoError(t, err) + + referenceDocument, err := automerge.NewReference(ctx, actor(77)) + require.NoError(t, err) + closeDocument(t, referenceDocument) + + nativeSync, err := nativeDocument.NewSyncState(ctx) + require.NoError(t, err) + closeSyncState(t, nativeSync) + + referenceSync, err := referenceDocument.NewSyncState(ctx) + require.NoError(t, err) + closeSyncState(t, referenceSync) + synchronize(t, nativeSync, referenceSync) + + referenceText, err := referenceDocument.Text(ctx, "body") + require.NoError(t, err) + + for iteration := range 20 { + nativeValue, err := nativeText.String(ctx) + require.NoError(t, err) + + nativeOffset := utf16Offsets([]rune(nativeValue)) + require.NoError( + t, + nativeText.Splice( + ctx, + nativeOffset[len(nativeOffset)-1], + 0, + "N", + ), + ) + _, err = nativeDocument.Commit(ctx, "Native edit", commitTime) + require.NoError(t, err) + + referenceValue, err := referenceText.String(ctx) + require.NoError(t, err) + + referenceOffset := utf16Offsets([]rune(referenceValue)) + require.NoError( + t, + referenceText.Splice( + ctx, + referenceOffset[len(referenceOffset)-1], + 0, + "R", + ), + ) + _, err = referenceDocument.Commit(ctx, "Reference edit", commitTime) + require.NoError(t, err) + + synchronize(t, nativeSync, referenceSync) + + nativeValue, err = nativeText.String(ctx) + require.NoError(t, err) + referenceValue, err = referenceText.String(ctx) + require.NoError(t, err) + assert.Equal(t, referenceValue, nativeValue, "iteration %d", iteration) + } +} + +func TestPureGoDocument_ReferenceEditsWhileMessageInFlight(t *testing.T) { + t.Parallel() + + ctx := context.Background() + client, err := automerge.NewReference(ctx, actor(79)) + require.NoError(t, err) + closeDocument(t, client) + clientText, err := client.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, clientText.Splice(ctx, 0, 0, "A")) + _, err = client.Commit(ctx, "initial", commitTime) + require.NoError(t, err) + + server, err := automerge.NewPureGo(ctx, actor(80)) + require.NoError(t, err) + closeDocument(t, server) + + clientSync, err := client.NewSyncState(ctx) + require.NoError(t, err) + closeSyncState(t, clientSync) + + serverSync, err := server.NewSyncState(ctx) + require.NoError(t, err) + closeSyncState(t, serverSync) + synchronize(t, clientSync, serverSync) + + require.NoError(t, clientText.Splice(ctx, 1, 0, "B")) + _, err = client.Commit(ctx, "first edit", commitTime) + require.NoError(t, err) + first, ok, err := clientSync.GenerateMessage(ctx) + require.NoError(t, err) + require.True(t, ok) + + require.NoError(t, clientText.Splice(ctx, 2, 0, "C")) + _, err = client.Commit(ctx, "second edit", commitTime) + require.NoError(t, err) + + require.NoError(t, serverSync.ReceiveMessage(ctx, first)) + ack, ok, err := serverSync.GenerateMessage(ctx) + require.NoError(t, err) + require.True(t, ok) + require.NoError(t, clientSync.ReceiveMessage(ctx, ack)) + + second, ok, err := clientSync.GenerateMessage(ctx) + require.NoError(t, err) + require.True(t, ok) + require.NoError(t, serverSync.ReceiveMessage(ctx, second)) + synchronize(t, clientSync, serverSync) + + serverText, err := server.Text(ctx, "body") + require.NoError(t, err) + value, err := serverText.String(ctx) + require.NoError(t, err) + assert.Equal(t, "ABC", value) +} diff --git a/pkg/automerge/native_differential_test.go b/pkg/automerge/native_differential_test.go new file mode 100644 index 0000000000..2b585dc192 --- /dev/null +++ b/pkg/automerge/native_differential_test.go @@ -0,0 +1,123 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package automerge_test + +import ( + "context" + "fmt" + "math/rand" + "testing" + "unicode/utf16" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/automerge" +) + +func TestPureGoDocument_RandomTextParity(t *testing.T) { + t.Parallel() + + const ( + histories = 20 + steps = 30 + ) + + characters := []rune("abcXYZ😀é") + + for history := range histories { + random := rand.New(rand.NewSource(int64(history + 1))) + ctx := context.Background() + nativeDocument, err := automerge.NewPureGo(ctx, actor(byte(80+history))) + require.NoError(t, err) + closeDocument(t, nativeDocument) + nativeText, err := nativeDocument.CreateText(ctx, "body") + require.NoError(t, err) + + referenceDocument, err := automerge.NewReference(ctx, actor(byte(80+history))) + require.NoError(t, err) + closeDocument(t, referenceDocument) + referenceText, err := referenceDocument.CreateText(ctx, "body") + require.NoError(t, err) + + var model []rune + for step := range steps { + offsets := utf16Offsets(model) + + var ( + index uint32 + deleteCount int32 + insert string + ) + + if len(model) > 0 && random.Intn(3) == 0 { + position := random.Intn(len(model)) + index = offsets[position] + deleteCount = int32(offsets[position+1] - offsets[position]) + model = append(model[:position], model[position+1:]...) + } else { + position := random.Intn(len(model) + 1) + index = offsets[position] + character := characters[random.Intn(len(characters))] + insert = string(character) + + model = append(model, 0) + copy(model[position+1:], model[position:]) + model[position] = character + } + + require.NoError(t, nativeText.Splice(ctx, index, deleteCount, insert)) + require.NoError(t, referenceText.Splice(ctx, index, deleteCount, insert)) + + message := fmt.Sprintf("history %d step %d", history, step) + _, err = nativeDocument.Commit(ctx, message, commitTime) + require.NoError(t, err) + _, err = referenceDocument.Commit(ctx, message, commitTime) + require.NoError(t, err) + + nativeValue, err := nativeText.String(ctx) + require.NoError(t, err) + referenceValue, err := referenceText.String(ctx) + require.NoError(t, err) + assert.Equal(t, string(model), nativeValue) + assert.Equal(t, referenceValue, nativeValue) + } + + data, err := nativeDocument.Save(ctx) + require.NoError(t, err) + loaded, err := automerge.LoadReference(ctx, data, actor(byte(120+history))) + require.NoError(t, err) + closeDocument(t, loaded) + loadedText, err := loaded.Text(ctx, "body") + require.NoError(t, err) + loadedValue, err := loadedText.String(ctx) + require.NoError(t, err) + assert.Equal(t, string(model), loadedValue) + } +} + +func utf16Offsets(value []rune) []uint32 { + offsets := make([]uint32, len(value)+1) + for i, character := range value { + offsets[i+1] = offsets[i] + uint32(len(utf16.Encode([]rune{character}))) + } + + return offsets +} diff --git a/pkg/automerge/prosemirror/render.go b/pkg/automerge/prosemirror/render.go new file mode 100644 index 0000000000..9c36142c8c --- /dev/null +++ b/pkg/automerge/prosemirror/render.go @@ -0,0 +1,408 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package prosemirror + +import ( + "encoding/json" + "fmt" + "slices" + "sort" + + "go.probo.inc/probo/pkg/automerge" +) + +type ( + node struct { + Type string `json:"type"` + Attrs map[string]any `json:"attrs,omitempty"` + Content []node `json:"content,omitempty"` + Text string `json:"text,omitempty"` + Marks []mark `json:"marks,omitempty"` + } + + mark struct { + Type string `json:"type"` + Attrs map[string]any `json:"attrs,omitempty"` + } + + block struct { + Type string + Parents []string + Attrs map[string]any + Content []node + } +) + +const ( + blockTypeBlockquote = "blockquote" + blockTypeCode = "code-block" + blockTypeHardBreak = "hard-break" + blockTypeHeading = "heading" + blockTypeHorizontalRule = "horizontal-rule" + blockTypeOrderedListItem = "ordered-list-item" + blockTypeParagraph = "paragraph" + blockTypeTable = "table" + blockTypeTableCell = "table-cell" + blockTypeTableHeader = "table-header" + blockTypeTableRow = "table-row" + blockTypeUnorderedListItem = "unordered-list-item" +) + +func Render(spans []automerge.Span) (string, error) { + blocks, err := collectBlocks(spans) + if err != nil { + return "", err + } + + content, consumed, err := renderBlocks(blocks, nil) + if err != nil { + return "", err + } + + if consumed != len(blocks) { + return "", fmt.Errorf("cannot render Automerge blocks: consumed %d of %d", consumed, len(blocks)) + } + + if len(content) == 0 { + content = []node{{Type: "paragraph"}} + } + + data, err := json.Marshal(node{Type: "doc", Content: content}) + if err != nil { + return "", fmt.Errorf("cannot marshal ProseMirror document: %w", err) + } + + return string(data), nil +} + +func collectBlocks(spans []automerge.Span) ([]block, error) { + blocks := make([]block, 0) + + for i, span := range spans { + switch span.Type { + case automerge.SpanTypeBlock: + blockType, ok := span.Block["type"].(string) + if !ok || blockType == "" { + return nil, fmt.Errorf("cannot collect Automerge block span %d: missing type", i) + } + + if blockType == blockTypeHardBreak { + if len(blocks) == 0 { + return nil, fmt.Errorf("cannot collect Automerge hard break span %d without a parent block", i) + } + + blocks[len(blocks)-1].Content = append( + blocks[len(blocks)-1].Content, + node{Type: "hardBreak"}, + ) + + continue + } + + parents, err := stringSlice(span.Block["parents"]) + if err != nil { + return nil, fmt.Errorf("cannot collect Automerge block span %d parents: %w", i, err) + } + + attrs, _ := span.Block["attrs"].(map[string]any) + blocks = append( + blocks, + block{ + Type: blockType, + Parents: parents, + Attrs: attrs, + }, + ) + case automerge.SpanTypeText: + if len(blocks) == 0 { + blocks = append(blocks, block{Type: blockTypeParagraph}) + } + + marks, err := renderMarks(span.Marks) + if err != nil { + return nil, fmt.Errorf("cannot collect Automerge text span %d marks: %w", i, err) + } + + if span.Text != "" { + blocks[len(blocks)-1].Content = append( + blocks[len(blocks)-1].Content, + node{ + Type: "text", + Text: span.Text, + Marks: marks, + }, + ) + } + default: + return nil, fmt.Errorf("cannot collect Automerge span %d: unknown type %q", i, span.Type) + } + } + + return blocks, nil +} + +func renderBlocks(blocks []block, parents []string) ([]node, int, error) { + content := make([]node, 0) + consumed := 0 + + for consumed < len(blocks) { + current := blocks[consumed] + if !slices.Equal(current.Parents, parents) { + break + } + + childParents := append(append([]string(nil), parents...), current.Type) + + childEnd := consumed + 1 + for childEnd < len(blocks) && hasPrefix(blocks[childEnd].Parents, childParents) { + childEnd++ + } + + children, childConsumed, err := renderBlocks(blocks[consumed+1:childEnd], childParents) + if err != nil { + return nil, 0, err + } + + if childConsumed != childEnd-consumed-1 { + return nil, 0, fmt.Errorf("cannot render children of Automerge block %q", current.Type) + } + + rendered, listType, err := renderBlock(current, children) + if err != nil { + return nil, 0, err + } + + if listType != "" { + if len(content) > 0 && content[len(content)-1].Type == listType { + content[len(content)-1].Content = append(content[len(content)-1].Content, rendered) + } else { + content = append( + content, + node{ + Type: listType, + Content: []node{rendered}, + }, + ) + } + } else { + content = append(content, rendered) + } + + consumed = childEnd + } + + return content, consumed, nil +} + +func renderBlock(source block, children []node) (node, string, error) { + switch source.Type { + case blockTypeParagraph: + if len(children) > 0 { + return node{}, "", fmt.Errorf("paragraph Automerge block cannot contain child blocks") + } + + return node{Type: "paragraph", Content: source.Content}, "", nil + case blockTypeHeading: + if len(children) > 0 { + return node{}, "", fmt.Errorf("heading Automerge block cannot contain child blocks") + } + + level := intAttribute(source.Attrs, "level", 1) + if level < 1 || level > 6 { + level = 1 + } + + return node{ + Type: "heading", + Attrs: map[string]any{"level": level}, + Content: source.Content, + }, "", nil + case blockTypeCode: + if len(children) > 0 { + return node{}, "", fmt.Errorf("code Automerge block cannot contain child blocks") + } + + attrs := map[string]any{"language": nil} + if language, ok := source.Attrs["language"].(string); ok { + attrs["language"] = language + } + + return node{Type: "codeBlock", Attrs: attrs, Content: source.Content}, "", nil + case blockTypeHorizontalRule: + if len(source.Content) > 0 || len(children) > 0 { + return node{}, "", fmt.Errorf("horizontal rule Automerge block cannot contain content") + } + + return node{Type: "horizontalRule"}, "", nil + case blockTypeBlockquote: + paragraph := node{Type: "paragraph", Content: source.Content} + + return node{ + Type: "blockquote", + Content: append([]node{paragraph}, children...), + }, "", nil + case blockTypeOrderedListItem: + paragraph := node{Type: "paragraph", Content: source.Content} + + return node{ + Type: "listItem", + Content: append([]node{paragraph}, children...), + }, "orderedList", nil + case blockTypeUnorderedListItem: + paragraph := node{Type: "paragraph", Content: source.Content} + + return node{ + Type: "listItem", + Content: append([]node{paragraph}, children...), + }, "bulletList", nil + case blockTypeTable: + if len(source.Content) > 0 { + return node{}, "", fmt.Errorf("table Automerge block cannot contain inline content") + } + + return node{Type: "table", Content: children}, "", nil + case blockTypeTableRow: + if len(source.Content) > 0 { + return node{}, "", fmt.Errorf("table row Automerge block cannot contain inline content") + } + + return node{Type: "tableRow", Content: children}, "", nil + case blockTypeTableCell, blockTypeTableHeader: + content := children + if len(source.Content) > 0 || len(children) == 0 { + paragraph := node{Type: "paragraph", Content: source.Content} + content = append([]node{paragraph}, children...) + } + + attrs := map[string]any{ + "colspan": intAttribute(source.Attrs, "colspan", 1), + "rowspan": intAttribute(source.Attrs, "rowspan", 1), + "colwidth": intSliceAttribute(source.Attrs, "colwidth"), + } + + nodeType := "tableCell" + if source.Type == blockTypeTableHeader { + nodeType = "tableHeader" + } + + return node{ + Type: nodeType, + Attrs: attrs, + Content: content, + }, "", nil + default: + return node{}, "", fmt.Errorf("unsupported Automerge block type %q", source.Type) + } +} + +func renderMarks(values map[string]any) ([]mark, error) { + if len(values) == 0 { + return nil, nil + } + + names := make([]string, 0, len(values)) + for name := range values { + names = append(names, name) + } + + sort.Strings(names) + + marks := make([]mark, 0, len(names)) + for _, name := range names { + switch name { + case "strong": + marks = append(marks, mark{Type: "bold"}) + case "em": + marks = append(marks, mark{Type: "italic"}) + case "strike", "underline", "code": + marks = append(marks, mark{Type: name}) + case "link": + raw, ok := values[name].(string) + if !ok { + return nil, fmt.Errorf("link mark value must be a string") + } + + var attrs map[string]any + if err := json.Unmarshal([]byte(raw), &attrs); err != nil { + return nil, fmt.Errorf("cannot decode link mark: %w", err) + } + + marks = append(marks, mark{Type: "link", Attrs: attrs}) + default: + return nil, fmt.Errorf("unsupported Automerge mark %q", name) + } + } + + return marks, nil +} + +func stringSlice(value any) ([]string, error) { + if value == nil { + return nil, nil + } + + values, ok := value.([]any) + if !ok { + return nil, fmt.Errorf("expected an array") + } + + result := make([]string, len(values)) + for i, value := range values { + result[i], ok = value.(string) + if !ok { + return nil, fmt.Errorf("value %d is not a string", i) + } + } + + return result, nil +} + +func intAttribute(attrs map[string]any, name string, fallback int) int { + value, ok := attrs[name].(float64) + if !ok { + return fallback + } + + return int(value) +} + +func intSliceAttribute(attrs map[string]any, name string) []int { + values, ok := attrs[name].([]any) + if !ok { + return nil + } + + result := make([]int, 0, len(values)) + for _, value := range values { + number, ok := value.(float64) + if !ok { + return nil + } + + result = append(result, int(number)) + } + + return result +} + +func hasPrefix(values, prefix []string) bool { + return len(values) >= len(prefix) && slices.Equal(values[:len(prefix)], prefix) +} diff --git a/pkg/automerge/prosemirror/render_test.go b/pkg/automerge/prosemirror/render_test.go new file mode 100644 index 0000000000..ffe3064e9f --- /dev/null +++ b/pkg/automerge/prosemirror/render_test.go @@ -0,0 +1,393 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package prosemirror_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/automerge" + automergeprosemirror "go.probo.inc/probo/pkg/automerge/prosemirror" + "go.probo.inc/probo/pkg/prosemirror" +) + +func TestRender_RichText(t *testing.T) { + t.Parallel() + + content, err := automergeprosemirror.Render( + []automerge.Span{ + { + Type: automerge.SpanTypeBlock, + Block: map[string]any{ + "type": "heading", + "parents": []any{}, + "attrs": map[string]any{"level": float64(2)}, + }, + }, + { + Type: automerge.SpanTypeText, + Text: "Policy", + Marks: map[string]any{"strong": true}, + }, + { + Type: automerge.SpanTypeBlock, + Block: map[string]any{ + "type": "paragraph", + "parents": []any{}, + "attrs": map[string]any{}, + }, + }, + { + Type: automerge.SpanTypeText, + Text: "Read ", + }, + { + Type: automerge.SpanTypeText, + Text: "more", + Marks: map[string]any{ + "link": `{"href":"https://example.com","title":"Example"}`, + }, + }, + }, + ) + + require.NoError(t, err) + assert.JSONEq( + t, + `{ + "type": "doc", + "content": [ + { + "type": "heading", + "attrs": {"level": 2}, + "content": [{ + "type": "text", + "text": "Policy", + "marks": [{"type": "bold"}] + }] + }, + { + "type": "paragraph", + "content": [ + {"type": "text", "text": "Read "}, + { + "type": "text", + "text": "more", + "marks": [{ + "type": "link", + "attrs": { + "href": "https://example.com", + "title": "Example" + } + }] + } + ] + } + ] + }`, + content, + ) + + _, err = prosemirror.Parse(content) + require.NoError(t, err) +} + +func TestRender_NestedList(t *testing.T) { + t.Parallel() + + content, err := automergeprosemirror.Render( + []automerge.Span{ + { + Type: automerge.SpanTypeBlock, + Block: map[string]any{ + "type": "unordered-list-item", + "parents": []any{}, + }, + }, + {Type: automerge.SpanTypeText, Text: "Outer"}, + { + Type: automerge.SpanTypeBlock, + Block: map[string]any{ + "type": "unordered-list-item", + "parents": []any{"unordered-list-item"}, + }, + }, + {Type: automerge.SpanTypeText, Text: "Inner"}, + }, + ) + + require.NoError(t, err) + assert.JSONEq( + t, + `{ + "type": "doc", + "content": [{ + "type": "bulletList", + "content": [{ + "type": "listItem", + "content": [ + { + "type": "paragraph", + "content": [{"type": "text", "text": "Outer"}] + }, + { + "type": "bulletList", + "content": [{ + "type": "listItem", + "content": [{ + "type": "paragraph", + "content": [{"type": "text", "text": "Inner"}] + }] + }] + } + ] + }] + }] + }`, + content, + ) + + _, err = prosemirror.Parse(content) + require.NoError(t, err) +} + +func TestRender_HardBreakAndHorizontalRule(t *testing.T) { + t.Parallel() + + content, err := automergeprosemirror.Render( + []automerge.Span{ + { + Type: automerge.SpanTypeBlock, + Block: map[string]any{ + "type": "paragraph", + "parents": []any{}, + }, + }, + {Type: automerge.SpanTypeText, Text: "A"}, + { + Type: automerge.SpanTypeBlock, + Block: map[string]any{ + "type": "hard-break", + "parents": []any{"paragraph"}, + "isEmbed": true, + }, + }, + {Type: automerge.SpanTypeText, Text: "B"}, + { + Type: automerge.SpanTypeBlock, + Block: map[string]any{ + "type": "horizontal-rule", + "parents": []any{}, + "isEmbed": true, + }, + }, + }, + ) + + require.NoError(t, err) + assert.JSONEq( + t, + `{ + "type": "doc", + "content": [ + { + "type": "paragraph", + "content": [ + {"type": "text", "text": "A"}, + {"type": "hardBreak"}, + {"type": "text", "text": "B"} + ] + }, + {"type": "horizontalRule"} + ] + }`, + content, + ) + + _, err = prosemirror.Parse(content) + require.NoError(t, err) +} + +func TestRender_TableStructure(t *testing.T) { + t.Parallel() + + content, err := automergeprosemirror.Render( + []automerge.Span{ + tableBlock("table", nil, nil), + tableBlock("table-row", []any{"table"}, nil), + tableBlock( + "table-header", + []any{"table", "table-row"}, + map[string]any{ + "colspan": float64(1), + "rowspan": float64(1), + "colwidth": nil, + }, + ), + {Type: automerge.SpanTypeText, Text: "A"}, + tableBlock( + "table-cell", + []any{"table", "table-row"}, + map[string]any{ + "colspan": float64(2), + "rowspan": float64(1), + "colwidth": []any{float64(100), float64(120)}, + }, + ), + {Type: automerge.SpanTypeText, Text: "B"}, + tableBlock("table-row", []any{"table"}, nil), + tableBlock( + "table-cell", + []any{"table", "table-row"}, + map[string]any{ + "colspan": float64(1), + "rowspan": float64(1), + "colwidth": nil, + }, + ), + {Type: automerge.SpanTypeText, Text: "C"}, + }, + ) + + require.NoError(t, err) + assert.JSONEq( + t, + `{ + "type": "doc", + "content": [{ + "type": "table", + "content": [ + { + "type": "tableRow", + "content": [ + { + "type": "tableHeader", + "attrs": { + "colspan": 1, + "rowspan": 1, + "colwidth": null + }, + "content": [{ + "type": "paragraph", + "content": [{"type": "text", "text": "A"}] + }] + }, + { + "type": "tableCell", + "attrs": { + "colspan": 2, + "rowspan": 1, + "colwidth": [100, 120] + }, + "content": [{ + "type": "paragraph", + "content": [{"type": "text", "text": "B"}] + }] + } + ] + }, + { + "type": "tableRow", + "content": [{ + "type": "tableCell", + "attrs": { + "colspan": 1, + "rowspan": 1, + "colwidth": null + }, + "content": [{ + "type": "paragraph", + "content": [{"type": "text", "text": "C"}] + }] + }] + } + ] + }] + }`, + content, + ) + + _, err = prosemirror.Parse(content) + require.NoError(t, err) +} + +func TestRender_CodeBlockLanguage(t *testing.T) { + t.Parallel() + + content, err := automergeprosemirror.Render( + []automerge.Span{ + { + Type: automerge.SpanTypeBlock, + Block: map[string]any{ + "type": "code-block", + "parents": []any{}, + "attrs": map[string]any{ + "language": "mermaid", + }, + }, + }, + { + Type: automerge.SpanTypeText, + Text: "graph TD; A-->B", + }, + }, + ) + + require.NoError(t, err) + assert.JSONEq( + t, + `{ + "type": "doc", + "content": [{ + "type": "codeBlock", + "attrs": {"language": "mermaid"}, + "content": [{ + "type": "text", + "text": "graph TD; A-->B" + }] + }] + }`, + content, + ) + + _, err = prosemirror.Parse(content) + require.NoError(t, err) +} + +func tableBlock( + blockType string, + parents []any, + attrs map[string]any, +) automerge.Span { + if attrs == nil { + attrs = map[string]any{} + } + + return automerge.Span{ + Type: automerge.SpanTypeBlock, + Block: map[string]any{ + "type": blockType, + "parents": parents, + "attrs": attrs, + }, + } +} diff --git a/pkg/automerge/rich_text.go b/pkg/automerge/rich_text.go new file mode 100644 index 0000000000..22d0cb4dc3 --- /dev/null +++ b/pkg/automerge/rich_text.go @@ -0,0 +1,89 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package automerge + +import ( + "context" + "encoding/json" + "fmt" +) + +type ( + SpanType string + + Span struct { + Type SpanType + Text string + Marks map[string]any + Block map[string]any + } + + encodedSpan struct { + Type SpanType `json:"type"` + Value json.RawMessage `json:"value"` + Marks map[string]any `json:"marks"` + } +) + +const ( + SpanTypeBlock SpanType = "block" + SpanTypeText SpanType = "text" +) + +func (t *Text) Spans(ctx context.Context) ([]Span, error) { + t.document.mu.Lock() + defer t.document.mu.Unlock() + + if t.document.closed { + return nil, ErrClosed + } + + data, err := t.document.backend.TextSpans(ctx, t.handle) + if err != nil { + return nil, fmt.Errorf("cannot read Automerge rich-text spans: %w", err) + } + + var encoded []encodedSpan + if err := json.Unmarshal(data, &encoded); err != nil { + return nil, fmt.Errorf("cannot decode Automerge rich-text spans: %w", err) + } + + spans := make([]Span, len(encoded)) + for i, source := range encoded { + spans[i].Type = source.Type + spans[i].Marks = source.Marks + + switch source.Type { + case SpanTypeText: + if err := json.Unmarshal(source.Value, &spans[i].Text); err != nil { + return nil, fmt.Errorf("cannot decode Automerge text span %d: %w", i, err) + } + case SpanTypeBlock: + if err := json.Unmarshal(source.Value, &spans[i].Block); err != nil { + return nil, fmt.Errorf("cannot decode Automerge block span %d: %w", i, err) + } + default: + return nil, fmt.Errorf("cannot decode Automerge span %d: unknown type %q", i, source.Type) + } + } + + return spans, nil +} diff --git a/pkg/coredata/document_version_automerge_change.go b/pkg/coredata/document_version_automerge_change.go new file mode 100644 index 0000000000..b2b3b08be4 --- /dev/null +++ b/pkg/coredata/document_version_automerge_change.go @@ -0,0 +1,167 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package coredata + +import ( + "context" + "fmt" + "maps" + "time" + + "github.com/jackc/pgx/v5" + "go.gearno.de/kit/pg" + "go.probo.inc/probo/pkg/gid" +) + +type ( + DocumentVersionAutomergeChange struct { + DocumentVersionID gid.GID `db:"document_version_id"` + OrganizationID gid.GID `db:"organization_id"` + Revision int64 `db:"revision"` + ChangeHash []byte `db:"change_hash"` + ChangeBytes []byte `db:"change_bytes"` + CreatedAt time.Time `db:"created_at"` + } + + DocumentVersionAutomergeChanges []*DocumentVersionAutomergeChange +) + +func (c DocumentVersionAutomergeChange) Insert( + ctx context.Context, + tx pg.Tx, + scope Scoper, +) error { + q := ` +INSERT INTO document_version_automerge_changes ( + tenant_id, + document_version_id, + organization_id, + revision, + change_hash, + change_bytes, + created_at +) +VALUES ( + @tenant_id, + @document_version_id, + @organization_id, + @revision, + @change_hash, + @change_bytes, + @created_at +) +ON CONFLICT (document_version_id, change_hash) DO NOTHING +` + args := pgx.StrictNamedArgs{ + "tenant_id": scope.GetTenantID(), + "document_version_id": c.DocumentVersionID, + "organization_id": c.OrganizationID, + "revision": c.Revision, + "change_hash": c.ChangeHash, + "change_bytes": c.ChangeBytes, + "created_at": c.CreatedAt, + } + + if _, err := tx.Exec(ctx, q, args); err != nil { + return fmt.Errorf("cannot insert document version Automerge change: %w", err) + } + + return nil +} + +func (cs *DocumentVersionAutomergeChanges) LoadAfterRevision( + ctx context.Context, + conn pg.Querier, + scope Scoper, + documentVersionID gid.GID, + afterRevision int64, + limit int, +) error { + q := ` +SELECT + document_version_id, + organization_id, + revision, + change_hash, + change_bytes, + created_at +FROM + document_version_automerge_changes +WHERE + %s + AND document_version_id = @document_version_id + AND revision > @after_revision +ORDER BY revision ASC +LIMIT @limit +` + q = fmt.Sprintf(q, scope.SQLFragment()) + args := pgx.StrictNamedArgs{ + "document_version_id": documentVersionID, + "after_revision": afterRevision, + "limit": limit, + } + maps.Copy(args, scope.SQLArguments()) + + rows, err := conn.Query(ctx, q, args) + if err != nil { + return fmt.Errorf("cannot query document version Automerge changes: %w", err) + } + + changes, err := pgx.CollectRows( + rows, + pgx.RowToAddrOfStructByName[DocumentVersionAutomergeChange], + ) + if err != nil { + return fmt.Errorf("cannot collect document version Automerge changes: %w", err) + } + + *cs = changes + + return nil +} + +func DeleteDocumentVersionAutomergeChangesThroughRevision( + ctx context.Context, + tx pg.Tx, + scope Scoper, + documentVersionID gid.GID, + revision int64, +) error { + q := ` +DELETE FROM document_version_automerge_changes +WHERE + %s + AND document_version_id = @document_version_id + AND revision <= @revision +` + q = fmt.Sprintf(q, scope.SQLFragment()) + args := pgx.StrictNamedArgs{ + "document_version_id": documentVersionID, + "revision": revision, + } + maps.Copy(args, scope.SQLArguments()) + + if _, err := tx.Exec(ctx, q, args); err != nil { + return fmt.Errorf("cannot delete compacted document version Automerge changes: %w", err) + } + + return nil +} diff --git a/pkg/coredata/document_version_automerge_state.go b/pkg/coredata/document_version_automerge_state.go new file mode 100644 index 0000000000..57ce4df4c3 --- /dev/null +++ b/pkg/coredata/document_version_automerge_state.go @@ -0,0 +1,295 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package coredata + +import ( + "context" + "errors" + "fmt" + "maps" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "go.gearno.de/kit/pg" + "go.probo.inc/probo/pkg/gid" +) + +type ( + DocumentVersionAutomergeState struct { + DocumentVersionID gid.GID `db:"document_version_id"` + OrganizationID gid.GID `db:"organization_id"` + Snapshot []byte `db:"snapshot"` + Heads []byte `db:"heads"` + Revision int64 `db:"revision"` + SnapshotRevision int64 `db:"snapshot_revision"` + ChangeRevision int64 `db:"change_revision"` + Seeded bool `db:"seeded"` + SeedClaimedAt *time.Time `db:"seed_claimed_at"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` + } +) + +func (s *DocumentVersionAutomergeState) LoadByDocumentVersionID( + ctx context.Context, + conn pg.Querier, + scope Scoper, + documentVersionID gid.GID, +) error { + return s.loadByDocumentVersionID(ctx, conn, scope, documentVersionID, false) +} + +func (s *DocumentVersionAutomergeState) LoadByDocumentVersionIDForUpdate( + ctx context.Context, + tx pg.Tx, + scope Scoper, + documentVersionID gid.GID, +) error { + return s.loadByDocumentVersionID(ctx, tx, scope, documentVersionID, true) +} + +func (s *DocumentVersionAutomergeState) loadByDocumentVersionID( + ctx context.Context, + conn pg.Querier, + scope Scoper, + documentVersionID gid.GID, + forUpdate bool, +) error { + q := ` +SELECT + document_version_id, + organization_id, + snapshot, + heads, + revision, + snapshot_revision, + change_revision, + seeded, + seed_claimed_at, + created_at, + updated_at +FROM + document_version_automerge_states +WHERE + %s + AND document_version_id = @document_version_id +` + if forUpdate { + q += "FOR UPDATE\n" + } + + q = fmt.Sprintf(q, scope.SQLFragment()) + + args := pgx.StrictNamedArgs{ + "document_version_id": documentVersionID, + } + maps.Copy(args, scope.SQLArguments()) + + rows, err := conn.Query(ctx, q, args) + if err != nil { + return fmt.Errorf("cannot query document version Automerge state: %w", err) + } + + state, err := pgx.CollectExactlyOneRow( + rows, + pgx.RowToStructByName[DocumentVersionAutomergeState], + ) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return ErrResourceNotFound + } + + return fmt.Errorf("cannot collect document version Automerge state: %w", err) + } + + *s = state + + return nil +} + +func (s DocumentVersionAutomergeState) Insert( + ctx context.Context, + tx pg.Tx, + scope Scoper, +) error { + q := ` +INSERT INTO document_version_automerge_states ( + tenant_id, + document_version_id, + organization_id, + snapshot, + heads, + revision, + snapshot_revision, + change_revision, + seeded, + seed_claimed_at, + created_at, + updated_at +) +VALUES ( + @tenant_id, + @document_version_id, + @organization_id, + @snapshot, + @heads, + @revision, + @snapshot_revision, + @change_revision, + @seeded, + @seed_claimed_at, + @created_at, + @updated_at +) +` + args := pgx.StrictNamedArgs{ + "tenant_id": scope.GetTenantID(), + "document_version_id": s.DocumentVersionID, + "organization_id": s.OrganizationID, + "snapshot": s.Snapshot, + "heads": s.Heads, + "revision": s.Revision, + "snapshot_revision": s.SnapshotRevision, + "change_revision": s.ChangeRevision, + "seeded": s.Seeded, + "seed_claimed_at": s.SeedClaimedAt, + "created_at": s.CreatedAt, + "updated_at": s.UpdatedAt, + } + + if _, err := tx.Exec(ctx, q, args); err != nil { + if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok { + if pgErr.Code == "23505" && pgErr.ConstraintName == "document_version_automerge_states_pkey" { + return ErrResourceAlreadyExists + } + } + + return fmt.Errorf("cannot insert document version Automerge state: %w", err) + } + + return nil +} + +func (s DocumentVersionAutomergeState) InsertIfAbsent( + ctx context.Context, + tx pg.Tx, + scope Scoper, +) (bool, error) { + q := ` +INSERT INTO document_version_automerge_states ( + tenant_id, + document_version_id, + organization_id, + snapshot, + heads, + revision, + snapshot_revision, + change_revision, + seeded, + seed_claimed_at, + created_at, + updated_at +) +VALUES ( + @tenant_id, + @document_version_id, + @organization_id, + @snapshot, + @heads, + @revision, + @snapshot_revision, + @change_revision, + @seeded, + @seed_claimed_at, + @created_at, + @updated_at +) +ON CONFLICT (document_version_id) DO NOTHING +` + args := pgx.StrictNamedArgs{ + "tenant_id": scope.GetTenantID(), + "document_version_id": s.DocumentVersionID, + "organization_id": s.OrganizationID, + "snapshot": s.Snapshot, + "heads": s.Heads, + "revision": s.Revision, + "snapshot_revision": s.SnapshotRevision, + "change_revision": s.ChangeRevision, + "seeded": s.Seeded, + "seed_claimed_at": s.SeedClaimedAt, + "created_at": s.CreatedAt, + "updated_at": s.UpdatedAt, + } + + result, err := tx.Exec(ctx, q, args) + if err != nil { + return false, fmt.Errorf("cannot insert document version Automerge state if absent: %w", err) + } + + return result.RowsAffected() == 1, nil +} + +func (s DocumentVersionAutomergeState) Update( + ctx context.Context, + tx pg.Tx, + scope Scoper, +) error { + q := ` +UPDATE document_version_automerge_states SET + snapshot = @snapshot, + heads = @heads, + revision = @revision, + snapshot_revision = @snapshot_revision, + change_revision = @change_revision, + seeded = @seeded, + seed_claimed_at = @seed_claimed_at, + updated_at = @updated_at +WHERE + %s + AND document_version_id = @document_version_id +` + q = fmt.Sprintf(q, scope.SQLFragment()) + + args := pgx.StrictNamedArgs{ + "document_version_id": s.DocumentVersionID, + "snapshot": s.Snapshot, + "heads": s.Heads, + "revision": s.Revision, + "snapshot_revision": s.SnapshotRevision, + "change_revision": s.ChangeRevision, + "seeded": s.Seeded, + "seed_claimed_at": s.SeedClaimedAt, + "updated_at": s.UpdatedAt, + } + maps.Copy(args, scope.SQLArguments()) + + result, err := tx.Exec(ctx, q, args) + if err != nil { + return fmt.Errorf("cannot update document version Automerge state: %w", err) + } + + if result.RowsAffected() == 0 { + return ErrResourceNotFound + } + + return nil +} diff --git a/pkg/coredata/document_version_collaboration_presence.go b/pkg/coredata/document_version_collaboration_presence.go new file mode 100644 index 0000000000..79cdc0268b --- /dev/null +++ b/pkg/coredata/document_version_collaboration_presence.go @@ -0,0 +1,233 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package coredata + +import ( + "context" + "fmt" + "maps" + "time" + + "github.com/jackc/pgx/v5" + "go.gearno.de/kit/pg" + "go.probo.inc/probo/pkg/gid" +) + +type ( + DocumentVersionCollaborationPresence struct { + ConnectionID string `db:"connection_id"` + DocumentVersionID gid.GID `db:"document_version_id"` + OrganizationID gid.GID `db:"organization_id"` + IdentityID gid.GID `db:"identity_id"` + AnchorPosition int `db:"anchor_position"` + HeadPosition int `db:"head_position"` + UpdatedAt time.Time `db:"updated_at"` + } + + DocumentVersionCollaborationPresences []*DocumentVersionCollaborationPresence +) + +func (p DocumentVersionCollaborationPresence) Insert( + ctx context.Context, + tx pg.Tx, + scope Scoper, +) error { + q := ` +INSERT INTO document_version_collaboration_presences ( + tenant_id, + connection_id, + document_version_id, + organization_id, + identity_id, + anchor_position, + head_position, + updated_at +) +VALUES ( + @tenant_id, + @connection_id, + @document_version_id, + @organization_id, + @identity_id, + @anchor_position, + @head_position, + @updated_at +) +` + args := pgx.StrictNamedArgs{ + "tenant_id": scope.GetTenantID(), + "connection_id": p.ConnectionID, + "document_version_id": p.DocumentVersionID, + "organization_id": p.OrganizationID, + "identity_id": p.IdentityID, + "anchor_position": p.AnchorPosition, + "head_position": p.HeadPosition, + "updated_at": p.UpdatedAt, + } + + if _, err := tx.Exec(ctx, q, args); err != nil { + return fmt.Errorf("cannot insert document collaboration presence: %w", err) + } + + return nil +} + +func (p DocumentVersionCollaborationPresence) Update( + ctx context.Context, + tx pg.Tx, + scope Scoper, +) error { + q := ` +UPDATE document_version_collaboration_presences SET + anchor_position = @anchor_position, + head_position = @head_position, + updated_at = @updated_at +WHERE + %s + AND connection_id = @connection_id +` + q = fmt.Sprintf(q, scope.SQLFragment()) + args := pgx.StrictNamedArgs{ + "connection_id": p.ConnectionID, + "anchor_position": p.AnchorPosition, + "head_position": p.HeadPosition, + "updated_at": p.UpdatedAt, + } + maps.Copy(args, scope.SQLArguments()) + + result, err := tx.Exec(ctx, q, args) + if err != nil { + return fmt.Errorf("cannot update document collaboration presence: %w", err) + } + + if result.RowsAffected() == 0 { + return ErrResourceNotFound + } + + return nil +} + +func (ps *DocumentVersionCollaborationPresences) Load( + ctx context.Context, + conn pg.Querier, + scope Scoper, + documentVersionID gid.GID, + excludeConnectionID string, + updatedAfter time.Time, + limit int, +) error { + q := ` +SELECT + connection_id, + document_version_id, + organization_id, + identity_id, + anchor_position, + head_position, + updated_at +FROM + document_version_collaboration_presences +WHERE + %s + AND document_version_id = @document_version_id + AND connection_id <> @exclude_connection_id + AND updated_at > @updated_after +ORDER BY updated_at DESC +LIMIT @limit +` + q = fmt.Sprintf(q, scope.SQLFragment()) + args := pgx.StrictNamedArgs{ + "document_version_id": documentVersionID, + "exclude_connection_id": excludeConnectionID, + "updated_after": updatedAfter, + "limit": limit, + } + maps.Copy(args, scope.SQLArguments()) + + rows, err := conn.Query(ctx, q, args) + if err != nil { + return fmt.Errorf("cannot query document collaboration presences: %w", err) + } + + presences, err := pgx.CollectRows( + rows, + pgx.RowToAddrOfStructByName[DocumentVersionCollaborationPresence], + ) + if err != nil { + return fmt.Errorf("cannot collect document collaboration presences: %w", err) + } + + *ps = presences + + return nil +} + +func (p DocumentVersionCollaborationPresence) Delete( + ctx context.Context, + tx pg.Tx, + scope Scoper, +) error { + q := ` +DELETE FROM document_version_collaboration_presences +WHERE + %s + AND connection_id = @connection_id +` + q = fmt.Sprintf(q, scope.SQLFragment()) + args := pgx.StrictNamedArgs{ + "connection_id": p.ConnectionID, + } + maps.Copy(args, scope.SQLArguments()) + + if _, err := tx.Exec(ctx, q, args); err != nil { + return fmt.Errorf("cannot delete document collaboration presence: %w", err) + } + + return nil +} + +func DeleteExpiredDocumentVersionCollaborationPresences( + ctx context.Context, + tx pg.Tx, + scope Scoper, + documentVersionID gid.GID, + updatedBefore time.Time, +) error { + q := ` +DELETE FROM document_version_collaboration_presences +WHERE + %s + AND document_version_id = @document_version_id + AND updated_at < @updated_before +` + q = fmt.Sprintf(q, scope.SQLFragment()) + args := pgx.StrictNamedArgs{ + "document_version_id": documentVersionID, + "updated_before": updatedBefore, + } + maps.Copy(args, scope.SQLArguments()) + + if _, err := tx.Exec(ctx, q, args); err != nil { + return fmt.Errorf("cannot delete expired document collaboration presences: %w", err) + } + + return nil +} diff --git a/pkg/coredata/migrations/20260807T125930Z.sql b/pkg/coredata/migrations/20260807T125930Z.sql new file mode 100644 index 0000000000..fde298531c --- /dev/null +++ b/pkg/coredata/migrations/20260807T125930Z.sql @@ -0,0 +1,31 @@ +-- Copyright (c) 2026 Probo Inc . +-- +-- Permission is hereby granted, free of charge, to any person obtaining a copy +-- of this software and associated documentation files (the "Software"), to deal +-- in the Software without restriction, including without limitation the rights +-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +-- copies of the Software, and to permit persons to whom the Software is +-- furnished to do so, subject to the following conditions: +-- +-- The above copyright notice and this permission notice shall be included in +-- all copies or substantial portions of the Software. +-- +-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +-- SOFTWARE. + +CREATE TABLE document_version_automerge_states ( + tenant_id TEXT NOT NULL, + document_version_id TEXT PRIMARY KEY REFERENCES document_versions(id) ON DELETE CASCADE, + organization_id TEXT NOT NULL REFERENCES organizations(id), + snapshot BYTEA NOT NULL, + revision BIGINT NOT NULL, + seeded BOOLEAN NOT NULL, + seed_claimed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); diff --git a/pkg/coredata/migrations/20260807T135955Z.sql b/pkg/coredata/migrations/20260807T135955Z.sql new file mode 100644 index 0000000000..de15b24f2a --- /dev/null +++ b/pkg/coredata/migrations/20260807T135955Z.sql @@ -0,0 +1,33 @@ +-- Copyright (c) 2026 Probo Inc . +-- +-- Permission is hereby granted, free of charge, to any person obtaining a copy +-- of this software and associated documentation files (the "Software"), to deal +-- in the Software without restriction, including without limitation the rights +-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +-- copies of the Software, and to permit persons to whom the Software is +-- furnished to do so, subject to the following conditions: +-- +-- The above copyright notice and this permission notice shall be included in +-- all copies or substantial portions of the Software. +-- +-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +-- SOFTWARE. + +CREATE TABLE document_version_collaboration_presences ( + tenant_id TEXT NOT NULL, + connection_id TEXT PRIMARY KEY, + document_version_id TEXT NOT NULL REFERENCES document_versions(id) ON DELETE CASCADE, + organization_id TEXT NOT NULL REFERENCES organizations(id), + identity_id TEXT NOT NULL REFERENCES identities(id) ON DELETE CASCADE, + anchor_position INTEGER NOT NULL, + head_position INTEGER NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX document_version_collaboration_presences_version_idx + ON document_version_collaboration_presences (tenant_id, document_version_id, updated_at); diff --git a/pkg/coredata/migrations/20260808T131031Z.sql b/pkg/coredata/migrations/20260808T131031Z.sql new file mode 100644 index 0000000000..1fc39644c7 --- /dev/null +++ b/pkg/coredata/migrations/20260808T131031Z.sql @@ -0,0 +1,58 @@ +-- Copyright (c) 2026 Probo Inc . +-- +-- Permission is hereby granted, free of charge, to any person obtaining a copy +-- of this software and associated documentation files (the "Software"), to deal +-- in the Software without restriction, including without limitation the rights +-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +-- copies of the Software, and to permit persons to whom the Software is +-- furnished to do so, subject to the following conditions: +-- +-- The above copyright notice and this permission notice shall be included in +-- all copies or substantial portions of the Software. +-- +-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +-- SOFTWARE. + +ALTER TABLE document_version_automerge_states + ADD COLUMN snapshot_revision BIGINT; + +ALTER TABLE document_version_automerge_states + ADD COLUMN change_revision BIGINT; + +ALTER TABLE document_version_automerge_states + ADD COLUMN heads BYTEA; + +UPDATE document_version_automerge_states +SET snapshot_revision = revision; + +UPDATE document_version_automerge_states +SET change_revision = revision; + +UPDATE document_version_automerge_states +SET heads = ''::BYTEA; + +ALTER TABLE document_version_automerge_states + ALTER COLUMN snapshot_revision SET NOT NULL; + +ALTER TABLE document_version_automerge_states + ALTER COLUMN change_revision SET NOT NULL; + +ALTER TABLE document_version_automerge_states + ALTER COLUMN heads SET NOT NULL; + +CREATE TABLE document_version_automerge_changes ( + tenant_id TEXT NOT NULL, + document_version_id TEXT NOT NULL REFERENCES document_versions(id) ON DELETE CASCADE, + organization_id TEXT NOT NULL REFERENCES organizations(id), + revision BIGINT NOT NULL, + change_hash BYTEA NOT NULL CHECK (octet_length(change_hash) = 32), + change_bytes BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (document_version_id, change_hash), + UNIQUE (document_version_id, revision) +); diff --git a/pkg/pgnotify/listener.go b/pkg/pgnotify/listener.go new file mode 100644 index 0000000000..6b8597685e --- /dev/null +++ b/pkg/pgnotify/listener.go @@ -0,0 +1,101 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package pgnotify + +import ( + "context" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + "go.gearno.de/kit/log" +) + +type Listener struct { + config *pgx.ConnConfig + channel string + logger *log.Logger + handler func(string) +} + +const reconnectInterval = time.Second + +func NewListener( + config *pgx.ConnConfig, + channel string, + logger *log.Logger, + handler func(string), +) *Listener { + return &Listener{ + config: config.Copy(), + channel: channel, + logger: logger, + handler: handler, + } +} + +func (l *Listener) Run(ctx context.Context) error { + for { + if err := l.listen(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + + l.logger.WarnCtx( + ctx, + "PostgreSQL notification listener disconnected", + log.Error(err), + ) + } + + timer := time.NewTimer(reconnectInterval) + select { + case <-ctx.Done(): + timer.Stop() + + return nil + case <-timer.C: + } + } +} + +func (l *Listener) listen(ctx context.Context) error { + connection, err := pgx.ConnectConfig(ctx, l.config) + if err != nil { + return fmt.Errorf("cannot connect PostgreSQL notification listener: %w", err) + } + + defer func() { _ = connection.Close(context.Background()) }() + + channel := pgx.Identifier{l.channel}.Sanitize() + if _, err := connection.Exec(ctx, "LISTEN "+channel); err != nil { + return fmt.Errorf("cannot listen for PostgreSQL notifications: %w", err) + } + + for { + notification, err := connection.WaitForNotification(ctx) + if err != nil { + return fmt.Errorf("cannot wait for PostgreSQL notification: %w", err) + } + + l.handler(notification.Payload) + } +} diff --git a/pkg/probo/document_collaboration_presence_service.go b/pkg/probo/document_collaboration_presence_service.go new file mode 100644 index 0000000000..20b1265365 --- /dev/null +++ b/pkg/probo/document_collaboration_presence_service.go @@ -0,0 +1,166 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package probo + +import ( + "context" + "errors" + "fmt" + "time" + + "go.gearno.de/kit/pg" + "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/gid" +) + +const ( + documentCollaborationPresenceLimit = 100 + documentCollaborationPresenceTTL = 15 * time.Second +) + +type ( + DocumentCollaborationPresence struct { + ConnectionID string + IdentityID gid.GID + AnchorPosition int + HeadPosition int + } +) + +func (s *DocumentService) SaveCollaborationPresence( + ctx context.Context, + scope coredata.Scoper, + documentVersionID gid.GID, + identityID gid.GID, + connectionID string, + anchorPosition int, + headPosition int, +) error { + if anchorPosition < 0 || headPosition < 0 { + return fmt.Errorf("document collaboration presence positions cannot be negative") + } + + return s.svc.pg.WithTx( + ctx, + func(ctx context.Context, tx pg.Tx) error { + version, err := s.loadEditableCollaborationVersion( + ctx, + tx, + scope, + documentVersionID, + ) + if err != nil { + return err + } + + now := time.Now() + + presence := coredata.DocumentVersionCollaborationPresence{ + ConnectionID: connectionID, + DocumentVersionID: documentVersionID, + OrganizationID: version.OrganizationID, + IdentityID: identityID, + AnchorPosition: anchorPosition, + HeadPosition: headPosition, + UpdatedAt: now, + } + if err := presence.Update(ctx, tx, scope); err != nil { + if !errors.Is(err, coredata.ErrResourceNotFound) { + return fmt.Errorf("cannot update document collaboration presence: %w", err) + } + + if err := presence.Insert(ctx, tx, scope); err != nil { + return fmt.Errorf("cannot insert document collaboration presence: %w", err) + } + } + + if err := coredata.DeleteExpiredDocumentVersionCollaborationPresences( + ctx, + tx, + scope, + documentVersionID, + now.Add(-time.Hour), + ); err != nil { + return fmt.Errorf("cannot clean document collaboration presences: %w", err) + } + + return nil + }, + ) +} + +func (s *DocumentService) ListCollaborationPresences( + ctx context.Context, + scope coredata.Scoper, + documentVersionID gid.GID, + excludeConnectionID string, +) ([]DocumentCollaborationPresence, error) { + var stored coredata.DocumentVersionCollaborationPresences + + if err := s.svc.pg.WithConn( + ctx, + func(ctx context.Context, conn pg.Querier) error { + return stored.Load( + ctx, + conn, + scope, + documentVersionID, + excludeConnectionID, + time.Now().Add(-documentCollaborationPresenceTTL), + documentCollaborationPresenceLimit, + ) + }, + ); err != nil { + return nil, fmt.Errorf("cannot load document collaboration presences: %w", err) + } + + presences := make([]DocumentCollaborationPresence, len(stored)) + for i, presence := range stored { + presences[i] = DocumentCollaborationPresence{ + ConnectionID: presence.ConnectionID, + IdentityID: presence.IdentityID, + AnchorPosition: presence.AnchorPosition, + HeadPosition: presence.HeadPosition, + } + } + + return presences, nil +} + +func (s *DocumentService) DeleteCollaborationPresence( + ctx context.Context, + scope coredata.Scoper, + connectionID string, +) error { + return s.svc.pg.WithTx( + ctx, + func(ctx context.Context, tx pg.Tx) error { + presence := coredata.DocumentVersionCollaborationPresence{ + ConnectionID: connectionID, + } + if err := presence.Delete(ctx, tx, scope); err != nil { + return fmt.Errorf("cannot delete document collaboration presence: %w", err) + } + + return nil + }, + ) +} diff --git a/pkg/probo/document_collaboration_service.go b/pkg/probo/document_collaboration_service.go new file mode 100644 index 0000000000..7d4b0706f6 --- /dev/null +++ b/pkg/probo/document_collaboration_service.go @@ -0,0 +1,802 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package probo + +import ( + "context" + "errors" + "fmt" + "slices" + "time" + "unicode/utf8" + + "github.com/jackc/pgx/v5" + "go.gearno.de/kit/pg" + "go.probo.inc/probo/pkg/automerge" + automergeprosemirror "go.probo.inc/probo/pkg/automerge/prosemirror" + "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/prosemirror" + "go.probo.inc/probo/pkg/realtime" +) + +const ( + documentCollaborationChangeBatchSize = 1000 + documentCollaborationCompactionChanges = 500 + documentCollaborationSeedLease = 30 * time.Second + documentCollaborationSnapshotMaxBytes = 8 * 1024 * 1024 +) + +type ( + DocumentCollaboration struct { + Document *automerge.Document + Revision int64 + SeedContent string + NeedsSeed bool + } + + ErrDocumentCollaborationStateTooLarge struct { + Size int + } + + DocumentCollaborationTextEdit struct { + ExpectedRevision int64 + Index uint32 + Cursor automerge.Cursor + DeleteCount int32 + Text string + } +) + +var ( + ErrDocumentCollaborationNotSeeded = errors.New("document collaboration is not initialized") + ErrDocumentCollaborationStale = errors.New("document collaboration changed; read it again before editing") +) + +func (e ErrDocumentCollaborationStateTooLarge) Error() string { + return fmt.Sprintf( + "document collaboration state is too large: %d bytes exceeds %d bytes", + e.Size, + documentCollaborationSnapshotMaxBytes, + ) +} + +func (s *DocumentService) OpenCollaboration( + ctx context.Context, + scope coredata.Scoper, + documentVersionID gid.GID, +) (*DocumentCollaboration, error) { + var ( + snapshot []byte + revision int64 + snapshotRevision int64 + changeRevision int64 + seedContent string + needsSeed bool + ) + + err := s.svc.pg.WithTx( + ctx, + func(ctx context.Context, tx pg.Tx) error { + version, err := s.loadEditableCollaborationVersion(ctx, tx, scope, documentVersionID) + if err != nil { + return err + } + + seedContent = version.Content + + state := &coredata.DocumentVersionAutomergeState{} + + err = state.LoadByDocumentVersionIDForUpdate(ctx, tx, scope, documentVersionID) + if err == nil { + snapshot = append([]byte(nil), state.Snapshot...) + + needsSeed, err = claimCollaborationSeed(ctx, tx, scope, state, time.Now()) + if err != nil { + return fmt.Errorf("cannot claim document collaboration seed: %w", err) + } + + revision = state.Revision + snapshotRevision = state.SnapshotRevision + changeRevision = state.ChangeRevision + + return nil + } + + if !errors.Is(err, coredata.ErrResourceNotFound) { + return fmt.Errorf("cannot load document collaboration state: %w", err) + } + + document, err := newAutomergeDocument(ctx) + if err != nil { + return fmt.Errorf("cannot create empty collaboration document: %w", err) + } + + defer func() { _ = document.Close(context.Background()) }() + + snapshot, err = document.Save(ctx) + if err != nil { + return fmt.Errorf("cannot save empty collaboration document: %w", err) + } + + now := time.Now() + state = &coredata.DocumentVersionAutomergeState{ + DocumentVersionID: documentVersionID, + OrganizationID: version.OrganizationID, + Snapshot: snapshot, + Heads: encodeAutomergeHeads(nil), + Revision: 1, + SnapshotRevision: 1, + ChangeRevision: 1, + Seeded: false, + SeedClaimedAt: new(now), + CreatedAt: now, + UpdatedAt: now, + } + + inserted, err := state.InsertIfAbsent(ctx, tx, scope) + if err != nil { + return fmt.Errorf("cannot initialize document collaboration state: %w", err) + } + + if !inserted { + if err := state.LoadByDocumentVersionIDForUpdate( + ctx, + tx, + scope, + documentVersionID, + ); err != nil { + return fmt.Errorf("cannot load concurrently initialized collaboration state: %w", err) + } + + snapshot = append([]byte(nil), state.Snapshot...) + + needsSeed, err = claimCollaborationSeed(ctx, tx, scope, state, now) + if err != nil { + return fmt.Errorf("cannot claim concurrently initialized collaboration seed: %w", err) + } + } else { + needsSeed = true + } + + revision = state.Revision + snapshotRevision = state.SnapshotRevision + changeRevision = state.ChangeRevision + + return nil + }, + ) + if err != nil { + return nil, err + } + + document, err := loadAutomergeDocument(ctx, snapshot) + if err != nil { + return nil, fmt.Errorf("cannot open collaboration snapshot: %w", err) + } + + if err := s.loadCollaborationChanges( + ctx, + scope, + documentVersionID, + document, + snapshotRevision, + changeRevision, + ); err != nil { + _ = document.Close(context.Background()) + + return nil, fmt.Errorf("cannot load collaboration changes: %w", err) + } + + return &DocumentCollaboration{ + Document: document, + Revision: revision, + SeedContent: seedContent, + NeedsSeed: needsSeed, + }, nil +} + +func (s *DocumentService) PersistCollaboration( + ctx context.Context, + scope coredata.Scoper, + documentVersionID gid.GID, + document *automerge.Document, +) (int64, error) { + localHeads, err := document.Heads(ctx) + if err != nil { + return 0, fmt.Errorf("cannot read local collaboration heads: %w", err) + } + + localSnapshot, err := document.Save(ctx) + if err != nil { + return 0, fmt.Errorf("cannot save local collaboration document: %w", err) + } + + if len(localSnapshot) > documentCollaborationSnapshotMaxBytes { + return 0, &ErrDocumentCollaborationStateTooLarge{Size: len(localSnapshot)} + } + + var ( + canonicalChangesForLocal [][]byte + revision int64 + ) + + err = s.svc.pg.WithTx( + ctx, + func(ctx context.Context, tx pg.Tx) error { + version, err := s.loadEditableCollaborationVersion( + ctx, + tx, + scope, + documentVersionID, + ) + if err != nil { + return err + } + + state := &coredata.DocumentVersionAutomergeState{} + if err := state.LoadByDocumentVersionIDForUpdate( + ctx, + tx, + scope, + documentVersionID, + ); err != nil { + return fmt.Errorf("cannot lock document collaboration state: %w", err) + } + + canonical, err := loadAutomergeDocument(ctx, state.Snapshot) + if err != nil { + return fmt.Errorf("cannot load canonical collaboration document: %w", err) + } + + defer func() { _ = canonical.Close(context.Background()) }() + + if err := s.loadCollaborationChanges( + ctx, + scope, + documentVersionID, + canonical, + state.SnapshotRevision, + state.ChangeRevision, + ); err != nil { + return fmt.Errorf("cannot load canonical collaboration changes: %w", err) + } + + local, err := loadAutomergeDocument(ctx, localSnapshot) + if err != nil { + return fmt.Errorf("cannot load local collaboration document: %w", err) + } + + defer func() { _ = local.Close(context.Background()) }() + + before, err := canonical.Heads(ctx) + if err != nil { + return fmt.Errorf("cannot read canonical collaboration heads: %w", err) + } + + if _, err := canonical.Merge(ctx, local); err != nil { + return fmt.Errorf("cannot merge collaboration changes: %w", err) + } + + after, err := canonical.Heads(ctx) + if err != nil { + return fmt.Errorf("cannot read merged collaboration heads: %w", err) + } + + incrementalChanges, err := canonical.ChangesSince(ctx, before) + if err != nil { + return fmt.Errorf("cannot read merged collaboration changes: %w", err) + } + + localChanges, err := canonical.ChangesSince(ctx, localHeads) + if err != nil { + return fmt.Errorf("cannot read canonical changes for local document: %w", err) + } + + canonicalChangesForLocal = make([][]byte, len(localChanges)) + for i, change := range localChanges { + canonicalChangesForLocal[i] = change.Bytes + } + + seeded := state.Seeded || len(after) > 0 + if !slices.Equal(before, after) || seeded != state.Seeded { + now := time.Now() + nextChangeRevision := state.ChangeRevision + + for _, change := range incrementalChanges { + nextChangeRevision++ + + storedChange := coredata.DocumentVersionAutomergeChange{ + DocumentVersionID: documentVersionID, + OrganizationID: version.OrganizationID, + Revision: nextChangeRevision, + ChangeHash: append([]byte(nil), change.Hash[:]...), + ChangeBytes: change.Bytes, + CreatedAt: now, + } + if err := storedChange.Insert(ctx, tx, scope); err != nil { + return fmt.Errorf( + "cannot append document collaboration change: %w", + err, + ) + } + } + + state.Revision++ + state.ChangeRevision = nextChangeRevision + state.Heads = encodeAutomergeHeads(after) + + if state.ChangeRevision-state.SnapshotRevision >= + documentCollaborationCompactionChanges { + snapshot, err := canonical.Save(ctx) + if err != nil { + return fmt.Errorf("cannot compact collaboration snapshot: %w", err) + } + + if len(snapshot) > documentCollaborationSnapshotMaxBytes { + return &ErrDocumentCollaborationStateTooLarge{Size: len(snapshot)} + } + + state.Snapshot = snapshot + state.SnapshotRevision = state.ChangeRevision + + if err := coredata.DeleteDocumentVersionAutomergeChangesThroughRevision( + ctx, + tx, + scope, + documentVersionID, + state.SnapshotRevision, + ); err != nil { + return fmt.Errorf("cannot delete compacted collaboration changes: %w", err) + } + } + + state.Seeded = seeded + if seeded { + state.SeedClaimedAt = nil + } + + state.UpdatedAt = now + if err := state.Update(ctx, tx, scope); err != nil { + return fmt.Errorf("cannot persist document collaboration state: %w", err) + } + + if _, err := tx.Exec( + ctx, + `SELECT pg_notify(@channel, @payload)`, + pgx.StrictNamedArgs{ + "channel": realtime.DocumentCollaborationChannel, + "payload": documentVersionID.String(), + }, + ); err != nil { + return fmt.Errorf("cannot notify document collaboration change: %w", err) + } + + if seeded { + content, err := materializeCollaboration(ctx, canonical) + if err != nil { + return fmt.Errorf("cannot materialize document collaboration: %w", err) + } + + req := UpdateDocumentRequest{ + DocumentID: version.DocumentID, + Content: new(content), + } + if err := req.Validate(); err != nil { + return fmt.Errorf("cannot validate materialized collaboration: %w", err) + } + + version.Content = content + + version.UpdatedAt = now + if err := version.Update(ctx, tx, scope); err != nil { + return fmt.Errorf("cannot update materialized document version: %w", err) + } + } + } + + revision = state.Revision + + return nil + }, + ) + if err != nil { + return 0, err + } + + if err := document.ApplyChanges(ctx, canonicalChangesForLocal); err != nil { + return 0, fmt.Errorf("cannot refresh local collaboration document: %w", err) + } + + return revision, nil +} + +func (s *DocumentService) ReleaseCollaborationSeed( + ctx context.Context, + scope coredata.Scoper, + documentVersionID gid.GID, +) error { + return s.svc.pg.WithTx( + ctx, + func(ctx context.Context, tx pg.Tx) error { + state := &coredata.DocumentVersionAutomergeState{} + if err := state.LoadByDocumentVersionIDForUpdate( + ctx, + tx, + scope, + documentVersionID, + ); err != nil { + return fmt.Errorf("cannot lock document collaboration seed: %w", err) + } + + if state.Seeded || state.SeedClaimedAt == nil { + return nil + } + + state.SeedClaimedAt = nil + state.Revision++ + + state.UpdatedAt = time.Now() + if err := state.Update(ctx, tx, scope); err != nil { + return fmt.Errorf("cannot release document collaboration seed: %w", err) + } + + return nil + }, + ) +} + +func (s *DocumentService) RefreshCollaboration( + ctx context.Context, + scope coredata.Scoper, + documentVersionID gid.GID, + document *automerge.Document, + knownRevision int64, +) (int64, bool, error) { + state := &coredata.DocumentVersionAutomergeState{} + + err := s.svc.pg.WithConn( + ctx, + func(ctx context.Context, conn pg.Querier) error { + if err := state.LoadByDocumentVersionID( + ctx, + conn, + scope, + documentVersionID, + ); err != nil { + return fmt.Errorf("cannot load document collaboration state: %w", err) + } + + return nil + }, + ) + if err != nil { + return 0, false, err + } + + if state.Revision == knownRevision { + return knownRevision, false, nil + } + + canonical, err := loadAutomergeDocument(ctx, state.Snapshot) + if err != nil { + return 0, false, fmt.Errorf("cannot load refreshed collaboration document: %w", err) + } + + defer func() { _ = canonical.Close(context.Background()) }() + + if err := s.loadCollaborationChanges( + ctx, + scope, + documentVersionID, + canonical, + state.SnapshotRevision, + state.ChangeRevision, + ); err != nil { + return 0, false, fmt.Errorf("cannot load refreshed collaboration changes: %w", err) + } + + if _, err := document.Merge(ctx, canonical); err != nil { + return 0, false, fmt.Errorf("cannot merge refreshed collaboration document: %w", err) + } + + return state.Revision, true, nil +} + +func (s *DocumentService) ApplyCollaborationTextEdit( + ctx context.Context, + scope coredata.Scoper, + documentVersionID gid.GID, + edit DocumentCollaborationTextEdit, +) (int64, error) { + if edit.DeleteCount < 0 { + return 0, fmt.Errorf("collaboration text delete count cannot be negative") + } + + collaboration, err := s.OpenCollaboration(ctx, scope, documentVersionID) + if err != nil { + return 0, fmt.Errorf("cannot open collaboration for text edit: %w", err) + } + + defer func() { _ = collaboration.Document.Close(context.Background()) }() + + if collaboration.NeedsSeed { + return 0, ErrDocumentCollaborationNotSeeded + } + + if edit.ExpectedRevision != collaboration.Revision { + return 0, ErrDocumentCollaborationStale + } + + text, err := collaboration.Document.Text(ctx, "body") + if err != nil { + return 0, fmt.Errorf("cannot get collaboration body: %w", err) + } + + index := edit.Index + if len(edit.Cursor) > 0 { + index, err = text.CursorPosition(ctx, edit.Cursor) + if err != nil { + return 0, fmt.Errorf("cannot resolve collaboration text cursor: %w", err) + } + } + + if err := text.Splice(ctx, index, edit.DeleteCount, edit.Text); err != nil { + return 0, fmt.Errorf("cannot apply collaboration text edit: %w", err) + } + + content, err := text.String(ctx) + if err != nil { + return 0, fmt.Errorf("cannot read edited collaboration body: %w", err) + } + + if utf8.RuneCountInString(content) > documentContentMaxTextLength { + return 0, fmt.Errorf( + "collaboration text exceeds maximum length of %d characters", + documentContentMaxTextLength, + ) + } + + if _, err := collaboration.Document.Commit(ctx, "Agent edit", time.Now()); err != nil { + return 0, fmt.Errorf("cannot commit collaboration text edit: %w", err) + } + + revision, err := s.PersistCollaboration( + ctx, + scope, + documentVersionID, + collaboration.Document, + ) + if err != nil { + return 0, fmt.Errorf("cannot persist collaboration text edit: %w", err) + } + + return revision, nil +} + +func (s *DocumentService) ReadCollaborationText( + ctx context.Context, + scope coredata.Scoper, + documentVersionID gid.GID, +) (string, int64, error) { + collaboration, err := s.OpenCollaboration(ctx, scope, documentVersionID) + if err != nil { + return "", 0, fmt.Errorf("cannot open collaboration for reading: %w", err) + } + + defer func() { _ = collaboration.Document.Close(context.Background()) }() + + if collaboration.NeedsSeed { + return "", 0, ErrDocumentCollaborationNotSeeded + } + + text, err := collaboration.Document.Text(ctx, "body") + if err != nil { + return "", 0, fmt.Errorf("cannot get collaboration body for reading: %w", err) + } + + content, err := text.String(ctx) + if err != nil { + return "", 0, fmt.Errorf("cannot read collaboration body: %w", err) + } + + return content, collaboration.Revision, nil +} + +func (s *DocumentService) loadEditableCollaborationVersion( + ctx context.Context, + conn pg.Querier, + scope coredata.Scoper, + documentVersionID gid.GID, +) (*coredata.DocumentVersion, error) { + version := &coredata.DocumentVersion{} + if err := version.LoadByID(ctx, conn, scope, documentVersionID); err != nil { + return nil, fmt.Errorf("cannot load collaboration document version: %w", err) + } + + if version.Status != coredata.DocumentVersionStatusDraft { + return nil, &ErrDocumentVersionNotDraft{} + } + + document := &coredata.Document{} + if err := document.LoadByID(ctx, conn, scope, version.DocumentID); err != nil { + return nil, fmt.Errorf("cannot load collaboration document: %w", err) + } + + if document.ArchivedAt != nil { + return nil, &ErrDocumentArchived{} + } + + if document.WriteMode == coredata.DocumentWriteModeGenerated { + return nil, &ErrDocumentVersionGenerated{} + } + + return version, nil +} + +func newAutomergeDocument(ctx context.Context) (*automerge.Document, error) { + actorID, err := automerge.NewActorID() + if err != nil { + return nil, fmt.Errorf("cannot generate collaboration actor ID: %w", err) + } + + document, err := automerge.New(ctx, actorID) + if err != nil { + return nil, fmt.Errorf("cannot create Automerge document: %w", err) + } + + return document, nil +} + +func loadAutomergeDocument(ctx context.Context, snapshot []byte) (*automerge.Document, error) { + actorID, err := automerge.NewActorID() + if err != nil { + return nil, fmt.Errorf("cannot generate collaboration actor ID: %w", err) + } + + document, err := automerge.Load(ctx, snapshot, actorID) + if err != nil { + return nil, fmt.Errorf("cannot load Automerge document: %w", err) + } + + return document, nil +} + +func (s *DocumentService) loadCollaborationChanges( + ctx context.Context, + scope coredata.Scoper, + documentVersionID gid.GID, + document *automerge.Document, + snapshotRevision int64, + latestRevision int64, +) error { + currentRevision := snapshotRevision + + for currentRevision < latestRevision { + var batch coredata.DocumentVersionAutomergeChanges + + if err := s.svc.pg.WithConn( + ctx, + func(ctx context.Context, conn pg.Querier) error { + return batch.LoadAfterRevision( + ctx, + conn, + scope, + documentVersionID, + currentRevision, + documentCollaborationChangeBatchSize, + ) + }, + ); err != nil { + return fmt.Errorf("cannot load collaboration change batch: %w", err) + } + + if len(batch) == 0 { + return fmt.Errorf( + "collaboration change log ends at revision %d, expected %d", + currentRevision, + latestRevision, + ) + } + + changes := make([][]byte, len(batch)) + for i, change := range batch { + if change.Revision != currentRevision+1 { + return fmt.Errorf( + "collaboration change revision is %d, expected %d", + change.Revision, + currentRevision+1, + ) + } + + changes[i] = change.ChangeBytes + currentRevision = change.Revision + } + + if err := document.ApplyChanges(ctx, changes); err != nil { + return fmt.Errorf("cannot apply collaboration change batch: %w", err) + } + } + + return nil +} + +func encodeAutomergeHeads(heads []automerge.Hash) []byte { + encoded := make([]byte, 0, len(heads)*32) + for _, head := range heads { + encoded = append(encoded, head[:]...) + } + + return encoded +} + +func claimCollaborationSeed( + ctx context.Context, + tx pg.Tx, + scope coredata.Scoper, + state *coredata.DocumentVersionAutomergeState, + now time.Time, +) (bool, error) { + if state.Seeded { + return false, nil + } + + if state.SeedClaimedAt != nil && state.SeedClaimedAt.After(now.Add(-documentCollaborationSeedLease)) { + return false, nil + } + + state.SeedClaimedAt = new(now) + state.Revision++ + + state.UpdatedAt = now + if err := state.Update(ctx, tx, scope); err != nil { + return false, fmt.Errorf("cannot update document collaboration seed claim: %w", err) + } + + return true, nil +} + +func materializeCollaboration( + ctx context.Context, + document *automerge.Document, +) (string, error) { + text, err := document.Text(ctx, "body") + if err != nil { + return "", fmt.Errorf("cannot get collaboration body: %w", err) + } + + spans, err := text.Spans(ctx) + if err != nil { + return "", fmt.Errorf("cannot get collaboration spans: %w", err) + } + + content, err := automergeprosemirror.Render(spans) + if err != nil { + return "", fmt.Errorf("cannot render collaboration spans: %w", err) + } + + content, err = prosemirror.SanitizeDocumentJSON(content) + if err != nil { + return "", fmt.Errorf("cannot sanitize collaboration content: %w", err) + } + + return content, nil +} diff --git a/pkg/probo/document_collaboration_service_test.go b/pkg/probo/document_collaboration_service_test.go new file mode 100644 index 0000000000..bf5a94407a --- /dev/null +++ b/pkg/probo/document_collaboration_service_test.go @@ -0,0 +1,53 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package probo + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/automerge" +) + +func TestEncodeAutomergeHeads_EmptyIsNotNull(t *testing.T) { + t.Parallel() + + encoded := encodeAutomergeHeads(nil) + + require.NotNil(t, encoded) + assert.Empty(t, encoded) +} + +func TestEncodeAutomergeHeads_ConcatenatesHashes(t *testing.T) { + t.Parallel() + + var first, second automerge.Hash + + first[0] = 1 + second[31] = 2 + + encoded := encodeAutomergeHeads([]automerge.Hash{first, second}) + + require.Len(t, encoded, 64) + assert.Equal(t, byte(1), encoded[0]) + assert.Equal(t, byte(2), encoded[63]) +} diff --git a/pkg/probo/document_collaboration_tools.go b/pkg/probo/document_collaboration_tools.go new file mode 100644 index 0000000000..95001aa8ae --- /dev/null +++ b/pkg/probo/document_collaboration_tools.go @@ -0,0 +1,116 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package probo + +import ( + "context" + "encoding/base64" + "fmt" + + "go.probo.inc/probo/pkg/agent" + "go.probo.inc/probo/pkg/automerge" + "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/gid" +) + +type ( + readLiveDocumentParams struct{} + + readLiveDocumentResult struct { + Text string `json:"text"` + Revision int64 `json:"revision"` + } + + editLiveDocumentParams struct { + ExpectedRevision int64 `json:"expected_revision" jsonschema:"Revision returned by read_live_document. The edit is rejected if the document changed."` + Index uint32 `json:"index" jsonschema:"UTF-16 sequence position at which to apply the edit."` + Cursor string `json:"cursor,omitempty" jsonschema:"Optional base64 stable Automerge cursor. When provided it takes precedence over index."` + DeleteCount int32 `json:"delete_count" jsonschema:"Number of UTF-16 sequence units to delete."` + Text string `json:"text" jsonschema:"Text to insert at the edit position."` + } + + editLiveDocumentResult struct { + Revision int64 `json:"revision"` + } +) + +func (s *DocumentService) CollaborationTools( + scope coredata.Scoper, + documentVersionID gid.GID, +) []agent.Tool { + return []agent.Tool{ + agent.FunctionTool( + "read_live_document", + "Read the current collaborative document text and revision before proposing an edit.", + func(ctx context.Context, _ readLiveDocumentParams) (agent.ToolResult, error) { + text, revision, err := s.ReadCollaborationText( + ctx, + scope, + documentVersionID, + ) + if err != nil { + return agent.ToolResult{}, fmt.Errorf("cannot read live document: %w", err) + } + + return agent.ResultJSON( + readLiveDocumentResult{ + Text: text, + Revision: revision, + }, + ), nil + }, + ), + agent.FunctionTool( + "edit_live_document_text", + "Edit the collaborative document using the revision returned by read_live_document. Re-read and retry if the document changed concurrently.", + func(ctx context.Context, params editLiveDocumentParams) (agent.ToolResult, error) { + var cursor automerge.Cursor + + if params.Cursor != "" { + decoded, err := base64.StdEncoding.DecodeString(params.Cursor) + if err != nil { + return agent.ToolResult{}, fmt.Errorf("cannot decode live document cursor: %w", err) + } + + cursor = automerge.Cursor(decoded) + } + + revision, err := s.ApplyCollaborationTextEdit( + ctx, + scope, + documentVersionID, + DocumentCollaborationTextEdit{ + ExpectedRevision: params.ExpectedRevision, + Index: params.Index, + Cursor: cursor, + DeleteCount: params.DeleteCount, + Text: params.Text, + }, + ) + if err != nil { + return agent.ToolResult{}, fmt.Errorf("cannot edit live document: %w", err) + } + + return agent.ResultJSON(editLiveDocumentResult{Revision: revision}), nil + }, + ), + } +} diff --git a/pkg/probod/probod.go b/pkg/probod/probod.go index d752c0eaaa..899d3001fb 100644 --- a/pkg/probod/probod.go +++ b/pkg/probod/probod.go @@ -74,7 +74,9 @@ import ( "go.probo.inc/probo/pkg/itam" "go.probo.inc/probo/pkg/mailer" "go.probo.inc/probo/pkg/mailman" + "go.probo.inc/probo/pkg/pgnotify" "go.probo.inc/probo/pkg/probo" + "go.probo.inc/probo/pkg/realtime" "go.probo.inc/probo/pkg/resourcealias" "go.probo.inc/probo/pkg/riskmanagement" "go.probo.inc/probo/pkg/securecookie" @@ -269,6 +271,20 @@ func (impl *Implm) Run( return fmt.Errorf("cannot create pg client: %w", err) } + collaborationEvents := realtime.NewEvents() + + notificationConfig, err := impl.cfg.Pg.ConnectionConfig() + if err != nil { + return fmt.Errorf("cannot configure PostgreSQL notifications: %w", err) + } + + collaborationListener := pgnotify.NewListener( + notificationConfig, + realtime.DocumentCollaborationChannel, + l.Named("document-collaboration-listener"), + collaborationEvents.Publish, + ) + pepper, err := impl.cfg.Auth.GetPepperBytes() if err != nil { rootSpan.RecordError(err) @@ -787,30 +803,35 @@ func (impl *Implm) Run( l.Named("itam"), ) + apiServerCtx, stopApiServer := context.WithCancel(context.Background()) + defer stopApiServer() + serverHandler, err := server.NewServer( server.Config{ - AllowedOrigins: impl.cfg.Api.Cors.AllowedOrigins, - ExtraHeaderFields: impl.cfg.Api.ExtraHeaderFields, - Probo: proboService, - ResourceAlias: resourceAliasService, - File: fileManagerService, - IAM: iamService, - Visitor: visitorService, - ESign: esignService, - Management: managementService, - CertManager: certManagerService, - AccessReview: accessReviewService, - AgentRun: agentRunService, - Mailman: mailmanService, - CookieBanner: cookieBannerService, - Geoloc: geolocService, - ThirdParty: thirdPartyService, - RiskManagement: riskManagementService, - ITAM: itamService, - Slack: slackService, - ConnectorRegistry: defaultConnectorRegistry, - ProviderRegistry: providerRegistry, - BaseURL: baseURL, + AllowedOrigins: impl.cfg.Api.Cors.AllowedOrigins, + ExtraHeaderFields: impl.cfg.Api.ExtraHeaderFields, + Probo: proboService, + CollaborationEvents: collaborationEvents, + ShutdownContext: apiServerCtx, + ResourceAlias: resourceAliasService, + File: fileManagerService, + IAM: iamService, + Visitor: visitorService, + ESign: esignService, + Management: managementService, + CertManager: certManagerService, + AccessReview: accessReviewService, + AgentRun: agentRunService, + Mailman: mailmanService, + CookieBanner: cookieBannerService, + Geoloc: geolocService, + ThirdParty: thirdPartyService, + RiskManagement: riskManagementService, + ITAM: itamService, + Slack: slackService, + ConnectorRegistry: defaultConnectorRegistry, + ProviderRegistry: providerRegistry, + BaseURL: baseURL, GraphQLLimits: gqlutils.Limits{ ParserTokenLimit: impl.cfg.Api.GraphQL.ParserTokenLimit, ComplexityLimit: impl.cfg.Api.GraphQL.ComplexityLimit, @@ -853,8 +874,16 @@ func (impl *Implm) Run( return fmt.Errorf("cannot create compliance portal handler: %w", err) } - apiServerCtx, stopApiServer := context.WithCancel(context.Background()) - defer stopApiServer() + notificationCtx, stopNotifications := context.WithCancel(context.Background()) + defer stopNotifications() + + wg.Go( + func() { + if err := collaborationListener.Run(notificationCtx); err != nil { + cancel(fmt.Errorf("document collaboration listener crashed: %w", err)) + } + }, + ) wg.Go( func() { @@ -1220,6 +1249,7 @@ func (impl *Implm) Run( <-ctx.Done() stopApiServer() + stopNotifications() stopCompliancePortalServer() stopWebhookWorker() stopESignService() diff --git a/pkg/probodconfig/pg_config.go b/pkg/probodconfig/pg_config.go index c96c35b1f4..319e1d01fb 100644 --- a/pkg/probodconfig/pg_config.go +++ b/pkg/probodconfig/pg_config.go @@ -21,10 +21,15 @@ package probodconfig import ( + "crypto/tls" "crypto/x509" "encoding/pem" + "fmt" + "net" + "strconv" "time" + "github.com/jackc/pgx/v5" "go.gearno.de/kit/pg" ) @@ -128,3 +133,72 @@ func (cfg PgConfig) Options(options ...pg.Option) []pg.Option { return opts } + +func (cfg PgConfig) ConnectionConfig() (*pgx.ConnConfig, error) { + config, err := pgx.ParseConfig("postgres://localhost?sslmode=disable") + if err != nil { + return nil, fmt.Errorf("cannot create base PostgreSQL connection config: %w", err) + } + + host, portValue, err := net.SplitHostPort(cfg.Addr) + if err != nil { + return nil, fmt.Errorf("cannot parse PostgreSQL address: %w", err) + } + + port, err := strconv.ParseUint(portValue, 10, 16) + if err != nil { + return nil, fmt.Errorf("cannot parse PostgreSQL port: %w", err) + } + + config.Host = host + config.Port = uint16(port) + config.User = cfg.Username + config.Password = cfg.Password + config.Database = cfg.Database + + if cfg.CACertBundle != "" { + certificates := parseCertificates(cfg.CACertBundle) + if len(certificates) == 0 { + return nil, fmt.Errorf("PostgreSQL CA certificate bundle has no certificates") + } + + roots := x509.NewCertPool() + for _, certificate := range certificates { + roots.AddCert(certificate) + } + + config.TLSConfig = &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: roots, + ServerName: host, + } + } + + return config, nil +} + +func parseCertificates(bundle string) []*x509.Certificate { + var certificates []*x509.Certificate + + pemData := []byte(bundle) + + for len(pemData) > 0 { + block, remaining := pem.Decode(pemData) + pemData = remaining + + if block == nil { + break + } + + if block.Type != "CERTIFICATE" { + continue + } + + certificate, err := x509.ParseCertificate(block.Bytes) + if err == nil { + certificates = append(certificates, certificate) + } + } + + return certificates +} diff --git a/pkg/probodconfig/pg_config_test.go b/pkg/probodconfig/pg_config_test.go new file mode 100644 index 0000000000..988dfe2a21 --- /dev/null +++ b/pkg/probodconfig/pg_config_test.go @@ -0,0 +1,49 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package probodconfig_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/probodconfig" +) + +func TestPgConfig_ConnectionConfig(t *testing.T) { + t.Parallel() + + input := probodconfig.PgConfig{ + Addr: "database.internal:5433", + Username: "probod", + Password: "secret", + Database: "probod_test", + } + config, err := input.ConnectionConfig() + + require.NoError(t, err) + assert.Equal(t, "database.internal", config.Host) + assert.Equal(t, uint16(5433), config.Port) + assert.Equal(t, "probod", config.User) + assert.Equal(t, "secret", config.Password) + assert.Equal(t, "probod_test", config.Database) + assert.Nil(t, config.TLSConfig) +} diff --git a/pkg/realtime/events.go b/pkg/realtime/events.go new file mode 100644 index 0000000000..fc68d264e9 --- /dev/null +++ b/pkg/realtime/events.go @@ -0,0 +1,68 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package realtime + +import "sync" + +const DocumentCollaborationChannel = "document_collaboration_changed" + +type ( + Handler func(payload string) + + Events struct { + mu sync.RWMutex + handlers map[uint64]Handler + nextHandler uint64 + } +) + +func NewEvents() *Events { + return &Events{handlers: make(map[uint64]Handler)} +} + +func (e *Events) Subscribe(handler Handler) func() { + e.mu.Lock() + id := e.nextHandler + e.nextHandler++ + e.handlers[id] = handler + e.mu.Unlock() + + return func() { + e.mu.Lock() + delete(e.handlers, id) + e.mu.Unlock() + } +} + +func (e *Events) Publish(payload string) { + e.mu.RLock() + + handlers := make([]Handler, 0, len(e.handlers)) + for _, handler := range e.handlers { + handlers = append(handlers, handler) + } + + e.mu.RUnlock() + + for _, handler := range handlers { + handler(payload) + } +} diff --git a/pkg/realtime/events_test.go b/pkg/realtime/events_test.go new file mode 100644 index 0000000000..317060b669 --- /dev/null +++ b/pkg/realtime/events_test.go @@ -0,0 +1,46 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package realtime_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.probo.inc/probo/pkg/realtime" +) + +func TestEvents_SubscribeAndUnsubscribe(t *testing.T) { + t.Parallel() + + events := realtime.NewEvents() + + var received []string + + unsubscribe := events.Subscribe(func(payload string) { + received = append(received, payload) + }) + + events.Publish("first") + unsubscribe() + events.Publish("second") + + assert.Equal(t, []string{"first"}, received) +} diff --git a/pkg/server/api/api.go b/pkg/server/api/api.go index 12c7406cec..ca103e58d1 100644 --- a/pkg/server/api/api.go +++ b/pkg/server/api/api.go @@ -21,6 +21,7 @@ package api import ( + "context" "errors" "fmt" "net/http" @@ -46,6 +47,7 @@ import ( "go.probo.inc/probo/pkg/itam" "go.probo.inc/probo/pkg/mailman" "go.probo.inc/probo/pkg/probo" + "go.probo.inc/probo/pkg/realtime" "go.probo.inc/probo/pkg/resourcealias" "go.probo.inc/probo/pkg/riskmanagement" "go.probo.inc/probo/pkg/saferedirect" @@ -64,32 +66,34 @@ import ( type ( Config struct { - BaseURL *baseurl.BaseURL - AllowedOrigins []string - Probo *probo.Service - ResourceAlias *resourcealias.Service - File *filemanager.Service - IAM *iam.Service - Visitor *visitor.Service - ESign *esign.Service - Management *management.Service - CertManager *certmanager.Service - AccessReview *accessreview.Service - AgentRun *agentrun.Service - Slack *slack.Service - Mailman *mailman.Service - CookieBanner *cookiebanner.Service - Geoloc *geoloc.Service - ThirdParty *thirdparty.Service - RiskManagement *riskmanagement.Service - ITAM *itam.Service - Cookie securecookie.Config - TokenSecret string - ConnectorRegistry *connector.ConnectorRegistry - ProviderRegistry *provider.Registry - CustomDomainCname string - GraphQLLimits gqlutils.Limits - Logger *log.Logger + BaseURL *baseurl.BaseURL + AllowedOrigins []string + Probo *probo.Service + CollaborationEvents *realtime.Events + ShutdownContext context.Context + ResourceAlias *resourcealias.Service + File *filemanager.Service + IAM *iam.Service + Visitor *visitor.Service + ESign *esign.Service + Management *management.Service + CertManager *certmanager.Service + AccessReview *accessreview.Service + AgentRun *agentrun.Service + Slack *slack.Service + Mailman *mailman.Service + CookieBanner *cookiebanner.Service + Geoloc *geoloc.Service + ThirdParty *thirdparty.Service + RiskManagement *riskmanagement.Service + ITAM *itam.Service + Cookie securecookie.Config + TokenSecret string + ConnectorRegistry *connector.ConnectorRegistry + ProviderRegistry *provider.Registry + CustomDomainCname string + GraphQLLimits gqlutils.Limits + Logger *log.Logger } MCPConfig struct { @@ -204,6 +208,8 @@ func NewServer(cfg Config) (*Server, error) { consoleHandler: console_v1.NewMux( cfg.Logger.Named("console.v1"), cfg.Probo, + cfg.CollaborationEvents, + cfg.ShutdownContext, cfg.ResourceAlias, cfg.IAM, cfg.ESign, @@ -219,6 +225,7 @@ func NewServer(cfg Config) (*Server, error) { cfg.ProviderRegistry, cfg.File, cfg.BaseURL, + cfg.AllowedOrigins, cfg.CustomDomainCname, cfg.ThirdParty, cfg.RiskManagement, diff --git a/pkg/server/api/console/v1/document_collaboration_handler.go b/pkg/server/api/console/v1/document_collaboration_handler.go new file mode 100644 index 0000000000..98ee5c392a --- /dev/null +++ b/pkg/server/api/console/v1/document_collaboration_handler.go @@ -0,0 +1,686 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package console_v1 + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "time" + + "github.com/coder/websocket" + "github.com/go-chi/chi/v5" + "go.gearno.de/kit/httpserver" + "go.gearno.de/kit/log" + "go.probo.inc/probo/pkg/automerge" + "go.probo.inc/probo/pkg/baseurl" + "go.probo.inc/probo/pkg/bearertoken" + "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/iam" + "go.probo.inc/probo/pkg/probo" + "go.probo.inc/probo/pkg/server/api/authn" + "go.probo.inc/probo/pkg/server/jsonx" +) + +const ( + documentCollaborationProtocol = "automerge-sync-v1" + documentCollaborationMessageMaxBytes = 1024 * 1024 + documentCollaborationRefreshInterval = 500 * time.Millisecond + documentCollaborationWriteTimeout = 10 * time.Second +) + +type ( + documentCollaborationHandler struct { + logger *log.Logger + probo *probo.Service + iam *iam.Service + baseURL *baseurl.BaseURL + allowedOrigins []string + hub *documentCollaborationHub + shutdown context.Context + } + + documentCollaborationHandshake struct { + Type string `json:"type"` + Version int `json:"version"` + Revision int64 `json:"revision"` + NeedsSeed bool `json:"needsSeed"` + SeedContent string `json:"seedContent,omitempty"` + ConnectionID string `json:"connectionId"` + } + + documentCollaborationIncoming struct { + MessageType websocket.MessageType + Data []byte + Err error + } + + documentCollaborationPresenceInput struct { + Type string `json:"type"` + AnchorPosition int `json:"anchorPosition"` + HeadPosition int `json:"headPosition"` + } + + documentCollaborationPresence struct { + ConnectionID string `json:"connectionId"` + IdentityID string `json:"identityId"` + AnchorPosition int `json:"anchorPosition"` + HeadPosition int `json:"headPosition"` + } + + documentCollaborationPresenceSnapshot struct { + Type string `json:"type"` + Presences []documentCollaborationPresence `json:"presences"` + } +) + +func (h *documentCollaborationHandler) handle(w http.ResponseWriter, r *http.Request) { + documentVersionIDString := chi.URLParam(r, "documentVersionID") + + documentVersionID, err := gid.ParseGID(documentVersionIDString) + if err != nil || documentVersionID.EntityType() != coredata.DocumentVersionEntityType { + jsonx.RenderNotFound(w, fmt.Errorf("document version not found")) + return + } + + scope, err := h.authorize(r.Context(), documentVersionID) + if err != nil { + h.renderAuthorizationError(w, err) + return + } + + identity := authn.IdentityFromContext(r.Context()) + + connectionID, err := newDocumentCollaborationConnectionID() + if err != nil { + jsonx.RenderInternalServerError(w) + return + } + + lease, err := h.hub.acquire( + r.Context(), + scope, + documentVersionID, + connectionID, + ) + if err != nil { + h.renderServiceError(w, r, documentVersionIDString, err) + return + } + + defer lease.Close() + + collaboration := lease.Collaboration() + + defer func() { + deleteCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := h.probo.Documents.DeleteCollaborationPresence( + deleteCtx, + scope, + connectionID, + ); err != nil { + h.logger.WarnCtx( + deleteCtx, + "cannot delete document collaboration presence", + log.Error(err), + log.String("document_version_id", documentVersionIDString), + ) + } + }() + + if lease.SeedOwner() { + defer func() { + releaseCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := h.probo.Documents.ReleaseCollaborationSeed( + releaseCtx, + scope, + documentVersionID, + ); err != nil { + h.logger.WarnCtx( + releaseCtx, + "cannot release document collaboration seed", + log.Error(err), + log.String("document_version_id", documentVersionIDString), + ) + } + }() + } + + connection, err := websocket.Accept( + w, + r, + &websocket.AcceptOptions{ + Subprotocols: []string{documentCollaborationProtocol}, + OriginPatterns: h.allowedOrigins, + CompressionMode: websocket.CompressionDisabled, + InsecureSkipVerify: false, + }, + ) + if err != nil { + h.logger.WarnCtx( + r.Context(), + "cannot accept document collaboration connection", + log.Error(err), + log.String("document_version_id", documentVersionIDString), + ) + + return + } + + defer func() { + _ = connection.Close(websocket.StatusNormalClosure, "") + }() + + connection.SetReadLimit(documentCollaborationMessageMaxBytes) + + if connection.Subprotocol() != documentCollaborationProtocol { + _ = connection.Close(websocket.StatusPolicyViolation, "required subprotocol not negotiated") + return + } + + syncState, err := collaboration.Document.NewSyncState(r.Context()) + if err != nil { + h.closeWithError(r.Context(), connection, documentVersionIDString, err) + return + } + + defer func() { _ = syncState.Close(context.Background()) }() + + seedContent := "" + if lease.SeedOwner() { + seedContent = collaboration.SeedContent + } + + handshake, err := json.Marshal( + documentCollaborationHandshake{ + Type: "ready", + Version: 1, + Revision: lease.Revision(), + NeedsSeed: lease.SeedOwner(), + SeedContent: seedContent, + ConnectionID: connectionID, + }, + ) + if err != nil { + h.closeWithError(r.Context(), connection, documentVersionIDString, err) + return + } + + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + + if err := writeCollaborationMessage( + ctx, + connection, + websocket.MessageText, + handshake, + ); err != nil { + return + } + + if err := sendAvailableSyncMessages(ctx, connection, syncState); err != nil { + h.closeWithError(ctx, connection, documentVersionIDString, err) + return + } + + incoming := make(chan documentCollaborationIncoming, 1) + go readCollaborationMessages(ctx, connection, incoming) + + revision := lease.Revision() + + ticker := time.NewTicker(documentCollaborationRefreshInterval) + defer ticker.Stop() + + for { + select { + case <-h.shutdown.Done(): + _ = connection.CloseNow() + + return + case <-ctx.Done(): + return + case message := <-incoming: + if message.Err != nil { + if websocket.CloseStatus(message.Err) == websocket.StatusNormalClosure || + websocket.CloseStatus(message.Err) == websocket.StatusGoingAway { + return + } + + h.logger.WarnCtx( + ctx, + "document collaboration connection closed with read error", + log.Error(message.Err), + log.String("document_version_id", documentVersionIDString), + ) + + return + } + + if message.MessageType != websocket.MessageBinary { + if message.MessageType != websocket.MessageText { + _ = connection.Close(websocket.StatusUnsupportedData, "unsupported message type") + return + } + + var presence documentCollaborationPresenceInput + if err := json.Unmarshal(message.Data, &presence); err != nil || + presence.Type != "presence" { + _ = connection.Close(websocket.StatusInvalidFramePayloadData, "invalid presence message") + return + } + + lease.UpdatePresence( + documentCollaborationPresence{ + ConnectionID: connectionID, + IdentityID: identity.ID.String(), + AnchorPosition: presence.AnchorPosition, + HeadPosition: presence.HeadPosition, + }, + ) + + if err := h.probo.Documents.SaveCollaborationPresence( + ctx, + scope, + documentVersionID, + identity.ID, + connectionID, + presence.AnchorPosition, + presence.HeadPosition, + ); err != nil { + h.closeWithError(ctx, connection, documentVersionIDString, err) + return + } + + continue + } + + if err := syncState.ReceiveMessage(ctx, message.Data); err != nil { + h.closeWithError(ctx, connection, documentVersionIDString, err) + return + } + + lease.NotifyPeers() + + if err := sendAvailableSyncMessages(ctx, connection, syncState); err != nil { + h.closeWithError(ctx, connection, documentVersionIDString, err) + + return + } + + if err := lease.PersistError(); err != nil { + h.closeWithError(ctx, connection, documentVersionIDString, err) + + return + } + + lease.SchedulePersist() + case <-ticker.C: + if err := lease.PersistError(); err != nil { + h.closeWithError(ctx, connection, documentVersionIDString, err) + + return + } + + var changed bool + + revision, changed, err = h.probo.Documents.RefreshCollaboration( + ctx, + scope, + documentVersionID, + collaboration.Document, + revision, + ) + if err != nil { + h.closeWithError(ctx, connection, documentVersionIDString, err) + return + } + + if changed { + lease.SetRevision(revision) + + if err := sendAvailableSyncMessages(ctx, connection, syncState); err != nil { + h.closeWithError(ctx, connection, documentVersionIDString, err) + return + } + } + + if err := h.sendPresences( + ctx, + connection, + scope, + documentVersionID, + connectionID, + ); err != nil { + h.closeWithError(ctx, connection, documentVersionIDString, err) + return + } + case wake := <-lease.Wake: + if wake.presence { + if err := writeCollaborationPresences( + ctx, + connection, + wake.presences, + ); err != nil { + h.closeWithError(ctx, connection, documentVersionIDString, err) + + return + } + + continue + } + + if err := lease.PersistError(); err != nil { + h.closeWithError(ctx, connection, documentVersionIDString, err) + + return + } + + if wake.refresh { + var changed bool + + revision, changed, err = h.probo.Documents.RefreshCollaboration( + ctx, + scope, + documentVersionID, + collaboration.Document, + revision, + ) + if err != nil { + h.closeWithError(ctx, connection, documentVersionIDString, err) + + return + } + + lease.SetRevision(revision) + + if !changed { + continue + } + } else { + revision = lease.Revision() + } + + if err := sendAvailableSyncMessages(ctx, connection, syncState); err != nil { + h.closeWithError(ctx, connection, documentVersionIDString, err) + + return + } + } + } +} + +func (h *documentCollaborationHandler) sendPresences( + ctx context.Context, + connection *websocket.Conn, + scope coredata.Scoper, + documentVersionID gid.GID, + connectionID string, +) error { + stored, err := h.probo.Documents.ListCollaborationPresences( + ctx, + scope, + documentVersionID, + connectionID, + ) + if err != nil { + return fmt.Errorf("cannot list document collaboration presences: %w", err) + } + + presences := make([]documentCollaborationPresence, len(stored)) + for i, presence := range stored { + presences[i] = documentCollaborationPresence{ + ConnectionID: presence.ConnectionID, + IdentityID: presence.IdentityID.String(), + AnchorPosition: presence.AnchorPosition, + HeadPosition: presence.HeadPosition, + } + } + + return writeCollaborationPresences(ctx, connection, presences) +} + +func writeCollaborationPresences( + ctx context.Context, + connection *websocket.Conn, + presences []documentCollaborationPresence, +) error { + data, err := json.Marshal( + documentCollaborationPresenceSnapshot{ + Type: "presence", + Presences: presences, + }, + ) + if err != nil { + return fmt.Errorf("cannot marshal document collaboration presences: %w", err) + } + + if err := writeCollaborationMessage(ctx, connection, websocket.MessageText, data); err != nil { + return fmt.Errorf("cannot send document collaboration presences: %w", err) + } + + return nil +} + +func newDocumentCollaborationConnectionID() (string, error) { + var value [16]byte + if _, err := rand.Read(value[:]); err != nil { + return "", fmt.Errorf("cannot generate document collaboration connection ID: %w", err) + } + + return hex.EncodeToString(value[:]), nil +} + +func (h *documentCollaborationHandler) authorize( + ctx context.Context, + documentVersionID gid.GID, +) (*coredata.Scope, error) { + versionScope, err := h.authorizeResource( + ctx, + documentVersionID, + probo.ActionDocumentVersionGet, + ) + if err != nil { + return nil, fmt.Errorf("cannot authorize document version: %w", err) + } + + version, err := h.probo.Documents.GetVersion(ctx, versionScope, documentVersionID) + if err != nil { + return nil, fmt.Errorf("cannot load authorized document version: %w", err) + } + + scope, err := h.authorizeResource(ctx, version.DocumentID, probo.ActionDocumentUpdate) + if err != nil { + return nil, fmt.Errorf("cannot authorize document update: %w", err) + } + + return scope, nil +} + +func (h *documentCollaborationHandler) authorizeResource( + ctx context.Context, + resourceID gid.GID, + action string, +) (*coredata.Scope, error) { + identity := authn.IdentityFromContext(ctx) + session := authn.SessionFromContext(ctx) + + params := iam.AuthorizeParams{ + Principal: identity.ID, + Resource: resourceID, + Action: action, + ResourceAttributes: make(map[string]string), + } + if session != nil { + params.Session = &session.ID + } + + scope, err := h.iam.Authorizer.Authorize(ctx, params) + if err != nil { + return nil, fmt.Errorf("cannot authorize resource: %w", err) + } + + return scope, nil +} + +func (h *documentCollaborationHandler) renderAuthorizationError(w http.ResponseWriter, err error) { + if scopeErr, ok := errors.AsType[*iam.ErrInsufficientOAuth2Scope](err); ok { + bearertoken.SetBearerInsufficientScope(w, h.baseURL, scopeErr.Scopes...) + jsonx.RenderForbidden(w) + + return + } + + if _, ok := errors.AsType[*iam.ErrInsufficientPermissions](err); ok { + jsonx.RenderForbidden(w) + return + } + + jsonx.RenderNotFound(w, fmt.Errorf("document version not found")) +} + +func (h *documentCollaborationHandler) renderServiceError( + w http.ResponseWriter, + r *http.Request, + documentVersionID string, + err error, +) { + if errors.Is(err, coredata.ErrResourceNotFound) { + jsonx.RenderNotFound(w, fmt.Errorf("document version not found")) + return + } + + if _, ok := errors.AsType[*probo.ErrDocumentVersionNotDraft](err); ok { + httpserver.RenderError(w, http.StatusConflict, err) + return + } + + if _, ok := errors.AsType[*probo.ErrDocumentArchived](err); ok { + httpserver.RenderError(w, http.StatusConflict, err) + return + } + + if _, ok := errors.AsType[*probo.ErrDocumentVersionGenerated](err); ok { + httpserver.RenderError(w, http.StatusConflict, err) + return + } + + h.logger.ErrorCtx( + r.Context(), + "cannot open document collaboration", + log.Error(err), + log.String("document_version_id", documentVersionID), + ) + jsonx.RenderInternalServerError(w) +} + +func (h *documentCollaborationHandler) closeWithError( + ctx context.Context, + connection *websocket.Conn, + documentVersionID string, + err error, +) { + h.logger.ErrorCtx( + ctx, + "document collaboration failed", + log.Error(err), + log.String("document_version_id", documentVersionID), + ) + + _ = connection.Close(websocket.StatusInternalError, "collaboration failed") +} + +func readCollaborationMessages( + ctx context.Context, + connection *websocket.Conn, + incoming chan<- documentCollaborationIncoming, +) { + for { + messageType, data, err := connection.Read(ctx) + + message := documentCollaborationIncoming{ + MessageType: messageType, + Data: data, + Err: err, + } + select { + case incoming <- message: + case <-ctx.Done(): + return + } + + if err != nil { + return + } + } +} + +func sendAvailableSyncMessages( + ctx context.Context, + connection *websocket.Conn, + syncState *automerge.SyncState, +) error { + for range 100 { + message, ok, err := syncState.GenerateMessage(ctx) + if err != nil { + return fmt.Errorf("cannot generate sync message: %w", err) + } + + if !ok { + return nil + } + + if err := writeCollaborationMessage( + ctx, + connection, + websocket.MessageBinary, + message, + ); err != nil { + return fmt.Errorf("cannot write sync message: %w", err) + } + } + + return fmt.Errorf("cannot generate sync messages: protocol did not quiesce") +} + +func writeCollaborationMessage( + ctx context.Context, + connection *websocket.Conn, + messageType websocket.MessageType, + data []byte, +) error { + writeCtx, cancel := context.WithTimeout(ctx, documentCollaborationWriteTimeout) + defer cancel() + + if err := connection.Write(writeCtx, messageType, data); err != nil { + return fmt.Errorf("cannot write collaboration message: %w", err) + } + + return nil +} diff --git a/pkg/server/api/console/v1/document_collaboration_hub.go b/pkg/server/api/console/v1/document_collaboration_hub.go new file mode 100644 index 0000000000..7705d9d173 --- /dev/null +++ b/pkg/server/api/console/v1/document_collaboration_hub.go @@ -0,0 +1,407 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package console_v1 + +import ( + "context" + "fmt" + "sort" + "sync" + "sync/atomic" + "time" + + "go.probo.inc/probo/pkg/automerge" + "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/probo" + "go.probo.inc/probo/pkg/realtime" +) + +type ( + documentCollaborationWake struct { + refresh bool + presence bool + presences []documentCollaborationPresence + } + + documentCollaborationRoomPeer struct { + connectionID string + wake chan documentCollaborationWake + } + + documentCollaborationDocuments interface { + OpenCollaboration( + context.Context, + coredata.Scoper, + gid.GID, + ) (*probo.DocumentCollaboration, error) + PersistCollaboration( + context.Context, + coredata.Scoper, + gid.GID, + *automerge.Document, + ) (int64, error) + } + + documentCollaborationHub struct { + mu sync.Mutex + documents documentCollaborationDocuments + rooms map[gid.GID]*documentCollaborationRoom + } + + documentCollaborationRoom struct { + mu sync.Mutex + collaboration *probo.DocumentCollaboration + documents documentCollaborationDocuments + scope coredata.Scoper + versionID gid.GID + revision atomic.Int64 + peers map[uint64]documentCollaborationRoomPeer + presences map[string]documentCollaborationPresence + nextPeerID uint64 + dirty chan struct{} + stop chan struct{} + done chan struct{} + persistErr error + } + + documentCollaborationRoomLease struct { + hub *documentCollaborationHub + documentVersionID gid.GID + room *documentCollaborationRoom + peerID uint64 + Wake <-chan documentCollaborationWake + seedOwner bool + once sync.Once + } +) + +const ( + documentCollaborationPersistDebounce = 50 * time.Millisecond + documentCollaborationPersistTimeout = 5 * time.Second +) + +func newDocumentCollaborationHub( + documents documentCollaborationDocuments, + events *realtime.Events, +) *documentCollaborationHub { + hub := &documentCollaborationHub{ + documents: documents, + rooms: make(map[gid.GID]*documentCollaborationRoom), + } + if events != nil { + events.Subscribe(hub.notifyExternal) + } + + return hub +} + +func (h *documentCollaborationHub) acquire( + ctx context.Context, + scope coredata.Scoper, + documentVersionID gid.GID, + connectionID string, +) (*documentCollaborationRoomLease, error) { + h.mu.Lock() + if room := h.rooms[documentVersionID]; room != nil { + lease := h.addPeerLocked(documentVersionID, room, connectionID) + h.mu.Unlock() + + return lease, nil + } + + h.mu.Unlock() + + collaboration, err := h.documents.OpenCollaboration( + ctx, + scope, + documentVersionID, + ) + if err != nil { + return nil, fmt.Errorf("cannot open collaboration room: %w", err) + } + + room := &documentCollaborationRoom{ + collaboration: collaboration, + documents: h.documents, + scope: scope, + versionID: documentVersionID, + peers: make(map[uint64]documentCollaborationRoomPeer), + presences: make(map[string]documentCollaborationPresence), + dirty: make(chan struct{}, 1), + stop: make(chan struct{}), + done: make(chan struct{}), + } + room.revision.Store(collaboration.Revision) + + h.mu.Lock() + if existing := h.rooms[documentVersionID]; existing != nil { + _ = collaboration.Document.Close(context.Background()) + lease := h.addPeerLocked(documentVersionID, existing, connectionID) + h.mu.Unlock() + + return lease, nil + } + + h.rooms[documentVersionID] = room + lease := h.addPeerLocked(documentVersionID, room, connectionID) + lease.seedOwner = collaboration.NeedsSeed + + go room.run() + h.mu.Unlock() + + return lease, nil +} + +func (h *documentCollaborationHub) addPeerLocked( + documentVersionID gid.GID, + room *documentCollaborationRoom, + connectionID string, +) *documentCollaborationRoomLease { + room.mu.Lock() + peerID := room.nextPeerID + room.nextPeerID++ + wake := make(chan documentCollaborationWake, 8) + room.peers[peerID] = documentCollaborationRoomPeer{ + connectionID: connectionID, + wake: wake, + } + room.mu.Unlock() + + return &documentCollaborationRoomLease{ + hub: h, + documentVersionID: documentVersionID, + room: room, + peerID: peerID, + Wake: wake, + } +} + +func (l *documentCollaborationRoomLease) Collaboration() *probo.DocumentCollaboration { + return l.room.collaboration +} + +func (l *documentCollaborationRoomLease) Revision() int64 { + return l.room.revision.Load() +} + +func (l *documentCollaborationRoomLease) SeedOwner() bool { + return l.seedOwner +} + +func (l *documentCollaborationRoomLease) SetRevision(revision int64) { + l.room.revision.Store(revision) +} + +func (l *documentCollaborationRoomLease) NotifyPeers() { + l.room.mu.Lock() + defer l.room.mu.Unlock() + + for peerID, peer := range l.room.peers { + if peerID == l.peerID { + continue + } + + select { + case peer.wake <- documentCollaborationWake{}: + default: + } + } +} + +func (l *documentCollaborationRoomLease) UpdatePresence( + presence documentCollaborationPresence, +) { + l.room.mu.Lock() + defer l.room.mu.Unlock() + + l.room.presences[presence.ConnectionID] = presence + l.room.notifyPresenceLocked() +} + +func (r *documentCollaborationRoom) notifyPresenceLocked() { + for _, peer := range r.peers { + presences := make([]documentCollaborationPresence, 0, len(r.presences)) + for connectionID, presence := range r.presences { + if connectionID != peer.connectionID { + presences = append(presences, presence) + } + } + + sort.Slice(presences, func(i, j int) bool { + return presences[i].ConnectionID < presences[j].ConnectionID + }) + + select { + case peer.wake <- documentCollaborationWake{ + presence: true, + presences: presences, + }: + default: + } + } +} + +func (h *documentCollaborationHub) notifyExternal(payload string) { + documentVersionID, err := gid.ParseGID(payload) + if err != nil || documentVersionID.EntityType() != coredata.DocumentVersionEntityType { + return + } + + h.mu.Lock() + room := h.rooms[documentVersionID] + h.mu.Unlock() + + if room == nil { + return + } + + room.mu.Lock() + defer room.mu.Unlock() + + for _, peer := range room.peers { + select { + case peer.wake <- documentCollaborationWake{refresh: true}: + default: + } + } +} + +func (l *documentCollaborationRoomLease) SchedulePersist() { + select { + case l.room.dirty <- struct{}{}: + default: + } +} + +func (l *documentCollaborationRoomLease) PersistError() error { + l.room.mu.Lock() + defer l.room.mu.Unlock() + + return l.room.persistErr +} + +func (l *documentCollaborationRoomLease) Close() { + l.once.Do(func() { + l.hub.mu.Lock() + l.room.mu.Lock() + peer := l.room.peers[l.peerID] + delete(l.room.peers, l.peerID) + delete(l.room.presences, peer.connectionID) + + empty := len(l.room.peers) == 0 + if !empty { + l.room.notifyPresenceLocked() + } + + l.room.mu.Unlock() + + if !empty { + l.hub.mu.Unlock() + + return + } + + delete(l.hub.rooms, l.documentVersionID) + l.hub.mu.Unlock() + + if l.room.stop != nil { + close(l.room.stop) + <-l.room.done + } + + _ = l.room.collaboration.Document.Close(context.Background()) + }) +} + +func (r *documentCollaborationRoom) run() { + defer close(r.done) + + var timer *time.Timer + + dirty := false + + for { + var timerChannel <-chan time.Time + if timer != nil { + timerChannel = timer.C + } + + select { + case <-r.dirty: + dirty = true + + if timer == nil { + timer = time.NewTimer(documentCollaborationPersistDebounce) + } else { + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + + timer.Reset(documentCollaborationPersistDebounce) + } + case <-timerChannel: + if dirty { + r.persist() + + dirty = false + } + + timer = nil + case <-r.stop: + if timer != nil { + timer.Stop() + } + + if dirty { + r.persist() + } + + return + } + } +} + +func (r *documentCollaborationRoom) persist() { + ctx, cancel := context.WithTimeout( + context.Background(), + documentCollaborationPersistTimeout, + ) + defer cancel() + + revision, err := r.documents.PersistCollaboration( + ctx, + r.scope, + r.versionID, + r.collaboration.Document, + ) + + r.mu.Lock() + defer r.mu.Unlock() + + r.persistErr = err + if err == nil { + r.revision.Store(revision) + } +} diff --git a/pkg/server/api/console/v1/document_collaboration_hub_test.go b/pkg/server/api/console/v1/document_collaboration_hub_test.go new file mode 100644 index 0000000000..3e5fb58d85 --- /dev/null +++ b/pkg/server/api/console/v1/document_collaboration_hub_test.go @@ -0,0 +1,197 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package console_v1 + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/automerge" + "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/probo" +) + +type fakeDocumentCollaborationDocuments struct { + collaboration *probo.DocumentCollaboration + persisted atomic.Int64 + persistedCh chan struct{} +} + +func (f *fakeDocumentCollaborationDocuments) OpenCollaboration( + context.Context, + coredata.Scoper, + gid.GID, +) (*probo.DocumentCollaboration, error) { + return f.collaboration, nil +} + +func (f *fakeDocumentCollaborationDocuments) PersistCollaboration( + context.Context, + coredata.Scoper, + gid.GID, + *automerge.Document, +) (int64, error) { + revision := f.persisted.Add(1) + 1 + + select { + case f.persistedCh <- struct{}{}: + default: + } + + return revision, nil +} + +func TestDocumentCollaborationRoom_NotifiesOtherPeers(t *testing.T) { + t.Parallel() + + tenantID := gid.NewTenantID() + versionID := gid.New(tenantID, coredata.DocumentVersionEntityType) + document, err := automerge.New(context.Background(), automerge.ActorID{1}) + require.NoError(t, err) + + room := &documentCollaborationRoom{ + collaboration: &probo.DocumentCollaboration{ + Document: document, + Revision: 1, + }, + peers: make(map[uint64]documentCollaborationRoomPeer), + presences: make(map[string]documentCollaborationPresence), + } + room.revision.Store(1) + hub := &documentCollaborationHub{ + rooms: map[gid.GID]*documentCollaborationRoom{ + versionID: room, + }, + } + + hub.mu.Lock() + first := hub.addPeerLocked(versionID, room, "first") + second := hub.addPeerLocked(versionID, room, "second") + hub.mu.Unlock() + + first.SetRevision(2) + first.NotifyPeers() + + assert.Equal(t, int64(2), second.Revision()) + + select { + case <-second.Wake: + default: + require.Fail(t, "second peer was not notified") + } + + select { + case <-first.Wake: + require.Fail(t, "originating peer must not notify itself") + default: + } + + hub.notifyExternal(versionID.String()) + + for _, wakeChannel := range []<-chan documentCollaborationWake{ + first.Wake, + second.Wake, + } { + select { + case wake := <-wakeChannel: + assert.True(t, wake.refresh) + default: + require.Fail(t, "external notification did not wake peer") + } + } + + first.UpdatePresence( + documentCollaborationPresence{ + ConnectionID: "first", + IdentityID: "identity", + AnchorPosition: 3, + HeadPosition: 4, + }, + ) + sourcePresence := <-first.Wake + assert.True(t, sourcePresence.presence) + assert.Empty(t, sourcePresence.presences) + + remotePresence := <-second.Wake + assert.True(t, remotePresence.presence) + require.Len(t, remotePresence.presences, 1) + assert.Equal(t, "first", remotePresence.presences[0].ConnectionID) + assert.Equal(t, 3, remotePresence.presences[0].AnchorPosition) + + first.Close() + assert.Contains(t, hub.rooms, versionID) + second.Close() + assert.NotContains(t, hub.rooms, versionID) +} + +func TestDocumentCollaborationRoom_DebouncesPersistence(t *testing.T) { + t.Parallel() + + tenantID := gid.NewTenantID() + versionID := gid.New(tenantID, coredata.DocumentVersionEntityType) + document, err := automerge.New(context.Background(), automerge.ActorID{2}) + require.NoError(t, err) + + documents := &fakeDocumentCollaborationDocuments{ + collaboration: &probo.DocumentCollaboration{ + Document: document, + Revision: 1, + }, + persistedCh: make(chan struct{}, 1), + } + room := &documentCollaborationRoom{ + collaboration: documents.collaboration, + documents: documents, + scope: coredata.NewScope(tenantID), + versionID: versionID, + peers: make(map[uint64]documentCollaborationRoomPeer), + presences: make(map[string]documentCollaborationPresence), + dirty: make(chan struct{}, 1), + stop: make(chan struct{}), + done: make(chan struct{}), + } + + room.revision.Store(1) + go room.run() + + lease := &documentCollaborationRoomLease{room: room} + for range 20 { + lease.SchedulePersist() + } + + select { + case <-documents.persistedCh: + case <-time.After(time.Second): + require.Fail(t, "collaboration room did not persist") + } + + assert.Equal(t, int64(1), documents.persisted.Load()) + assert.Equal(t, int64(2), room.revision.Load()) + + close(room.stop) + <-room.done + require.NoError(t, document.Close(context.Background())) +} diff --git a/pkg/server/api/console/v1/resolver.go b/pkg/server/api/console/v1/resolver.go index 4217d0d67a..5856315625 100644 --- a/pkg/server/api/console/v1/resolver.go +++ b/pkg/server/api/console/v1/resolver.go @@ -49,6 +49,7 @@ import ( "go.probo.inc/probo/pkg/itam" "go.probo.inc/probo/pkg/mailman" "go.probo.inc/probo/pkg/probo" + "go.probo.inc/probo/pkg/realtime" "go.probo.inc/probo/pkg/resourcealias" "go.probo.inc/probo/pkg/riskmanagement" "go.probo.inc/probo/pkg/saferedirect" @@ -91,6 +92,8 @@ type ( func NewMux( logger *log.Logger, proboSvc *probo.Service, + collaborationEvents *realtime.Events, + shutdownContext context.Context, resourceAliasSvc *resourcealias.Service, iamSvc *iam.Service, esignSvc *esign.Service, @@ -106,6 +109,7 @@ func NewMux( providerRegistry *provider.Registry, fileManagerSvc *filemanager.Service, baseURL *baseurl.BaseURL, + allowedOrigins []string, customDomainCname string, thirdPartySvc *thirdparty.Service, riskManagementSvc *riskmanagement.Service, @@ -140,6 +144,23 @@ func NewMux( itamSvc, ) + if shutdownContext == nil { + shutdownContext = context.Background() + } + + collaborationHandler := &documentCollaborationHandler{ + logger: logger, + probo: proboSvc, + iam: iamSvc, + baseURL: baseURL, + allowedOrigins: allowedOrigins, + hub: newDocumentCollaborationHub( + proboSvc.Documents, + collaborationEvents, + ), + shutdown: shutdownContext, + } + r.Group(func(r chi.Router) { r.Use(authn.NewSessionMiddleware(iamSvc, cookieConfig)) r.Use(authn.NewAPIKeyMiddleware(iamSvc, tokenSecret)) @@ -148,6 +169,10 @@ func NewMux( r.Use(dataloader.NewMiddleware(proboSvc, iamSvc, cookieBannerSvc, thirdPartySvc)) r.Handle("/graphql", graphqlHandler) + r.Get( + "/document-versions/{documentVersionID}/sync", + collaborationHandler.handle, + ) r.Get( "/connectors/initiate", diff --git a/pkg/server/server.go b/pkg/server/server.go index fc5d081569..7ddc3ce180 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -21,6 +21,7 @@ package server import ( + "context" "net/http" "github.com/go-chi/chi/v5" @@ -42,6 +43,7 @@ import ( "go.probo.inc/probo/pkg/itam" "go.probo.inc/probo/pkg/mailman" "go.probo.inc/probo/pkg/probo" + "go.probo.inc/probo/pkg/realtime" "go.probo.inc/probo/pkg/resourcealias" "go.probo.inc/probo/pkg/riskmanagement" "go.probo.inc/probo/pkg/securecookie" @@ -56,33 +58,35 @@ import ( ) type Config struct { - BaseURL *baseurl.BaseURL - AllowedOrigins []string - ExtraHeaderFields map[string]string - Probo *probo.Service - ResourceAlias *resourcealias.Service - File *filemanager.Service - IAM *iam.Service - Visitor *visitor.Service - ESign *esign.Service - Management *management.Service - CertManager *certmanager.Service - AccessReview *accessreview.Service - AgentRun *agentrun.Service - Slack *slack.Service - Mailman *mailman.Service - CookieBanner *cookiebanner.Service - Geoloc *geoloc.Service - ThirdParty *thirdparty.Service - RiskManagement *riskmanagement.Service - ITAM *itam.Service - Cookie securecookie.Config - TokenSecret string - ConnectorRegistry *connector.ConnectorRegistry - ProviderRegistry *provider.Registry - CustomDomainCname string - GraphQLLimits gqlutils.Limits - Logger *log.Logger + BaseURL *baseurl.BaseURL + AllowedOrigins []string + ExtraHeaderFields map[string]string + Probo *probo.Service + CollaborationEvents *realtime.Events + ShutdownContext context.Context + ResourceAlias *resourcealias.Service + File *filemanager.Service + IAM *iam.Service + Visitor *visitor.Service + ESign *esign.Service + Management *management.Service + CertManager *certmanager.Service + AccessReview *accessreview.Service + AgentRun *agentrun.Service + Slack *slack.Service + Mailman *mailman.Service + CookieBanner *cookiebanner.Service + Geoloc *geoloc.Service + ThirdParty *thirdparty.Service + RiskManagement *riskmanagement.Service + ITAM *itam.Service + Cookie securecookie.Config + TokenSecret string + ConnectorRegistry *connector.ConnectorRegistry + ProviderRegistry *provider.Registry + CustomDomainCname string + GraphQLLimits gqlutils.Limits + Logger *log.Logger } type Server struct { @@ -100,32 +104,34 @@ type Server struct { func NewServer(cfg Config) (*Server, error) { apiCfg := api.Config{ - BaseURL: cfg.BaseURL, - AllowedOrigins: cfg.AllowedOrigins, - Probo: cfg.Probo, - ResourceAlias: cfg.ResourceAlias, - File: cfg.File, - IAM: cfg.IAM, - Visitor: cfg.Visitor, - ESign: cfg.ESign, - Management: cfg.Management, - CertManager: cfg.CertManager, - AccessReview: cfg.AccessReview, - AgentRun: cfg.AgentRun, - Slack: cfg.Slack, - Mailman: cfg.Mailman, - CookieBanner: cfg.CookieBanner, - Geoloc: cfg.Geoloc, - ThirdParty: cfg.ThirdParty, - RiskManagement: cfg.RiskManagement, - ITAM: cfg.ITAM, - Cookie: cfg.Cookie, - TokenSecret: cfg.TokenSecret, - ConnectorRegistry: cfg.ConnectorRegistry, - ProviderRegistry: cfg.ProviderRegistry, - CustomDomainCname: cfg.CustomDomainCname, - GraphQLLimits: cfg.GraphQLLimits, - Logger: cfg.Logger.Named("api"), + BaseURL: cfg.BaseURL, + AllowedOrigins: cfg.AllowedOrigins, + Probo: cfg.Probo, + CollaborationEvents: cfg.CollaborationEvents, + ShutdownContext: cfg.ShutdownContext, + ResourceAlias: cfg.ResourceAlias, + File: cfg.File, + IAM: cfg.IAM, + Visitor: cfg.Visitor, + ESign: cfg.ESign, + Management: cfg.Management, + CertManager: cfg.CertManager, + AccessReview: cfg.AccessReview, + AgentRun: cfg.AgentRun, + Slack: cfg.Slack, + Mailman: cfg.Mailman, + CookieBanner: cfg.CookieBanner, + Geoloc: cfg.Geoloc, + ThirdParty: cfg.ThirdParty, + RiskManagement: cfg.RiskManagement, + ITAM: cfg.ITAM, + Cookie: cfg.Cookie, + TokenSecret: cfg.TokenSecret, + ConnectorRegistry: cfg.ConnectorRegistry, + ProviderRegistry: cfg.ProviderRegistry, + CustomDomainCname: cfg.CustomDomainCname, + GraphQLLimits: cfg.GraphQLLimits, + Logger: cfg.Logger.Named("api"), } apiServer, err := api.NewServer(apiCfg)