diff --git a/pkg/automerge/PARITY_PLAN.md b/pkg/automerge/PARITY_PLAN.md index e203ac0b17..f4394d6f59 100644 --- a/pkg/automerge/PARITY_PLAN.md +++ b/pkg/automerge/PARITY_PLAN.md @@ -67,7 +67,8 @@ patches, and native reproduces that reference behavior exactly. DEFLATE compression on save is now covered: the native save compresses change chunks whose body reaches the reference DEFLATE_MIN_SIZE threshold (small changes -stay byte-identical), and SaveNoCompress mirrors AutoCommit::save_nocompress. +stay byte-identical), and `Save(ctx, NoCompress())` mirrors +AutoCommit::save_nocompress. Transaction isolation is now covered: `isolate` pins reads and writes to a historical frontier using derived concurrency actors (matching Rust's @@ -79,11 +80,46 @@ remains, because it asserts the exact incremental patch stream produced by Rust' patch log across isolate/integrate, which native's state-comparison diff does not reproduce. -Orphan retention across save/load is now covered without a full document-chunk -encoder: the native save appends retained orphan changes and the native load -falls back to a dependency-tolerant path that applies every change whose -dependencies are satisfiable and queues the rest (still failing a load that can -apply nothing, so a bare orphan without a base is rejected as before). +Orphan retention across save/load is covered: the native save appends retained +orphan changes and the native load falls back to a dependency-tolerant path that +applies every change whose dependencies are satisfiable and queues the rest +(still failing a load that can apply nothing, so a bare orphan without a base is +rejected as before). + +Snapshot writing is implemented and is now what `Save` produces, matching the +Rust and JavaScript `save()` semantics. It compacts the whole history into one +document chunk, followed by any retained orphan changes as trailing change +chunks, and DEFLATEs individual columns above a size threshold. It is gated on +byte identity with the reference for the same history across linear text, map +puts and deletes, counters, marks and unmarks, and text deletion; re-encoding a +reference-written snapshot reproduces that file exactly for the official fixture +and for reference histories covering nested objects, lists and a merged +multi-actor graph. Compaction matters for size as well as parity: a 200-commit +typing history is 21816 bytes as the old change stream and 400 bytes compacted. + +`Save` falls back to the faithful change stream (the loaded base plus each change +chunk since) when a history cannot be compacted: while isolated, or when the +change graph is not internally consistent. That stream is also what preserves +loaded bytes verbatim, so a document is never rewritten in a lossy way when it +cannot be safely compacted. `SaveIncremental` is unaffected: `Save` leaves the +incremental cursor at the end exactly as the stream save did. + +Compressed columns are not byte-identical to the reference because the DEFLATE +implementations differ, so the byte-identity gate holds only for histories small +enough that no column crosses the threshold; above it both files are valid and +load equally, and compression is a size optimization the decoder reverses on the +way back in. + +Two smaller differences remain. The reference `am_save_no_orphans` shim sets +`deflate: false`, while Rust's own `SaveOptions::default()` compresses, so +`Save(ctx, DiscardOrphans())` diverges from the shim rather than from Rust; +correcting it means rebuilding the WASM oracle. Unknown columns survive a normal +load +because the loaded bytes are kept verbatim, and `EncodeDocument` writes them back +to the table they came from when a document is re-encoded unmodified, but they +cannot be carried across a compaction of a mutated history because their rows no +longer line up with the recomputed columns; this matches Rust, which also drops +them across re-serialization. The V2 sync internals are now covered: the empty-message codec round-trips to the reference wire bytes, and Bloom false-positive recovery is verified on both @@ -101,9 +137,9 @@ JavaScript binding-type helpers (ImmutableString/RawString, legacy Text-as-array proxy/change-callback) are likewise recorded as api-convenience or language-specific rather than interop-required. -## Known native defects (found by parity reproduction, fix pending) +## Native defects found by parity reproduction (resolved) -**Mark boundaries: fixed for the reported case, deeper cases still diverge.** +**Mark boundaries: matches the reference, including the error paths.** The originally reported defect is fixed. Mark begin and end operations now hold positions in the sequence order, insertions (including the mark boundaries themselves) resolve their anchors through a port of the reference's insert @@ -139,24 +175,29 @@ checks span consolidation and text) drove a series of fixes this pass: operation reusing that counter is not mistaken for it. These closed the common cases. The dangling begin the reference leaves behind -when a mark is applied with an out-of-range end boundary is now largely handled: -the mark call fails, but the begin was already recorded, and it then covers text +when a mark is applied with an out-of-range end boundary is now handled in full. +The mark call fails, but the begin was already recorded, and it then covers text according to its expand direction. A leftward-expanding begin (expand "before" or "both") sorts after same-anchor insertions by descending operation ID, so its -walk index lands past text it should cover; `richTextMarks` now starts such a +walk index lands past text it should cover; `richTextMarks` starts such a dangling begin at the position just after the begin's own anchor element (or the -document start for a head anchor) rather than at its walk index. This is the exact -range the reference produces: for `mark(0,3,before)` on empty text, split at 0, -insert `"w"` at 0, both engines now report `"w"` bold (`bold 0..2`). - -`TestRustText_DanglingMarkBoundaries` gates four delta-debugged reproducers that -previously diverged. A randomized value-level sweep including out-of-range -boundaries and all four expand modes dropped from roughly 7% divergence to about -3%; the same sweep restricted to in-range marks diverges on none. What remains are -deeper interactions — several overlapping dangling begins whose anchor elements -have since been deleted — where native over-extends one of the marks. These are -strictly an error path (no valid caller marks past the end of the text) and never -arise from the frontend, which clamps mark ranges to the text length. +document start for a head anchor) rather than at its walk index. + +The last remaining divergences were not in span computation but in authoring: a +split block did not resolve its insertion anchor against neighbouring mark +boundaries the way a text insertion does. A block inserted next to a dangling +begin therefore landed on the wrong side of it, and every insertion anchored +after that block inherited the mistake, so the marks the following text carried +diverged from the reference in both directions (a mark dropped, or a mark leaking +past a block). `SplitBlock` now resolves its anchor through the same insert query +as `Splice`. + +`TestRustText_DanglingMarkBoundaries` gates eleven delta-debugged reproducers, +and `TestRustText_MarkValuesMatchReferenceUnderErrors` compares marked spans run +for run against the reference across two thousand randomized scenarios that +include out-of-range boundaries and every expand mode. A wider sweep of +twenty-four thousand scenarios across six seeds, up to seventeen steps each, found +no divergence. **Concurrent re-encoding is now byte-identical.** Assigning the value a key already resolves to used to skip writing an operation. That is correct for an @@ -191,6 +232,7 @@ These do not block Go/Rust engine parity but remain tracked: Implement and verify: - complete document, change, compressed-change, and bundle parsing; +- canonical document-chunk encoding, what Save now writes (done); - canonical encoding for every operation and scalar column; - expanded/compressed change byte and hash stability; - 64-bit object IDs and actor tables referenced only by deletes; diff --git a/pkg/automerge/automerge.go b/pkg/automerge/automerge.go index e0ad566295..d312f43401 100644 --- a/pkg/automerge/automerge.go +++ b/pkg/automerge/automerge.go @@ -77,9 +77,7 @@ type ( engine interface { Close(context.Context) error - Save(context.Context) ([]byte, error) - SaveWithOptions(context.Context, bool) ([]byte, error) - SaveNoCompress(context.Context) ([]byte, error) + Save(context.Context, bool, bool) ([]byte, error) Isolate(context.Context, [][32]byte) error Integrate(context.Context) error Stats(context.Context) ([]byte, error) @@ -179,104 +177,113 @@ func NewActorID() (ActorID, error) { // New creates an empty document using the native Go engine. func New(ctx context.Context, actorID ActorID) (*Document, error) { - return NewPureGo(ctx, actorID) -} - -// NewReference creates an empty document using the official WASM reference -// engine. It is retained as a differential oracle for the native engine. -func NewReference(ctx context.Context, actorID ActorID) (*Document, error) { - b, err := reference.New(ctx) + b, err := native.NewEngine(ctx) if err != nil { - return nil, fmt.Errorf("cannot create Automerge engine: %w", err) + return nil, fmt.Errorf("cannot create native Automerge engine: %w", err) } if err := b.SetActor(ctx, actorID[:]); err != nil { _ = b.Close(ctx) - return nil, fmt.Errorf("cannot initialize Automerge actor: %w", err) + return nil, fmt.Errorf("cannot initialize native Automerge actor: %w", err) } return &Document{engine: b}, nil } -// NewPureGo creates an empty document using the experimental native Go engine. -// -// The native engine is intended for differential testing until its complete -// feature surface reaches parity with the reference engine. -func NewPureGo(ctx context.Context, actorID ActorID) (*Document, error) { - b, err := native.NewEngine(ctx) +// NewReference creates an empty document using the official WASM reference +// engine. It exists as a differential oracle for the native engine and is +// intended for tests, not production use. +func NewReference(ctx context.Context, actorID ActorID) (*Document, error) { + b, err := reference.New(ctx) if err != nil { - return nil, fmt.Errorf("cannot create native Automerge engine: %w", err) + return nil, fmt.Errorf("cannot create Automerge engine: %w", err) } if err := b.SetActor(ctx, actorID[:]); err != nil { _ = b.Close(ctx) - return nil, fmt.Errorf("cannot initialize native Automerge actor: %w", err) + return nil, fmt.Errorf("cannot initialize Automerge actor: %w", err) } return &Document{engine: b}, nil } -// Load creates a document using the native Go engine and assigns a new writer. -func Load(ctx context.Context, data []byte, actorID ActorID) (*Document, error) { - return LoadPureGo(ctx, data, actorID) +// LoadOption configures how Load interprets stored data. +type LoadOption func(*loadConfig) + +type loadConfig struct { + convertStringsToText bool } -// LoadReference loads a document using the official WASM reference engine. -func LoadReference( +// ConvertStringsToText converts every string scalar stored in a map or list +// into a text object as the document loads, mirroring Rust's +// StringMigration::ConvertToText load option. +func ConvertStringsToText() LoadOption { + return func(c *loadConfig) { c.convertStringsToText = true } +} + +// Load creates a document from stored data using the native Go engine and +// assigns a new writer. +func Load( ctx context.Context, data []byte, actorID ActorID, + options ...LoadOption, ) (*Document, error) { - b, err := reference.Load(ctx, data) + config := loadConfig{} + for _, option := range options { + option(&config) + } + + b, err := native.LoadEngine(ctx, data) if err != nil { - return nil, fmt.Errorf("cannot load Automerge engine: %w", err) + return nil, fmt.Errorf("cannot load native Automerge engine: %w", err) } if err := b.SetActor(ctx, actorID[:]); err != nil { _ = b.Close(ctx) - return nil, fmt.Errorf("cannot assign loaded Automerge actor: %w", err) + return nil, fmt.Errorf("cannot assign native Automerge actor: %w", err) } - return &Document{engine: b}, nil -} - -// LoadConvertingStrings loads a document with the native engine, converting -// every string scalar stored in a map or list into a text object. It mirrors -// the Rust StringMigration::ConvertToText load option. -func LoadConvertingStrings( - ctx context.Context, - data []byte, - actorID ActorID, -) (*Document, error) { - document, err := LoadPureGo(ctx, data, actorID) - if err != nil { - return nil, err - } + document := &Document{engine: b} - if err := document.convertStringsToText(ctx); err != nil { - _ = document.Close(ctx) + if config.convertStringsToText { + if err := document.convertStringsToText(ctx); err != nil { + _ = document.Close(ctx) - return nil, err + return nil, err + } } return document, nil } -// LoadReferenceConvertingStrings loads a document with the reference engine and -// the string-to-text migration applied. -func LoadReferenceConvertingStrings( +// LoadReference loads a document using the official WASM reference engine. Like +// NewReference it is intended for tests, not production use. +func LoadReference( ctx context.Context, data []byte, actorID ActorID, + options ...LoadOption, ) (*Document, error) { - b, err := reference.LoadConvertingStrings(ctx, data) + config := loadConfig{} + for _, option := range options { + option(&config) + } + + load := reference.Load + if config.convertStringsToText { + // The reference applies the migration during load through its own WASM + // entry point rather than as a post-load pass. + load = reference.LoadConvertingStrings + } + + b, err := load(ctx, data) if err != nil { return nil, fmt.Errorf("cannot load Automerge engine: %w", err) } if err := b.SetActor(ctx, actorID[:]); err != nil { _ = b.Close(ctx) - return nil, fmt.Errorf("cannot assign loaded Automerge actor: %w", err) } @@ -333,7 +340,7 @@ func convertMapStrings(ctx context.Context, object *Object) (bool, error) { return false, err } - handle, err := text.Text() + handle, err := text.Text(ctx) if err != nil { return false, err } @@ -382,7 +389,7 @@ func convertListStrings(ctx context.Context, object *Object) (bool, error) { return false, err } - handle, err := text.Text() + handle, err := text.Text(ctx) if err != nil { return false, err } @@ -412,26 +419,7 @@ func convertListStrings(ctx context.Context, object *Object) (bool, error) { return changed, nil } -// LoadPureGo loads Automerge data using the experimental native Go engine. -func LoadPureGo( - ctx context.Context, - data []byte, - actorID ActorID, -) (*Document, error) { - b, err := native.LoadEngine(ctx, data) - if err != nil { - return nil, fmt.Errorf("cannot load native Automerge engine: %w", err) - } - - if err := b.SetActor(ctx, actorID[:]); err != nil { - _ = b.Close(ctx) - return nil, fmt.Errorf("cannot assign native Automerge actor: %w", err) - } - - return &Document{engine: b}, nil -} - -// Close releases the document's WASM module instance. +// Close releases the engine resources held by the document. func (d *Document) Close(ctx context.Context) error { d.mu.Lock() defer d.mu.Unlock() @@ -449,32 +437,33 @@ func (d *Document) Close(ctx context.Context) error { return nil } -// Save serializes the complete Automerge history. -func (d *Document) Save(ctx context.Context) ([]byte, error) { - d.mu.Lock() - defer d.mu.Unlock() +// SaveOption configures how Save serializes a document. +type SaveOption func(*saveConfig) - if d.closed { - return nil, ErrClosed - } +type saveConfig struct { + retainOrphans bool + compress bool +} - data, err := d.engine.Save(ctx) - if err != nil { - return nil, fmt.Errorf("cannot save Automerge document: %w", err) - } +// NoCompress disables DEFLATE compression of the saved document. The default is +// to compress, which the reference's save_nocompress also opts out of; the +// uncompressed form is mainly useful for comparing sizes or debugging. +func NoCompress() SaveOption { + return func(c *saveConfig) { c.compress = false } +} - return data, nil +// DiscardOrphans drops orphan changes (changes whose dependencies are missing) +// instead of retaining them. Retaining them, the default, preserves them across +// a save/load round trip so they resolve once their dependencies arrive; +// discarding drops them permanently. It mirrors Rust's SaveOptions.retain_orphans. +func DiscardOrphans() SaveOption { + return func(c *saveConfig) { c.retainOrphans = false } } -// SaveWithOptions serializes the document, choosing whether to retain orphan -// changes (changes whose dependencies are missing). Retaining them, the default -// for Save, preserves them across a save/load round trip so they can be resolved -// once their dependencies arrive; discarding them drops them permanently. It -// mirrors the Rust SaveOptions.retain_orphans flag. -func (d *Document) SaveWithOptions( - ctx context.Context, - retainOrphans bool, -) ([]byte, error) { +// Save serializes the complete Automerge history as a compacted document. By +// default it compresses and retains orphan changes; pass NoCompress or +// DiscardOrphans to change that. +func (d *Document) Save(ctx context.Context, options ...SaveOption) ([]byte, error) { d.mu.Lock() defer d.mu.Unlock() @@ -482,7 +471,12 @@ func (d *Document) SaveWithOptions( return nil, ErrClosed } - data, err := d.engine.SaveWithOptions(ctx, retainOrphans) + config := saveConfig{retainOrphans: true, compress: true} + for _, option := range options { + option(&config) + } + + data, err := d.engine.Save(ctx, config.retainOrphans, config.compress) if err != nil { return nil, fmt.Errorf("cannot save Automerge document: %w", err) } @@ -526,26 +520,6 @@ func (d *Document) Integrate(ctx context.Context) error { return nil } -// SaveNoCompress serializes the document without DEFLATE-compressing its change -// data. The default Save compresses large change chunks; this variant is useful -// for comparing compressed and uncompressed sizes. It mirrors the Rust -// AutoCommit::save_nocompress API. -func (d *Document) SaveNoCompress(ctx context.Context) ([]byte, error) { - d.mu.Lock() - defer d.mu.Unlock() - - if d.closed { - return nil, ErrClosed - } - - data, err := d.engine.SaveNoCompress(ctx) - if err != nil { - return nil, fmt.Errorf("cannot save Automerge document: %w", err) - } - - return data, nil -} - // Stats reports aggregate document statistics. type Stats struct { NumChanges uint64 `json:"numChanges"` @@ -586,7 +560,7 @@ func (d *Document) Fork( return nil, ErrClosed } - data, err := d.engine.Save(ctx) + data, err := d.engine.Save(ctx, true, true) if err != nil { d.mu.Unlock() return nil, fmt.Errorf("cannot save Automerge fork source: %w", err) @@ -922,7 +896,7 @@ func (d *Document) ChangesSince( // the document. func (d *Document) ApplyChanges( ctx context.Context, - changes [][]byte, + changes []Change, ) error { d.mu.Lock() defer d.mu.Unlock() @@ -936,7 +910,12 @@ func (d *Document) ApplyChanges( return fmt.Errorf("automerge engine does not accept incremental changes") } - if err := applier.ApplyChanges(ctx, changes); err != nil { + raw := make([][]byte, len(changes)) + for i, change := range changes { + raw[i] = change.Bytes + } + + if err := applier.ApplyChanges(ctx, raw); err != nil { return fmt.Errorf("cannot apply incremental Automerge changes: %w", err) } @@ -1247,3 +1226,8 @@ func (s *SyncState) Save(ctx context.Context) ([]byte, error) { func (h Hash) String() string { return hex.EncodeToString(h[:]) } + +// String returns the lowercase hexadecimal actor ID. +func (a ActorID) String() string { + return hex.EncodeToString(a[:]) +} diff --git a/pkg/automerge/compressed_save_parity_test.go b/pkg/automerge/compressed_save_parity_test.go index b719322ca0..2233fe695c 100644 --- a/pkg/automerge/compressed_save_parity_test.go +++ b/pkg/automerge/compressed_save_parity_test.go @@ -65,7 +65,7 @@ func TestRustTest_CompressedDocCols(t *testing.T) { _, err = document.Commit(ctx, "list", commitTime) require.NoError(t, err) - uncompressed, err := document.SaveNoCompress(ctx) + uncompressed, err := document.Save(ctx, automerge.NoCompress()) require.NoError(t, err) compressed, err := document.Save(ctx) diff --git a/pkg/automerge/conformance_test.go b/pkg/automerge/conformance_test.go index 64a5d52aea..e399c83d52 100644 --- a/pkg/automerge/conformance_test.go +++ b/pkg/automerge/conformance_test.go @@ -319,7 +319,7 @@ func TestConformance_GoReadsJavaScriptRichTextSpans(t *testing.T) { assert.Equal(t, "Policy", spans[1].Text) assert.Equal(t, true, spans[1].Marks["strong"]) - nativeDocument, err := automerge.LoadPureGo(context.Background(), data, actor(14)) + nativeDocument, err := automerge.Load(context.Background(), data, actor(14)) require.NoError(t, err) closeDocument(t, nativeDocument) nativeText, err := nativeDocument.Text(context.Background(), "body") @@ -501,9 +501,21 @@ func TestConformance_NativeConcurrentChangesConverge(t *testing.T) { require.NoError(t, err) _, err = backend.Merge(context.Background(), rawChanges[1]) require.NoError(t, err) - saved, err := backend.Save(context.Background()) + + before, err := backend.Heads(context.Background()) + require.NoError(t, err) + + saved, err := backend.Save(context.Background(), true, true) + require.NoError(t, err) + + // Save now writes a compacted document rather than embedding change bytes, so + // the guarantee is that reloading it reproduces the same frontier. + reloaded, err := native.LoadEngine(context.Background(), saved) + require.NoError(t, err) + + after, err := reloaded.Heads(context.Background()) require.NoError(t, err) - assert.True(t, bytes.Contains(saved, rawChanges[1])) + assert.Equal(t, before, after) } func TestConformance_NativeSyncMessageRoundTrip(t *testing.T) { diff --git a/pkg/automerge/convert_string_to_text_parity_test.go b/pkg/automerge/convert_string_to_text_parity_test.go index 6e8c63ae26..9172d1a05d 100644 --- a/pkg/automerge/convert_string_to_text_parity_test.go +++ b/pkg/automerge/convert_string_to_text_parity_test.go @@ -77,8 +77,18 @@ func loadConvertingEngines() []struct { name string load func(context.Context, []byte, automerge.ActorID) (*automerge.Document, error) }{ - {"native", automerge.LoadConvertingStrings}, - {"reference", automerge.LoadReferenceConvertingStrings}, + { + "native", + func(ctx context.Context, data []byte, actorID automerge.ActorID) (*automerge.Document, error) { + return automerge.Load(ctx, data, actorID, automerge.ConvertStringsToText()) + }, + }, + { + "reference", + func(ctx context.Context, data []byte, actorID automerge.ActorID) (*automerge.Document, error) { + return automerge.LoadReference(ctx, data, actorID, automerge.ConvertStringsToText()) + }, + }, } } @@ -112,7 +122,7 @@ func TestRustConvert_StringsInMapsAreConvertedToText(t *testing.T) { require.NoError(t, err) assert.Equal(t, automerge.ObjectTypeText, object.Type) - text, err := object.Text() + text, err := object.Text(ctx) require.NoError(t, err) value, err := text.String(ctx) require.NoError(t, err) @@ -150,7 +160,7 @@ func TestRustConvert_StringsInListsAreConvertedToText(t *testing.T) { require.NoError(t, err) assert.Equal(t, automerge.ObjectTypeText, element.Type) - text, err := element.Text() + text, err := element.Text(ctx) require.NoError(t, err) value, err := text.String(ctx) require.NoError(t, err) diff --git a/pkg/automerge/core_model_test.go b/pkg/automerge/core_model_test.go index 4c52ff2740..f1303b4e6e 100644 --- a/pkg/automerge/core_model_test.go +++ b/pkg/automerge/core_model_test.go @@ -904,7 +904,7 @@ func TestDocument_WrongObjectOperationsMatchReference(t *testing.T) { require.NoError(t, err) assert.Zero(t, length) - _, err = listObject.Text() + _, err = listObject.Text(ctx) require.Error(t, err) }) } diff --git a/pkg/automerge/diff_parity_test.go b/pkg/automerge/diff_parity_test.go index b2d8e97505..adf00cdcdb 100644 --- a/pkg/automerge/diff_parity_test.go +++ b/pkg/automerge/diff_parity_test.go @@ -99,7 +99,7 @@ func TestRustDiff_ReverseDeletionOfObjectInList(t *testing.T) { require.NoError(t, list.InsertScalar(ctx, 0, automerge.Scalar{Type: automerge.ScalarTypeString, String: "a"})) text, err := list.InsertObject(ctx, 1, automerge.ObjectTypeText) require.NoError(t, err) - textValue, err := text.Text() + textValue, err := text.Text(ctx) require.NoError(t, err) require.NoError(t, textValue.Splice(ctx, 0, 0, "b")) require.NoError(t, list.InsertScalar(ctx, 2, automerge.Scalar{Type: automerge.ScalarTypeString, String: "c"})) @@ -150,7 +150,7 @@ func TestRustDiff_ReverseDeletionOfObjectInMap(t *testing.T) { require.NoError(t, mapObject.PutScalar(ctx, "a", automerge.Scalar{Type: automerge.ScalarTypeString, String: "a"})) textB, err := mapObject.CreateObject(ctx, "b", automerge.ObjectTypeText) require.NoError(t, err) - textBValue, err := textB.Text() + textBValue, err := textB.Text(ctx) require.NoError(t, err) require.NoError(t, textBValue.Splice(ctx, 0, 0, "b")) require.NoError(t, mapObject.PutScalar(ctx, "c", automerge.Scalar{Type: automerge.ScalarTypeString, String: "c"})) diff --git a/pkg/automerge/document_save_parity_test.go b/pkg/automerge/document_save_parity_test.go new file mode 100644 index 0000000000..5365021f4a --- /dev/null +++ b/pkg/automerge/document_save_parity_test.go @@ -0,0 +1,215 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package automerge_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/automerge" +) + +// documentSaveScenario applies the same history to whichever engine it is given +// so a native snapshot can be held against the reference's byte for byte. +type documentSaveScenario struct { + name string + apply func(t *testing.T, ctx context.Context, document *automerge.Document) +} + +func documentSaveScenarios() []documentSaveScenario { + base := time.Unix(1786147200, 0).UTC() + + return []documentSaveScenario{ + { + name: "linear text", + apply: func(t *testing.T, ctx context.Context, document *automerge.Document) { + text, err := document.CreateText(ctx, "body") + require.NoError(t, err) + + for i := range 5 { + require.NoError(t, text.Splice(ctx, uint32(i), 0, "x")) + _, err = document.Commit(ctx, "edit", base.Add(time.Duration(i)*time.Second)) + require.NoError(t, err) + } + }, + }, + { + name: "map puts and delete", + apply: func(t *testing.T, ctx context.Context, document *automerge.Document) { + require.NoError(t, document.PutScalar(ctx, "title", + automerge.Scalar{Type: automerge.ScalarTypeString, String: "first"})) + require.NoError(t, document.PutScalar(ctx, "keep", + automerge.Scalar{Type: automerge.ScalarTypeString, String: "value"})) + _, err := document.Commit(ctx, "one", base) + require.NoError(t, err) + + require.NoError(t, document.PutScalar(ctx, "title", + automerge.Scalar{Type: automerge.ScalarTypeString, String: "second"})) + _, err = document.Commit(ctx, "two", base.Add(time.Second)) + require.NoError(t, err) + + require.NoError(t, document.Root().DeleteKey(ctx, "title")) + _, err = document.Commit(ctx, "three", base.Add(2*time.Second)) + require.NoError(t, err) + }, + }, + { + name: "counter increment", + apply: func(t *testing.T, ctx context.Context, document *automerge.Document) { + require.NoError(t, document.PutScalar(ctx, "counter", + automerge.Scalar{Type: automerge.ScalarTypeCounter, Int: 5})) + _, err := document.Commit(ctx, "create", base) + require.NoError(t, err) + + require.NoError(t, document.Root().Increment(ctx, "counter", 3)) + _, err = document.Commit(ctx, "bump", base.Add(time.Second)) + require.NoError(t, err) + }, + }, + { + name: "marks and unmarks", + apply: func(t *testing.T, ctx context.Context, document *automerge.Document) { + text, err := document.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 0, 0, "hello brave world")) + _, err = document.Commit(ctx, "write", base) + require.NoError(t, err) + + require.NoError(t, text.Mark(ctx, 0, 5, "strong", + automerge.Scalar{Type: automerge.ScalarTypeBoolean, Bool: true}, + automerge.MarkExpandBoth)) + _, err = document.Commit(ctx, "mark", base.Add(time.Second)) + require.NoError(t, err) + + require.NoError(t, text.Unmark(ctx, 1, 3, "strong", automerge.MarkExpandNone)) + _, err = document.Commit(ctx, "unmark", base.Add(2*time.Second)) + require.NoError(t, err) + }, + }, + { + name: "text with deletion", + apply: func(t *testing.T, ctx context.Context, document *automerge.Document) { + text, err := document.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 0, 0, "hello brave world")) + _, err = document.Commit(ctx, "write", base) + require.NoError(t, err) + + require.NoError(t, text.Splice(ctx, 5, 6, "")) + _, err = document.Commit(ctx, "trim", base.Add(time.Second)) + require.NoError(t, err) + }, + }, + } +} + +// TestDocumentSave_MatchesReferenceBytes requires the compacted snapshot the +// native engine writes to be the file the reference writes for the same +// history. Byte identity is the only check that proves the operation-set order, +// the column layout and the frontier all agree. +func TestDocumentSave_MatchesReferenceBytes(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + for _, scenario := range documentSaveScenarios() { + t.Run(scenario.name, func(t *testing.T) { + t.Parallel() + + native, err := automerge.New(ctx, actor(41)) + require.NoError(t, err) + closeDocument(t, native) + + reference, err := automerge.NewReference(ctx, actor(41)) + require.NoError(t, err) + closeDocument(t, reference) + + scenario.apply(t, ctx, native) + scenario.apply(t, ctx, reference) + + expected, err := reference.Save(ctx) + require.NoError(t, err) + + actual, err := native.Save(ctx) + require.NoError(t, err) + + assert.Equal(t, expected, actual) + }) + } +} + +// TestDocumentSave_ReloadsIntoTheSameHistory checks a compacted snapshot is a +// complete history rather than only the right bytes: it must reload with every +// change reachable and be accepted by the reference. +func TestDocumentSave_ReloadsIntoTheSameHistory(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + for _, scenario := range documentSaveScenarios() { + t.Run(scenario.name, func(t *testing.T) { + t.Parallel() + + document, err := automerge.New(ctx, actor(42)) + require.NoError(t, err) + closeDocument(t, document) + + scenario.apply(t, ctx, document) + + heads, err := document.Heads(ctx) + require.NoError(t, err) + + original, err := document.ChangesSince(ctx, nil) + require.NoError(t, err) + + snapshot, err := document.Save(ctx) + require.NoError(t, err) + + reloaded, err := automerge.Load(ctx, snapshot, actor(43)) + require.NoError(t, err) + closeDocument(t, reloaded) + + reloadedHeads, err := reloaded.Heads(ctx) + require.NoError(t, err) + assert.Equal(t, heads, reloadedHeads) + + changes, err := reloaded.ChangesSince(ctx, nil) + require.NoError(t, err) + require.Len(t, changes, len(original)) + + for i := range changes { + assert.Equal(t, original[i].Hash, changes[i].Hash, "change %d", i) + assert.Equal(t, original[i].Bytes, changes[i].Bytes, "change %d bytes", i) + } + + adopted, err := automerge.LoadReference(ctx, snapshot, actor(44)) + require.NoError(t, err) + closeDocument(t, adopted) + + adoptedHeads, err := adopted.Heads(ctx) + require.NoError(t, err) + assert.Equal(t, heads, adoptedHeads) + }) + } +} diff --git a/pkg/automerge/hydrate_test.go b/pkg/automerge/hydrate_test.go index 5e1951964f..72cac8ce66 100644 --- a/pkg/automerge/hydrate_test.go +++ b/pkg/automerge/hydrate_test.go @@ -249,7 +249,7 @@ func TestDocument_HydrateSpliceMatchesReference(t *testing.T) { textObject, err := list.ObjectAt(ctx, 2) require.NoError(t, err) - text, err := textObject.Text() + text, err := textObject.Text(ctx) require.NoError(t, err) textValue, err := text.String(ctx) require.NoError(t, err) @@ -305,7 +305,7 @@ func assertHydratedDocument( require.NoError(t, err) nestedTextObject, err := nestedMap.Object(ctx, "nested") require.NoError(t, err) - nestedText, err := nestedTextObject.Text() + nestedText, err := nestedTextObject.Text(ctx) require.NoError(t, err) nestedValue, err := nestedText.String(ctx) require.NoError(t, err) diff --git a/pkg/automerge/internal/native/ARCHITECTURE.md b/pkg/automerge/internal/native/ARCHITECTURE.md index b273f10599..302147437f 100644 --- a/pkg/automerge/internal/native/ARCHITECTURE.md +++ b/pkg/automerge/internal/native/ARCHITECTURE.md @@ -21,13 +21,13 @@ types or storage details. | Rich-text state | `rich_text_state.go` | Span/mark state machines, mark anchors and UTF-16 ranges | | Hydration | `hydrate_state.go` | Recursive map/list materialization | | Storage facade | `storage.go` | Delegation to the independent storage and encoding packages | -| Shared model facade | `types.go` | Aliases to the independent shared types package | +| Shared model facade | `model.go` | Aliases to the independent operation-set model package | ## Internal package boundaries | Package | Responsibility | |---|---| -| `internal/types` | Dependency-free actor, operation, change, object, scalar and chunk model | +| `internal/opset` | Dependency-free actor, operation, change, object, scalar and chunk model | | `internal/encoding` | Bounded binary reader, ULEB128 and length-prefixed primitives | | `internal/storage` | Automerge chunk/column encoding, decoding and graph validation | | `internal/sync` | V1/V2 sync message wire codec and resource limits | @@ -45,7 +45,7 @@ native Engine methods ↓ State / sequence / rich-text state ↓ -internal/types ← internal/encoding ← internal/storage +internal/opset ← internal/encoding ← internal/storage ↑ internal/sync ``` diff --git a/pkg/automerge/internal/native/document.go b/pkg/automerge/internal/native/document.go new file mode 100644 index 0000000000..19dfd8e2e4 --- /dev/null +++ b/pkg/automerge/internal/native/document.go @@ -0,0 +1,189 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package native + +import "slices" + +// compact serializes the whole history as one document chunk, the form save() +// produces in the other implementations, followed by any retained orphan changes +// as trailing change chunks (which is how a snapshot carries changes it cannot +// place in the operation set because their dependencies are missing). +// +// It reports ok=false, rather than an error, when the history cannot be +// compacted: while isolated, where the pinned view is not the whole history, or +// when the change graph is not internally consistent. The caller then falls back +// to the faithful change stream, which preserves every byte that was loaded. +func (b *Engine) compact(retainOrphans, deflate bool) ([]byte, bool, error) { + if b.isolationActive { + return nil, false, nil + } + + changes, ok := b.state.allChanges() + if !ok { + return nil, false, nil + } + + document := &Document{ + Changes: make([]Change, 0, len(changes)), + Heads: b.state.Heads(), + } + for _, change := range changes { + document.Changes = append(document.Changes, *change) + } + + data, err := EncodeDocument(document, b.state.documentOperationOrder(), deflate) + if err != nil { + return nil, false, err + } + + if retainOrphans { + for _, change := range orderedQueuedChanges(b.queuedChanges) { + data = append(data, maybeCompressChangeChunk(change.Raw, deflate)...) + } + } + + return data, true, nil +} + +// documentOperationOrder returns the operation-set order a document chunk is +// written in: the root map first, then every object in identifier order, with a +// map's operations grouped by property and a sequence's following the order a +// reader sees. Deletes are left out because a snapshot records them only as +// successors of what they removed. +func (s *State) documentOperationOrder() []OpID { + order := make([]OpID, 0, len(s.operations)) + + for _, object := range s.documentObjects() { + if object.IsRoot || isMapObject(s.operations[object.OpID].Action) { + order = append(order, s.mapObjectOrder(object)...) + + continue + } + + order = append(order, s.sequenceObjectOrder(object)...) + } + + return order +} + +// documentObjects lists the root map followed by every object the history +// creates, ordered by the identifier of the operation that made it. +func (s *State) documentObjects() []ObjectID { + objects := make([]ObjectID, 0) + + for id, operation := range s.operations { + if isObjectAction(operation.Action) { + objects = append(objects, ObjectID{OpID: id}) + } + } + + slices.SortFunc(objects, func(left, right ObjectID) int { + return left.OpID.Compare(right.OpID) + }) + + return append([]ObjectID{RootObject()}, objects...) +} + +func (s *State) mapObjectOrder(object ObjectID) []OpID { + byProperty := make(map[string][]OpID) + + for id, operation := range s.operations { + if operation.Object != object || + operation.Key.Property == nil || + operation.Action == ActionDelete { + continue + } + + property := *operation.Key.Property + byProperty[property] = append(byProperty[property], id) + } + + properties := make([]string, 0, len(byProperty)) + for property := range byProperty { + properties = append(properties, property) + } + + slices.Sort(properties) + + order := make([]OpID, 0, len(s.operations)) + + for _, property := range properties { + identifiers := byProperty[property] + + slices.SortFunc(identifiers, func(left, right OpID) int { + return left.Compare(right) + }) + + order = append(order, identifiers...) + } + + return order +} + +func (s *State) sequenceObjectOrder(object ObjectID) []OpID { + // Operations that address an element rather than create it, such as an + // overwrite, follow the element they target. + byElement := make(map[OpID][]OpID) + + for id, operation := range s.operations { + if operation.Object != object || + operation.Insert || + operation.Key.Element == nil || + operation.Action == ActionDelete { + continue + } + + element := *operation.Key.Element + byElement[element] = append(byElement[element], id) + } + + for element := range byElement { + slices.SortFunc(byElement[element], func(left, right OpID) int { + return left.Compare(right) + }) + } + + elements := s.insertOrder(object.OpID) + order := make([]OpID, 0, len(elements)) + + for _, element := range elements { + if operation, ok := s.operations[element]; ok && operation.Action != ActionDelete { + order = append(order, element) + } + + order = append(order, byElement[element]...) + } + + return order +} + +func isObjectAction(action Action) bool { + switch action { + case ActionMakeMap, ActionMakeList, ActionMakeText, ActionMakeTable: + return true + default: + return false + } +} + +func isMapObject(action Action) bool { + return action == ActionMakeMap || action == ActionMakeTable +} diff --git a/pkg/automerge/internal/native/engine.go b/pkg/automerge/internal/native/engine.go index 06e1bfdd70..29eaa1170c 100644 --- a/pkg/automerge/internal/native/engine.go +++ b/pkg/automerge/internal/native/engine.go @@ -59,6 +59,23 @@ type Engine struct { // the current heads, matching the reference's patch-log output across // isolate/integrate rather than a direct state comparison. isolationDiffTargets [][][32]byte + + // revision increases on every change to the committed history or the + // retained orphan set. The compacted save is cached against it so repeated + // saves of an unchanged document skip rebuilding the whole columnar + // document, which the collaboration snapshot path does on every request. + revision uint64 + saveCache saveCacheEntry +} + +// saveCacheEntry memoizes one compacted save. It is valid only for the exact +// revision and option combination it was built from. +type saveCacheEntry struct { + revision uint64 + retainOrphans bool + compress bool + valid bool + data []byte } type nativeSyncState struct { @@ -287,25 +304,18 @@ func (b *Engine) Close(context.Context) error { return nil } -func (b *Engine) Save(ctx context.Context) ([]byte, error) { - return b.save(ctx, true, true) -} - -// SaveNoCompress serializes the document without DEFLATE-compressing any change -// chunks, mirroring Rust's AutoCommit::save_nocompress. -func (b *Engine) SaveNoCompress(ctx context.Context) ([]byte, error) { - return b.save(ctx, true, false) -} - -// SaveWithOptions serializes the document, optionally appending retained orphan -// changes (queued changes whose dependencies are still missing) so they survive -// a save/load round trip. It mirrors the Rust SaveOptions.retain_orphans flag; -// the reference retains orphans by default. -func (b *Engine) SaveWithOptions( +// Save serializes the whole history as one compacted document chunk, the form +// save() produces in the Rust and JavaScript implementations. It replaces the +// change-by-change stream Go used to write, which grew without bound as a +// history accumulated commits. retainOrphans keeps queued changes whose +// dependencies are still missing so they survive a save/load round trip, and +// compress DEFLATEs the document columns and any trailing change chunks. +func (b *Engine) Save( ctx context.Context, retainOrphans bool, + compress bool, ) ([]byte, error) { - return b.save(ctx, retainOrphans, true) + return b.save(ctx, retainOrphans, compress) } func (b *Engine) save( @@ -319,6 +329,71 @@ func (b *Engine) save( } } + // Rebuilding the columnar document is by far the costliest part of a save, so + // an unchanged document returns the bytes built last time. The cache is keyed + // by the mutation revision and the option combination, and it is invalidated + // implicitly because every committed change advances the revision. + if cached, ok := b.cachedSave(retainOrphans, deflate); ok { + b.saveCursor = len(b.appended) + + return cached, nil + } + + // A compacted document chunk is the form every other implementation writes + // and is dramatically smaller than the change stream for a long history. It + // leaves the incremental cursor at the end, exactly as the stream save did, + // so a following SaveIncremental still emits only later changes. + if data, ok, err := b.compact(retainOrphans, deflate); err != nil { + return nil, err + } else if ok { + b.saveCursor = len(b.appended) + b.storeSave(retainOrphans, deflate, data) + + return data, nil + } + + data, err := b.streamSave(retainOrphans, deflate) + if err != nil { + return nil, err + } + + b.storeSave(retainOrphans, deflate, data) + + return data, nil +} + +// cachedSave returns a copy of the previously built save when it is still valid +// for this revision and option combination. A copy is returned because callers +// own the bytes and may retain or mutate them. +func (b *Engine) cachedSave(retainOrphans, compress bool) ([]byte, bool) { + if !b.saveCache.valid || + b.saveCache.revision != b.revision || + b.saveCache.retainOrphans != retainOrphans || + b.saveCache.compress != compress { + return nil, false + } + + return append([]byte(nil), b.saveCache.data...), true +} + +// storeSave records a freshly built save so an unchanged document can return it +// without rebuilding. The stored bytes are copied so a caller mutating the +// returned slice cannot corrupt the cache. +func (b *Engine) storeSave(retainOrphans, compress bool, data []byte) { + b.saveCache = saveCacheEntry{ + revision: b.revision, + retainOrphans: retainOrphans, + compress: compress, + valid: true, + data: append([]byte(nil), data...), + } +} + +// streamSave serializes the history as the loaded base followed by each change +// chunk since. It preserves the loaded bytes verbatim, including columns this +// version does not understand, and is the fallback when a history cannot be +// compacted (while isolated, or when the change graph is inconsistent). +func (b *Engine) streamSave(retainOrphans, deflate bool) ([]byte, error) { total := len(b.base) for _, change := range b.appended { total += len(change) diff --git a/pkg/automerge/internal/native/engine_helpers.go b/pkg/automerge/internal/native/engine_helpers.go index b0c14c164a..cc30fc2c8a 100644 --- a/pkg/automerge/internal/native/engine_helpers.go +++ b/pkg/automerge/internal/native/engine_helpers.go @@ -29,6 +29,7 @@ import ( "encoding/json" "fmt" "math" + "sort" ) func (b *Engine) addPending(operation Operation) error { @@ -426,62 +427,51 @@ func richTextPosition( return nil, previous, nil } +// sequenceRange resolves a UTF-16 index and delete count against the visible +// sequence using the precomputed cumulative offsets, so a splice locates its +// position by binary search rather than walking the whole sequence. offsets has +// one entry per element plus a trailing total, where offsets[i] is the width +// before element i. func sequenceRange( sequence []Operation, + offsets []uint32, index uint32, deleteCount uint32, ) (int, int, *OpID, error) { - position := uint32(0) - start := -1 - - var ( - previous *OpID - previousValue OpID - ) - - for i, operation := range sequence { - if position == index { - start = i - break - } - - length := elementLength(operation) - if position+length > index { - // UTF-16 callers can address the middle of a surrogate pair. - // Upstream Rust advances such a position to the boundary after - // the character rather than rejecting the edit. - position += length - previousValue = operation.ID - previous = &previousValue - start = i + 1 - - break - } - - position += length - previousValue = operation.ID - previous = &previousValue + total := offsets[len(offsets)-1] + if index > total { + return 0, 0, nil, fmt.Errorf("text index %d is out of bounds", index) } - if start == -1 { - if position != index { - return 0, 0, nil, fmt.Errorf("text index %d is out of bounds", index) - } - - start = len(sequence) + // Find the element whose starting offset is the last one at or before index. + // When that offset equals index the insertion sits on the boundary before + // the element; when it is smaller the index fell inside the element (a UTF-16 + // caller addressing the middle of a surrogate pair), so advance to the + // boundary after it, matching the reference. + boundary := sort.Search(len(offsets), func(i int) bool { return offsets[i] > index }) + start := boundary - 1 + if offsets[start] < index { + start++ } - target := position + deleteCount - - end := start - for end < len(sequence) && position < target { - position += elementLength(sequence[end]) - end++ + var previous *OpID + if start > 0 { + previousValue := sequence[start-1].ID + previous = &previousValue } // A deletion that runs past the end of the sequence is clamped to the // remaining elements rather than rejected, matching the reference, whose - // splice stops once there are no more elements to delete. + // splice stops once there are no more elements to delete. end is the first + // element boundary at or past the deletion target. + target := offsets[start] + deleteCount + end := start + sort.Search(len(sequence)-start+1, func(i int) bool { + return offsets[start+i] >= target + }) + if end > len(sequence) { + end = len(sequence) + } + return start, end, previous, nil } diff --git a/pkg/automerge/internal/native/frontier_test.go b/pkg/automerge/internal/native/frontier_test.go index 3c0ddb51d4..58eef36303 100644 --- a/pkg/automerge/internal/native/frontier_test.go +++ b/pkg/automerge/internal/native/frontier_test.go @@ -99,6 +99,105 @@ func TestNewStateFromDocument_RebuildsInconsistentFrontier(t *testing.T) { assert.True(t, ok, "changesSince(own heads) must succeed after rebuild") } +// TestChangesSince_DegradesToReachablePrefix is the regression for the wedge +// where one unreachable ancestor failed the whole read. A branched history has +// an intact branch and a branch whose middle change is removed; the intact +// branch must still come back as a consistent, replayable prefix while the +// broken branch is dropped, and completeness must report false. +func TestChangesSince_DegradesToReachablePrefix(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + actorBytes := func(id byte) []byte { + return []byte{id, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + } + + // Shared base commit. + base, err := NewEngine(ctx) + require.NoError(t, err) + require.NoError(t, base.SetActor(ctx, actorBytes(0x10))) + + handle, err := base.PutText(ctx, 0, "body") + require.NoError(t, err) + require.NoError(t, base.SpliceText(ctx, handle, 0, 0, "a")) + _, err = base.Commit(ctx, "base", time.Unix(0, 0)) + require.NoError(t, err) + + shared, err := base.Save(ctx, true, true) + require.NoError(t, err) + + // A branch two commits deep, authored by a second actor. + deep, err := LoadEngine(ctx, shared) + require.NoError(t, err) + require.NoError(t, deep.SetActor(ctx, actorBytes(0x20))) + + deepHandle, _, err := deep.GetObject(ctx, 0, "body") + require.NoError(t, err) + require.NoError(t, deep.SpliceText(ctx, deepHandle, 1, 0, "b")) + _, err = deep.Commit(ctx, "deep-1", time.Unix(1, 0)) + require.NoError(t, err) + require.NoError(t, deep.SpliceText(ctx, deepHandle, 2, 0, "c")) + _, err = deep.Commit(ctx, "deep-2", time.Unix(2, 0)) + require.NoError(t, err) + + deepSave, err := deep.Save(ctx, true, true) + require.NoError(t, err) + + // The base adds its own branch commit, then merges the deep branch, so the + // frontier holds two heads: the base branch and the deep branch. + require.NoError(t, base.SpliceText(ctx, handle, 1, 0, "z")) + _, err = base.Commit(ctx, "base-2", time.Unix(3, 0)) + require.NoError(t, err) + _, err = base.Merge(ctx, deepSave) + require.NoError(t, err) + + all, complete := base.state.changesSince(nil) + require.True(t, complete) + require.Len(t, all, 4) + + // Remove the deep branch's first change, the ancestor of its head. + deepActor, err := NewActorID(actorBytes(0x20)) + require.NoError(t, err) + + var removed ChangeHash + + for hash, change := range base.state.changes { + if change.Actor == deepActor && change.Sequence == 1 { + removed = hash + } + } + + delete(base.state.changes, removed) + + changes, complete := base.state.changesSince(nil) + assert.False(t, complete, "the broken branch leaves the walk incomplete") + assert.Len(t, changes, 2, "the intact branch is still emitted as a prefix") + + for i, change := range changes { + require.NotNil(t, change.Hash) + assert.NotEqual(t, removed, *change.Hash) + + // Every emitted change's dependencies precede it, so the prefix replays. + for _, dependency := range change.Dependencies { + found := false + for _, earlier := range changes[:i] { + if earlier.Hash != nil && *earlier.Hash == dependency { + found = true + } + } + + assert.True(t, found, "dependency of an emitted change must precede it") + } + } + + // The engine method must not wedge: it returns the reachable prefix. + raw, hashes, err := base.ChangesSince(ctx, nil) + require.NoError(t, err) + assert.Len(t, raw, 2) + assert.Len(t, hashes, 2) +} + // TestChangesSince_ToleratesUnknownBaseline mirrors Rust's get_changes, which // takes have_deps by value and never errors: an unknown baseline excludes // nothing and the full history is returned. @@ -117,8 +216,9 @@ func TestChangesSince_ToleratesUnknownBaseline(t *testing.T) { } // TestChangesSince_ToleratesFrontierWithMissingChange guards the frontier walk: -// a head recorded without a retrievable change contributes nothing and must not -// abort the incremental computation. +// a head recorded without a retrievable change must not abort the computation. +// The reachable changes are still returned so an incremental read keeps working, +// and completeness reports false so sync knows to fall back to a full document. func TestChangesSince_ToleratesFrontierWithMissingChange(t *testing.T) { t.Parallel() @@ -129,7 +229,7 @@ func TestChangesSince_ToleratesFrontierWithMissingChange(t *testing.T) { phantom[0] = 0xEF backend.state.heads[phantom] = struct{}{} - changes, ok := backend.state.changesSince(nil) - require.True(t, ok) - assert.Len(t, changes, 2) + changes, complete := backend.state.changesSince(nil) + assert.False(t, complete, "an unretrievable head leaves the walk incomplete") + assert.Len(t, changes, 2, "the reachable changes are still returned") } diff --git a/pkg/automerge/internal/native/history.go b/pkg/automerge/internal/native/history.go index 404ff65452..efe08562bb 100644 --- a/pkg/automerge/internal/native/history.go +++ b/pkg/automerge/internal/native/history.go @@ -109,10 +109,13 @@ func (b *Engine) ChangesSince( knownHeads[i] = ChangeHash(head) } - changes, ok := b.state.changesSince(knownHeads) - if !ok { - return nil, nil, fmt.Errorf("cannot compute changes from unknown heads") - } + // A change in the frontier's ancestry may occasionally be unreachable, for + // example after a merge that rebuilt the graph. changesSince then returns the + // consistent, replayable prefix it can produce rather than nothing: reporting + // every change that can be emitted keeps collaboration alive, where failing + // the whole read would wedge the document on every request. A change that is + // dropped has no bytes to return in any case. + changes, _ := b.state.changesSince(knownHeads) raw := make([][]byte, len(changes)) @@ -167,6 +170,11 @@ func (b *Engine) Merge(ctx context.Context, data []byte) ([][32]byte, error) { return nil, err } + // A merge may apply nothing when every change is already present, but bumping + // unconditionally only risks an extra rebuild on the next save, never a stale + // one, and it keeps every apply path covered by a single line. + b.revision++ + if len(b.state.Heads()) == 0 && len(b.pending) == 0 { state, err := NewStateFromDocument(document) if err != nil { diff --git a/pkg/automerge/internal/native/incremental_sync_test.go b/pkg/automerge/internal/native/incremental_sync_test.go index 10c6be82de..fa224da8b0 100644 --- a/pkg/automerge/internal/native/incremental_sync_test.go +++ b/pkg/automerge/internal/native/incremental_sync_test.go @@ -79,7 +79,7 @@ func TestBackendSync_SendsOnlyChangesSinceRemoteHeads(t *testing.T) { require.Len(t, secondDocument.Changes, 1) assert.Equal(t, "second", secondDocument.Changes[0].Message) - fullDocument, err := backend.Save(ctx) + fullDocument, err := backend.Save(ctx, true, true) require.NoError(t, err) assert.Less(t, len(secondMessage), len(fullDocument)) } diff --git a/pkg/automerge/internal/native/merge_test.go b/pkg/automerge/internal/native/merge_test.go index 2f9d403ae7..a5f2ff755d 100644 --- a/pkg/automerge/internal/native/merge_test.go +++ b/pkg/automerge/internal/native/merge_test.go @@ -41,7 +41,7 @@ func TestBackendMerge_AppliesReversedDependentChanges(t *testing.T) { require.NoError(t, base.SpliceText(ctx, text, 0, 0, "A")) _, err = base.Commit(ctx, "base", time.Unix(1, 0)) require.NoError(t, err) - baseData, err := base.Save(ctx) + baseData, err := base.Save(ctx, true, true) require.NoError(t, err) source, err := LoadEngine(ctx, baseData) diff --git a/pkg/automerge/internal/native/model.go b/pkg/automerge/internal/native/model.go new file mode 100644 index 0000000000..dd72604218 --- /dev/null +++ b/pkg/automerge/internal/native/model.go @@ -0,0 +1,70 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package native + +import "go.probo.inc/probo/pkg/automerge/internal/opset" + +type ( + ActorID = opset.ActorID + ChangeHash = opset.ChangeHash + OpID = opset.OpID + ObjectID = opset.ObjectID + Key = opset.Key + Action = opset.Action + ScalarType = opset.ScalarType + Scalar = opset.Scalar + Operation = opset.Operation + Change = opset.Change + ChunkType = opset.ChunkType + RawColumn = opset.RawColumn + Document = opset.Document +) + +const ( + ActionMakeMap = opset.ActionMakeMap + ActionSet = opset.ActionSet + ActionMakeList = opset.ActionMakeList + ActionDelete = opset.ActionDelete + ActionMakeText = opset.ActionMakeText + ActionIncrement = opset.ActionIncrement + ActionMakeTable = opset.ActionMakeTable + ActionMark = opset.ActionMark + + ScalarNull = opset.ScalarNull + ScalarFalse = opset.ScalarFalse + ScalarTrue = opset.ScalarTrue + ScalarUint = opset.ScalarUint + ScalarInt = opset.ScalarInt + ScalarFloat64 = opset.ScalarFloat64 + ScalarString = opset.ScalarString + ScalarBytes = opset.ScalarBytes + ScalarCounter = opset.ScalarCounter + ScalarTimestamp = opset.ScalarTimestamp + + ChunkDocument = opset.ChunkDocument + ChunkChange = opset.ChunkChange + ChunkCompressedChange = opset.ChunkCompressedChange +) + +var ( + NewActorID = opset.NewActorID + RootObject = opset.RootObject +) diff --git a/pkg/automerge/internal/native/rich_text.go b/pkg/automerge/internal/native/rich_text.go index a08042f6ae..861617dadc 100644 --- a/pkg/automerge/internal/native/rich_text.go +++ b/pkg/automerge/internal/native/rich_text.go @@ -98,8 +98,9 @@ func (b *Engine) SpliceText( // blocks, so walk the full visible element sequence (text and block markers) // rather than the text-only view. sequence := b.state.sequenceElements(object.OpID) + offsets := b.state.sequenceOffsets(object.OpID, sequence) - start, end, previous, err := sequenceRange(sequence, index, uint32(deleteCount)) + start, end, previous, err := sequenceRange(sequence, offsets, index, uint32(deleteCount)) if err != nil { return err } @@ -811,6 +812,13 @@ func (b *Engine) SplitBlock( key.Element = new(*previous) } + // A block marker shares the unified rich-text sequence with text, so it must + // resolve its anchor against neighbouring mark boundaries the same way a text + // insertion does. Without this, a block inserted next to a mark boundary lands + // on the wrong side of it, which then misplaces later insertions and makes the + // marks they should or should not carry diverge from the reference. + key = b.state.insertAnchorKey(object.OpID, key) + operation := Operation{ ID: b.nextOperationID(), Object: object, diff --git a/pkg/automerge/internal/native/rich_text_state.go b/pkg/automerge/internal/native/rich_text_state.go index b987d937b6..7ac665eb08 100644 --- a/pkg/automerge/internal/native/rich_text_state.go +++ b/pkg/automerge/internal/native/rich_text_state.go @@ -119,20 +119,12 @@ func (s *State) insertAnchorKey(object OpID, base Key) Key { return base } - found := false - - for i, id := range order { - if id == *base.Element { - start = i + 1 - found = true - - break - } - } - - if !found { + position, ok := s.insertOrderPositions(object)[*base.Element] + if !ok { return base } + + start = position + 1 } type candidate struct { diff --git a/pkg/automerge/internal/native/sequence_state.go b/pkg/automerge/internal/native/sequence_state.go index 938b320c6b..9a423086c1 100644 --- a/pkg/automerge/internal/native/sequence_state.go +++ b/pkg/automerge/internal/native/sequence_state.go @@ -155,12 +155,15 @@ func (s *State) spliceInsertOrder(operation Operation) { if operation.Key.IsHead { s.insertOrderCache[object] = append([]OpID{operation.ID}, order...) + // A prepend shifts every position, so the index is rebuilt on demand. + delete(s.insertOrderPositionCache, object) return } if operation.Key.Element == nil { delete(s.insertOrderCache, object) + delete(s.insertOrderPositionCache, object) return } @@ -170,6 +173,13 @@ func (s *State) spliceInsertOrder(operation Operation) { if len(order) > 0 && order[len(order)-1] == anchor { s.insertOrderCache[object] = append(order, operation.ID) + // Appending at the end keeps every existing index, so extend the position + // index in step to keep sequential typing constant time. + if positions, ok := s.insertOrderPositionCache[object]; ok && + len(positions) == len(order) { + positions[operation.ID] = len(order) + } + return } @@ -184,12 +194,15 @@ func (s *State) spliceInsertOrder(operation Operation) { updated = append(updated, operation.ID) updated = append(updated, order[position:]...) s.insertOrderCache[object] = updated + // An insertion in the middle shifts later positions; rebuild on demand. + delete(s.insertOrderPositionCache, object) return } // The anchor is not present in the cached order; rebuild lazily. delete(s.insertOrderCache, object) + delete(s.insertOrderPositionCache, object) } func (s *State) sequenceValues(object OpID) []sequenceValue { @@ -260,6 +273,7 @@ func (s *State) updateSequenceValues(operation Operation) { if !appended { delete(s.sequenceValuesCache, object) delete(s.sequenceElementsCache, object) + delete(s.sequenceOffsetCache, object) return } @@ -273,7 +287,57 @@ func (s *State) updateSequenceValues(operation Operation) { if cached, ok := s.sequenceElementsCache[object]; ok { s.sequenceElementsCache[object] = append(cached, operation) + + // Extend the offset index in step so appending stays constant time. A + // valid offset slice for N elements holds N+1 entries, so it lines up + // with the pre-append element count; the new element starts at the + // previous total width. + if offsets, ok := s.sequenceOffsetCache[object]; ok && + len(offsets) == len(cached)+1 { + s.sequenceOffsetCache[object] = append( + offsets, + offsets[len(offsets)-1]+elementLength(operation), + ) + } + } +} + +// sequenceOffsets returns the cumulative UTF-16 width before each element of the +// given sequence, with a trailing entry holding the total width. It is cached +// and rebuilt whenever it does not line up with the elements, so a text index +// can be resolved by binary search rather than a linear walk. +func (s *State) sequenceOffsets(object OpID, elements []Operation) []uint32 { + if cached, ok := s.sequenceOffsetCache[object]; ok && len(cached) == len(elements)+1 { + return cached } + + offsets := make([]uint32, len(elements)+1) + for i, operation := range elements { + offsets[i+1] = offsets[i] + elementLength(operation) + } + + s.sequenceOffsetCache[object] = offsets + + return offsets +} + +// insertOrderPositions returns each insert-order operation's index. Insert order +// only ever grows, so a length mismatch is a sufficient staleness check. +func (s *State) insertOrderPositions(object OpID) map[OpID]int { + order := s.insertOrder(object) + + if cached, ok := s.insertOrderPositionCache[object]; ok && len(cached) == len(order) { + return cached + } + + positions := make(map[OpID]int, len(order)) + for index, id := range order { + positions[id] = index + } + + s.insertOrderPositionCache[object] = positions + + return positions } // elementValueWinners returns, for every list element that has been assigned a diff --git a/pkg/automerge/internal/native/state.go b/pkg/automerge/internal/native/state.go index 20c6037b88..60ecd5ac97 100644 --- a/pkg/automerge/internal/native/state.go +++ b/pkg/automerge/internal/native/state.go @@ -52,6 +52,18 @@ type ( sequenceValuesCache map[OpID][]sequenceValue sequenceElementsCache map[OpID][]Operation + // sequenceOffsetCache holds the cumulative UTF-16 width before each + // element of sequenceElementsCache (with a trailing total), so a text + // index resolves to an element by binary search instead of a linear + // walk. It is kept in step with the elements cache and guarded by length. + sequenceOffsetCache map[OpID][]uint32 + + // insertOrderPositionCache maps each insert-order operation to its index, + // so an insertion anchor resolves in constant time instead of scanning + // the order. Insert order only ever grows, so a length guard is enough to + // detect staleness. + insertOrderPositionCache map[OpID]map[OpID]int + // mapKeyIndex groups operation IDs by the map property they address so // reading a key does not scan the whole operation set. It is built on // first use and then maintained as operations are applied. @@ -73,22 +85,34 @@ type ( func NewState() *State { return &State{ - changes: make(map[ChangeHash]*Change), - actorSequence: make(map[ActorID]uint64), - operations: make(map[OpID]Operation), - superseded: make(map[OpID]struct{}), - heads: make(map[ChangeHash]struct{}), - sequenceCache: make(map[OpID][]Operation), - insertOrderCache: make(map[OpID][]OpID), - sequenceValuesCache: make(map[OpID][]sequenceValue), - sequenceElementsCache: make(map[OpID][]Operation), - mapKeyIndex: make(map[ObjectID]map[string][]OpID), + changes: make(map[ChangeHash]*Change), + actorSequence: make(map[ActorID]uint64), + operations: make(map[OpID]Operation), + superseded: make(map[OpID]struct{}), + heads: make(map[ChangeHash]struct{}), + sequenceCache: make(map[OpID][]Operation), + insertOrderCache: make(map[OpID][]OpID), + sequenceValuesCache: make(map[OpID][]sequenceValue), + sequenceElementsCache: make(map[OpID][]Operation), + sequenceOffsetCache: make(map[OpID][]uint32), + insertOrderPositionCache: make(map[OpID]map[OpID]int), + mapKeyIndex: make(map[ObjectID]map[string][]OpID), } } func NewStateFromDocument(document *Document) (*State, error) { state := NewState() + // Presize the operation and change maps so loading a large document does not + // rehash them repeatedly as it inserts every operation. + operationCount := 0 + for i := range document.Changes { + operationCount += len(document.Changes[i].Operations) + } + + state.operations = make(map[OpID]Operation, operationCount) + state.changes = make(map[ChangeHash]*Change, len(document.Changes)) + for i := range document.Changes { change := &document.Changes[i] if change.Sequence > state.actorSequence[change.Actor] { @@ -200,8 +224,10 @@ func (s *State) ApplyChange(change *Change) error { if !operation.Object.IsRoot { delete(s.sequenceCache, operation.Object.OpID) delete(s.insertOrderCache, operation.Object.OpID) + delete(s.insertOrderPositionCache, operation.Object.OpID) delete(s.sequenceValuesCache, operation.Object.OpID) delete(s.sequenceElementsCache, operation.Object.OpID) + delete(s.sequenceOffsetCache, operation.Object.OpID) } s.indexMapKeyOperation(operation) @@ -563,58 +589,99 @@ func (s *State) hasDependencies(change *Change) bool { return true } +// changesSince returns the changes reachable from the current frontier that the +// baseline heads do not already cover, in dependency order, ready to replay. +// +// The result is always a consistent prefix: a change is emitted only once every +// one of its ancestors has been emitted or is already known to the baseline, so +// a caller never receives a change whose dependency it was not also given. The +// second return reports whether that prefix is complete. It is false when some +// change in the frontier's ancestry could not be produced, either because it is +// absent from the graph or because its original bytes are unavailable. +// +// Completeness is a signal, not a gate. Sync uses it to fall back to sending a +// whole document, which reproduces changes even when the in-memory graph is +// inconsistent. Incremental reads use the prefix regardless, because returning +// every change that can be produced keeps a document usable where failing the +// whole read would wedge it: a change that cannot be emitted has no bytes to +// return anyway. func (s *State) changesSince(heads []ChangeHash) ([]*Change, bool) { known := s.changeClosure(heads) + const ( + visiting = iota + reachable + unreachable + ) + ordered := make([]*Change, 0) - visited := make(map[ChangeHash]struct{}) + status := make(map[ChangeHash]int) var visit func(ChangeHash) bool visit = func(hash ChangeHash) bool { - if _, ok := visited[hash]; ok { + if state, ok := status[hash]; ok { + // A change still on the stack cannot be depended upon to be complete + // yet, but treating the cycle edge as reachable avoids excluding the + // whole branch over a graph that should never contain a cycle anyway. + return state != unreachable + } + + // The baseline closure is transitively closed, so everything below a change + // the peer already holds is also held and need not be walked. + if _, ok := known[hash]; ok { + status[hash] = reachable + return true } - visited[hash] = struct{}{} + status[hash] = visiting change, ok := s.changes[hash] if !ok { + status[hash] = unreachable + return false } + complete := true + for _, dependency := range change.Dependencies { if !visit(dependency) { - return false + complete = false } } - if _, ok := known[hash]; ok { - return true - } + // A change is emittable only when every ancestor is, so the result stays a + // replayable prefix, and only when its bytes exist to be returned. + if !complete || len(change.Raw) == 0 { + status[hash] = unreachable - if len(change.Raw) == 0 { return false } ordered = append(ordered, change) + status[hash] = reachable return true } + complete := true + for _, head := range s.Heads() { - // A frontier head whose change is not retrievable contributes nothing and - // must not abort the incremental computation for the whole document. + // A frontier head whose change is not retrievable contributes nothing. if _, ok := s.changes[head]; !ok { + complete = false + continue } if !visit(head) { - return nil, false + complete = false } } - return ordered, true + return ordered, complete } func (s *State) allChanges() ([]*Change, bool) { diff --git a/pkg/automerge/internal/native/storage.go b/pkg/automerge/internal/native/storage.go index 08d51a917c..7d6b3b0a06 100644 --- a/pkg/automerge/internal/native/storage.go +++ b/pkg/automerge/internal/native/storage.go @@ -30,6 +30,7 @@ var ( DecodePartial = internalstorage.DecodePartial DecodeIncremental = internalstorage.DecodeIncremental EncodeChange = internalstorage.EncodeChange + EncodeDocument = internalstorage.EncodeDocument ) func deflate(data []byte) ([]byte, error) { return internalstorage.Deflate(data) } diff --git a/pkg/automerge/internal/native/sync_engine.go b/pkg/automerge/internal/native/sync_engine.go index 42f9c8dfe1..561505ea80 100644 --- a/pkg/automerge/internal/native/sync_engine.go +++ b/pkg/automerge/internal/native/sync_engine.go @@ -230,7 +230,7 @@ func (b *Engine) GenerateSyncMessage( ) } } else { - document, err := b.Save(ctx) + document, err := b.Save(ctx, true, true) if err != nil { return nil, false, err } diff --git a/pkg/automerge/internal/native/transaction.go b/pkg/automerge/internal/native/transaction.go index b65078a6f5..637033e758 100644 --- a/pkg/automerge/internal/native/transaction.go +++ b/pkg/automerge/internal/native/transaction.go @@ -92,6 +92,7 @@ func (b *Engine) Isolate(ctx context.Context, heads [][32]byte) error { b.state = pinned b.actor = isolationActor(full, pinned, baseActor) b.nextOp = full.maxOpGlobal() + 1 + b.revision++ b.isolationDiffTargets = append( b.isolationDiffTargets, @@ -134,6 +135,7 @@ func (b *Engine) Integrate(ctx context.Context) error { b.fullState = nil b.isolationActive = false b.nextOp = b.state.maxOpGlobal() + 1 + b.revision++ return nil } @@ -215,6 +217,7 @@ func (b *Engine) Commit( b.appended = append(b.appended, raw) b.pending = nil + b.revision++ return [32]byte(*change.Hash), nil } @@ -257,6 +260,7 @@ func (b *Engine) EmptyCommit( } b.appended = append(b.appended, raw) + b.revision++ return [32]byte(*change.Hash), nil } @@ -291,6 +295,7 @@ func (b *Engine) Rollback(ctx context.Context) (uint64, error) { b.pending = nil b.objects = map[uint32]ObjectID{0: RootObject()} b.nextHandle = 1 + b.revision++ return cancelled, nil } diff --git a/pkg/automerge/internal/native/types.go b/pkg/automerge/internal/native/types.go deleted file mode 100644 index 671f34e5a3..0000000000 --- a/pkg/automerge/internal/native/types.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2026 Probo Inc . -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package native - -import "go.probo.inc/probo/pkg/automerge/internal/types" - -type ( - ActorID = types.ActorID - ChangeHash = types.ChangeHash - OpID = types.OpID - ObjectID = types.ObjectID - Key = types.Key - Action = types.Action - ScalarType = types.ScalarType - Scalar = types.Scalar - Operation = types.Operation - Change = types.Change - ChunkType = types.ChunkType - RawColumn = types.RawColumn - Document = types.Document -) - -const ( - ActionMakeMap = types.ActionMakeMap - ActionSet = types.ActionSet - ActionMakeList = types.ActionMakeList - ActionDelete = types.ActionDelete - ActionMakeText = types.ActionMakeText - ActionIncrement = types.ActionIncrement - ActionMakeTable = types.ActionMakeTable - ActionMark = types.ActionMark - - ScalarNull = types.ScalarNull - ScalarFalse = types.ScalarFalse - ScalarTrue = types.ScalarTrue - ScalarUint = types.ScalarUint - ScalarInt = types.ScalarInt - ScalarFloat64 = types.ScalarFloat64 - ScalarString = types.ScalarString - ScalarBytes = types.ScalarBytes - ScalarCounter = types.ScalarCounter - ScalarTimestamp = types.ScalarTimestamp - - ChunkDocument = types.ChunkDocument - ChunkChange = types.ChunkChange - ChunkCompressedChange = types.ChunkCompressedChange -) - -var ( - NewActorID = types.NewActorID - RootObject = types.RootObject -) diff --git a/pkg/automerge/internal/types/types.go b/pkg/automerge/internal/opset/opset.go similarity index 95% rename from pkg/automerge/internal/types/types.go rename to pkg/automerge/internal/opset/opset.go index 8dbf513847..5c8619978c 100644 --- a/pkg/automerge/internal/types/types.go +++ b/pkg/automerge/internal/opset/opset.go @@ -18,10 +18,11 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -// Package types defines the shared Automerge CRDT and storage model. -// It is dependency-free so storage, sync, and native execution packages can -// exchange changes without importing one another. -package types +// Package opset defines the shared Automerge operation-set model: actors, +// operation IDs, operations, changes, scalars, objects and the validated +// document history. It is dependency-free so the storage, sync, and native +// execution packages can exchange changes without importing one another. +package opset import ( "bytes" diff --git a/pkg/automerge/internal/reference/reference.go b/pkg/automerge/internal/reference/reference.go index 8e5635f521..9c6946d979 100644 --- a/pkg/automerge/internal/reference/reference.go +++ b/pkg/automerge/internal/reference/reference.go @@ -194,46 +194,26 @@ func (b *Engine) Close(ctx context.Context) error { return nil } -func (b *Engine) Save(ctx context.Context) ([]byte, error) { - if err := b.run(ctx, "am_save"); err != nil { - return nil, fmt.Errorf("cannot save reference document: %w", err) - } - - output, err := b.output(ctx) - if err != nil { - return nil, fmt.Errorf("cannot read saved reference document: %w", err) - } - - return output, nil -} - -// SaveNoCompress serializes the document without DEFLATE compression, mirroring -// AutoCommit::save_nocompress. -func (b *Engine) SaveNoCompress(ctx context.Context) ([]byte, error) { - if err := b.run(ctx, "am_save_nocompress"); err != nil { - return nil, fmt.Errorf("cannot save reference document: %w", err) - } - - output, err := b.output(ctx) - if err != nil { - return nil, fmt.Errorf("cannot read saved reference document: %w", err) - } - - return output, nil -} - -// SaveWithOptions serializes the document, optionally discarding orphan changes. -// Retaining orphans uses the default am_save; discarding them uses -// save_with_options(retain_orphans=false). -func (b *Engine) SaveWithOptions( +// Save serializes the document. retainOrphans keeps changes whose dependencies +// are missing; compress DEFLATEs the output. The reference exposes three save +// entry points, so the flag combination maps onto the closest one: the default +// am_save (retain, compress), am_save_nocompress (retain, no compress), and +// am_save_no_orphans (discard orphans, which also does not compress). +func (b *Engine) Save( ctx context.Context, retainOrphans bool, + compress bool, ) ([]byte, error) { - if retainOrphans { - return b.Save(ctx) + function := "am_save" + + switch { + case !retainOrphans: + function = "am_save_no_orphans" + case !compress: + function = "am_save_nocompress" } - if err := b.run(ctx, "am_save_no_orphans"); err != nil { + if err := b.run(ctx, function); err != nil { return nil, fmt.Errorf("cannot save reference document: %w", err) } diff --git a/pkg/automerge/internal/storage/columns.go b/pkg/automerge/internal/storage/columns.go index fdcb4601cc..d6ac719c49 100644 --- a/pkg/automerge/internal/storage/columns.go +++ b/pkg/automerge/internal/storage/columns.go @@ -27,6 +27,7 @@ import ( "fmt" "io" "math" + "slices" "unicode/utf8" ) @@ -222,6 +223,8 @@ func decodeRLE[T any](data []byte, decodeValue func(*reader) (T, error)) ([]opti return nil, err } + values = slices.Grow(values, int(count)) + for range count { value, err := decodeValue(r) if err != nil { @@ -241,10 +244,16 @@ func appendRepeated[T any](values *[]optional[T], value optional[T], count uint6 return err } + // Grow once for the whole run. A long run (an entire column of the same + // value, common for text where every operation shares an object and action) + // otherwise reallocated the slice repeatedly as it doubled. + grown := slices.Grow(*values, int(count)) for range count { - *values = append(*values, value) + grown = append(grown, value) } + *values = grown + return nil } diff --git a/pkg/automerge/internal/storage/decode.go b/pkg/automerge/internal/storage/decode.go index e39f04fe2e..c9e0db2b55 100644 --- a/pkg/automerge/internal/storage/decode.go +++ b/pkg/automerge/internal/storage/decode.go @@ -274,6 +274,11 @@ func decodeDocumentChunk(document *Document, data []byte) error { return fmt.Errorf("cannot decode operations: %w", err) } + operations, err = restoreChangeOperations(operations) + if err != nil { + return fmt.Errorf("cannot restore change operations: %w", err) + } + if err := assignOperations(changes, operations); err != nil { return fmt.Errorf("cannot assign operations: %w", err) } @@ -307,6 +312,10 @@ func decodeDocumentChunk(document *Document, data []byte) error { return err } + if err := reconstructSnapshotChanges(changes); err != nil { + return err + } + document.Actors = actors document.Heads = heads document.Changes = changes @@ -316,6 +325,168 @@ func decodeDocumentChunk(document *Document, data []byte) error { return nil } +// restoreChangeOperations turns a snapshot's operation view back into the one a +// change carries. A snapshot records, for every surviving operation, the +// operations that superseded it, and it drops delete operations entirely because +// they survive only as those successor entries. A change instead names each +// operation's predecessors and spells its deletes out, so rebuilding a change +// means inverting the successor lists and recreating the deletes they imply. +// +// Successors are left in place: the engine materializes a loaded snapshot from +// them, and re-encoding a change only ever reads predecessors. +func restoreChangeOperations(operations []Operation) ([]Operation, error) { + stored := make(map[OpID]struct{}, len(operations)) + for _, operation := range operations { + stored[operation.ID] = struct{}{} + } + + predecessors := make(map[OpID][]OpID) + for _, operation := range operations { + for _, successor := range operation.Successors { + predecessors[successor] = append(predecessors[successor], operation.ID) + } + } + + for identifier := range predecessors { + slices.SortFunc(predecessors[identifier], func(left, right OpID) int { + return left.Compare(right) + }) + } + + for i := range operations { + operations[i].Predecessors = predecessors[operations[i].ID] + } + + deletes := make([]Operation, 0) + + for identifier, superseded := range predecessors { + if _, ok := stored[identifier]; ok { + continue + } + + // Only a delete leaves no operation of its own behind, and every operation + // it removed shares the object and key it targeted. + source := operationByID(operations, superseded[0]) + if source == nil { + return nil, fmt.Errorf( + "operation %s@%d supersedes nothing that the snapshot retains", + identifier.Actor, + identifier.Counter, + ) + } + + deletes = append(deletes, Operation{ + ID: identifier, + Object: source.Object, + Key: supersededKey(source), + Action: ActionDelete, + Predecessors: superseded, + }) + } + + slices.SortFunc(deletes, func(left, right Operation) int { + return left.ID.Compare(right.ID) + }) + + return append(operations, deletes...), nil +} + +// supersededKey names the location an operation occupies, which is what an +// operation superseding it addresses. A map operation is addressed by its +// property, while a sequence operation is addressed by the element identifier: +// an insertion creates the element it is named by, and any later operation on +// that element already carries it. +func supersededKey(operation *Operation) Key { + if operation.Key.Property != nil || !operation.Insert { + return operation.Key + } + + element := operation.ID + + return Key{Element: &element} +} + +func operationByID(operations []Operation, identifier OpID) *Operation { + for i := range operations { + if operations[i].ID == identifier { + return &operations[i] + } + } + + return nil +} + +// reconstructSnapshotChanges restores the change-chunk identity of every change +// in a document chunk. Snapshots store the frontier hashes only and reference +// ancestry by column index, so every non-head change decodes without a hash and +// without its original bytes. Re-encoding each change once its dependencies are +// known recovers both, which keeps the change graph whole: without it only the +// heads are addressable and any walk of their ancestry hits a missing change. +// +// Dependencies are rebuilt in the stored index order because the encoder writes +// dependency hashes in slice order, so that order is what the original hash was +// computed over. +func reconstructSnapshotChanges(changes []Change) error { + resolved := make([]bool, len(changes)) + remaining := len(changes) + + for remaining > 0 { + progressed := false + + for i := range changes { + change := &changes[i] + + if resolved[i] || !dependenciesResolved(change, resolved) { + continue + } + + change.Dependencies = make([]ChangeHash, 0, len(change.DependencyIndexes)) + for _, index := range change.DependencyIndexes { + change.Dependencies = append(change.Dependencies, *changes[index].Hash) + } + + recorded := change.Hash + change.Hash = nil + + if _, err := EncodeChange(change); err != nil { + return fmt.Errorf("cannot rebuild snapshot change %d: %w", i, err) + } + + // A recorded hash only exists for frontier changes. Disagreeing with it + // means the rebuilt bytes are not the ones the writer hashed, so the + // whole graph would be keyed by identifiers no peer shares. + if recorded != nil && *recorded != *change.Hash { + return fmt.Errorf( + "snapshot change %d rebuilds to hash %s but the document records %s", + i, + change.Hash, + recorded, + ) + } + + resolved[i] = true + remaining-- + progressed = true + } + + if !progressed { + return fmt.Errorf("snapshot dependency graph cannot be ordered") + } + } + + return nil +} + +func dependenciesResolved(change *Change, resolved []bool) bool { + for _, index := range change.DependencyIndexes { + if !resolved[index] { + return false + } + } + + return true +} + func decodeDocumentChanges( columns map[uint32]column, actors []ActorID, @@ -745,7 +916,11 @@ func decodeOperations( } } - if markExpand[i].valid { + // Expand is a property of a mark, and its column is dense because booleans + // cannot be null. A document chunk shares one column across every change, + // so keeping the flag on ordinary operations would make a change that never + // carried an expand column re-encode with one and hash differently. + if markExpand[i].valid && operations[i].Action == ActionMark { value := markExpand[i].value operations[i].MarkExpand = &value } diff --git a/pkg/automerge/internal/storage/decode_test.go b/pkg/automerge/internal/storage/decode_test.go index dc07d1d894..4cdc1573af 100644 --- a/pkg/automerge/internal/storage/decode_test.go +++ b/pkg/automerge/internal/storage/decode_test.go @@ -263,7 +263,7 @@ func TestDecode_ReferenceBackendDocument(t *testing.T) { require.NoError(t, backend.PutString(ctx, 0, "policy", "approved")) _, err = backend.Commit(ctx, "reference fixture", time.Unix(1_700_000_000, 0)) require.NoError(t, err) - data, err := backend.Save(ctx) + data, err := backend.Save(ctx, true, true) require.NoError(t, err) document, err := Decode(data) @@ -286,7 +286,7 @@ func TestDecode_ReferenceBackendConcurrentGraph(t *testing.T) { require.NoError(t, base.PutString(ctx, 0, "base", "value")) _, err = base.Commit(ctx, "base", time.Unix(1, 0)) require.NoError(t, err) - baseData, err := base.Save(ctx) + baseData, err := base.Save(ctx, true, true) require.NoError(t, err) left, err := reference.Load(ctx, baseData) @@ -308,12 +308,12 @@ func TestDecode_ReferenceBackendConcurrentGraph(t *testing.T) { require.NoError(t, right.PutString(ctx, 0, "right", "value")) _, err = right.Commit(ctx, "right", time.Unix(3, 0)) require.NoError(t, err) - rightData, err := right.Save(ctx) + rightData, err := right.Save(ctx, true, true) require.NoError(t, err) _, err = left.Merge(ctx, rightData) require.NoError(t, err) - mergedData, err := left.Save(ctx) + mergedData, err := left.Save(ctx, true, true) require.NoError(t, err) document, err := Decode(mergedData) diff --git a/pkg/automerge/internal/storage/encode.go b/pkg/automerge/internal/storage/encode.go index ba82a47135..1e67e6750d 100644 --- a/pkg/automerge/internal/storage/encode.go +++ b/pkg/automerge/internal/storage/encode.go @@ -214,8 +214,10 @@ func encodeOperationColumns( predCounters = append(predCounters, some(int64(predecessor.Counter))) } - if operation.MarkExpand != nil { - markExpands[i] = *operation.MarkExpand + // An all-false expand column carries no information and is left out, so a + // mark that expands in neither direction encodes without one. + if operation.MarkExpand != nil && *operation.MarkExpand { + markExpands[i] = true hasMarkExpand = true } diff --git a/pkg/automerge/internal/storage/encode_document.go b/pkg/automerge/internal/storage/encode_document.go new file mode 100644 index 0000000000..72ef49a151 --- /dev/null +++ b/pkg/automerge/internal/storage/encode_document.go @@ -0,0 +1,669 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package storage + +import ( + "bytes" + "crypto/sha256" + "fmt" + "slices" +) + +// assembleChunk frames a chunk body: the magic bytes, the first four bytes of +// the digest covering the typed and length-prefixed body, then that body. +func assembleChunk(kind ChunkType, body []byte) []byte { + length := appendULEB(nil, uint64(len(body))) + + digestInput := make([]byte, 0, 1+len(length)+len(body)) + digestInput = append(digestInput, byte(kind)) + digestInput = append(digestInput, length...) + digestInput = append(digestInput, body...) + + digest := sha256.Sum256(digestInput) + + chunk := make([]byte, 0, 4+4+1+len(length)+len(body)) + chunk = append(chunk, magic[:]...) + chunk = append(chunk, digest[:4]...) + chunk = append(chunk, byte(kind)) + chunk = append(chunk, length...) + chunk = append(chunk, body...) + + return chunk +} + +// EncodeDocument serializes a whole history as one compacted document chunk, +// the form Rust and JavaScript write from save(). +// +// A document chunk is not a concatenation of changes. It stores the operation +// set once, in operation-set order rather than per change, and reduces each +// change to a row of metadata whose ancestry is expressed as indexes into that +// same table. Deletes disappear into the successor lists of the operations they +// removed, and every operation's predecessors are recovered from those lists on +// the way back in. +// +// order gives the operation-set sequence: object by object, and within an +// object the order a reader would see. The caller owns that sequence because it +// requires the sequence state only the engine maintains. Operations named by +// order must exist in the history, and deletes must be left out of it. +// +// compress DEFLATEs individual columns above a size threshold, which is what +// save() does and save_nocompress() does not. The compressed bytes are not +// byte-identical to the reference because the DEFLATE implementations differ, so +// byte identity only holds for histories small enough that no column crosses the +// threshold; compression is a size optimization, and every column round-trips +// because the decoder inflates any column whose specification carries the +// compressed bit. +func EncodeDocument(document *Document, order []OpID, compress bool) ([]byte, error) { + changes, err := documentChangeOrder(document) + if err != nil { + return nil, err + } + + operations, err := documentOperations(changes, order) + if err != nil { + return nil, err + } + + actors := documentActorTable(changes, operations) + actorIndexes := make(map[ActorID]uint64, len(actors)) + + for i, actor := range actors { + actorIndexes[actor] = uint64(i) + } + + heads, headIndexes, err := documentHeads(changes) + if err != nil { + return nil, err + } + + changeColumns, err := encodeDocumentChangeColumns(changes, actorIndexes) + if err != nil { + return nil, err + } + + operationColumns, err := encodeDocumentOperationColumns(operations, actorIndexes) + if err != nil { + return nil, err + } + + changeColumns = compressColumns(sortColumns( + append(changeColumns, retainedColumns(document, changeColumnSpecifications)...), + ), compress) + operationColumns = compressColumns(sortColumns( + append(operationColumns, retainedColumns(document, operationColumnSpecifications)...), + ), compress) + + var body []byte + + body = appendULEB(body, uint64(len(actors))) + for _, actor := range actors { + body = appendLengthPrefixedNative(body, actor.Bytes()) + } + + body = appendULEB(body, uint64(len(heads))) + for _, head := range heads { + body = append(body, head[:]...) + } + + // Both column sets are described before either is written, so the metadata + // has to be laid out ahead of the data it measures. + body = appendColumnMetadata(body, changeColumns) + body = appendColumnMetadata(body, operationColumns) + body = appendColumnData(body, changeColumns) + body = appendColumnData(body, operationColumns) + + for _, index := range headIndexes { + body = appendULEB(body, index) + } + + return assembleChunk(ChunkDocument, body), nil +} + +// documentChangeOrder returns the changes in dependency order. A snapshot may +// legally store them in any order, but writing ancestors first keeps the index +// references pointing backwards, which is what every other implementation emits +// and what makes the result readable in one pass. +func documentChangeOrder(document *Document) ([]*Change, error) { + byHash := make(map[ChangeHash]*Change, len(document.Changes)) + + for i := range document.Changes { + change := &document.Changes[i] + if change.Hash == nil { + return nil, fmt.Errorf("change %d cannot be written without a hash", i) + } + + byHash[*change.Hash] = change + } + + ordered := make([]*Change, 0, len(document.Changes)) + placed := make(map[ChangeHash]struct{}, len(document.Changes)) + + var place func(*Change) error + + place = func(change *Change) error { + if _, ok := placed[*change.Hash]; ok { + return nil + } + + // Claim the change before descending so a cycle is reported rather than + // followed forever. + placed[*change.Hash] = struct{}{} + + for _, dependency := range change.Dependencies { + parent, ok := byHash[dependency] + if !ok { + return fmt.Errorf( + "change %s depends on %s which the history does not hold", + change.Hash, + dependency, + ) + } + + if err := place(parent); err != nil { + return err + } + } + + ordered = append(ordered, change) + + return nil + } + + for i := range document.Changes { + if err := place(&document.Changes[i]); err != nil { + return nil, err + } + } + + return ordered, nil +} + +// documentOperations collects the operations to store, in the given order, with +// each one's successors derived from the predecessors recorded across the whole +// history. Deletes are dropped: they exist in the result only as the successor +// entries they contribute. +func documentOperations(changes []*Change, order []OpID) ([]Operation, error) { + sources := make(map[OpID]*Operation) + + for _, change := range changes { + for i := range change.Operations { + operation := &change.Operations[i] + if _, ok := sources[operation.ID]; ok { + return nil, fmt.Errorf("operation %s@%d occurs twice", + operation.ID.Actor, operation.ID.Counter) + } + + sources[operation.ID] = operation + } + } + + successors := make(map[OpID][]OpID) + + for _, change := range changes { + for _, operation := range change.Operations { + for _, predecessor := range operation.Predecessors { + successors[predecessor] = append(successors[predecessor], operation.ID) + } + } + } + + for identifier := range successors { + slices.SortFunc(successors[identifier], func(left, right OpID) int { + return left.Compare(right) + }) + } + + operations := make([]Operation, 0, len(order)) + + for _, identifier := range order { + source, ok := sources[identifier] + if !ok { + return nil, fmt.Errorf( + "operation %s@%d is ordered but absent from the history", + identifier.Actor, + identifier.Counter, + ) + } + + if source.Action == ActionDelete { + return nil, fmt.Errorf( + "operation %s@%d is a delete and cannot be stored", + identifier.Actor, + identifier.Counter, + ) + } + + stored := *source + stored.Predecessors = nil + stored.Successors = successors[identifier] + + operations = append(operations, stored) + } + + return operations, nil +} + +func documentActorTable(changes []*Change, operations []Operation) []ActorID { + seen := make(map[ActorID]struct{}) + + add := func(actor ActorID) { + if actor != "" { + seen[actor] = struct{}{} + } + } + + for _, change := range changes { + add(change.Actor) + } + + for _, operation := range operations { + add(operation.ID.Actor) + + if !operation.Object.IsRoot { + add(operation.Object.OpID.Actor) + } + + if operation.Key.Element != nil { + add(operation.Key.Element.Actor) + } + + for _, successor := range operation.Successors { + add(successor.Actor) + } + } + + actors := make([]ActorID, 0, len(seen)) + for actor := range seen { + actors = append(actors, actor) + } + + slices.SortFunc(actors, func(left, right ActorID) int { + return left.Compare(right) + }) + + return actors +} + +// documentHeads returns the frontier and the index of each head, which is how a +// snapshot names its heads. +func documentHeads(changes []*Change) ([]ChangeHash, []uint64, error) { + indexes := make(map[ChangeHash]uint64, len(changes)) + dependedOn := make(map[ChangeHash]struct{}, len(changes)) + + for i, change := range changes { + indexes[*change.Hash] = uint64(i) + + for _, dependency := range change.Dependencies { + dependedOn[dependency] = struct{}{} + } + } + + heads := make([]ChangeHash, 0) + + for _, change := range changes { + if _, ok := dependedOn[*change.Hash]; !ok { + heads = append(heads, *change.Hash) + } + } + + slices.SortFunc(heads, func(left, right ChangeHash) int { + return bytes.Compare(left[:], right[:]) + }) + + headIndexes := make([]uint64, len(heads)) + for i, head := range heads { + headIndexes[i] = indexes[head] + } + + return heads, headIndexes, nil +} + +func encodeDocumentChangeColumns( + changes []*Change, + actorIndexes map[ActorID]uint64, +) ([]encodedColumn, error) { + count := len(changes) + + indexes := make(map[ChangeHash]uint64, count) + for i, change := range changes { + indexes[*change.Hash] = uint64(i) + } + + var ( + actors = make([]optional[uint64], count) + sequences = make([]optional[int64], count) + maxOps = make([]optional[int64], count) + times = make([]optional[int64], count) + messages = make([]optional[string], count) + dependencySize = make([]optional[uint64], count) + dependencies []optional[int64] + extraMetadata = make([]optional[uint64], count) + extraData []byte + ) + + for i, change := range changes { + actorIndex, ok := actorIndexes[change.Actor] + if !ok { + return nil, fmt.Errorf("change %d actor is not in the actor table", i) + } + + actors[i] = some(actorIndex) + sequences[i] = some(int64(change.Sequence)) + maxOps[i] = some(int64(change.MaxOp)) + times[i] = some(change.Time) + + if change.Message != "" { + messages[i] = some(change.Message) + } + + dependencySize[i] = some(uint64(len(change.Dependencies))) + + for _, dependency := range change.Dependencies { + index, ok := indexes[dependency] + if !ok { + return nil, fmt.Errorf("change %d depends on an absent change", i) + } + + dependencies = append(dependencies, some(int64(index))) + } + + metadata, data, err := encodeScalar(changeExtra(change)) + if err != nil { + return nil, fmt.Errorf("cannot encode change %d extra: %w", i, err) + } + + extraMetadata[i] = metadata + extraData = append(extraData, data...) + } + + columns := []encodedColumn{ + {specification: 1, data: encodeRLE(actors, appendULEB)}, + {specification: 3, data: encodeDelta(sequences)}, + {specification: 19, data: encodeDelta(maxOps)}, + {specification: 35, data: encodeDelta(times)}, + {specification: 53, data: encodeStrings(messages)}, + {specification: 64, data: encodeRLE(dependencySize, appendULEB)}, + {specification: 67, data: encodeDelta(dependencies)}, + {specification: 86, data: encodeRLE(extraMetadata, appendULEB)}, + {specification: 87, data: extraData}, + } + + return withData(columns), nil +} + +// changeExtra reports the change's extra payload as the scalar a snapshot +// stores. A change chunk keeps the payload as trailing bytes, so the two forms +// have to be reconciled in whichever direction carries the value. +func changeExtra(change *Change) *Scalar { + if len(change.ExtraBytes) > 0 { + return &Scalar{Type: ScalarBytes, Bytes: change.ExtraBytes} + } + + if change.Extra != nil { + return change.Extra + } + + // The payload is a byte string even when a change carries none, so an absent + // one is empty rather than null. + return &Scalar{Type: ScalarBytes} +} + +func encodeDocumentOperationColumns( + operations []Operation, + actorIndexes map[ActorID]uint64, +) ([]encodedColumn, error) { + count := len(operations) + + var ( + idActors = make([]optional[uint64], count) + idCounters = make([]optional[int64], count) + objectActors = make([]optional[uint64], count) + objectCounters = make([]optional[uint64], count) + keyActors = make([]optional[uint64], count) + keyCounters = make([]optional[int64], count) + keyStrings = make([]optional[string], count) + inserts = make([]bool, count) + actions = make([]optional[uint64], count) + valueMetadata = make([]optional[uint64], count) + valueData []byte + successorSize = make([]optional[uint64], count) + successorActors []optional[uint64] + successorCounter []optional[int64] + markExpands = make([]bool, count) + hasMarkExpand bool + markNames = make([]optional[string], count) + ) + + for i, operation := range operations { + index, ok := actorIndexes[operation.ID.Actor] + if !ok { + return nil, fmt.Errorf("operation %d actor is not in the actor table", i) + } + + idActors[i] = some(index) + idCounters[i] = some(int64(operation.ID.Counter)) + + if !operation.Object.IsRoot { + index, ok := actorIndexes[operation.Object.OpID.Actor] + if !ok { + return nil, fmt.Errorf("operation %d object actor is unknown", i) + } + + objectActors[i] = some(index) + objectCounters[i] = some(operation.Object.OpID.Counter) + } + + switch { + case operation.Key.Property != nil: + keyStrings[i] = some(*operation.Key.Property) + case operation.Key.IsHead: + keyCounters[i] = some(int64(0)) + case operation.Key.Element != nil: + index, ok := actorIndexes[operation.Key.Element.Actor] + if !ok { + return nil, fmt.Errorf("operation %d key actor is unknown", i) + } + + keyActors[i] = some(index) + keyCounters[i] = some(int64(operation.Key.Element.Counter)) + default: + return nil, fmt.Errorf("operation %d has no key", i) + } + + inserts[i] = operation.Insert + actions[i] = some(uint64(operation.Action)) + + metadata, data, err := encodeScalar(operation.Value) + if err != nil { + return nil, fmt.Errorf("cannot encode operation %d value: %w", i, err) + } + + valueMetadata[i] = metadata + valueData = append(valueData, data...) + + successorSize[i] = some(uint64(len(operation.Successors))) + + for _, successor := range operation.Successors { + index, ok := actorIndexes[successor.Actor] + if !ok { + return nil, fmt.Errorf("operation %d successor actor is unknown", i) + } + + successorActors = append(successorActors, some(index)) + successorCounter = append(successorCounter, some(int64(successor.Counter))) + } + + // Expand only means anything on a mark, and an all-false column is left + // out, so the flag is written only where it is actually set. + if operation.MarkExpand != nil && *operation.MarkExpand { + markExpands[i] = true + hasMarkExpand = true + } + + if operation.MarkName != nil { + markNames[i] = some(*operation.MarkName) + } + } + + var markExpandData []byte + if hasMarkExpand { + markExpandData = encodeBooleans(markExpands) + } + + columns := []encodedColumn{ + {specification: 1, data: encodeRLE(objectActors, appendULEB)}, + {specification: 2, data: encodeRLE(objectCounters, appendULEB)}, + {specification: 17, data: encodeRLE(keyActors, appendULEB)}, + {specification: 19, data: encodeDelta(keyCounters)}, + {specification: 21, data: encodeStrings(keyStrings)}, + {specification: 33, data: encodeRLE(idActors, appendULEB)}, + {specification: 35, data: encodeDelta(idCounters)}, + {specification: 52, data: encodeBooleans(inserts)}, + {specification: 66, data: encodeRLE(actions, appendULEB)}, + {specification: 86, data: encodeRLE(valueMetadata, appendULEB)}, + {specification: 87, data: valueData}, + {specification: 128, data: encodeRLE(successorSize, appendULEB)}, + {specification: 129, data: encodeRLE(successorActors, appendULEB)}, + {specification: 131, data: encodeDelta(successorCounter)}, + {specification: 148, data: markExpandData}, + {specification: 165, data: encodeStrings(markNames)}, + } + + return withData(columns), nil +} + +var ( + changeColumnSpecifications = []uint32{1, 3, 19, 35, 53, 64, 67, 86, 87} + operationColumnSpecifications = []uint32{ + 1, 2, 17, 19, 21, 33, 35, 52, 66, 86, 87, 128, 129, 131, 148, 165, + } +) + +// retainedColumns returns the columns a previous reader did not understand but +// kept, so writing a history back does not quietly drop what a newer version of +// the format put there. Each retained column is matched to the table it came +// from by its specification. +func retainedColumns(document *Document, known []uint32) []encodedColumn { + retained := make([]encodedColumn, 0) + + for _, column := range document.UnknownColumns { + normalized := column.Specification &^ 8 + if slices.Contains(known, normalized) || len(column.Data) == 0 { + continue + } + + retained = append(retained, encodedColumn{ + specification: normalized, + data: append([]byte(nil), column.Data...), + }) + } + + return retained +} + +// sortColumns puts columns in the strictly ascending order a reader requires, +// which both the metadata and the data must follow. Ordering is by the +// normalized specification so a column keeps its place whether or not it carries +// the compressed bit. +func sortColumns(columns []encodedColumn) []encodedColumn { + slices.SortFunc(columns, func(left, right encodedColumn) int { + leftSpec := left.specification &^ compressedColumnBit + rightSpec := right.specification &^ compressedColumnBit + + switch { + case leftSpec < rightSpec: + return -1 + case leftSpec > rightSpec: + return 1 + default: + return 0 + } + }) + + return columns +} + +// compressedColumnBit marks a column whose data is DEFLATE-compressed. A reader +// strips it to recover the logical specification and inflates the data. +const compressedColumnBit = 8 + +// columnDeflateMinSize is the smallest column worth compressing, matching the +// change-chunk threshold. Below it, compression tends to grow the data. +const columnDeflateMinSize = 250 + +// compressColumns DEFLATEs each column whose data is large enough to benefit, +// marking it with the compressed bit. A column that is already compressed (a +// retained unknown column) or that does not shrink is left untouched, so the +// result is never larger than the input. +func compressColumns(columns []encodedColumn, compress bool) []encodedColumn { + if !compress { + return columns + } + + for i := range columns { + column := &columns[i] + + if column.specification&compressedColumnBit != 0 || + len(column.data) < columnDeflateMinSize { + continue + } + + deflated, err := deflate(column.data) + if err != nil || len(deflated) >= len(column.data) { + continue + } + + column.specification |= compressedColumnBit + column.data = deflated + } + + return columns +} + +func withData(columns []encodedColumn) []encodedColumn { + filtered := make([]encodedColumn, 0, len(columns)) + + for _, column := range columns { + if len(column.data) > 0 { + filtered = append(filtered, column) + } + } + + return filtered +} + +func appendColumnMetadata(data []byte, columns []encodedColumn) []byte { + data = appendULEB(data, uint64(len(columns))) + for _, column := range columns { + data = appendULEB(data, uint64(column.specification)) + data = appendULEB(data, uint64(len(column.data))) + } + + return data +} + +func appendColumnData(data []byte, columns []encodedColumn) []byte { + for _, column := range columns { + data = append(data, column.data...) + } + + return data +} diff --git a/pkg/automerge/internal/storage/encode_document_reference_test.go b/pkg/automerge/internal/storage/encode_document_reference_test.go new file mode 100644 index 0000000000..f38fd9a679 --- /dev/null +++ b/pkg/automerge/internal/storage/encode_document_reference_test.go @@ -0,0 +1,246 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package storage + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/automerge/internal/reference" +) + +func boolScalar() []byte { return []byte(`{"type":"boolean","bool":true}`) } +func nullScalar() []byte { return []byte(`{"type":"null"}`) } +func counterScalar() []byte { return []byte(`{"type":"counter","int":5}`) } + +func stringScalar(value string) []byte { + return []byte(`{"type":"string","string":"` + value + `"}`) +} + +func newReference(t *testing.T, ctx context.Context, actor byte) *reference.Engine { + t.Helper() + + engine, err := reference.New(ctx) + require.NoError(t, err) + require.NoError(t, engine.SetActor(ctx, []byte{ + actor, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + })) + + return engine +} + +// assertSnapshotReencodes decodes a reference-written snapshot and requires the +// encoder to reproduce it byte for byte. +func assertSnapshotReencodes(t *testing.T, saved []byte) { + t.Helper() + + document, err := Decode(saved) + require.NoError(t, err) + + encoded, err := EncodeDocument(document, storedOperationOrder(t, saved), true) + require.NoError(t, err) + + assert.Equal(t, saved, encoded) +} + +// TestEncodeDocument_MatchesReferenceSnapshots pins snapshot writing against the +// reference implementation across the shapes whose storage form differs most +// from a change: deletes that survive only as successors, marks whose expand +// column is shared, counters, nested objects and several actors. +func TestEncodeDocument_MatchesReferenceSnapshots(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + for _, testCase := range []struct { + name string + build func(t *testing.T, engine *reference.Engine) []byte + }{ + { + name: "linear text history", + build: func(t *testing.T, engine *reference.Engine) []byte { + handle, err := engine.PutText(ctx, 0, "body") + require.NoError(t, err) + + for i := range 5 { + require.NoError(t, engine.SpliceText(ctx, handle, uint32(i), 0, "x")) + _, err = engine.Commit(ctx, "edit", time.Unix(int64(i+1), 0)) + require.NoError(t, err) + } + + saved, err := engine.Save(ctx, true, true) + require.NoError(t, err) + + return saved + }, + }, + { + name: "marks and unmarks", + build: func(t *testing.T, engine *reference.Engine) []byte { + handle, err := engine.PutText(ctx, 0, "body") + require.NoError(t, err) + require.NoError(t, engine.SpliceText(ctx, handle, 0, 0, "hello brave world")) + _, err = engine.Commit(ctx, "write", time.Unix(1, 0)) + require.NoError(t, err) + + require.NoError(t, engine.MarkText(ctx, handle, 0, 5, "strong", boolScalar(), "both")) + _, err = engine.Commit(ctx, "mark", time.Unix(2, 0)) + require.NoError(t, err) + + require.NoError(t, engine.MarkText(ctx, handle, 1, 3, "strong", nullScalar(), "none")) + _, err = engine.Commit(ctx, "unmark", time.Unix(3, 0)) + require.NoError(t, err) + + saved, err := engine.Save(ctx, true, true) + require.NoError(t, err) + + return saved + }, + }, + { + name: "deletes and overwrites", + build: func(t *testing.T, engine *reference.Engine) []byte { + require.NoError(t, engine.PutString(ctx, 0, "title", "first")) + require.NoError(t, engine.PutString(ctx, 0, "keep", "value")) + _, err := engine.Commit(ctx, "one", time.Unix(1, 0)) + require.NoError(t, err) + + require.NoError(t, engine.PutString(ctx, 0, "title", "second")) + _, err = engine.Commit(ctx, "two", time.Unix(2, 0)) + require.NoError(t, err) + + require.NoError(t, engine.DeleteMap(ctx, 0, "title")) + _, err = engine.Commit(ctx, "three", time.Unix(3, 0)) + require.NoError(t, err) + + saved, err := engine.Save(ctx, true, true) + require.NoError(t, err) + + return saved + }, + }, + { + name: "list with deletion and counter", + build: func(t *testing.T, engine *reference.Engine) []byte { + list, err := engine.PutObject(ctx, 0, "items", "list") + require.NoError(t, err) + require.NoError(t, engine.InsertScalar(ctx, list, 0, stringScalar("a"))) + require.NoError(t, engine.InsertScalar(ctx, list, 1, stringScalar("b"))) + require.NoError(t, engine.InsertScalar(ctx, list, 2, stringScalar("c"))) + require.NoError(t, engine.PutScalar(ctx, 0, "counter", counterScalar())) + _, err = engine.Commit(ctx, "build", time.Unix(1, 0)) + require.NoError(t, err) + + require.NoError(t, engine.DeleteSequence(ctx, list, 1)) + require.NoError(t, engine.Increment(ctx, 0, "counter", 3)) + _, err = engine.Commit(ctx, "trim", time.Unix(2, 0)) + require.NoError(t, err) + + saved, err := engine.Save(ctx, true, true) + require.NoError(t, err) + + return saved + }, + }, + { + name: "nested objects", + build: func(t *testing.T, engine *reference.Engine) []byte { + outer, err := engine.PutObject(ctx, 0, "outer", "map") + require.NoError(t, err) + + inner, err := engine.PutObject(ctx, outer, "inner", "list") + require.NoError(t, err) + require.NoError(t, engine.InsertScalar(ctx, inner, 0, stringScalar("deep"))) + + text, err := engine.PutText(ctx, outer, "note") + require.NoError(t, err) + require.NoError(t, engine.SpliceText(ctx, text, 0, 0, "nested")) + + _, err = engine.Commit(ctx, "nest", time.Unix(1, 0)) + require.NoError(t, err) + + saved, err := engine.Save(ctx, true, true) + require.NoError(t, err) + + return saved + }, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + assertSnapshotReencodes(t, testCase.build(t, newReference(t, ctx, 0x20))) + }) + } +} + +// TestEncodeDocument_MatchesReferenceConcurrentSnapshot covers a merged history, +// where several actors share the actor table and a change has more than one +// dependency. +func TestEncodeDocument_MatchesReferenceConcurrentSnapshot(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + first := newReference(t, ctx, 0x20) + + require.NoError(t, first.PutString(ctx, 0, "title", "one")) + + body, err := first.PutText(ctx, 0, "body") + require.NoError(t, err) + require.NoError(t, first.SpliceText(ctx, body, 0, 0, "hello")) + _, err = first.Commit(ctx, "first", time.Unix(1, 0)) + require.NoError(t, err) + + shared, err := first.Save(ctx, true, true) + require.NoError(t, err) + + second, err := reference.Load(ctx, shared) + require.NoError(t, err) + require.NoError(t, second.SetActor(ctx, []byte{ + 0x10, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + })) + + secondBody, _, err := second.GetObject(ctx, 0, "body") + require.NoError(t, err) + require.NoError(t, second.SpliceText(ctx, secondBody, 5, 0, " there")) + require.NoError(t, second.PutString(ctx, 0, "title", "two")) + _, err = second.Commit(ctx, "second", time.Unix(2, 0)) + require.NoError(t, err) + + secondSave, err := second.Save(ctx, true, true) + require.NoError(t, err) + + _, err = first.Merge(ctx, secondSave) + require.NoError(t, err) + + require.NoError(t, first.SpliceText(ctx, body, 0, 1, "")) + _, err = first.Commit(ctx, "third", time.Unix(3, 0)) + require.NoError(t, err) + + saved, err := first.Save(ctx, true, true) + require.NoError(t, err) + + assertSnapshotReencodes(t, saved) +} diff --git a/pkg/automerge/internal/storage/encode_document_test.go b/pkg/automerge/internal/storage/encode_document_test.go new file mode 100644 index 0000000000..97d08fecb7 --- /dev/null +++ b/pkg/automerge/internal/storage/encode_document_test.go @@ -0,0 +1,119 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package storage + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// storedOperationOrder reports the operation-set order a document chunk was +// written in, which is the order the encoder has to be given to reproduce it. +func storedOperationOrder(t *testing.T, data []byte) []OpID { + t.Helper() + + r := &reader{data: data} + + chunk, err := decodeChunk(r) + require.NoError(t, err) + require.Equal(t, ChunkDocument, chunk.kind) + + content := &reader{data: chunk.content} + + actors, err := decodeActorArray(content, true) + require.NoError(t, err) + + _, err = decodeHashArray(content, true) + require.NoError(t, err) + + changeMetadata, err := parseColumnMetadata(content, true) + require.NoError(t, err) + + operationMetadata, err := parseColumnMetadata(content, true) + require.NoError(t, err) + + _, err = readColumns(content, changeMetadata) + require.NoError(t, err) + + operationColumns, err := readColumns(content, operationMetadata) + require.NoError(t, err) + + operations, _, err := decodeOperations(operationColumns, actors, false, nil) + require.NoError(t, err) + + order := make([]OpID, len(operations)) + for i, operation := range operations { + order[i] = operation.ID + } + + return order +} + +// TestEncodeDocument_ReproducesOfficialSnapshotBytes is the byte-identity gate +// for snapshot writing. Re-encoding a snapshot the reference implementation +// wrote has to produce that same file, which pins the column layout, the actor +// and head tables, the extra payload and the chunk framing all at once. +func TestEncodeDocument_ReproducesOfficialSnapshotBytes(t *testing.T) { + t.Parallel() + + data := fixture(t, officialDocumentFixture) + + document, err := Decode(data) + require.NoError(t, err) + + encoded, err := EncodeDocument(document, storedOperationOrder(t, data), true) + require.NoError(t, err) + + assert.Equal(t, data, encoded, "re-encoded snapshot must match the original bytes") +} + +// TestEncodeDocument_RoundTripsThroughDecode checks the written snapshot reads +// back as the same history even where byte identity is not the subject. +func TestEncodeDocument_RoundTripsThroughDecode(t *testing.T) { + t.Parallel() + + data := fixture(t, officialDocumentFixture) + + document, err := Decode(data) + require.NoError(t, err) + + encoded, err := EncodeDocument(document, storedOperationOrder(t, data), true) + require.NoError(t, err) + + reloaded, err := Decode(encoded) + require.NoError(t, err) + + require.Len(t, reloaded.Changes, len(document.Changes)) + assert.Equal(t, document.Heads, reloaded.Heads) + assert.Equal(t, document.Actors, reloaded.Actors) + + for i := range document.Changes { + expected := &document.Changes[i] + actual := &reloaded.Changes[i] + + assert.Equal(t, expected.Hash, actual.Hash, "change %d hash", i) + assert.Equal(t, expected.Raw, actual.Raw, "change %d bytes", i) + assert.Equal(t, expected.Message, actual.Message, "change %d message", i) + assert.Equal(t, expected.Time, actual.Time, "change %d time", i) + } +} diff --git a/pkg/automerge/internal/storage/model.go b/pkg/automerge/internal/storage/model.go index d20f2f40aa..8b69df154a 100644 --- a/pkg/automerge/internal/storage/model.go +++ b/pkg/automerge/internal/storage/model.go @@ -22,51 +22,51 @@ // graph validation independently from the native execution engine. package storage -import "go.probo.inc/probo/pkg/automerge/internal/types" +import "go.probo.inc/probo/pkg/automerge/internal/opset" type ( - ActorID = types.ActorID - ChangeHash = types.ChangeHash - OpID = types.OpID - ObjectID = types.ObjectID - Key = types.Key - Action = types.Action - ScalarType = types.ScalarType - Scalar = types.Scalar - Operation = types.Operation - Change = types.Change - ChunkType = types.ChunkType - RawColumn = types.RawColumn - Document = types.Document + ActorID = opset.ActorID + ChangeHash = opset.ChangeHash + OpID = opset.OpID + ObjectID = opset.ObjectID + Key = opset.Key + Action = opset.Action + ScalarType = opset.ScalarType + Scalar = opset.Scalar + Operation = opset.Operation + Change = opset.Change + ChunkType = opset.ChunkType + RawColumn = opset.RawColumn + Document = opset.Document ) const ( - ActionMakeMap = types.ActionMakeMap - ActionSet = types.ActionSet - ActionMakeList = types.ActionMakeList - ActionDelete = types.ActionDelete - ActionMakeText = types.ActionMakeText - ActionIncrement = types.ActionIncrement - ActionMakeTable = types.ActionMakeTable - ActionMark = types.ActionMark + ActionMakeMap = opset.ActionMakeMap + ActionSet = opset.ActionSet + ActionMakeList = opset.ActionMakeList + ActionDelete = opset.ActionDelete + ActionMakeText = opset.ActionMakeText + ActionIncrement = opset.ActionIncrement + ActionMakeTable = opset.ActionMakeTable + ActionMark = opset.ActionMark - ScalarNull = types.ScalarNull - ScalarFalse = types.ScalarFalse - ScalarTrue = types.ScalarTrue - ScalarUint = types.ScalarUint - ScalarInt = types.ScalarInt - ScalarFloat64 = types.ScalarFloat64 - ScalarString = types.ScalarString - ScalarBytes = types.ScalarBytes - ScalarCounter = types.ScalarCounter - ScalarTimestamp = types.ScalarTimestamp + ScalarNull = opset.ScalarNull + ScalarFalse = opset.ScalarFalse + ScalarTrue = opset.ScalarTrue + ScalarUint = opset.ScalarUint + ScalarInt = opset.ScalarInt + ScalarFloat64 = opset.ScalarFloat64 + ScalarString = opset.ScalarString + ScalarBytes = opset.ScalarBytes + ScalarCounter = opset.ScalarCounter + ScalarTimestamp = opset.ScalarTimestamp - ChunkDocument = types.ChunkDocument - ChunkChange = types.ChunkChange - ChunkCompressedChange = types.ChunkCompressedChange + ChunkDocument = opset.ChunkDocument + ChunkChange = opset.ChunkChange + ChunkCompressedChange = opset.ChunkCompressedChange ) var ( - NewActorID = types.NewActorID - RootObject = types.RootObject + NewActorID = opset.NewActorID + RootObject = opset.RootObject ) diff --git a/pkg/automerge/internal/storage/validate.go b/pkg/automerge/internal/storage/validate.go index fc9849b74d..6fc85bfddf 100644 --- a/pkg/automerge/internal/storage/validate.go +++ b/pkg/automerge/internal/storage/validate.go @@ -490,6 +490,15 @@ func assignOperations(changes []Change, operations []Operation) error { byActor[actor] = indexes } + // Each change holds exactly the operations whose counter falls in its + // [StartOp, MaxOp] range, so its operation slice can be sized once here. + // Growing it by append instead reallocated repeatedly and dominated the cost + // of loading a large single-change document. + for i := range changes { + size := changes[i].MaxOp - changes[i].StartOp + 1 + changes[i].Operations = make([]Operation, 0, size) + } + for _, operation := range operations { indexes := byActor[operation.ID.Actor] @@ -534,6 +543,24 @@ func assignOperations(changes []Change, operations []Operation) error { }) } + // The per-actor bounds above are a permissive lower bound that only has to + // locate operations. A change's operations carry consecutive counters ending + // at maxOp, so the real start operation follows from the operation count, and + // re-encoding the change depends on it being exact. + for i := range changes { + count := uint64(len(changes[i].Operations)) + if count > changes[i].MaxOp { + return fmt.Errorf( + "change %d holds %d operations but ends at operation %d", + i, + count, + changes[i].MaxOp, + ) + } + + changes[i].StartOp = changes[i].MaxOp - count + 1 + } + return nil } @@ -739,21 +766,15 @@ func validateChangeChunksAfterSnapshot(document *Document) error { known := make(map[ChangeHash]struct{}) dependedOn := make(map[ChangeHash]struct{}) + // A change may legitimately appear both inside the snapshot and as a trailing + // change chunk, so repeats identify the same change rather than a conflict. for _, change := range document.Changes { if change.Hash != nil { - if _, exists := known[*change.Hash]; exists { - return fmt.Errorf("duplicate known change hash %s", change.Hash) - } - known[*change.Hash] = struct{}{} } } for _, change := range document.Changes { - if len(change.DependencyIndexes) > 0 { - continue - } - for _, dependency := range change.Dependencies { if _, ok := known[dependency]; !ok { return fmt.Errorf("change %s has missing dependency %s", change.Hash, dependency) diff --git a/pkg/automerge/invariants_test.go b/pkg/automerge/invariants_test.go index 9a1f6e6110..27a0730443 100644 --- a/pkg/automerge/invariants_test.go +++ b/pkg/automerge/invariants_test.go @@ -71,23 +71,23 @@ func TestDocument_AppliesDependentChangesInAnyOrder(t *testing.T) { target, err := automerge.Load(ctx, baseData, actor(102)) require.NoError(t, err) closeDocument(t, target) - require.NoError(t, target.ApplyChanges(ctx, [][]byte{changes[1].Bytes})) + require.NoError(t, target.ApplyChanges(ctx, []automerge.Change{changes[1]})) missing, err := target.MissingDependencies( ctx, []automerge.Hash{childHash}, ) require.NoError(t, err) assert.Equal(t, []automerge.Hash{parentHash}, missing) - require.NoError(t, target.ApplyChanges(ctx, [][]byte{changes[0].Bytes})) + require.NoError(t, target.ApplyChanges(ctx, []automerge.Change{changes[0]})) missing, err = target.MissingDependencies( ctx, []automerge.Hash{childHash}, ) require.NoError(t, err) assert.Empty(t, missing) - require.NoError(t, target.ApplyChanges(ctx, [][]byte{ - changes[0].Bytes, - changes[1].Bytes, + require.NoError(t, target.ApplyChanges(ctx, []automerge.Change{ + changes[0], + changes[1], })) targetText, err := target.Text(ctx, "body") @@ -118,8 +118,8 @@ func TestDocument_InvalidChangesDoNotMutateState(t *testing.T) { headsBefore, err := document.Heads(ctx) require.NoError(t, err) - err = document.ApplyChanges(ctx, [][]byte{ - []byte("invalid"), + err = document.ApplyChanges(ctx, []automerge.Change{ + {Bytes: []byte("invalid")}, }) require.Error(t, err) diff --git a/pkg/automerge/js_text_parity_test.go b/pkg/automerge/js_text_parity_test.go index 3515ce3896..238399800c 100644 --- a/pkg/automerge/js_text_parity_test.go +++ b/pkg/automerge/js_text_parity_test.go @@ -197,7 +197,7 @@ func TestJSText_SplicingIntoArrays(t *testing.T) { require.NoError(t, err) textObject, err := inner.ObjectAt(ctx, 0) require.NoError(t, err) - text, err := textObject.Text() + text, err := textObject.Text(ctx) require.NoError(t, err) require.NoError(t, text.Splice(ctx, 0, 0, "Hello ")) diff --git a/pkg/automerge/marks_dangling_parity_test.go b/pkg/automerge/marks_dangling_parity_test.go index fd3f9a6d4e..2a91a132ce 100644 --- a/pkg/automerge/marks_dangling_parity_test.go +++ b/pkg/automerge/marks_dangling_parity_test.go @@ -23,6 +23,7 @@ package automerge_test import ( "context" "fmt" + "math/rand" "sort" "strings" "testing" @@ -63,10 +64,13 @@ func (s markScenarioStep) String() string { // before. Native now starts a leftward-expanding dangling begin at the position // just after its own anchor, matching the reference. // -// Every case here is minimized by delta debugging from randomized differential -// runs. Deeper dangling-begin interactions (multiple overlapping marks whose -// anchors have since been deleted) still diverge on a small fraction of -// randomized error-path scenarios and are tracked in PARITY_PLAN.md. +// The block-boundary and multi-dangling cases below were minimized by delta +// debugging from randomized differential runs. They all shared one cause: a +// split block did not resolve its insertion anchor against neighbouring mark +// boundaries the way a text insertion does, so a block, and every insertion +// anchored after it, landed on the wrong side of a dangling begin. SplitBlock +// now resolves its anchor identically to Splice, and a value-level randomized +// sweep including out-of-range marks and every expand mode no longer diverges. func TestRustText_DanglingMarkBoundaries(t *testing.T) { t.Parallel() @@ -112,6 +116,71 @@ func TestRustText_DanglingMarkBoundaries(t *testing.T) { {kind: "insert", index: 4, value: "J"}, }, }, + { + name: "two dangling begins both survive", + steps: []markScenarioStep{ + {kind: "mark", index: 0, end: 3, name: "italic", expand: automerge.MarkExpandBoth}, + {kind: "mark", index: 0, end: 3, name: "bold", expand: automerge.MarkExpandBoth}, + {kind: "split", index: 0}, + {kind: "insert", index: 0, value: "u"}, + }, + }, + { + name: "overlapping begins with mixed expand both survive", + steps: []markScenarioStep{ + {kind: "mark", index: 0, end: 1, name: "italic", expand: automerge.MarkExpandBoth}, + {kind: "mark", index: 0, end: 1, name: "underline", expand: automerge.MarkExpandBefore}, + {kind: "split", index: 0}, + {kind: "insert", index: 0, value: "vJ"}, + }, + }, + { + name: "dangling begin after a full delete and resplit", + steps: []markScenarioStep{ + {kind: "split", index: 0}, + {kind: "mark", index: 1, end: 4, name: "bold", expand: automerge.MarkExpandBoth}, + {kind: "delete", index: 0, count: 4}, + {kind: "split", index: 0}, + {kind: "insert", index: 0, value: "rG"}, + }, + }, + { + name: "none expand mark does not leak past a block", + steps: []markScenarioStep{ + {kind: "split", index: 0}, + {kind: "mark", index: 0, end: 1, name: "bold", expand: automerge.MarkExpandNone}, + {kind: "split", index: 1}, + {kind: "insert", index: 1, value: "Plk"}, + }, + }, + { + name: "before expand mark does not leak across a block", + steps: []markScenarioStep{ + {kind: "split", index: 0}, + {kind: "mark", index: 0, end: 1, name: "bold", expand: automerge.MarkExpandBefore}, + {kind: "split", index: 1}, + {kind: "insert", index: 1, value: "lMf"}, + }, + }, + { + name: "before expand mark bounded by two blocks", + steps: []markScenarioStep{ + {kind: "split", index: 0}, + {kind: "mark", index: 0, end: 1, name: "bold", expand: automerge.MarkExpandBefore}, + {kind: "split", index: 1}, + {kind: "split", index: 1}, + {kind: "insert", index: 2, value: "ed"}, + }, + }, + { + name: "before expand mark across a block at head", + steps: []markScenarioStep{ + {kind: "split", index: 0}, + {kind: "mark", index: 0, end: 1, name: "italic", expand: automerge.MarkExpandBefore}, + {kind: "split", index: 0}, + {kind: "insert", index: 0, value: "C"}, + }, + }, } for _, tt := range tests { @@ -127,6 +196,90 @@ func TestRustText_DanglingMarkBoundaries(t *testing.T) { } } +// TestRustText_MarkValuesMatchReferenceUnderErrors is the standing gate for the +// dangling-mark behavior: many randomized scenarios, deliberately including marks +// whose end boundary runs past the text and every expand mode, must produce the +// exact same marked spans on the native and reference engines. This is stronger +// than the consolidation-and-text invariants of marks_are_okay because it +// compares the mark values run for run, which is what caught the block-anchor +// divergence this gate now protects against. +func TestRustText_MarkValuesMatchReferenceUnderErrors(t *testing.T) { + t.Parallel() + + ctx := context.Background() + random := rand.New(rand.NewSource(0x1e3779b97f4a7c15)) + + const scenarios = 2000 + + for scenario := range scenarios { + steps := randomDanglingMarkSteps(random) + + reference := runMarkScenario(t, ctx, rustParityEngines()[1], steps) + native := runMarkScenario(t, ctx, rustParityEngines()[0], steps) + + require.Equalf(t, reference, native, + "scenario %d diverged; steps: %s", scenario, renderMarkScenario(steps)) + } +} + +// randomDanglingMarkSteps builds a random editing scenario over text, list and +// mark operations. Marks may target an end boundary past the current length so +// the error path that leaves a dangling begin is exercised, and every expand +// mode appears. +func randomDanglingMarkSteps(random *rand.Rand) []markScenarioStep { + names := []string{"bold", "italic", "underline"} + expands := []automerge.MarkExpand{ + automerge.MarkExpandNone, + automerge.MarkExpandBefore, + automerge.MarkExpandAfter, + automerge.MarkExpandBoth, + } + + steps := 3 + random.Intn(10) + out := make([]markScenarioStep, 0, steps) + length := 0 + + for range steps { + switch random.Intn(4) { + case 0: + index := random.Intn(length + 1) + value := randomLetters(random, 1+random.Intn(3)) + out = append(out, markScenarioStep{kind: "insert", index: uint32(index), value: value}) + length += len(value) + case 1: + if length == 0 { + continue + } + + index := random.Intn(length) + count := 1 + random.Intn(length-index) + out = append(out, markScenarioStep{kind: "delete", index: uint32(index), count: int32(count)}) + length -= count + case 2: + index := random.Intn(length + 1) + out = append(out, markScenarioStep{kind: "split", index: uint32(index)}) + length++ + case 3: + if length == 0 { + continue + } + + index := random.Intn(length) + // end may exceed the length so the dangling-begin path is covered. + end := index + 1 + random.Intn(length+2-index) + out = append(out, markScenarioStep{ + kind: "mark", + index: uint32(index), + end: uint32(end), + name: names[random.Intn(len(names))], + expand: expands[random.Intn(len(expands))], + }) + } + } + + return out +} + func runMarkScenario( t *testing.T, ctx context.Context, diff --git a/pkg/automerge/native_backend_test.go b/pkg/automerge/native_backend_test.go index 07396356fc..088a02bc00 100644 --- a/pkg/automerge/native_backend_test.go +++ b/pkg/automerge/native_backend_test.go @@ -34,7 +34,7 @@ func TestPureGoDocument_ReferenceLoadsNativeHistory(t *testing.T) { t.Parallel() ctx := context.Background() - nativeDocument, err := automerge.NewPureGo(ctx, actor(40)) + nativeDocument, err := automerge.New(ctx, actor(40)) require.NoError(t, err) closeDocument(t, nativeDocument) text, err := nativeDocument.CreateText(ctx, "body") @@ -59,7 +59,7 @@ func TestPureGoDocument_ExtendsReferenceSnapshot(t *testing.T) { t.Parallel() ctx := context.Background() - nativeDocument, err := automerge.LoadPureGo(ctx, newBaseDocument(t), actor(42)) + nativeDocument, err := automerge.Load(ctx, newBaseDocument(t), actor(42)) require.NoError(t, err) closeDocument(t, nativeDocument) text, err := nativeDocument.Text(ctx, "body") @@ -84,7 +84,7 @@ func TestPureGoDocument_InsertsInsideReferenceText(t *testing.T) { t.Parallel() ctx := context.Background() - nativeDocument, err := automerge.LoadPureGo(ctx, newBaseDocument(t), actor(46)) + nativeDocument, err := automerge.Load(ctx, newBaseDocument(t), actor(46)) require.NoError(t, err) closeDocument(t, nativeDocument) text, err := nativeDocument.Text(ctx, "body") @@ -109,7 +109,7 @@ func TestPureGoDocument_DeletesReferenceText(t *testing.T) { t.Parallel() ctx := context.Background() - nativeDocument, err := automerge.LoadPureGo(ctx, newBaseDocument(t), actor(48)) + nativeDocument, err := automerge.Load(ctx, newBaseDocument(t), actor(48)) require.NoError(t, err) closeDocument(t, nativeDocument) text, err := nativeDocument.Text(ctx, "body") @@ -134,7 +134,7 @@ func TestPureGoDocument_EmptySnapshotLoadsInReference(t *testing.T) { t.Parallel() ctx := context.Background() - nativeDocument, err := automerge.NewPureGo(ctx, actor(44)) + nativeDocument, err := automerge.New(ctx, actor(44)) require.NoError(t, err) closeDocument(t, nativeDocument) data, err := nativeDocument.Save(ctx) @@ -153,7 +153,7 @@ func TestPureGoDocument_ReusesActorAfterLoad(t *testing.T) { ctx := context.Background() actorID := actor(55) - document, err := automerge.NewPureGo(ctx, actorID) + document, err := automerge.New(ctx, actorID) require.NoError(t, err) closeDocument(t, document) text, err := document.CreateText(ctx, "body") @@ -164,7 +164,7 @@ func TestPureGoDocument_ReusesActorAfterLoad(t *testing.T) { data, err := document.Save(ctx) require.NoError(t, err) - loaded, err := automerge.LoadPureGo(ctx, data, actorID) + loaded, err := automerge.Load(ctx, data, actorID) require.NoError(t, err) closeDocument(t, loaded) loadedText, err := loaded.Text(ctx, "body") @@ -189,7 +189,7 @@ func TestPureGoDocument_ConcurrentChangesConverge(t *testing.T) { t.Parallel() ctx := context.Background() - base, err := automerge.NewPureGo(ctx, actor(50)) + base, err := automerge.New(ctx, actor(50)) require.NoError(t, err) closeDocument(t, base) baseText, err := base.CreateText(ctx, "body") @@ -200,7 +200,7 @@ func TestPureGoDocument_ConcurrentChangesConverge(t *testing.T) { baseData, err := base.Save(ctx) require.NoError(t, err) - left, err := automerge.LoadPureGo(ctx, baseData, actor(51)) + left, err := automerge.Load(ctx, baseData, actor(51)) require.NoError(t, err) closeDocument(t, left) leftText, err := left.Text(ctx, "body") @@ -209,7 +209,7 @@ func TestPureGoDocument_ConcurrentChangesConverge(t *testing.T) { _, err = left.Commit(ctx, "Edit left", commitTime) require.NoError(t, err) - right, err := automerge.LoadPureGo(ctx, baseData, actor(52)) + right, err := automerge.Load(ctx, baseData, actor(52)) require.NoError(t, err) closeDocument(t, right) rightText, err := right.Text(ctx, "body") @@ -241,7 +241,7 @@ func TestPureGoDocument_CursorMatchesReference(t *testing.T) { ctx := context.Background() baseData := newBaseDocument(t) - nativeDocument, err := automerge.LoadPureGo(ctx, baseData, actor(60)) + nativeDocument, err := automerge.Load(ctx, baseData, actor(60)) require.NoError(t, err) closeDocument(t, nativeDocument) nativeText, err := nativeDocument.Text(ctx, "body") @@ -269,7 +269,7 @@ func TestPureGoDocument_DeletedCursorMatchesReference(t *testing.T) { ctx := context.Background() baseData := newBaseDocument(t) - nativeDocument, err := automerge.LoadPureGo(ctx, baseData, actor(62)) + nativeDocument, err := automerge.Load(ctx, baseData, actor(62)) require.NoError(t, err) closeDocument(t, nativeDocument) nativeText, err := nativeDocument.Text(ctx, "body") @@ -309,7 +309,7 @@ func TestPureGoDocument_UTF16CursorBoundariesMatchReference(t *testing.T) { baseData, err := base.Save(ctx) require.NoError(t, err) - nativeDocument, err := automerge.LoadPureGo(ctx, baseData, actor(123)) + nativeDocument, err := automerge.Load(ctx, baseData, actor(123)) require.NoError(t, err) closeDocument(t, nativeDocument) nativeText, err := nativeDocument.Text(ctx, "body") @@ -687,7 +687,7 @@ func TestPureGoDocument_SynchronizesWithNativePeer(t *testing.T) { t.Parallel() ctx := context.Background() - left, err := automerge.NewPureGo(ctx, actor(70)) + left, err := automerge.New(ctx, actor(70)) require.NoError(t, err) closeDocument(t, left) text, err := left.CreateText(ctx, "body") @@ -696,7 +696,7 @@ func TestPureGoDocument_SynchronizesWithNativePeer(t *testing.T) { _, err = left.Commit(ctx, "Create sync body", commitTime) require.NoError(t, err) - right, err := automerge.NewPureGo(ctx, actor(71)) + right, err := automerge.New(ctx, actor(71)) require.NoError(t, err) closeDocument(t, right) @@ -721,7 +721,7 @@ func TestPureGoDocument_SyncWaitsForPeerResponse(t *testing.T) { t.Parallel() ctx := context.Background() - document, err := automerge.NewPureGo(ctx, actor(78)) + document, err := automerge.New(ctx, actor(78)) require.NoError(t, err) closeDocument(t, document) text, err := document.CreateText(ctx, "body") @@ -748,7 +748,7 @@ func TestPureGoDocument_SynchronizesWithReferencePeer(t *testing.T) { t.Parallel() ctx := context.Background() - left, err := automerge.NewPureGo(ctx, actor(72)) + left, err := automerge.New(ctx, actor(72)) require.NoError(t, err) closeDocument(t, left) text, err := left.CreateText(ctx, "body") @@ -791,7 +791,7 @@ func TestPureGoDocument_ReceivesReferenceSync(t *testing.T) { _, err = left.Commit(ctx, "Create reference body", commitTime) require.NoError(t, err) - right, err := automerge.NewPureGo(ctx, actor(75)) + right, err := automerge.New(ctx, actor(75)) require.NoError(t, err) closeDocument(t, right) @@ -816,7 +816,7 @@ func TestPureGoDocument_RepeatedMixedPeerSync(t *testing.T) { t.Parallel() ctx := context.Background() - nativeDocument, err := automerge.NewPureGo(ctx, actor(76)) + nativeDocument, err := automerge.New(ctx, actor(76)) require.NoError(t, err) closeDocument(t, nativeDocument) nativeText, err := nativeDocument.CreateText(ctx, "body") @@ -896,7 +896,7 @@ func TestPureGoDocument_ReferenceEditsWhileMessageInFlight(t *testing.T) { _, err = client.Commit(ctx, "initial", commitTime) require.NoError(t, err) - server, err := automerge.NewPureGo(ctx, actor(80)) + server, err := automerge.New(ctx, actor(80)) require.NoError(t, err) closeDocument(t, server) diff --git a/pkg/automerge/native_differential_test.go b/pkg/automerge/native_differential_test.go index 240c27b273..3788810373 100644 --- a/pkg/automerge/native_differential_test.go +++ b/pkg/automerge/native_differential_test.go @@ -46,7 +46,7 @@ func TestPureGoDocument_RandomTextParity(t *testing.T) { for history := range histories { random := rand.New(rand.NewSource(int64(history + 1))) ctx := context.Background() - nativeDocument, err := automerge.NewPureGo(ctx, actor(byte(80+history))) + nativeDocument, err := automerge.New(ctx, actor(byte(80+history))) require.NoError(t, err) closeDocument(t, nativeDocument) nativeText, err := nativeDocument.CreateText(ctx, "body") @@ -127,7 +127,7 @@ func TestPureGoDocument_RandomConcurrentSyncParity(t *testing.T) { for history := range histories { random := rand.New(rand.NewSource(int64(10_000 + history))) - nativeDocument, err := automerge.NewPureGo( + nativeDocument, err := automerge.New( ctx, actor(byte(140+history)), ) diff --git a/pkg/automerge/object.go b/pkg/automerge/object.go index 50da14782c..0f6dddc4fa 100644 --- a/pkg/automerge/object.go +++ b/pkg/automerge/object.go @@ -532,7 +532,11 @@ func (o *Object) Keys(ctx context.Context) ([]string, error) { } // Text returns a collaborative text wrapper for a text object. -func (o *Object) Text() (*Text, error) { +func (o *Object) Text(ctx context.Context) (*Text, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if o.Type != ObjectTypeText { return nil, fmt.Errorf("automerge object is %q, not text", o.Type) } diff --git a/pkg/automerge/orphan_save_load_parity_test.go b/pkg/automerge/orphan_save_load_parity_test.go index 9584d84eec..233b8f1f2b 100644 --- a/pkg/automerge/orphan_save_load_parity_test.go +++ b/pkg/automerge/orphan_save_load_parity_test.go @@ -128,7 +128,7 @@ func TestRustOrphans_DiscardOrphans(t *testing.T) { doc, missing := orphanScenario(t, ctx, engine) closeDocument(t, doc) - saved, err := doc.SaveWithOptions(ctx, false) + saved, err := doc.Save(ctx, automerge.DiscardOrphans()) require.NoError(t, err) loaded, err := engine.load(ctx, saved, actor(0x03)) diff --git a/pkg/automerge/scalar.go b/pkg/automerge/scalar.go index dabbf08993..f8a14e06bc 100644 --- a/pkg/automerge/scalar.go +++ b/pkg/automerge/scalar.go @@ -66,6 +66,39 @@ const ( ScalarTypeTimestamp ScalarType = "timestamp" ) +// The constructors below build a Scalar with its type and matching field set +// together, so a caller cannot pair a type with the wrong field. + +// NullScalar returns the null scalar. +func NullScalar() Scalar { return Scalar{Type: ScalarTypeNull} } + +// BoolScalar returns a boolean scalar. +func BoolScalar(value bool) Scalar { return Scalar{Type: ScalarTypeBoolean, Bool: value} } + +// UintScalar returns an unsigned integer scalar. +func UintScalar(value uint64) Scalar { return Scalar{Type: ScalarTypeUint, Uint: value} } + +// IntScalar returns a signed integer scalar. +func IntScalar(value int64) Scalar { return Scalar{Type: ScalarTypeInt, Int: value} } + +// FloatScalar returns a 64-bit floating point scalar. +func FloatScalar(value float64) Scalar { return Scalar{Type: ScalarTypeFloat64, Float: value} } + +// StringScalar returns a string scalar. This stores an immutable string value; +// use a text object for collaboratively editable text. +func StringScalar(value string) Scalar { return Scalar{Type: ScalarTypeString, String: value} } + +// BytesScalar returns a byte string scalar. +func BytesScalar(value []byte) Scalar { return Scalar{Type: ScalarTypeBytes, Bytes: value} } + +// CounterScalar returns a counter scalar, whose value is the sum of every +// increment applied across the history. +func CounterScalar(value int64) Scalar { return Scalar{Type: ScalarTypeCounter, Int: value} } + +// TimestampScalar returns a timestamp scalar carrying milliseconds since the +// Unix epoch. +func TimestampScalar(millis int64) Scalar { return Scalar{Type: ScalarTypeTimestamp, Int: millis} } + // PutScalar assigns a typed scalar at a key in the root map. func (d *Document) PutScalar(ctx context.Context, key string, value Scalar) error { d.mu.Lock() diff --git a/pkg/automerge/scalar_constructors_test.go b/pkg/automerge/scalar_constructors_test.go new file mode 100644 index 0000000000..6f9f26139d --- /dev/null +++ b/pkg/automerge/scalar_constructors_test.go @@ -0,0 +1,55 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package automerge_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.probo.inc/probo/pkg/automerge" +) + +// TestScalarConstructors verifies each constructor pairs its type with the +// matching field, which is the misuse the plain struct literal invites. +func TestScalarConstructors(t *testing.T) { + t.Parallel() + + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeNull}, automerge.NullScalar()) + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeBoolean, Bool: true}, automerge.BoolScalar(true)) + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeUint, Uint: 7}, automerge.UintScalar(7)) + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeInt, Int: -7}, automerge.IntScalar(-7)) + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeFloat64, Float: 1.5}, automerge.FloatScalar(1.5)) + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeString, String: "x"}, automerge.StringScalar("x")) + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeBytes, Bytes: []byte{1, 2}}, automerge.BytesScalar([]byte{1, 2})) + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeCounter, Int: 5}, automerge.CounterScalar(5)) + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeTimestamp, Int: 1000}, automerge.TimestampScalar(1000)) +} + +// TestActorIDString checks the actor ID renders as lowercase hex like Hash. +func TestActorIDString(t *testing.T) { + t.Parallel() + + var actorID automerge.ActorID + actorID[0] = 0xab + actorID[15] = 0x01 + + assert.Equal(t, "ab000000000000000000000000000001", actorID.String()) +} diff --git a/pkg/automerge/snapshot_change_graph_test.go b/pkg/automerge/snapshot_change_graph_test.go new file mode 100644 index 0000000000..48e18cc559 --- /dev/null +++ b/pkg/automerge/snapshot_change_graph_test.go @@ -0,0 +1,206 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package automerge_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/automerge" +) + +// buildSnapshotHistory authors a multi-commit history exercising the operations +// a snapshot stores differently from a change: marks, whose expand column is +// shared across every change, and deletes, which a snapshot keeps only as +// successor entries on the operations they removed. +func buildSnapshotHistory( + t *testing.T, + ctx context.Context, + document *automerge.Document, +) { + t.Helper() + + base := time.Unix(1786147200, 0).UTC() + + text, err := document.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 0, 0, "hello brave world")) + _, err = document.Commit(ctx, "write", base) + require.NoError(t, err) + + require.NoError(t, text.Mark( + ctx, + 0, + 5, + "strong", + automerge.Scalar{Type: automerge.ScalarTypeBoolean, Bool: true}, + automerge.MarkExpandBoth, + )) + _, err = document.Commit(ctx, "mark", base.Add(time.Second)) + require.NoError(t, err) + + require.NoError(t, text.Unmark(ctx, 1, 3, "strong", automerge.MarkExpandNone)) + _, err = document.Commit(ctx, "unmark", base.Add(2*time.Second)) + require.NoError(t, err) + + require.NoError(t, text.Splice(ctx, 5, 6, "")) + _, err = document.Commit(ctx, "delete", base.Add(3*time.Second)) + require.NoError(t, err) + + require.NoError(t, document.PutScalar( + ctx, + "counter", + automerge.Scalar{Type: automerge.ScalarTypeCounter, Int: 5}, + )) + _, err = document.Commit(ctx, "counter", base.Add(4*time.Second)) + require.NoError(t, err) + + require.NoError(t, document.Root().Increment(ctx, "counter", 3)) + _, err = document.Commit(ctx, "increment", base.Add(5*time.Second)) + require.NoError(t, err) +} + +// TestLoadedSnapshotExposesEveryChange is the regression for the production +// outage where collaboration failed with "cannot compute changes from unknown +// heads" on every request for an affected document. +// +// A snapshot records hashes for the frontier only and names ancestry by column +// index, so before the decoder rebuilt them, every non-head change loaded +// without a hash and never entered the change graph. Reading the document still +// worked, which is why the corruption stayed invisible, but walking a head's +// ancestry immediately hit a change that was not there and the walk aborted. +func TestLoadedSnapshotExposesEveryChange(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + reference, err := automerge.NewReference(ctx, actor(11)) + require.NoError(t, err) + closeDocument(t, reference) + + buildSnapshotHistory(t, ctx, reference) + + // The browser client persists exactly this: a document chunk, not a stream of + // change chunks. + snapshot, err := reference.Save(ctx) + require.NoError(t, err) + + referenceHeads, err := reference.Heads(ctx) + require.NoError(t, err) + + loaded, err := automerge.Load(ctx, snapshot, actor(12)) + require.NoError(t, err) + closeDocument(t, loaded) + + loadedHeads, err := loaded.Heads(ctx) + require.NoError(t, err) + assert.Equal(t, referenceHeads, loadedHeads, "snapshot must load onto the same frontier") + + changes, err := loaded.ChangesSince(ctx, nil) + require.NoError(t, err, "every change in a loaded snapshot must be reachable") + assert.Len(t, changes, 6, "each commit must survive as an addressable change") + + // Replaying the rebuilt changes has to land on the same frontier, which only + // holds when each one carries the bytes the original writer hashed. + replayed, err := automerge.New(ctx, actor(13)) + require.NoError(t, err) + closeDocument(t, replayed) + + require.NoError(t, replayed.ApplyChanges(ctx, changes)) + + replayedHeads, err := replayed.Heads(ctx) + require.NoError(t, err) + assert.Equal(t, referenceHeads, replayedHeads, "rebuilt changes must reproduce the frontier") + + // Concatenated change chunks are a document the reference can load, so this + // proves Rust accepts the rebuilt bytes as the changes it originally wrote. + var concatenated []byte + for _, change := range changes { + concatenated = append(concatenated, change.Bytes...) + } + + roundTripped, err := automerge.LoadReference(ctx, concatenated, actor(14)) + require.NoError(t, err) + closeDocument(t, roundTripped) + + roundTrippedHeads, err := roundTripped.Heads(ctx) + require.NoError(t, err) + assert.Equal(t, referenceHeads, roundTrippedHeads) +} + +// TestSnapshotMergeReportsIncrementalChanges reproduces the collaboration +// service's persist path: a canonical document restored from a stored snapshot, +// merged with a peer's document, then asked for the changes the merge added. +func TestSnapshotMergeReportsIncrementalChanges(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + origin, err := automerge.NewReference(ctx, actor(21)) + require.NoError(t, err) + closeDocument(t, origin) + + buildSnapshotHistory(t, ctx, origin) + + snapshot, err := origin.Save(ctx) + require.NoError(t, err) + + canonical, err := automerge.Load(ctx, snapshot, actor(22)) + require.NoError(t, err) + closeDocument(t, canonical) + + peer, err := automerge.Load(ctx, snapshot, actor(23)) + require.NoError(t, err) + closeDocument(t, peer) + + peerText, err := peer.Text(ctx, "body") + require.NoError(t, err) + require.NoError(t, peerText.Splice(ctx, 0, 0, "new ")) + _, err = peer.Commit(ctx, "peer edit", time.Unix(1786147300, 0).UTC()) + require.NoError(t, err) + + before, err := canonical.Heads(ctx) + require.NoError(t, err) + + _, err = canonical.Merge(ctx, peer) + require.NoError(t, err) + + incremental, err := canonical.ChangesSince(ctx, before) + require.NoError(t, err, "merging a peer must not break incremental reads") + assert.Len(t, incremental, 1, "only the peer's commit is new") + + after, err := canonical.Heads(ctx) + require.NoError(t, err) + + peerHeads, err := peer.Heads(ctx) + require.NoError(t, err) + assert.Equal(t, peerHeads, after, "the merge must adopt the peer's frontier") + + canonicalText, err := canonical.Text(ctx, "body") + require.NoError(t, err) + + value, err := canonicalText.String(ctx) + require.NoError(t, err) + assert.Equal(t, "new hello world", value) +} diff --git a/pkg/automerge/sync_quiesce_test.go b/pkg/automerge/sync_quiesce_test.go index fe5e5c0c11..56e2695ec3 100644 --- a/pkg/automerge/sync_quiesce_test.go +++ b/pkg/automerge/sync_quiesce_test.go @@ -94,7 +94,7 @@ func TestSyncState_QuiescesWithOrphanedChange(t *testing.T) { // The child change depends on the base the host never received, so it is // retained as an orphan rather than applied. - require.NoError(t, orphanHost.ApplyChanges(ctx, [][]byte{childChanges[0].Bytes})) + require.NoError(t, orphanHost.ApplyChanges(ctx, []automerge.Change{childChanges[0]})) // A fresh peer with an empty document handshakes with the orphan host. peer, err := automerge.New(ctx, actor(3)) diff --git a/pkg/automerge/test_rs_core_parity_test.go b/pkg/automerge/test_rs_core_parity_test.go index 6e492762b5..f7bf572986 100644 --- a/pkg/automerge/test_rs_core_parity_test.go +++ b/pkg/automerge/test_rs_core_parity_test.go @@ -46,7 +46,7 @@ import ( type rustParityEngine struct { name string open func(context.Context, automerge.ActorID) (*automerge.Document, error) - load func(context.Context, []byte, automerge.ActorID) (*automerge.Document, error) + load func(context.Context, []byte, automerge.ActorID, ...automerge.LoadOption) (*automerge.Document, error) } func rustParityEngines() []rustParityEngine { @@ -1615,7 +1615,7 @@ func TestRust_SimpleBadSaveload(t *testing.T) { saved, err := doc.Save(ctx) require.NoError(t, err) - for _, load := range []func(context.Context, []byte, automerge.ActorID) (*automerge.Document, error){ + for _, load := range []func(context.Context, []byte, automerge.ActorID, ...automerge.LoadOption) (*automerge.Document, error){ automerge.Load, automerge.LoadReference, } { diff --git a/pkg/automerge/update_text_parity_test.go b/pkg/automerge/update_text_parity_test.go index a5e7ff26ac..1591d51850 100644 --- a/pkg/automerge/update_text_parity_test.go +++ b/pkg/automerge/update_text_parity_test.go @@ -60,7 +60,7 @@ func TestRustText_SimpleUpdateText(t *testing.T) { otherObject, err := other.Root().Object(ctx, "text") require.NoError(t, err) - otherText, err := otherObject.Text() + otherText, err := otherObject.Text(ctx) require.NoError(t, err) require.NoError(t, otherText.Update(ctx, "Goodbye, world!")) _, err = other.Commit(ctx, "goodbye", commitTime) @@ -112,7 +112,7 @@ func TestRustText_UpdateTextBigOleGraphemes(t *testing.T) { otherObject, err := other.Root().Object(ctx, "text") require.NoError(t, err) - otherText, err := otherObject.Text() + otherText, err := otherObject.Text(ctx) require.NoError(t, err) require.NoError(t, otherText.Update(ctx, "left👨‍👩‍👧right")) _, err = other.Commit(ctx, "girl", commitTime) diff --git a/pkg/probo/document_collaboration_service.go b/pkg/probo/document_collaboration_service.go index 7d4b0706f6..97aad7267c 100644 --- a/pkg/probo/document_collaboration_service.go +++ b/pkg/probo/document_collaboration_service.go @@ -235,7 +235,7 @@ func (s *DocumentService) PersistCollaboration( } var ( - canonicalChangesForLocal [][]byte + canonicalChangesForLocal []automerge.Change revision int64 ) @@ -311,10 +311,7 @@ func (s *DocumentService) PersistCollaboration( return fmt.Errorf("cannot read canonical changes for local document: %w", err) } - canonicalChangesForLocal = make([][]byte, len(localChanges)) - for i, change := range localChanges { - canonicalChangesForLocal[i] = change.Bytes - } + canonicalChangesForLocal = localChanges seeded := state.Seeded || len(after) > 0 if !slices.Equal(before, after) || seeded != state.Seeded { @@ -717,7 +714,7 @@ func (s *DocumentService) loadCollaborationChanges( ) } - changes := make([][]byte, len(batch)) + changes := make([]automerge.Change, len(batch)) for i, change := range batch { if change.Revision != currentRevision+1 { return fmt.Errorf( @@ -727,7 +724,7 @@ func (s *DocumentService) loadCollaborationChanges( ) } - changes[i] = change.ChangeBytes + changes[i] = automerge.Change{Bytes: change.ChangeBytes} currentRevision = change.Revision }