From 4d84feebc90f335d1f4b1d3aed1095fa3301ab9a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 10 Aug 2026 14:24:53 +0000 Subject: [PATCH 01/17] Replace generic types package with opset domain The shared CRDT model lived in a catch-all internal/types package, which drifts from the Rust/develer layout of dedicated, domain-named packages. Rename it to internal/opset since the model is precisely the Automerge operation-set vocabulary: actors, operation IDs, operations, changes, scalars, objects and the validated document history. Update the storage and native alias facades to re-export from opset, rename native's facade file from types.go to model.go for parity with storage, and refresh the architecture doc's boundary table and dependency graph. No behavior change: the model is byte-for-byte the same, and parity, storage and native tests stay green. Signed-off-by: Cursor Agent Co-authored-by: Bryan FRIMIN --- pkg/automerge/internal/native/ARCHITECTURE.md | 6 +- pkg/automerge/internal/native/model.go | 70 ++++++++++++++++++ pkg/automerge/internal/native/types.go | 70 ------------------ .../{types/types.go => opset/opset.go} | 9 ++- pkg/automerge/internal/storage/model.go | 74 +++++++++---------- 5 files changed, 115 insertions(+), 114 deletions(-) create mode 100644 pkg/automerge/internal/native/model.go delete mode 100644 pkg/automerge/internal/native/types.go rename pkg/automerge/internal/{types/types.go => opset/opset.go} (95%) diff --git a/pkg/automerge/internal/native/ARCHITECTURE.md b/pkg/automerge/internal/native/ARCHITECTURE.md index b273f10599..302147437f 100644 --- a/pkg/automerge/internal/native/ARCHITECTURE.md +++ b/pkg/automerge/internal/native/ARCHITECTURE.md @@ -21,13 +21,13 @@ types or storage details. | Rich-text state | `rich_text_state.go` | Span/mark state machines, mark anchors and UTF-16 ranges | | Hydration | `hydrate_state.go` | Recursive map/list materialization | | Storage facade | `storage.go` | Delegation to the independent storage and encoding packages | -| Shared model facade | `types.go` | Aliases to the independent shared types package | +| Shared model facade | `model.go` | Aliases to the independent operation-set model package | ## Internal package boundaries | Package | Responsibility | |---|---| -| `internal/types` | Dependency-free actor, operation, change, object, scalar and chunk model | +| `internal/opset` | Dependency-free actor, operation, change, object, scalar and chunk model | | `internal/encoding` | Bounded binary reader, ULEB128 and length-prefixed primitives | | `internal/storage` | Automerge chunk/column encoding, decoding and graph validation | | `internal/sync` | V1/V2 sync message wire codec and resource limits | @@ -45,7 +45,7 @@ native Engine methods ↓ State / sequence / rich-text state ↓ -internal/types ← internal/encoding ← internal/storage +internal/opset ← internal/encoding ← internal/storage ↑ internal/sync ``` diff --git a/pkg/automerge/internal/native/model.go b/pkg/automerge/internal/native/model.go new file mode 100644 index 0000000000..dd72604218 --- /dev/null +++ b/pkg/automerge/internal/native/model.go @@ -0,0 +1,70 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package native + +import "go.probo.inc/probo/pkg/automerge/internal/opset" + +type ( + ActorID = opset.ActorID + ChangeHash = opset.ChangeHash + OpID = opset.OpID + ObjectID = opset.ObjectID + Key = opset.Key + Action = opset.Action + ScalarType = opset.ScalarType + Scalar = opset.Scalar + Operation = opset.Operation + Change = opset.Change + ChunkType = opset.ChunkType + RawColumn = opset.RawColumn + Document = opset.Document +) + +const ( + ActionMakeMap = opset.ActionMakeMap + ActionSet = opset.ActionSet + ActionMakeList = opset.ActionMakeList + ActionDelete = opset.ActionDelete + ActionMakeText = opset.ActionMakeText + ActionIncrement = opset.ActionIncrement + ActionMakeTable = opset.ActionMakeTable + ActionMark = opset.ActionMark + + ScalarNull = opset.ScalarNull + ScalarFalse = opset.ScalarFalse + ScalarTrue = opset.ScalarTrue + ScalarUint = opset.ScalarUint + ScalarInt = opset.ScalarInt + ScalarFloat64 = opset.ScalarFloat64 + ScalarString = opset.ScalarString + ScalarBytes = opset.ScalarBytes + ScalarCounter = opset.ScalarCounter + ScalarTimestamp = opset.ScalarTimestamp + + ChunkDocument = opset.ChunkDocument + ChunkChange = opset.ChunkChange + ChunkCompressedChange = opset.ChunkCompressedChange +) + +var ( + NewActorID = opset.NewActorID + RootObject = opset.RootObject +) diff --git a/pkg/automerge/internal/native/types.go b/pkg/automerge/internal/native/types.go deleted file mode 100644 index 671f34e5a3..0000000000 --- a/pkg/automerge/internal/native/types.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2026 Probo Inc . -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package native - -import "go.probo.inc/probo/pkg/automerge/internal/types" - -type ( - ActorID = types.ActorID - ChangeHash = types.ChangeHash - OpID = types.OpID - ObjectID = types.ObjectID - Key = types.Key - Action = types.Action - ScalarType = types.ScalarType - Scalar = types.Scalar - Operation = types.Operation - Change = types.Change - ChunkType = types.ChunkType - RawColumn = types.RawColumn - Document = types.Document -) - -const ( - ActionMakeMap = types.ActionMakeMap - ActionSet = types.ActionSet - ActionMakeList = types.ActionMakeList - ActionDelete = types.ActionDelete - ActionMakeText = types.ActionMakeText - ActionIncrement = types.ActionIncrement - ActionMakeTable = types.ActionMakeTable - ActionMark = types.ActionMark - - ScalarNull = types.ScalarNull - ScalarFalse = types.ScalarFalse - ScalarTrue = types.ScalarTrue - ScalarUint = types.ScalarUint - ScalarInt = types.ScalarInt - ScalarFloat64 = types.ScalarFloat64 - ScalarString = types.ScalarString - ScalarBytes = types.ScalarBytes - ScalarCounter = types.ScalarCounter - ScalarTimestamp = types.ScalarTimestamp - - ChunkDocument = types.ChunkDocument - ChunkChange = types.ChunkChange - ChunkCompressedChange = types.ChunkCompressedChange -) - -var ( - NewActorID = types.NewActorID - RootObject = types.RootObject -) diff --git a/pkg/automerge/internal/types/types.go b/pkg/automerge/internal/opset/opset.go similarity index 95% rename from pkg/automerge/internal/types/types.go rename to pkg/automerge/internal/opset/opset.go index 8dbf513847..5c8619978c 100644 --- a/pkg/automerge/internal/types/types.go +++ b/pkg/automerge/internal/opset/opset.go @@ -18,10 +18,11 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -// Package types defines the shared Automerge CRDT and storage model. -// It is dependency-free so storage, sync, and native execution packages can -// exchange changes without importing one another. -package types +// Package opset defines the shared Automerge operation-set model: actors, +// operation IDs, operations, changes, scalars, objects and the validated +// document history. It is dependency-free so the storage, sync, and native +// execution packages can exchange changes without importing one another. +package opset import ( "bytes" diff --git a/pkg/automerge/internal/storage/model.go b/pkg/automerge/internal/storage/model.go index d20f2f40aa..8b69df154a 100644 --- a/pkg/automerge/internal/storage/model.go +++ b/pkg/automerge/internal/storage/model.go @@ -22,51 +22,51 @@ // graph validation independently from the native execution engine. package storage -import "go.probo.inc/probo/pkg/automerge/internal/types" +import "go.probo.inc/probo/pkg/automerge/internal/opset" type ( - ActorID = types.ActorID - ChangeHash = types.ChangeHash - OpID = types.OpID - ObjectID = types.ObjectID - Key = types.Key - Action = types.Action - ScalarType = types.ScalarType - Scalar = types.Scalar - Operation = types.Operation - Change = types.Change - ChunkType = types.ChunkType - RawColumn = types.RawColumn - Document = types.Document + ActorID = opset.ActorID + ChangeHash = opset.ChangeHash + OpID = opset.OpID + ObjectID = opset.ObjectID + Key = opset.Key + Action = opset.Action + ScalarType = opset.ScalarType + Scalar = opset.Scalar + Operation = opset.Operation + Change = opset.Change + ChunkType = opset.ChunkType + RawColumn = opset.RawColumn + Document = opset.Document ) const ( - ActionMakeMap = types.ActionMakeMap - ActionSet = types.ActionSet - ActionMakeList = types.ActionMakeList - ActionDelete = types.ActionDelete - ActionMakeText = types.ActionMakeText - ActionIncrement = types.ActionIncrement - ActionMakeTable = types.ActionMakeTable - ActionMark = types.ActionMark + ActionMakeMap = opset.ActionMakeMap + ActionSet = opset.ActionSet + ActionMakeList = opset.ActionMakeList + ActionDelete = opset.ActionDelete + ActionMakeText = opset.ActionMakeText + ActionIncrement = opset.ActionIncrement + ActionMakeTable = opset.ActionMakeTable + ActionMark = opset.ActionMark - ScalarNull = types.ScalarNull - ScalarFalse = types.ScalarFalse - ScalarTrue = types.ScalarTrue - ScalarUint = types.ScalarUint - ScalarInt = types.ScalarInt - ScalarFloat64 = types.ScalarFloat64 - ScalarString = types.ScalarString - ScalarBytes = types.ScalarBytes - ScalarCounter = types.ScalarCounter - ScalarTimestamp = types.ScalarTimestamp + ScalarNull = opset.ScalarNull + ScalarFalse = opset.ScalarFalse + ScalarTrue = opset.ScalarTrue + ScalarUint = opset.ScalarUint + ScalarInt = opset.ScalarInt + ScalarFloat64 = opset.ScalarFloat64 + ScalarString = opset.ScalarString + ScalarBytes = opset.ScalarBytes + ScalarCounter = opset.ScalarCounter + ScalarTimestamp = opset.ScalarTimestamp - ChunkDocument = types.ChunkDocument - ChunkChange = types.ChunkChange - ChunkCompressedChange = types.ChunkCompressedChange + ChunkDocument = opset.ChunkDocument + ChunkChange = opset.ChunkChange + ChunkCompressedChange = opset.ChunkCompressedChange ) var ( - NewActorID = types.NewActorID - RootObject = types.RootObject + NewActorID = opset.NewActorID + RootObject = opset.RootObject ) From 1271d93802be693fa59520b8056547138a5dc7f0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 10 Aug 2026 14:52:29 +0000 Subject: [PATCH 02/17] Rebuild change identity when loading a snapshot Collaboration failed permanently for affected documents with "cannot compute changes from unknown heads". Every request reloaded the stored snapshot, rebuilt the same broken change graph and failed again. A document chunk records hashes for the frontier only and names ancestry by column index, so every non-head change decoded without a hash and without the bytes its writer hashed. Only hashed changes entered the graph, which left the heads present and all their ancestors missing. Reading the document still worked because operations load separately, which is why the damage stayed invisible until a walk of a head's ancestry hit an absent change and aborted. Restore each change's identity while decoding the chunk: resolve dependency indexes into hashes, then re-encode the change to recover its hash and bytes, and refuse the document when a rebuilt frontier hash disagrees with the one recorded. Three encoding details had to match the writer for the bytes to come out identical: - startOp was a permissive per-actor lower bound that only had to locate operations. It is now derived from the operation count, which is exact because a change's counters are consecutive. - A snapshot drops delete operations and keeps them only as successor entries, and it names predecessors nowhere. Both are reconstructed, with a sequence delete keyed by the element it removes. - The expand column is shared across a snapshot and dense because booleans cannot be null, so it leaked onto ordinary operations and onto marks that expand in neither direction. It is now written only when some operation actually expands, matching the writer. No stored data was lost, so affected documents recover on their next load. The walk in changesSince now stops at a change the peer already holds instead of descending into ancestry it cannot need, so a single unavailable change can no longer fail an entire read. Signed-off-by: Cursor Agent Co-authored-by: Bryan FRIMIN --- pkg/automerge/internal/native/state.go | 11 +- pkg/automerge/internal/storage/decode.go | 177 +++++++++++++++- pkg/automerge/internal/storage/encode.go | 6 +- pkg/automerge/internal/storage/validate.go | 28 ++- pkg/automerge/snapshot_change_graph_test.go | 211 ++++++++++++++++++++ 5 files changed, 418 insertions(+), 15 deletions(-) create mode 100644 pkg/automerge/snapshot_change_graph_test.go diff --git a/pkg/automerge/internal/native/state.go b/pkg/automerge/internal/native/state.go index 20c6037b88..8596e5b788 100644 --- a/pkg/automerge/internal/native/state.go +++ b/pkg/automerge/internal/native/state.go @@ -578,6 +578,13 @@ func (s *State) changesSince(heads []ChangeHash) ([]*Change, bool) { visited[hash] = struct{}{} + // The baseline closure is transitively closed, so everything below a change + // the peer already holds is also held. Descending would add nothing and + // would fail the whole read if any ancestor were ever unavailable. + if _, ok := known[hash]; ok { + return true + } + change, ok := s.changes[hash] if !ok { return false @@ -589,10 +596,6 @@ func (s *State) changesSince(heads []ChangeHash) ([]*Change, bool) { } } - if _, ok := known[hash]; ok { - return true - } - if len(change.Raw) == 0 { return false } diff --git a/pkg/automerge/internal/storage/decode.go b/pkg/automerge/internal/storage/decode.go index e39f04fe2e..c9e0db2b55 100644 --- a/pkg/automerge/internal/storage/decode.go +++ b/pkg/automerge/internal/storage/decode.go @@ -274,6 +274,11 @@ func decodeDocumentChunk(document *Document, data []byte) error { return fmt.Errorf("cannot decode operations: %w", err) } + operations, err = restoreChangeOperations(operations) + if err != nil { + return fmt.Errorf("cannot restore change operations: %w", err) + } + if err := assignOperations(changes, operations); err != nil { return fmt.Errorf("cannot assign operations: %w", err) } @@ -307,6 +312,10 @@ func decodeDocumentChunk(document *Document, data []byte) error { return err } + if err := reconstructSnapshotChanges(changes); err != nil { + return err + } + document.Actors = actors document.Heads = heads document.Changes = changes @@ -316,6 +325,168 @@ func decodeDocumentChunk(document *Document, data []byte) error { return nil } +// restoreChangeOperations turns a snapshot's operation view back into the one a +// change carries. A snapshot records, for every surviving operation, the +// operations that superseded it, and it drops delete operations entirely because +// they survive only as those successor entries. A change instead names each +// operation's predecessors and spells its deletes out, so rebuilding a change +// means inverting the successor lists and recreating the deletes they imply. +// +// Successors are left in place: the engine materializes a loaded snapshot from +// them, and re-encoding a change only ever reads predecessors. +func restoreChangeOperations(operations []Operation) ([]Operation, error) { + stored := make(map[OpID]struct{}, len(operations)) + for _, operation := range operations { + stored[operation.ID] = struct{}{} + } + + predecessors := make(map[OpID][]OpID) + for _, operation := range operations { + for _, successor := range operation.Successors { + predecessors[successor] = append(predecessors[successor], operation.ID) + } + } + + for identifier := range predecessors { + slices.SortFunc(predecessors[identifier], func(left, right OpID) int { + return left.Compare(right) + }) + } + + for i := range operations { + operations[i].Predecessors = predecessors[operations[i].ID] + } + + deletes := make([]Operation, 0) + + for identifier, superseded := range predecessors { + if _, ok := stored[identifier]; ok { + continue + } + + // Only a delete leaves no operation of its own behind, and every operation + // it removed shares the object and key it targeted. + source := operationByID(operations, superseded[0]) + if source == nil { + return nil, fmt.Errorf( + "operation %s@%d supersedes nothing that the snapshot retains", + identifier.Actor, + identifier.Counter, + ) + } + + deletes = append(deletes, Operation{ + ID: identifier, + Object: source.Object, + Key: supersededKey(source), + Action: ActionDelete, + Predecessors: superseded, + }) + } + + slices.SortFunc(deletes, func(left, right Operation) int { + return left.ID.Compare(right.ID) + }) + + return append(operations, deletes...), nil +} + +// supersededKey names the location an operation occupies, which is what an +// operation superseding it addresses. A map operation is addressed by its +// property, while a sequence operation is addressed by the element identifier: +// an insertion creates the element it is named by, and any later operation on +// that element already carries it. +func supersededKey(operation *Operation) Key { + if operation.Key.Property != nil || !operation.Insert { + return operation.Key + } + + element := operation.ID + + return Key{Element: &element} +} + +func operationByID(operations []Operation, identifier OpID) *Operation { + for i := range operations { + if operations[i].ID == identifier { + return &operations[i] + } + } + + return nil +} + +// reconstructSnapshotChanges restores the change-chunk identity of every change +// in a document chunk. Snapshots store the frontier hashes only and reference +// ancestry by column index, so every non-head change decodes without a hash and +// without its original bytes. Re-encoding each change once its dependencies are +// known recovers both, which keeps the change graph whole: without it only the +// heads are addressable and any walk of their ancestry hits a missing change. +// +// Dependencies are rebuilt in the stored index order because the encoder writes +// dependency hashes in slice order, so that order is what the original hash was +// computed over. +func reconstructSnapshotChanges(changes []Change) error { + resolved := make([]bool, len(changes)) + remaining := len(changes) + + for remaining > 0 { + progressed := false + + for i := range changes { + change := &changes[i] + + if resolved[i] || !dependenciesResolved(change, resolved) { + continue + } + + change.Dependencies = make([]ChangeHash, 0, len(change.DependencyIndexes)) + for _, index := range change.DependencyIndexes { + change.Dependencies = append(change.Dependencies, *changes[index].Hash) + } + + recorded := change.Hash + change.Hash = nil + + if _, err := EncodeChange(change); err != nil { + return fmt.Errorf("cannot rebuild snapshot change %d: %w", i, err) + } + + // A recorded hash only exists for frontier changes. Disagreeing with it + // means the rebuilt bytes are not the ones the writer hashed, so the + // whole graph would be keyed by identifiers no peer shares. + if recorded != nil && *recorded != *change.Hash { + return fmt.Errorf( + "snapshot change %d rebuilds to hash %s but the document records %s", + i, + change.Hash, + recorded, + ) + } + + resolved[i] = true + remaining-- + progressed = true + } + + if !progressed { + return fmt.Errorf("snapshot dependency graph cannot be ordered") + } + } + + return nil +} + +func dependenciesResolved(change *Change, resolved []bool) bool { + for _, index := range change.DependencyIndexes { + if !resolved[index] { + return false + } + } + + return true +} + func decodeDocumentChanges( columns map[uint32]column, actors []ActorID, @@ -745,7 +916,11 @@ func decodeOperations( } } - if markExpand[i].valid { + // Expand is a property of a mark, and its column is dense because booleans + // cannot be null. A document chunk shares one column across every change, + // so keeping the flag on ordinary operations would make a change that never + // carried an expand column re-encode with one and hash differently. + if markExpand[i].valid && operations[i].Action == ActionMark { value := markExpand[i].value operations[i].MarkExpand = &value } diff --git a/pkg/automerge/internal/storage/encode.go b/pkg/automerge/internal/storage/encode.go index ba82a47135..1e67e6750d 100644 --- a/pkg/automerge/internal/storage/encode.go +++ b/pkg/automerge/internal/storage/encode.go @@ -214,8 +214,10 @@ func encodeOperationColumns( predCounters = append(predCounters, some(int64(predecessor.Counter))) } - if operation.MarkExpand != nil { - markExpands[i] = *operation.MarkExpand + // An all-false expand column carries no information and is left out, so a + // mark that expands in neither direction encodes without one. + if operation.MarkExpand != nil && *operation.MarkExpand { + markExpands[i] = true hasMarkExpand = true } diff --git a/pkg/automerge/internal/storage/validate.go b/pkg/automerge/internal/storage/validate.go index fc9849b74d..d32dc8c6e4 100644 --- a/pkg/automerge/internal/storage/validate.go +++ b/pkg/automerge/internal/storage/validate.go @@ -534,6 +534,24 @@ func assignOperations(changes []Change, operations []Operation) error { }) } + // The per-actor bounds above are a permissive lower bound that only has to + // locate operations. A change's operations carry consecutive counters ending + // at maxOp, so the real start operation follows from the operation count, and + // re-encoding the change depends on it being exact. + for i := range changes { + count := uint64(len(changes[i].Operations)) + if count > changes[i].MaxOp { + return fmt.Errorf( + "change %d holds %d operations but ends at operation %d", + i, + count, + changes[i].MaxOp, + ) + } + + changes[i].StartOp = changes[i].MaxOp - count + 1 + } + return nil } @@ -739,21 +757,15 @@ func validateChangeChunksAfterSnapshot(document *Document) error { known := make(map[ChangeHash]struct{}) dependedOn := make(map[ChangeHash]struct{}) + // A change may legitimately appear both inside the snapshot and as a trailing + // change chunk, so repeats identify the same change rather than a conflict. for _, change := range document.Changes { if change.Hash != nil { - if _, exists := known[*change.Hash]; exists { - return fmt.Errorf("duplicate known change hash %s", change.Hash) - } - known[*change.Hash] = struct{}{} } } for _, change := range document.Changes { - if len(change.DependencyIndexes) > 0 { - continue - } - for _, dependency := range change.Dependencies { if _, ok := known[dependency]; !ok { return fmt.Errorf("change %s has missing dependency %s", change.Hash, dependency) diff --git a/pkg/automerge/snapshot_change_graph_test.go b/pkg/automerge/snapshot_change_graph_test.go new file mode 100644 index 0000000000..ddd4e8bac5 --- /dev/null +++ b/pkg/automerge/snapshot_change_graph_test.go @@ -0,0 +1,211 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package automerge_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/automerge" +) + +// buildSnapshotHistory authors a multi-commit history exercising the operations +// a snapshot stores differently from a change: marks, whose expand column is +// shared across every change, and deletes, which a snapshot keeps only as +// successor entries on the operations they removed. +func buildSnapshotHistory( + t *testing.T, + ctx context.Context, + document *automerge.Document, +) { + t.Helper() + + base := time.Unix(1786147200, 0).UTC() + + text, err := document.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 0, 0, "hello brave world")) + _, err = document.Commit(ctx, "write", base) + require.NoError(t, err) + + require.NoError(t, text.Mark( + ctx, + 0, + 5, + "strong", + automerge.Scalar{Type: automerge.ScalarTypeBoolean, Bool: true}, + automerge.MarkExpandBoth, + )) + _, err = document.Commit(ctx, "mark", base.Add(time.Second)) + require.NoError(t, err) + + require.NoError(t, text.Unmark(ctx, 1, 3, "strong", automerge.MarkExpandNone)) + _, err = document.Commit(ctx, "unmark", base.Add(2*time.Second)) + require.NoError(t, err) + + require.NoError(t, text.Splice(ctx, 5, 6, "")) + _, err = document.Commit(ctx, "delete", base.Add(3*time.Second)) + require.NoError(t, err) + + require.NoError(t, document.PutScalar( + ctx, + "counter", + automerge.Scalar{Type: automerge.ScalarTypeCounter, Int: 5}, + )) + _, err = document.Commit(ctx, "counter", base.Add(4*time.Second)) + require.NoError(t, err) + + require.NoError(t, document.Root().Increment(ctx, "counter", 3)) + _, err = document.Commit(ctx, "increment", base.Add(5*time.Second)) + require.NoError(t, err) +} + +// TestLoadedSnapshotExposesEveryChange is the regression for the production +// outage where collaboration failed with "cannot compute changes from unknown +// heads" on every request for an affected document. +// +// A snapshot records hashes for the frontier only and names ancestry by column +// index, so before the decoder rebuilt them, every non-head change loaded +// without a hash and never entered the change graph. Reading the document still +// worked, which is why the corruption stayed invisible, but walking a head's +// ancestry immediately hit a change that was not there and the walk aborted. +func TestLoadedSnapshotExposesEveryChange(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + reference, err := automerge.NewReference(ctx, actor(11)) + require.NoError(t, err) + closeDocument(t, reference) + + buildSnapshotHistory(t, ctx, reference) + + // The browser client persists exactly this: a document chunk, not a stream of + // change chunks. + snapshot, err := reference.Save(ctx) + require.NoError(t, err) + + referenceHeads, err := reference.Heads(ctx) + require.NoError(t, err) + + loaded, err := automerge.Load(ctx, snapshot, actor(12)) + require.NoError(t, err) + closeDocument(t, loaded) + + loadedHeads, err := loaded.Heads(ctx) + require.NoError(t, err) + assert.Equal(t, referenceHeads, loadedHeads, "snapshot must load onto the same frontier") + + changes, err := loaded.ChangesSince(ctx, nil) + require.NoError(t, err, "every change in a loaded snapshot must be reachable") + assert.Len(t, changes, 6, "each commit must survive as an addressable change") + + // Replaying the rebuilt changes has to land on the same frontier, which only + // holds when each one carries the bytes the original writer hashed. + replayed, err := automerge.New(ctx, actor(13)) + require.NoError(t, err) + closeDocument(t, replayed) + + raw := make([][]byte, len(changes)) + for i, change := range changes { + raw[i] = change.Bytes + } + + require.NoError(t, replayed.ApplyChanges(ctx, raw)) + + replayedHeads, err := replayed.Heads(ctx) + require.NoError(t, err) + assert.Equal(t, referenceHeads, replayedHeads, "rebuilt changes must reproduce the frontier") + + // Concatenated change chunks are a document the reference can load, so this + // proves Rust accepts the rebuilt bytes as the changes it originally wrote. + var concatenated []byte + for _, change := range changes { + concatenated = append(concatenated, change.Bytes...) + } + + roundTripped, err := automerge.LoadReference(ctx, concatenated, actor(14)) + require.NoError(t, err) + closeDocument(t, roundTripped) + + roundTrippedHeads, err := roundTripped.Heads(ctx) + require.NoError(t, err) + assert.Equal(t, referenceHeads, roundTrippedHeads) +} + +// TestSnapshotMergeReportsIncrementalChanges reproduces the collaboration +// service's persist path: a canonical document restored from a stored snapshot, +// merged with a peer's document, then asked for the changes the merge added. +func TestSnapshotMergeReportsIncrementalChanges(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + origin, err := automerge.NewReference(ctx, actor(21)) + require.NoError(t, err) + closeDocument(t, origin) + + buildSnapshotHistory(t, ctx, origin) + + snapshot, err := origin.Save(ctx) + require.NoError(t, err) + + canonical, err := automerge.Load(ctx, snapshot, actor(22)) + require.NoError(t, err) + closeDocument(t, canonical) + + peer, err := automerge.Load(ctx, snapshot, actor(23)) + require.NoError(t, err) + closeDocument(t, peer) + + peerText, err := peer.Text(ctx, "body") + require.NoError(t, err) + require.NoError(t, peerText.Splice(ctx, 0, 0, "new ")) + _, err = peer.Commit(ctx, "peer edit", time.Unix(1786147300, 0).UTC()) + require.NoError(t, err) + + before, err := canonical.Heads(ctx) + require.NoError(t, err) + + _, err = canonical.Merge(ctx, peer) + require.NoError(t, err) + + incremental, err := canonical.ChangesSince(ctx, before) + require.NoError(t, err, "merging a peer must not break incremental reads") + assert.Len(t, incremental, 1, "only the peer's commit is new") + + after, err := canonical.Heads(ctx) + require.NoError(t, err) + + peerHeads, err := peer.Heads(ctx) + require.NoError(t, err) + assert.Equal(t, peerHeads, after, "the merge must adopt the peer's frontier") + + canonicalText, err := canonical.Text(ctx, "body") + require.NoError(t, err) + + value, err := canonicalText.String(ctx) + require.NoError(t, err) + assert.Equal(t, "new hello world", value) +} From e54dbe3a6b9d409f34c5d5ff36fbee0355dbde5e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 10 Aug 2026 16:22:48 +0000 Subject: [PATCH 03/17] Add a document-chunk encoder Go could read the compacted snapshot Rust and JavaScript write from save() but could never write one. Saving concatenated the existing base with every change chunk since, so a Go-authored history was stored as a stream of changes rather than a compacted operation set. That left the snapshot decoder one-sided, with no encoder to test it against, and it is why the decoder's ancestry loss went unnoticed for so long. Write the format properly. A document chunk stores the operation set once in operation-set order rather than per change, reduces each change to a row whose ancestry is an index into the same table, and lets deletes exist only as successor entries on the operations they removed. The encoder inverts predecessors into successors, drops deletes, orders changes so ancestors precede dependents, builds the sorted actor and head tables, and frames the chunk with its checksum. Two things the format keeps that Go was dropping are carried through: a change's extra payload, which a snapshot stores as a scalar while a change stores trailing bytes, and columns a reader did not understand, which are written back to the table they came from. Operation order is supplied by the caller because it depends on sequence state only the engine maintains, which keeps the CRDT ordering rules out of the storage layer. Byte identity is the gate rather than round-tripping: re-encoding a snapshot the reference wrote reproduces that file exactly, for the official fixture and for reference-authored histories covering marks and unmarks, deletes and overwrites, lists, counters, nested objects and a merged multi-actor history. Signed-off-by: Cursor Agent Co-authored-by: Bryan FRIMIN --- .../internal/storage/encode_document.go | 613 ++++++++++++++++++ .../storage/encode_document_reference_test.go | 246 +++++++ .../internal/storage/encode_document_test.go | 119 ++++ pkg/automerge/zz_probe_order_test.go | 140 ++++ 4 files changed, 1118 insertions(+) create mode 100644 pkg/automerge/internal/storage/encode_document.go create mode 100644 pkg/automerge/internal/storage/encode_document_reference_test.go create mode 100644 pkg/automerge/internal/storage/encode_document_test.go create mode 100644 pkg/automerge/zz_probe_order_test.go diff --git a/pkg/automerge/internal/storage/encode_document.go b/pkg/automerge/internal/storage/encode_document.go new file mode 100644 index 0000000000..7f5f3a43c4 --- /dev/null +++ b/pkg/automerge/internal/storage/encode_document.go @@ -0,0 +1,613 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package storage + +import ( + "bytes" + "crypto/sha256" + "fmt" + "slices" +) + +// assembleChunk frames a chunk body: the magic bytes, the first four bytes of +// the digest covering the typed and length-prefixed body, then that body. +func assembleChunk(kind ChunkType, body []byte) []byte { + length := appendULEB(nil, uint64(len(body))) + + digestInput := make([]byte, 0, 1+len(length)+len(body)) + digestInput = append(digestInput, byte(kind)) + digestInput = append(digestInput, length...) + digestInput = append(digestInput, body...) + + digest := sha256.Sum256(digestInput) + + chunk := make([]byte, 0, 4+4+1+len(length)+len(body)) + chunk = append(chunk, magic[:]...) + chunk = append(chunk, digest[:4]...) + chunk = append(chunk, byte(kind)) + chunk = append(chunk, length...) + chunk = append(chunk, body...) + + return chunk +} + +// EncodeDocument serializes a whole history as one compacted document chunk, +// the form Rust and JavaScript write from save(). +// +// A document chunk is not a concatenation of changes. It stores the operation +// set once, in operation-set order rather than per change, and reduces each +// change to a row of metadata whose ancestry is expressed as indexes into that +// same table. Deletes disappear into the successor lists of the operations they +// removed, and every operation's predecessors are recovered from those lists on +// the way back in. +// +// order gives the operation-set sequence: object by object, and within an +// object the order a reader would see. The caller owns that sequence because it +// requires the sequence state only the engine maintains. Operations named by +// order must exist in the history, and deletes must be left out of it. +func EncodeDocument(document *Document, order []OpID) ([]byte, error) { + changes, err := documentChangeOrder(document) + if err != nil { + return nil, err + } + + operations, err := documentOperations(changes, order) + if err != nil { + return nil, err + } + + actors := documentActorTable(changes, operations) + actorIndexes := make(map[ActorID]uint64, len(actors)) + + for i, actor := range actors { + actorIndexes[actor] = uint64(i) + } + + heads, headIndexes, err := documentHeads(changes) + if err != nil { + return nil, err + } + + changeColumns, err := encodeDocumentChangeColumns(changes, actorIndexes) + if err != nil { + return nil, err + } + + operationColumns, err := encodeDocumentOperationColumns(operations, actorIndexes) + if err != nil { + return nil, err + } + + changeColumns = sortColumns( + append(changeColumns, retainedColumns(document, changeColumnSpecifications)...), + ) + operationColumns = sortColumns( + append(operationColumns, retainedColumns(document, operationColumnSpecifications)...), + ) + + var body []byte + + body = appendULEB(body, uint64(len(actors))) + for _, actor := range actors { + body = appendLengthPrefixedNative(body, actor.Bytes()) + } + + body = appendULEB(body, uint64(len(heads))) + for _, head := range heads { + body = append(body, head[:]...) + } + + // Both column sets are described before either is written, so the metadata + // has to be laid out ahead of the data it measures. + body = appendColumnMetadata(body, changeColumns) + body = appendColumnMetadata(body, operationColumns) + body = appendColumnData(body, changeColumns) + body = appendColumnData(body, operationColumns) + + for _, index := range headIndexes { + body = appendULEB(body, index) + } + + return assembleChunk(ChunkDocument, body), nil +} + +// documentChangeOrder returns the changes in dependency order. A snapshot may +// legally store them in any order, but writing ancestors first keeps the index +// references pointing backwards, which is what every other implementation emits +// and what makes the result readable in one pass. +func documentChangeOrder(document *Document) ([]*Change, error) { + byHash := make(map[ChangeHash]*Change, len(document.Changes)) + + for i := range document.Changes { + change := &document.Changes[i] + if change.Hash == nil { + return nil, fmt.Errorf("change %d cannot be written without a hash", i) + } + + byHash[*change.Hash] = change + } + + ordered := make([]*Change, 0, len(document.Changes)) + placed := make(map[ChangeHash]struct{}, len(document.Changes)) + + var place func(*Change) error + + place = func(change *Change) error { + if _, ok := placed[*change.Hash]; ok { + return nil + } + + // Claim the change before descending so a cycle is reported rather than + // followed forever. + placed[*change.Hash] = struct{}{} + + for _, dependency := range change.Dependencies { + parent, ok := byHash[dependency] + if !ok { + return fmt.Errorf( + "change %s depends on %s which the history does not hold", + change.Hash, + dependency, + ) + } + + if err := place(parent); err != nil { + return err + } + } + + ordered = append(ordered, change) + + return nil + } + + for i := range document.Changes { + if err := place(&document.Changes[i]); err != nil { + return nil, err + } + } + + return ordered, nil +} + +// documentOperations collects the operations to store, in the given order, with +// each one's successors derived from the predecessors recorded across the whole +// history. Deletes are dropped: they exist in the result only as the successor +// entries they contribute. +func documentOperations(changes []*Change, order []OpID) ([]Operation, error) { + sources := make(map[OpID]*Operation) + + for _, change := range changes { + for i := range change.Operations { + operation := &change.Operations[i] + if _, ok := sources[operation.ID]; ok { + return nil, fmt.Errorf("operation %s@%d occurs twice", + operation.ID.Actor, operation.ID.Counter) + } + + sources[operation.ID] = operation + } + } + + successors := make(map[OpID][]OpID) + + for _, change := range changes { + for _, operation := range change.Operations { + for _, predecessor := range operation.Predecessors { + successors[predecessor] = append(successors[predecessor], operation.ID) + } + } + } + + for identifier := range successors { + slices.SortFunc(successors[identifier], func(left, right OpID) int { + return left.Compare(right) + }) + } + + operations := make([]Operation, 0, len(order)) + + for _, identifier := range order { + source, ok := sources[identifier] + if !ok { + return nil, fmt.Errorf( + "operation %s@%d is ordered but absent from the history", + identifier.Actor, + identifier.Counter, + ) + } + + if source.Action == ActionDelete { + return nil, fmt.Errorf( + "operation %s@%d is a delete and cannot be stored", + identifier.Actor, + identifier.Counter, + ) + } + + stored := *source + stored.Predecessors = nil + stored.Successors = successors[identifier] + + operations = append(operations, stored) + } + + return operations, nil +} + +func documentActorTable(changes []*Change, operations []Operation) []ActorID { + seen := make(map[ActorID]struct{}) + + add := func(actor ActorID) { + if actor != "" { + seen[actor] = struct{}{} + } + } + + for _, change := range changes { + add(change.Actor) + } + + for _, operation := range operations { + add(operation.ID.Actor) + + if !operation.Object.IsRoot { + add(operation.Object.OpID.Actor) + } + + if operation.Key.Element != nil { + add(operation.Key.Element.Actor) + } + + for _, successor := range operation.Successors { + add(successor.Actor) + } + } + + actors := make([]ActorID, 0, len(seen)) + for actor := range seen { + actors = append(actors, actor) + } + + slices.SortFunc(actors, func(left, right ActorID) int { + return left.Compare(right) + }) + + return actors +} + +// documentHeads returns the frontier and the index of each head, which is how a +// snapshot names its heads. +func documentHeads(changes []*Change) ([]ChangeHash, []uint64, error) { + indexes := make(map[ChangeHash]uint64, len(changes)) + dependedOn := make(map[ChangeHash]struct{}, len(changes)) + + for i, change := range changes { + indexes[*change.Hash] = uint64(i) + + for _, dependency := range change.Dependencies { + dependedOn[dependency] = struct{}{} + } + } + + heads := make([]ChangeHash, 0) + + for _, change := range changes { + if _, ok := dependedOn[*change.Hash]; !ok { + heads = append(heads, *change.Hash) + } + } + + slices.SortFunc(heads, func(left, right ChangeHash) int { + return bytes.Compare(left[:], right[:]) + }) + + headIndexes := make([]uint64, len(heads)) + for i, head := range heads { + headIndexes[i] = indexes[head] + } + + return heads, headIndexes, nil +} + +func encodeDocumentChangeColumns( + changes []*Change, + actorIndexes map[ActorID]uint64, +) ([]encodedColumn, error) { + count := len(changes) + + indexes := make(map[ChangeHash]uint64, count) + for i, change := range changes { + indexes[*change.Hash] = uint64(i) + } + + var ( + actors = make([]optional[uint64], count) + sequences = make([]optional[int64], count) + maxOps = make([]optional[int64], count) + times = make([]optional[int64], count) + messages = make([]optional[string], count) + dependencySize = make([]optional[uint64], count) + dependencies []optional[int64] + extraMetadata = make([]optional[uint64], count) + extraData []byte + ) + + for i, change := range changes { + actorIndex, ok := actorIndexes[change.Actor] + if !ok { + return nil, fmt.Errorf("change %d actor is not in the actor table", i) + } + + actors[i] = some(actorIndex) + sequences[i] = some(int64(change.Sequence)) + maxOps[i] = some(int64(change.MaxOp)) + times[i] = some(change.Time) + + if change.Message != "" { + messages[i] = some(change.Message) + } + + dependencySize[i] = some(uint64(len(change.Dependencies))) + + for _, dependency := range change.Dependencies { + index, ok := indexes[dependency] + if !ok { + return nil, fmt.Errorf("change %d depends on an absent change", i) + } + + dependencies = append(dependencies, some(int64(index))) + } + + metadata, data, err := encodeScalar(changeExtra(change)) + if err != nil { + return nil, fmt.Errorf("cannot encode change %d extra: %w", i, err) + } + + extraMetadata[i] = metadata + extraData = append(extraData, data...) + } + + columns := []encodedColumn{ + {specification: 1, data: encodeRLE(actors, appendULEB)}, + {specification: 3, data: encodeDelta(sequences)}, + {specification: 19, data: encodeDelta(maxOps)}, + {specification: 35, data: encodeDelta(times)}, + {specification: 53, data: encodeStrings(messages)}, + {specification: 64, data: encodeRLE(dependencySize, appendULEB)}, + {specification: 67, data: encodeDelta(dependencies)}, + {specification: 86, data: encodeRLE(extraMetadata, appendULEB)}, + {specification: 87, data: extraData}, + } + + return withData(columns), nil +} + +// changeExtra reports the change's extra payload as the scalar a snapshot +// stores. A change chunk keeps the payload as trailing bytes, so the two forms +// have to be reconciled in whichever direction carries the value. +func changeExtra(change *Change) *Scalar { + if len(change.ExtraBytes) > 0 { + return &Scalar{Type: ScalarBytes, Bytes: change.ExtraBytes} + } + + return change.Extra +} + +func encodeDocumentOperationColumns( + operations []Operation, + actorIndexes map[ActorID]uint64, +) ([]encodedColumn, error) { + count := len(operations) + + var ( + idActors = make([]optional[uint64], count) + idCounters = make([]optional[int64], count) + objectActors = make([]optional[uint64], count) + objectCounters = make([]optional[uint64], count) + keyActors = make([]optional[uint64], count) + keyCounters = make([]optional[int64], count) + keyStrings = make([]optional[string], count) + inserts = make([]bool, count) + actions = make([]optional[uint64], count) + valueMetadata = make([]optional[uint64], count) + valueData []byte + successorSize = make([]optional[uint64], count) + successorActors []optional[uint64] + successorCounter []optional[int64] + markExpands = make([]bool, count) + hasMarkExpand bool + markNames = make([]optional[string], count) + ) + + for i, operation := range operations { + index, ok := actorIndexes[operation.ID.Actor] + if !ok { + return nil, fmt.Errorf("operation %d actor is not in the actor table", i) + } + + idActors[i] = some(index) + idCounters[i] = some(int64(operation.ID.Counter)) + + if !operation.Object.IsRoot { + index, ok := actorIndexes[operation.Object.OpID.Actor] + if !ok { + return nil, fmt.Errorf("operation %d object actor is unknown", i) + } + + objectActors[i] = some(index) + objectCounters[i] = some(operation.Object.OpID.Counter) + } + + switch { + case operation.Key.Property != nil: + keyStrings[i] = some(*operation.Key.Property) + case operation.Key.IsHead: + keyCounters[i] = some(int64(0)) + case operation.Key.Element != nil: + index, ok := actorIndexes[operation.Key.Element.Actor] + if !ok { + return nil, fmt.Errorf("operation %d key actor is unknown", i) + } + + keyActors[i] = some(index) + keyCounters[i] = some(int64(operation.Key.Element.Counter)) + default: + return nil, fmt.Errorf("operation %d has no key", i) + } + + inserts[i] = operation.Insert + actions[i] = some(uint64(operation.Action)) + + metadata, data, err := encodeScalar(operation.Value) + if err != nil { + return nil, fmt.Errorf("cannot encode operation %d value: %w", i, err) + } + + valueMetadata[i] = metadata + valueData = append(valueData, data...) + + successorSize[i] = some(uint64(len(operation.Successors))) + + for _, successor := range operation.Successors { + index, ok := actorIndexes[successor.Actor] + if !ok { + return nil, fmt.Errorf("operation %d successor actor is unknown", i) + } + + successorActors = append(successorActors, some(index)) + successorCounter = append(successorCounter, some(int64(successor.Counter))) + } + + // Expand only means anything on a mark, and an all-false column is left + // out, so the flag is written only where it is actually set. + if operation.MarkExpand != nil && *operation.MarkExpand { + markExpands[i] = true + hasMarkExpand = true + } + + if operation.MarkName != nil { + markNames[i] = some(*operation.MarkName) + } + } + + var markExpandData []byte + if hasMarkExpand { + markExpandData = encodeBooleans(markExpands) + } + + columns := []encodedColumn{ + {specification: 1, data: encodeRLE(objectActors, appendULEB)}, + {specification: 2, data: encodeRLE(objectCounters, appendULEB)}, + {specification: 17, data: encodeRLE(keyActors, appendULEB)}, + {specification: 19, data: encodeDelta(keyCounters)}, + {specification: 21, data: encodeStrings(keyStrings)}, + {specification: 33, data: encodeRLE(idActors, appendULEB)}, + {specification: 35, data: encodeDelta(idCounters)}, + {specification: 52, data: encodeBooleans(inserts)}, + {specification: 66, data: encodeRLE(actions, appendULEB)}, + {specification: 86, data: encodeRLE(valueMetadata, appendULEB)}, + {specification: 87, data: valueData}, + {specification: 128, data: encodeRLE(successorSize, appendULEB)}, + {specification: 129, data: encodeRLE(successorActors, appendULEB)}, + {specification: 131, data: encodeDelta(successorCounter)}, + {specification: 148, data: markExpandData}, + {specification: 165, data: encodeStrings(markNames)}, + } + + return withData(columns), nil +} + +var ( + changeColumnSpecifications = []uint32{1, 3, 19, 35, 53, 64, 67, 86, 87} + operationColumnSpecifications = []uint32{ + 1, 2, 17, 19, 21, 33, 35, 52, 66, 86, 87, 128, 129, 131, 148, 165, + } +) + +// retainedColumns returns the columns a previous reader did not understand but +// kept, so writing a history back does not quietly drop what a newer version of +// the format put there. Each retained column is matched to the table it came +// from by its specification. +func retainedColumns(document *Document, known []uint32) []encodedColumn { + retained := make([]encodedColumn, 0) + + for _, column := range document.UnknownColumns { + normalized := column.Specification &^ 8 + if slices.Contains(known, normalized) || len(column.Data) == 0 { + continue + } + + retained = append(retained, encodedColumn{ + specification: normalized, + data: append([]byte(nil), column.Data...), + }) + } + + return retained +} + +// sortColumns puts columns in the strictly ascending specification order a +// reader requires, which both the metadata and the data must follow. +func sortColumns(columns []encodedColumn) []encodedColumn { + slices.SortFunc(columns, func(left, right encodedColumn) int { + switch { + case left.specification < right.specification: + return -1 + case left.specification > right.specification: + return 1 + default: + return 0 + } + }) + + return columns +} + +func withData(columns []encodedColumn) []encodedColumn { + filtered := make([]encodedColumn, 0, len(columns)) + + for _, column := range columns { + if len(column.data) > 0 { + filtered = append(filtered, column) + } + } + + return filtered +} + +func appendColumnMetadata(data []byte, columns []encodedColumn) []byte { + data = appendULEB(data, uint64(len(columns))) + for _, column := range columns { + data = appendULEB(data, uint64(column.specification)) + data = appendULEB(data, uint64(len(column.data))) + } + + return data +} + +func appendColumnData(data []byte, columns []encodedColumn) []byte { + for _, column := range columns { + data = append(data, column.data...) + } + + return data +} diff --git a/pkg/automerge/internal/storage/encode_document_reference_test.go b/pkg/automerge/internal/storage/encode_document_reference_test.go new file mode 100644 index 0000000000..0a59b41cd0 --- /dev/null +++ b/pkg/automerge/internal/storage/encode_document_reference_test.go @@ -0,0 +1,246 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package storage + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/automerge/internal/reference" +) + +func boolScalar() []byte { return []byte(`{"type":"boolean","bool":true}`) } +func nullScalar() []byte { return []byte(`{"type":"null"}`) } +func counterScalar() []byte { return []byte(`{"type":"counter","int":5}`) } + +func stringScalar(value string) []byte { + return []byte(`{"type":"string","string":"` + value + `"}`) +} + +func newReference(t *testing.T, ctx context.Context, actor byte) *reference.Engine { + t.Helper() + + engine, err := reference.New(ctx) + require.NoError(t, err) + require.NoError(t, engine.SetActor(ctx, []byte{ + actor, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + })) + + return engine +} + +// assertSnapshotReencodes decodes a reference-written snapshot and requires the +// encoder to reproduce it byte for byte. +func assertSnapshotReencodes(t *testing.T, saved []byte) { + t.Helper() + + document, err := Decode(saved) + require.NoError(t, err) + + encoded, err := EncodeDocument(document, storedOperationOrder(t, saved)) + require.NoError(t, err) + + assert.Equal(t, saved, encoded) +} + +// TestEncodeDocument_MatchesReferenceSnapshots pins snapshot writing against the +// reference implementation across the shapes whose storage form differs most +// from a change: deletes that survive only as successors, marks whose expand +// column is shared, counters, nested objects and several actors. +func TestEncodeDocument_MatchesReferenceSnapshots(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + for _, testCase := range []struct { + name string + build func(t *testing.T, engine *reference.Engine) []byte + }{ + { + name: "linear text history", + build: func(t *testing.T, engine *reference.Engine) []byte { + handle, err := engine.PutText(ctx, 0, "body") + require.NoError(t, err) + + for i := range 5 { + require.NoError(t, engine.SpliceText(ctx, handle, uint32(i), 0, "x")) + _, err = engine.Commit(ctx, "edit", time.Unix(int64(i+1), 0)) + require.NoError(t, err) + } + + saved, err := engine.Save(ctx) + require.NoError(t, err) + + return saved + }, + }, + { + name: "marks and unmarks", + build: func(t *testing.T, engine *reference.Engine) []byte { + handle, err := engine.PutText(ctx, 0, "body") + require.NoError(t, err) + require.NoError(t, engine.SpliceText(ctx, handle, 0, 0, "hello brave world")) + _, err = engine.Commit(ctx, "write", time.Unix(1, 0)) + require.NoError(t, err) + + require.NoError(t, engine.MarkText(ctx, handle, 0, 5, "strong", boolScalar(), "both")) + _, err = engine.Commit(ctx, "mark", time.Unix(2, 0)) + require.NoError(t, err) + + require.NoError(t, engine.MarkText(ctx, handle, 1, 3, "strong", nullScalar(), "none")) + _, err = engine.Commit(ctx, "unmark", time.Unix(3, 0)) + require.NoError(t, err) + + saved, err := engine.Save(ctx) + require.NoError(t, err) + + return saved + }, + }, + { + name: "deletes and overwrites", + build: func(t *testing.T, engine *reference.Engine) []byte { + require.NoError(t, engine.PutString(ctx, 0, "title", "first")) + require.NoError(t, engine.PutString(ctx, 0, "keep", "value")) + _, err := engine.Commit(ctx, "one", time.Unix(1, 0)) + require.NoError(t, err) + + require.NoError(t, engine.PutString(ctx, 0, "title", "second")) + _, err = engine.Commit(ctx, "two", time.Unix(2, 0)) + require.NoError(t, err) + + require.NoError(t, engine.DeleteMap(ctx, 0, "title")) + _, err = engine.Commit(ctx, "three", time.Unix(3, 0)) + require.NoError(t, err) + + saved, err := engine.Save(ctx) + require.NoError(t, err) + + return saved + }, + }, + { + name: "list with deletion and counter", + build: func(t *testing.T, engine *reference.Engine) []byte { + list, err := engine.PutObject(ctx, 0, "items", "list") + require.NoError(t, err) + require.NoError(t, engine.InsertScalar(ctx, list, 0, stringScalar("a"))) + require.NoError(t, engine.InsertScalar(ctx, list, 1, stringScalar("b"))) + require.NoError(t, engine.InsertScalar(ctx, list, 2, stringScalar("c"))) + require.NoError(t, engine.PutScalar(ctx, 0, "counter", counterScalar())) + _, err = engine.Commit(ctx, "build", time.Unix(1, 0)) + require.NoError(t, err) + + require.NoError(t, engine.DeleteSequence(ctx, list, 1)) + require.NoError(t, engine.Increment(ctx, 0, "counter", 3)) + _, err = engine.Commit(ctx, "trim", time.Unix(2, 0)) + require.NoError(t, err) + + saved, err := engine.Save(ctx) + require.NoError(t, err) + + return saved + }, + }, + { + name: "nested objects", + build: func(t *testing.T, engine *reference.Engine) []byte { + outer, err := engine.PutObject(ctx, 0, "outer", "map") + require.NoError(t, err) + + inner, err := engine.PutObject(ctx, outer, "inner", "list") + require.NoError(t, err) + require.NoError(t, engine.InsertScalar(ctx, inner, 0, stringScalar("deep"))) + + text, err := engine.PutText(ctx, outer, "note") + require.NoError(t, err) + require.NoError(t, engine.SpliceText(ctx, text, 0, 0, "nested")) + + _, err = engine.Commit(ctx, "nest", time.Unix(1, 0)) + require.NoError(t, err) + + saved, err := engine.Save(ctx) + require.NoError(t, err) + + return saved + }, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + assertSnapshotReencodes(t, testCase.build(t, newReference(t, ctx, 0x20))) + }) + } +} + +// TestEncodeDocument_MatchesReferenceConcurrentSnapshot covers a merged history, +// where several actors share the actor table and a change has more than one +// dependency. +func TestEncodeDocument_MatchesReferenceConcurrentSnapshot(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + first := newReference(t, ctx, 0x20) + + require.NoError(t, first.PutString(ctx, 0, "title", "one")) + + body, err := first.PutText(ctx, 0, "body") + require.NoError(t, err) + require.NoError(t, first.SpliceText(ctx, body, 0, 0, "hello")) + _, err = first.Commit(ctx, "first", time.Unix(1, 0)) + require.NoError(t, err) + + shared, err := first.Save(ctx) + require.NoError(t, err) + + second, err := reference.Load(ctx, shared) + require.NoError(t, err) + require.NoError(t, second.SetActor(ctx, []byte{ + 0x10, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + })) + + secondBody, _, err := second.GetObject(ctx, 0, "body") + require.NoError(t, err) + require.NoError(t, second.SpliceText(ctx, secondBody, 5, 0, " there")) + require.NoError(t, second.PutString(ctx, 0, "title", "two")) + _, err = second.Commit(ctx, "second", time.Unix(2, 0)) + require.NoError(t, err) + + secondSave, err := second.Save(ctx) + require.NoError(t, err) + + _, err = first.Merge(ctx, secondSave) + require.NoError(t, err) + + require.NoError(t, first.SpliceText(ctx, body, 0, 1, "")) + _, err = first.Commit(ctx, "third", time.Unix(3, 0)) + require.NoError(t, err) + + saved, err := first.Save(ctx) + require.NoError(t, err) + + assertSnapshotReencodes(t, saved) +} diff --git a/pkg/automerge/internal/storage/encode_document_test.go b/pkg/automerge/internal/storage/encode_document_test.go new file mode 100644 index 0000000000..4031cc87c8 --- /dev/null +++ b/pkg/automerge/internal/storage/encode_document_test.go @@ -0,0 +1,119 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package storage + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// storedOperationOrder reports the operation-set order a document chunk was +// written in, which is the order the encoder has to be given to reproduce it. +func storedOperationOrder(t *testing.T, data []byte) []OpID { + t.Helper() + + r := &reader{data: data} + + chunk, err := decodeChunk(r) + require.NoError(t, err) + require.Equal(t, ChunkDocument, chunk.kind) + + content := &reader{data: chunk.content} + + actors, err := decodeActorArray(content, true) + require.NoError(t, err) + + _, err = decodeHashArray(content, true) + require.NoError(t, err) + + changeMetadata, err := parseColumnMetadata(content, true) + require.NoError(t, err) + + operationMetadata, err := parseColumnMetadata(content, true) + require.NoError(t, err) + + _, err = readColumns(content, changeMetadata) + require.NoError(t, err) + + operationColumns, err := readColumns(content, operationMetadata) + require.NoError(t, err) + + operations, _, err := decodeOperations(operationColumns, actors, false, nil) + require.NoError(t, err) + + order := make([]OpID, len(operations)) + for i, operation := range operations { + order[i] = operation.ID + } + + return order +} + +// TestEncodeDocument_ReproducesOfficialSnapshotBytes is the byte-identity gate +// for snapshot writing. Re-encoding a snapshot the reference implementation +// wrote has to produce that same file, which pins the column layout, the actor +// and head tables, the extra payload and the chunk framing all at once. +func TestEncodeDocument_ReproducesOfficialSnapshotBytes(t *testing.T) { + t.Parallel() + + data := fixture(t, officialDocumentFixture) + + document, err := Decode(data) + require.NoError(t, err) + + encoded, err := EncodeDocument(document, storedOperationOrder(t, data)) + require.NoError(t, err) + + assert.Equal(t, data, encoded, "re-encoded snapshot must match the original bytes") +} + +// TestEncodeDocument_RoundTripsThroughDecode checks the written snapshot reads +// back as the same history even where byte identity is not the subject. +func TestEncodeDocument_RoundTripsThroughDecode(t *testing.T) { + t.Parallel() + + data := fixture(t, officialDocumentFixture) + + document, err := Decode(data) + require.NoError(t, err) + + encoded, err := EncodeDocument(document, storedOperationOrder(t, data)) + require.NoError(t, err) + + reloaded, err := Decode(encoded) + require.NoError(t, err) + + require.Len(t, reloaded.Changes, len(document.Changes)) + assert.Equal(t, document.Heads, reloaded.Heads) + assert.Equal(t, document.Actors, reloaded.Actors) + + for i := range document.Changes { + expected := &document.Changes[i] + actual := &reloaded.Changes[i] + + assert.Equal(t, expected.Hash, actual.Hash, "change %d hash", i) + assert.Equal(t, expected.Raw, actual.Raw, "change %d bytes", i) + assert.Equal(t, expected.Message, actual.Message, "change %d message", i) + assert.Equal(t, expected.Time, actual.Time, "change %d time", i) + } +} diff --git a/pkg/automerge/zz_probe_order_test.go b/pkg/automerge/zz_probe_order_test.go new file mode 100644 index 0000000000..e62be7f10c --- /dev/null +++ b/pkg/automerge/zz_probe_order_test.go @@ -0,0 +1,140 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package automerge_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/automerge/internal/reference" + "go.probo.inc/probo/pkg/automerge/internal/storage" +) + +func TestProbeDocumentOrdering(t *testing.T) { + ctx := context.Background() + + first, err := reference.New(ctx) + require.NoError(t, err) + require.NoError(t, first.SetActor(ctx, []byte{0x20, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16})) + + require.NoError(t, first.PutString(ctx, 0, "title", "one")) + + list, err := first.PutObject(ctx, 0, "items", "list") + require.NoError(t, err) + require.NoError(t, first.InsertScalar(ctx, list, 0, []byte(`{"type":"string","string":"a"}`))) + require.NoError(t, first.InsertScalar(ctx, list, 1, []byte(`{"type":"string","string":"b"}`))) + + body, err := first.PutText(ctx, 0, "body") + require.NoError(t, err) + require.NoError(t, first.SpliceText(ctx, body, 0, 0, "hello")) + _, err = first.Commit(ctx, "first", time.Unix(1, 0)) + require.NoError(t, err) + + shared, err := first.Save(ctx) + require.NoError(t, err) + + second, err := reference.Load(ctx, shared) + require.NoError(t, err) + require.NoError(t, second.SetActor(ctx, []byte{0x10, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16})) + + secondBody, _, err := second.GetObject(ctx, 0, "body") + require.NoError(t, err) + require.NoError(t, second.SpliceText(ctx, secondBody, 5, 0, " there")) + require.NoError(t, second.PutString(ctx, 0, "title", "two")) + _, err = second.Commit(ctx, "second", time.Unix(2, 0)) + require.NoError(t, err) + + secondSave, err := second.Save(ctx) + require.NoError(t, err) + + _, err = first.Merge(ctx, secondSave) + require.NoError(t, err) + + require.NoError(t, first.SpliceText(ctx, body, 0, 1, "")) + _, err = first.Commit(ctx, "third", time.Unix(3, 0)) + require.NoError(t, err) + + saved, err := first.Save(ctx) + require.NoError(t, err) + + document, err := storage.Decode(saved) + require.NoError(t, err) + + t.Logf("actors (in table order):") + + for i, actorID := range document.Actors { + t.Logf(" [%d] %s", i, actorID) + } + + t.Logf("changes (in stored order):") + + for i := range document.Changes { + change := &document.Changes[i] + t.Logf(" [%d] actor=%.4s seq=%d startOp=%d maxOp=%d deps=%v msg=%q", + i, change.Actor.String(), change.Sequence, change.StartOp, + change.MaxOp, change.DependencyIndexes, change.Message) + } + + t.Logf("operations (in stored order, reconstructed per change):") + + type row struct { + change int + op storage.Operation + } + + rows := make([]row, 0) + for i := range document.Changes { + for _, operation := range document.Changes[i].Operations { + rows = append(rows, row{change: i, op: operation}) + } + } + + for _, item := range rows { + t.Logf(" change=%d %s", item.change, formatOperation(item.op)) + } +} + +func formatOperation(operation storage.Operation) string { + object := "root" + if !operation.Object.IsRoot { + object = fmt.Sprintf("%.4s@%d", operation.Object.OpID.Actor.String(), operation.Object.OpID.Counter) + } + + key := "?" + + switch { + case operation.Key.Property != nil: + key = "prop:" + *operation.Key.Property + case operation.Key.IsHead: + key = "head" + case operation.Key.Element != nil: + key = fmt.Sprintf("elem:%.4s@%d", operation.Key.Element.Actor.String(), operation.Key.Element.Counter) + } + + return fmt.Sprintf( + "id=%.4s@%d action=%d obj=%s key=%s insert=%t succ=%v", + operation.ID.Actor.String(), operation.ID.Counter, + operation.Action, object, key, operation.Insert, operation.Successors, + ) +} From 2b76145cb949f4ebe7361117f9890ee4396fbf96 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 10 Aug 2026 16:22:57 +0000 Subject: [PATCH 04/17] Drop a scratch probe left in the encoder commit The file was used to learn the snapshot operation order and has no value now that the byte-identity tests cover it. Signed-off-by: Cursor Agent Co-authored-by: Bryan FRIMIN --- pkg/automerge/zz_probe_order_test.go | 140 --------------------------- 1 file changed, 140 deletions(-) delete mode 100644 pkg/automerge/zz_probe_order_test.go diff --git a/pkg/automerge/zz_probe_order_test.go b/pkg/automerge/zz_probe_order_test.go deleted file mode 100644 index e62be7f10c..0000000000 --- a/pkg/automerge/zz_probe_order_test.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright (c) 2026 Probo Inc . -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package automerge_test - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/require" - "go.probo.inc/probo/pkg/automerge/internal/reference" - "go.probo.inc/probo/pkg/automerge/internal/storage" -) - -func TestProbeDocumentOrdering(t *testing.T) { - ctx := context.Background() - - first, err := reference.New(ctx) - require.NoError(t, err) - require.NoError(t, first.SetActor(ctx, []byte{0x20, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16})) - - require.NoError(t, first.PutString(ctx, 0, "title", "one")) - - list, err := first.PutObject(ctx, 0, "items", "list") - require.NoError(t, err) - require.NoError(t, first.InsertScalar(ctx, list, 0, []byte(`{"type":"string","string":"a"}`))) - require.NoError(t, first.InsertScalar(ctx, list, 1, []byte(`{"type":"string","string":"b"}`))) - - body, err := first.PutText(ctx, 0, "body") - require.NoError(t, err) - require.NoError(t, first.SpliceText(ctx, body, 0, 0, "hello")) - _, err = first.Commit(ctx, "first", time.Unix(1, 0)) - require.NoError(t, err) - - shared, err := first.Save(ctx) - require.NoError(t, err) - - second, err := reference.Load(ctx, shared) - require.NoError(t, err) - require.NoError(t, second.SetActor(ctx, []byte{0x10, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16})) - - secondBody, _, err := second.GetObject(ctx, 0, "body") - require.NoError(t, err) - require.NoError(t, second.SpliceText(ctx, secondBody, 5, 0, " there")) - require.NoError(t, second.PutString(ctx, 0, "title", "two")) - _, err = second.Commit(ctx, "second", time.Unix(2, 0)) - require.NoError(t, err) - - secondSave, err := second.Save(ctx) - require.NoError(t, err) - - _, err = first.Merge(ctx, secondSave) - require.NoError(t, err) - - require.NoError(t, first.SpliceText(ctx, body, 0, 1, "")) - _, err = first.Commit(ctx, "third", time.Unix(3, 0)) - require.NoError(t, err) - - saved, err := first.Save(ctx) - require.NoError(t, err) - - document, err := storage.Decode(saved) - require.NoError(t, err) - - t.Logf("actors (in table order):") - - for i, actorID := range document.Actors { - t.Logf(" [%d] %s", i, actorID) - } - - t.Logf("changes (in stored order):") - - for i := range document.Changes { - change := &document.Changes[i] - t.Logf(" [%d] actor=%.4s seq=%d startOp=%d maxOp=%d deps=%v msg=%q", - i, change.Actor.String(), change.Sequence, change.StartOp, - change.MaxOp, change.DependencyIndexes, change.Message) - } - - t.Logf("operations (in stored order, reconstructed per change):") - - type row struct { - change int - op storage.Operation - } - - rows := make([]row, 0) - for i := range document.Changes { - for _, operation := range document.Changes[i].Operations { - rows = append(rows, row{change: i, op: operation}) - } - } - - for _, item := range rows { - t.Logf(" change=%d %s", item.change, formatOperation(item.op)) - } -} - -func formatOperation(operation storage.Operation) string { - object := "root" - if !operation.Object.IsRoot { - object = fmt.Sprintf("%.4s@%d", operation.Object.OpID.Actor.String(), operation.Object.OpID.Counter) - } - - key := "?" - - switch { - case operation.Key.Property != nil: - key = "prop:" + *operation.Key.Property - case operation.Key.IsHead: - key = "head" - case operation.Key.Element != nil: - key = fmt.Sprintf("elem:%.4s@%d", operation.Key.Element.Actor.String(), operation.Key.Element.Counter) - } - - return fmt.Sprintf( - "id=%.4s@%d action=%d obj=%s key=%s insert=%t succ=%v", - operation.ID.Actor.String(), operation.ID.Counter, - operation.Action, object, key, operation.Insert, operation.Successors, - ) -} From bcf94f0be6ca15db66544f9d716ab65765de7ace Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 10 Aug 2026 16:28:37 +0000 Subject: [PATCH 05/17] Compact the history when saving a document Saving concatenated the base with every change chunk since, so a Go history grew with each commit and never took the compacted shape the other implementations write. SaveDocument now writes the whole history as one document chunk. The operation-set order the format needs comes from the engine, which is the only place that knows it: the root map first, then every object by identifier, with a map's operations grouped by property and a sequence's in the order a reader sees, marks and tombstones included because insertions anchor to them. Deletes are left out, since a snapshot keeps them only as successors of what they removed. A change with no extra payload is stored as an empty byte string rather than null, which is what the reference writes and the last difference between the two encoders. The gate is byte identity against the reference for the same history, covering linear text, map puts and a delete, a counter increment, marks and unmarks, and text with a deletion. Each snapshot is also reloaded to confirm it carries the full history and is accepted by the reference. Signed-off-by: Cursor Agent Co-authored-by: Bryan FRIMIN --- pkg/automerge/automerge.go | 30 +++ pkg/automerge/document_save_parity_test.go | 215 ++++++++++++++++++ pkg/automerge/internal/native/document.go | 182 +++++++++++++++ pkg/automerge/internal/native/storage.go | 1 + .../internal/storage/encode_document.go | 8 +- 5 files changed, 435 insertions(+), 1 deletion(-) create mode 100644 pkg/automerge/document_save_parity_test.go create mode 100644 pkg/automerge/internal/native/document.go diff --git a/pkg/automerge/automerge.go b/pkg/automerge/automerge.go index e0ad566295..c0749d3daa 100644 --- a/pkg/automerge/automerge.go +++ b/pkg/automerge/automerge.go @@ -154,6 +154,10 @@ type ( changeApplier interface { ApplyChanges(context.Context, [][]byte) error } + + documentSaver interface { + SaveDocument(context.Context) ([]byte, error) + } ) var ( @@ -466,6 +470,32 @@ func (d *Document) Save(ctx context.Context) ([]byte, error) { return data, nil } +// SaveDocument serializes the history as a single compacted document chunk, +// the form save() produces in the Rust and JavaScript implementations. Save +// writes the history as a stream of changes instead, which stays byte-for-byte +// faithful to what was loaded but grows with every commit. +func (d *Document) SaveDocument(ctx context.Context) ([]byte, error) { + d.mu.Lock() + defer d.mu.Unlock() + + if d.closed { + return nil, ErrClosed + } + + compactor, ok := d.engine.(documentSaver) + if !ok { + // The reference engine only ever writes compacted documents. + return d.engine.Save(ctx) + } + + data, err := compactor.SaveDocument(ctx) + if err != nil { + return nil, fmt.Errorf("cannot save compacted Automerge document: %w", err) + } + + return data, nil +} + // SaveWithOptions serializes the document, choosing whether to retain orphan // changes (changes whose dependencies are missing). Retaining them, the default // for Save, preserves them across a save/load round trip so they can be resolved diff --git a/pkg/automerge/document_save_parity_test.go b/pkg/automerge/document_save_parity_test.go new file mode 100644 index 0000000000..ea6a6c0481 --- /dev/null +++ b/pkg/automerge/document_save_parity_test.go @@ -0,0 +1,215 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package automerge_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/automerge" +) + +// documentSaveScenario applies the same history to whichever engine it is given +// so a native snapshot can be held against the reference's byte for byte. +type documentSaveScenario struct { + name string + apply func(t *testing.T, ctx context.Context, document *automerge.Document) +} + +func documentSaveScenarios() []documentSaveScenario { + base := time.Unix(1786147200, 0).UTC() + + return []documentSaveScenario{ + { + name: "linear text", + apply: func(t *testing.T, ctx context.Context, document *automerge.Document) { + text, err := document.CreateText(ctx, "body") + require.NoError(t, err) + + for i := range 5 { + require.NoError(t, text.Splice(ctx, uint32(i), 0, "x")) + _, err = document.Commit(ctx, "edit", base.Add(time.Duration(i)*time.Second)) + require.NoError(t, err) + } + }, + }, + { + name: "map puts and delete", + apply: func(t *testing.T, ctx context.Context, document *automerge.Document) { + require.NoError(t, document.PutScalar(ctx, "title", + automerge.Scalar{Type: automerge.ScalarTypeString, String: "first"})) + require.NoError(t, document.PutScalar(ctx, "keep", + automerge.Scalar{Type: automerge.ScalarTypeString, String: "value"})) + _, err := document.Commit(ctx, "one", base) + require.NoError(t, err) + + require.NoError(t, document.PutScalar(ctx, "title", + automerge.Scalar{Type: automerge.ScalarTypeString, String: "second"})) + _, err = document.Commit(ctx, "two", base.Add(time.Second)) + require.NoError(t, err) + + require.NoError(t, document.Root().DeleteKey(ctx, "title")) + _, err = document.Commit(ctx, "three", base.Add(2*time.Second)) + require.NoError(t, err) + }, + }, + { + name: "counter increment", + apply: func(t *testing.T, ctx context.Context, document *automerge.Document) { + require.NoError(t, document.PutScalar(ctx, "counter", + automerge.Scalar{Type: automerge.ScalarTypeCounter, Int: 5})) + _, err := document.Commit(ctx, "create", base) + require.NoError(t, err) + + require.NoError(t, document.Root().Increment(ctx, "counter", 3)) + _, err = document.Commit(ctx, "bump", base.Add(time.Second)) + require.NoError(t, err) + }, + }, + { + name: "marks and unmarks", + apply: func(t *testing.T, ctx context.Context, document *automerge.Document) { + text, err := document.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 0, 0, "hello brave world")) + _, err = document.Commit(ctx, "write", base) + require.NoError(t, err) + + require.NoError(t, text.Mark(ctx, 0, 5, "strong", + automerge.Scalar{Type: automerge.ScalarTypeBoolean, Bool: true}, + automerge.MarkExpandBoth)) + _, err = document.Commit(ctx, "mark", base.Add(time.Second)) + require.NoError(t, err) + + require.NoError(t, text.Unmark(ctx, 1, 3, "strong", automerge.MarkExpandNone)) + _, err = document.Commit(ctx, "unmark", base.Add(2*time.Second)) + require.NoError(t, err) + }, + }, + { + name: "text with deletion", + apply: func(t *testing.T, ctx context.Context, document *automerge.Document) { + text, err := document.CreateText(ctx, "body") + require.NoError(t, err) + require.NoError(t, text.Splice(ctx, 0, 0, "hello brave world")) + _, err = document.Commit(ctx, "write", base) + require.NoError(t, err) + + require.NoError(t, text.Splice(ctx, 5, 6, "")) + _, err = document.Commit(ctx, "trim", base.Add(time.Second)) + require.NoError(t, err) + }, + }, + } +} + +// TestDocumentSave_MatchesReferenceBytes requires the compacted snapshot the +// native engine writes to be the file the reference writes for the same +// history. Byte identity is the only check that proves the operation-set order, +// the column layout and the frontier all agree. +func TestDocumentSave_MatchesReferenceBytes(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + for _, scenario := range documentSaveScenarios() { + t.Run(scenario.name, func(t *testing.T) { + t.Parallel() + + native, err := automerge.New(ctx, actor(41)) + require.NoError(t, err) + closeDocument(t, native) + + reference, err := automerge.NewReference(ctx, actor(41)) + require.NoError(t, err) + closeDocument(t, reference) + + scenario.apply(t, ctx, native) + scenario.apply(t, ctx, reference) + + expected, err := reference.Save(ctx) + require.NoError(t, err) + + actual, err := native.SaveDocument(ctx) + require.NoError(t, err) + + assert.Equal(t, expected, actual) + }) + } +} + +// TestDocumentSave_ReloadsIntoTheSameHistory checks a compacted snapshot is a +// complete history rather than only the right bytes: it must reload with every +// change reachable and be accepted by the reference. +func TestDocumentSave_ReloadsIntoTheSameHistory(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + for _, scenario := range documentSaveScenarios() { + t.Run(scenario.name, func(t *testing.T) { + t.Parallel() + + document, err := automerge.New(ctx, actor(42)) + require.NoError(t, err) + closeDocument(t, document) + + scenario.apply(t, ctx, document) + + heads, err := document.Heads(ctx) + require.NoError(t, err) + + original, err := document.ChangesSince(ctx, nil) + require.NoError(t, err) + + snapshot, err := document.SaveDocument(ctx) + require.NoError(t, err) + + reloaded, err := automerge.Load(ctx, snapshot, actor(43)) + require.NoError(t, err) + closeDocument(t, reloaded) + + reloadedHeads, err := reloaded.Heads(ctx) + require.NoError(t, err) + assert.Equal(t, heads, reloadedHeads) + + changes, err := reloaded.ChangesSince(ctx, nil) + require.NoError(t, err) + require.Len(t, changes, len(original)) + + for i := range changes { + assert.Equal(t, original[i].Hash, changes[i].Hash, "change %d", i) + assert.Equal(t, original[i].Bytes, changes[i].Bytes, "change %d bytes", i) + } + + adopted, err := automerge.LoadReference(ctx, snapshot, actor(44)) + require.NoError(t, err) + closeDocument(t, adopted) + + adoptedHeads, err := adopted.Heads(ctx) + require.NoError(t, err) + assert.Equal(t, heads, adoptedHeads) + }) + } +} diff --git a/pkg/automerge/internal/native/document.go b/pkg/automerge/internal/native/document.go new file mode 100644 index 0000000000..cdb78f0c2e --- /dev/null +++ b/pkg/automerge/internal/native/document.go @@ -0,0 +1,182 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package native + +import ( + "context" + "fmt" + "slices" + "time" +) + +// SaveDocument serializes the whole history as one compacted document chunk, +// the form save() produces in the other implementations. It replaces the +// change-by-change stream a plain save writes. +func (b *Engine) SaveDocument(ctx context.Context) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + if len(b.pending) > 0 { + if _, err := b.Commit(ctx, "", time.Time{}); err != nil { + return nil, err + } + } + + changes, ok := b.state.allChanges() + if !ok { + return nil, fmt.Errorf("cannot enumerate changes for a document save") + } + + document := &Document{Changes: make([]Change, 0, len(changes))} + for _, change := range changes { + document.Changes = append(document.Changes, *change) + } + + document.Heads = b.state.Heads() + + return EncodeDocument(document, b.state.documentOperationOrder()) +} + +// documentOperationOrder returns the operation-set order a document chunk is +// written in: the root map first, then every object in identifier order, with a +// map's operations grouped by property and a sequence's following the order a +// reader sees. Deletes are left out because a snapshot records them only as +// successors of what they removed. +func (s *State) documentOperationOrder() []OpID { + order := make([]OpID, 0, len(s.operations)) + + for _, object := range s.documentObjects() { + if object.IsRoot || isMapObject(s.operations[object.OpID].Action) { + order = append(order, s.mapObjectOrder(object)...) + + continue + } + + order = append(order, s.sequenceObjectOrder(object)...) + } + + return order +} + +// documentObjects lists the root map followed by every object the history +// creates, ordered by the identifier of the operation that made it. +func (s *State) documentObjects() []ObjectID { + objects := make([]ObjectID, 0) + + for id, operation := range s.operations { + if isObjectAction(operation.Action) { + objects = append(objects, ObjectID{OpID: id}) + } + } + + slices.SortFunc(objects, func(left, right ObjectID) int { + return left.OpID.Compare(right.OpID) + }) + + return append([]ObjectID{RootObject()}, objects...) +} + +func (s *State) mapObjectOrder(object ObjectID) []OpID { + byProperty := make(map[string][]OpID) + + for id, operation := range s.operations { + if operation.Object != object || + operation.Key.Property == nil || + operation.Action == ActionDelete { + continue + } + + property := *operation.Key.Property + byProperty[property] = append(byProperty[property], id) + } + + properties := make([]string, 0, len(byProperty)) + for property := range byProperty { + properties = append(properties, property) + } + + slices.Sort(properties) + + order := make([]OpID, 0, len(s.operations)) + + for _, property := range properties { + identifiers := byProperty[property] + + slices.SortFunc(identifiers, func(left, right OpID) int { + return left.Compare(right) + }) + + order = append(order, identifiers...) + } + + return order +} + +func (s *State) sequenceObjectOrder(object ObjectID) []OpID { + // Operations that address an element rather than create it, such as an + // overwrite, follow the element they target. + byElement := make(map[OpID][]OpID) + + for id, operation := range s.operations { + if operation.Object != object || + operation.Insert || + operation.Key.Element == nil || + operation.Action == ActionDelete { + continue + } + + element := *operation.Key.Element + byElement[element] = append(byElement[element], id) + } + + for element := range byElement { + slices.SortFunc(byElement[element], func(left, right OpID) int { + return left.Compare(right) + }) + } + + elements := s.insertOrder(object.OpID) + order := make([]OpID, 0, len(elements)) + + for _, element := range elements { + if operation, ok := s.operations[element]; ok && operation.Action != ActionDelete { + order = append(order, element) + } + + order = append(order, byElement[element]...) + } + + return order +} + +func isObjectAction(action Action) bool { + switch action { + case ActionMakeMap, ActionMakeList, ActionMakeText, ActionMakeTable: + return true + default: + return false + } +} + +func isMapObject(action Action) bool { + return action == ActionMakeMap || action == ActionMakeTable +} diff --git a/pkg/automerge/internal/native/storage.go b/pkg/automerge/internal/native/storage.go index 08d51a917c..7d6b3b0a06 100644 --- a/pkg/automerge/internal/native/storage.go +++ b/pkg/automerge/internal/native/storage.go @@ -30,6 +30,7 @@ var ( DecodePartial = internalstorage.DecodePartial DecodeIncremental = internalstorage.DecodeIncremental EncodeChange = internalstorage.EncodeChange + EncodeDocument = internalstorage.EncodeDocument ) func deflate(data []byte) ([]byte, error) { return internalstorage.Deflate(data) } diff --git a/pkg/automerge/internal/storage/encode_document.go b/pkg/automerge/internal/storage/encode_document.go index 7f5f3a43c4..cf538dbe2c 100644 --- a/pkg/automerge/internal/storage/encode_document.go +++ b/pkg/automerge/internal/storage/encode_document.go @@ -409,7 +409,13 @@ func changeExtra(change *Change) *Scalar { return &Scalar{Type: ScalarBytes, Bytes: change.ExtraBytes} } - return change.Extra + if change.Extra != nil { + return change.Extra + } + + // The payload is a byte string even when a change carries none, so an absent + // one is empty rather than null. + return &Scalar{Type: ScalarBytes} } func encodeDocumentOperationColumns( From 1f4fa866b71f92c3ed0432502778de5a4c6f0735 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 10 Aug 2026 16:30:34 +0000 Subject: [PATCH 06/17] Record the snapshot encoder in the parity plan Note that snapshot writing exists and how it is gated, that Save still writes the change stream, and the two storage differences that remain: the save_no_orphans shim disabling compression where Rust does not, and unknown columns being unpreservable on the dependency-tolerant load path that rebuilds its base from applied changes. Signed-off-by: Cursor Agent Co-authored-by: Bryan FRIMIN --- pkg/automerge/PARITY_PLAN.md | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/pkg/automerge/PARITY_PLAN.md b/pkg/automerge/PARITY_PLAN.md index e203ac0b17..f911b520a6 100644 --- a/pkg/automerge/PARITY_PLAN.md +++ b/pkg/automerge/PARITY_PLAN.md @@ -79,11 +79,32 @@ remains, because it asserts the exact incremental patch stream produced by Rust' patch log across isolate/integrate, which native's state-comparison diff does not reproduce. -Orphan retention across save/load is now covered without a full document-chunk -encoder: the native save appends retained orphan changes and the native load -falls back to a dependency-tolerant path that applies every change whose -dependencies are satisfiable and queues the rest (still failing a load that can -apply nothing, so a bare orphan without a base is rejected as before). +Orphan retention across save/load is covered: the native save appends retained +orphan changes and the native load falls back to a dependency-tolerant path that +applies every change whose dependencies are satisfiable and queues the rest +(still failing a load that can apply nothing, so a bare orphan without a base is +rejected as before). + +Snapshot writing is now implemented. `SaveDocument` compacts the whole history +into one document chunk and is gated on byte identity with the reference for the +same history across linear text, map puts and deletes, counters, marks and +unmarks, and text deletion; re-encoding a reference-written snapshot reproduces +that file exactly for the official fixture and for reference histories covering +nested objects, lists and a merged multi-actor graph. Compaction matters for +size as well as parity: a 200-commit typing history is 21816 bytes as a change +stream and 400 bytes compacted. + +`Save` still writes the change stream, so nothing that depends on preserving the +loaded bytes changes. Switching the default is a separate decision because +`SaveIncremental` tracks its cursor against the appended change list. + +Two storage differences remain. The reference `am_save_no_orphans` shim sets +`deflate: false`, while Rust's own `SaveOptions::default()` compresses, so +`SaveWithOptions(false)` diverges from the shim rather than from Rust; correcting +it means rebuilding the WASM oracle. Unknown columns survive a normal load +because the loaded bytes are kept verbatim, and `EncodeDocument` writes them back +to the table they came from, but the dependency-tolerant load path rebuilds its +base from applied change bytes and cannot preserve a wrapper it did not retain. The V2 sync internals are now covered: the empty-message codec round-trips to the reference wire bytes, and Bloom false-positive recovery is verified on both @@ -191,6 +212,7 @@ These do not block Go/Rust engine parity but remain tracked: Implement and verify: - complete document, change, compressed-change, and bundle parsing; +- canonical document-chunk encoding, byte-identical to the reference (done); - canonical encoding for every operation and scalar column; - expanded/compressed change byte and hash stability; - 64-bit object IDs and actor tables referenced only by deletes; From a555147a38f5e58fb205a9e6fc7003b8c72bcfb8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 07:01:47 +0000 Subject: [PATCH 07/17] Degrade incremental reads instead of wedging ChangesSince failed the whole read when any change in the frontier's ancestry was unreachable, and because every collaboration request recomputes it, one unreachable change wedged the document permanently. The decoder fix removed the cause, but a single bad change should never again be able to take a document offline. changesSince now returns the consistent, replayable prefix it can produce rather than nothing: a change is emitted only once all of its ancestors are, so a caller never receives a change whose dependency it was not also given. The second return becomes a completeness signal rather than a hard failure. Sync keeps using it to fall back to sending a whole document, which is unchanged and still optimal, while the incremental read returns the prefix regardless, since a change that cannot be emitted has no bytes to return in any case. A branched regression drops the ancestor of one head and asserts the intact branch still comes back as a dependency-ordered prefix while completeness reports false and the engine method does not error. Signed-off-by: Cursor Agent Co-authored-by: Bryan FRIMIN --- .../internal/native/frontier_test.go | 110 +++++++++++++++++- pkg/automerge/internal/native/history.go | 11 +- pkg/automerge/internal/native/state.go | 64 +++++++--- 3 files changed, 163 insertions(+), 22 deletions(-) diff --git a/pkg/automerge/internal/native/frontier_test.go b/pkg/automerge/internal/native/frontier_test.go index 3c0ddb51d4..a67ce79802 100644 --- a/pkg/automerge/internal/native/frontier_test.go +++ b/pkg/automerge/internal/native/frontier_test.go @@ -99,6 +99,105 @@ func TestNewStateFromDocument_RebuildsInconsistentFrontier(t *testing.T) { assert.True(t, ok, "changesSince(own heads) must succeed after rebuild") } +// TestChangesSince_DegradesToReachablePrefix is the regression for the wedge +// where one unreachable ancestor failed the whole read. A branched history has +// an intact branch and a branch whose middle change is removed; the intact +// branch must still come back as a consistent, replayable prefix while the +// broken branch is dropped, and completeness must report false. +func TestChangesSince_DegradesToReachablePrefix(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + actorBytes := func(id byte) []byte { + return []byte{id, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + } + + // Shared base commit. + base, err := NewEngine(ctx) + require.NoError(t, err) + require.NoError(t, base.SetActor(ctx, actorBytes(0x10))) + + handle, err := base.PutText(ctx, 0, "body") + require.NoError(t, err) + require.NoError(t, base.SpliceText(ctx, handle, 0, 0, "a")) + _, err = base.Commit(ctx, "base", time.Unix(0, 0)) + require.NoError(t, err) + + shared, err := base.Save(ctx) + require.NoError(t, err) + + // A branch two commits deep, authored by a second actor. + deep, err := LoadEngine(ctx, shared) + require.NoError(t, err) + require.NoError(t, deep.SetActor(ctx, actorBytes(0x20))) + + deepHandle, _, err := deep.GetObject(ctx, 0, "body") + require.NoError(t, err) + require.NoError(t, deep.SpliceText(ctx, deepHandle, 1, 0, "b")) + _, err = deep.Commit(ctx, "deep-1", time.Unix(1, 0)) + require.NoError(t, err) + require.NoError(t, deep.SpliceText(ctx, deepHandle, 2, 0, "c")) + _, err = deep.Commit(ctx, "deep-2", time.Unix(2, 0)) + require.NoError(t, err) + + deepSave, err := deep.Save(ctx) + require.NoError(t, err) + + // The base adds its own branch commit, then merges the deep branch, so the + // frontier holds two heads: the base branch and the deep branch. + require.NoError(t, base.SpliceText(ctx, handle, 1, 0, "z")) + _, err = base.Commit(ctx, "base-2", time.Unix(3, 0)) + require.NoError(t, err) + _, err = base.Merge(ctx, deepSave) + require.NoError(t, err) + + all, complete := base.state.changesSince(nil) + require.True(t, complete) + require.Len(t, all, 4) + + // Remove the deep branch's first change, the ancestor of its head. + deepActor, err := NewActorID(actorBytes(0x20)) + require.NoError(t, err) + + var removed ChangeHash + + for hash, change := range base.state.changes { + if change.Actor == deepActor && change.Sequence == 1 { + removed = hash + } + } + + delete(base.state.changes, removed) + + changes, complete := base.state.changesSince(nil) + assert.False(t, complete, "the broken branch leaves the walk incomplete") + assert.Len(t, changes, 2, "the intact branch is still emitted as a prefix") + + for i, change := range changes { + require.NotNil(t, change.Hash) + assert.NotEqual(t, removed, *change.Hash) + + // Every emitted change's dependencies precede it, so the prefix replays. + for _, dependency := range change.Dependencies { + found := false + for _, earlier := range changes[:i] { + if earlier.Hash != nil && *earlier.Hash == dependency { + found = true + } + } + + assert.True(t, found, "dependency of an emitted change must precede it") + } + } + + // The engine method must not wedge: it returns the reachable prefix. + raw, hashes, err := base.ChangesSince(ctx, nil) + require.NoError(t, err) + assert.Len(t, raw, 2) + assert.Len(t, hashes, 2) +} + // TestChangesSince_ToleratesUnknownBaseline mirrors Rust's get_changes, which // takes have_deps by value and never errors: an unknown baseline excludes // nothing and the full history is returned. @@ -117,8 +216,9 @@ func TestChangesSince_ToleratesUnknownBaseline(t *testing.T) { } // TestChangesSince_ToleratesFrontierWithMissingChange guards the frontier walk: -// a head recorded without a retrievable change contributes nothing and must not -// abort the incremental computation. +// a head recorded without a retrievable change must not abort the computation. +// The reachable changes are still returned so an incremental read keeps working, +// and completeness reports false so sync knows to fall back to a full document. func TestChangesSince_ToleratesFrontierWithMissingChange(t *testing.T) { t.Parallel() @@ -129,7 +229,7 @@ func TestChangesSince_ToleratesFrontierWithMissingChange(t *testing.T) { phantom[0] = 0xEF backend.state.heads[phantom] = struct{}{} - changes, ok := backend.state.changesSince(nil) - require.True(t, ok) - assert.Len(t, changes, 2) + changes, complete := backend.state.changesSince(nil) + assert.False(t, complete, "an unretrievable head leaves the walk incomplete") + assert.Len(t, changes, 2, "the reachable changes are still returned") } diff --git a/pkg/automerge/internal/native/history.go b/pkg/automerge/internal/native/history.go index 404ff65452..0f2685d911 100644 --- a/pkg/automerge/internal/native/history.go +++ b/pkg/automerge/internal/native/history.go @@ -109,10 +109,13 @@ func (b *Engine) ChangesSince( knownHeads[i] = ChangeHash(head) } - changes, ok := b.state.changesSince(knownHeads) - if !ok { - return nil, nil, fmt.Errorf("cannot compute changes from unknown heads") - } + // A change in the frontier's ancestry may occasionally be unreachable, for + // example after a merge that rebuilt the graph. changesSince then returns the + // consistent, replayable prefix it can produce rather than nothing: reporting + // every change that can be emitted keeps collaboration alive, where failing + // the whole read would wedge the document on every request. A change that is + // dropped has no bytes to return in any case. + changes, _ := b.state.changesSince(knownHeads) raw := make([][]byte, len(changes)) diff --git a/pkg/automerge/internal/native/state.go b/pkg/automerge/internal/native/state.go index 8596e5b788..6166ed9234 100644 --- a/pkg/automerge/internal/native/state.go +++ b/pkg/automerge/internal/native/state.go @@ -563,61 +563,99 @@ func (s *State) hasDependencies(change *Change) bool { return true } +// changesSince returns the changes reachable from the current frontier that the +// baseline heads do not already cover, in dependency order, ready to replay. +// +// The result is always a consistent prefix: a change is emitted only once every +// one of its ancestors has been emitted or is already known to the baseline, so +// a caller never receives a change whose dependency it was not also given. The +// second return reports whether that prefix is complete. It is false when some +// change in the frontier's ancestry could not be produced, either because it is +// absent from the graph or because its original bytes are unavailable. +// +// Completeness is a signal, not a gate. Sync uses it to fall back to sending a +// whole document, which reproduces changes even when the in-memory graph is +// inconsistent. Incremental reads use the prefix regardless, because returning +// every change that can be produced keeps a document usable where failing the +// whole read would wedge it: a change that cannot be emitted has no bytes to +// return anyway. func (s *State) changesSince(heads []ChangeHash) ([]*Change, bool) { known := s.changeClosure(heads) + const ( + visiting = iota + reachable + unreachable + ) + ordered := make([]*Change, 0) - visited := make(map[ChangeHash]struct{}) + status := make(map[ChangeHash]int) var visit func(ChangeHash) bool visit = func(hash ChangeHash) bool { - if _, ok := visited[hash]; ok { - return true + if state, ok := status[hash]; ok { + // A change still on the stack cannot be depended upon to be complete + // yet, but treating the cycle edge as reachable avoids excluding the + // whole branch over a graph that should never contain a cycle anyway. + return state != unreachable } - visited[hash] = struct{}{} - // The baseline closure is transitively closed, so everything below a change - // the peer already holds is also held. Descending would add nothing and - // would fail the whole read if any ancestor were ever unavailable. + // the peer already holds is also held and need not be walked. if _, ok := known[hash]; ok { + status[hash] = reachable + return true } + status[hash] = visiting + change, ok := s.changes[hash] if !ok { + status[hash] = unreachable + return false } + complete := true + for _, dependency := range change.Dependencies { if !visit(dependency) { - return false + complete = false } } - if len(change.Raw) == 0 { + // A change is emittable only when every ancestor is, so the result stays a + // replayable prefix, and only when its bytes exist to be returned. + if !complete || len(change.Raw) == 0 { + status[hash] = unreachable + return false } ordered = append(ordered, change) + status[hash] = reachable return true } + complete := true + for _, head := range s.Heads() { - // A frontier head whose change is not retrievable contributes nothing and - // must not abort the incremental computation for the whole document. + // A frontier head whose change is not retrievable contributes nothing. if _, ok := s.changes[head]; !ok { + complete = false + continue } if !visit(head) { - return nil, false + complete = false } } - return ordered, true + return ordered, complete } func (s *State) allChanges() ([]*Change, bool) { From 4bce2acb7a1cb29d2d6d06ab1fe4af0550f3bbe1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 07:12:33 +0000 Subject: [PATCH 08/17] Compact by default when saving Save wrote the history as a stream of change chunks, a Go-specific deviation that grew without bound and left the document-chunk encoder exercised only by tests. It now writes the compacted document chunk the Rust and JavaScript save() produce, so a long history is stored in a fraction of the space and Go finally matches the reference save format. The whole history becomes one document chunk, followed by any retained orphan changes as trailing change chunks so nothing an orphan carried is lost. Columns above a size threshold are DEFLATE-compressed, which the decoder already reverses. When a history cannot be compacted, while isolated or when the change graph is inconsistent, Save falls back to the faithful change stream that preserves the loaded bytes verbatim, so a document is never rewritten in a lossy way. The incremental cursor is left at the end exactly as before, so SaveIncremental is unaffected. Because Save now compacts, the separate SaveDocument method it briefly had is redundant and is removed; its parity gate now covers Save. The conformance merge check no longer asserts the saved bytes embed a change verbatim, which was the old stream behavior, and instead requires the reloaded snapshot to reproduce the same frontier. Compressed columns are not byte-identical to the reference because the DEFLATE implementations differ, so the byte-identity tests stay on histories below the threshold; above it both files load equally. Signed-off-by: Cursor Agent Co-authored-by: Bryan FRIMIN --- pkg/automerge/PARITY_PLAN.md | 47 ++++++++----- pkg/automerge/automerge.go | 30 -------- pkg/automerge/conformance_test.go | 14 +++- pkg/automerge/document_save_parity_test.go | 4 +- pkg/automerge/internal/native/document.go | 51 ++++++++------ pkg/automerge/internal/native/engine.go | 36 ++++++++-- .../internal/storage/encode_document.go | 68 ++++++++++++++++--- .../storage/encode_document_reference_test.go | 2 +- .../internal/storage/encode_document_test.go | 4 +- 9 files changed, 166 insertions(+), 90 deletions(-) diff --git a/pkg/automerge/PARITY_PLAN.md b/pkg/automerge/PARITY_PLAN.md index f911b520a6..0f67025544 100644 --- a/pkg/automerge/PARITY_PLAN.md +++ b/pkg/automerge/PARITY_PLAN.md @@ -85,26 +85,39 @@ applies every change whose dependencies are satisfiable and queues the rest (still failing a load that can apply nothing, so a bare orphan without a base is rejected as before). -Snapshot writing is now implemented. `SaveDocument` compacts the whole history -into one document chunk and is gated on byte identity with the reference for the -same history across linear text, map puts and deletes, counters, marks and -unmarks, and text deletion; re-encoding a reference-written snapshot reproduces -that file exactly for the official fixture and for reference histories covering -nested objects, lists and a merged multi-actor graph. Compaction matters for -size as well as parity: a 200-commit typing history is 21816 bytes as a change -stream and 400 bytes compacted. - -`Save` still writes the change stream, so nothing that depends on preserving the -loaded bytes changes. Switching the default is a separate decision because -`SaveIncremental` tracks its cursor against the appended change list. - -Two storage differences remain. The reference `am_save_no_orphans` shim sets +Snapshot writing is implemented and is now what `Save` produces, matching the +Rust and JavaScript `save()` semantics. It compacts the whole history into one +document chunk, followed by any retained orphan changes as trailing change +chunks, and DEFLATEs individual columns above a size threshold. It is gated on +byte identity with the reference for the same history across linear text, map +puts and deletes, counters, marks and unmarks, and text deletion; re-encoding a +reference-written snapshot reproduces that file exactly for the official fixture +and for reference histories covering nested objects, lists and a merged +multi-actor graph. Compaction matters for size as well as parity: a 200-commit +typing history is 21816 bytes as the old change stream and 400 bytes compacted. + +`Save` falls back to the faithful change stream (the loaded base plus each change +chunk since) when a history cannot be compacted: while isolated, or when the +change graph is not internally consistent. That stream is also what preserves +loaded bytes verbatim, so a document is never rewritten in a lossy way when it +cannot be safely compacted. `SaveIncremental` is unaffected: `Save` leaves the +incremental cursor at the end exactly as the stream save did. + +Compressed columns are not byte-identical to the reference because the DEFLATE +implementations differ, so the byte-identity gate holds only for histories small +enough that no column crosses the threshold; above it both files are valid and +load equally, and compression is a size optimization the decoder reverses on the +way back in. + +Two smaller differences remain. The reference `am_save_no_orphans` shim sets `deflate: false`, while Rust's own `SaveOptions::default()` compresses, so `SaveWithOptions(false)` diverges from the shim rather than from Rust; correcting it means rebuilding the WASM oracle. Unknown columns survive a normal load because the loaded bytes are kept verbatim, and `EncodeDocument` writes them back -to the table they came from, but the dependency-tolerant load path rebuilds its -base from applied change bytes and cannot preserve a wrapper it did not retain. +to the table they came from when a document is re-encoded unmodified, but they +cannot be carried across a compaction of a mutated history because their rows no +longer line up with the recomputed columns; this matches Rust, which also drops +them across re-serialization. The V2 sync internals are now covered: the empty-message codec round-trips to the reference wire bytes, and Bloom false-positive recovery is verified on both @@ -212,7 +225,7 @@ These do not block Go/Rust engine parity but remain tracked: Implement and verify: - complete document, change, compressed-change, and bundle parsing; -- canonical document-chunk encoding, byte-identical to the reference (done); +- canonical document-chunk encoding, what Save now writes (done); - canonical encoding for every operation and scalar column; - expanded/compressed change byte and hash stability; - 64-bit object IDs and actor tables referenced only by deletes; diff --git a/pkg/automerge/automerge.go b/pkg/automerge/automerge.go index c0749d3daa..e0ad566295 100644 --- a/pkg/automerge/automerge.go +++ b/pkg/automerge/automerge.go @@ -154,10 +154,6 @@ type ( changeApplier interface { ApplyChanges(context.Context, [][]byte) error } - - documentSaver interface { - SaveDocument(context.Context) ([]byte, error) - } ) var ( @@ -470,32 +466,6 @@ func (d *Document) Save(ctx context.Context) ([]byte, error) { return data, nil } -// SaveDocument serializes the history as a single compacted document chunk, -// the form save() produces in the Rust and JavaScript implementations. Save -// writes the history as a stream of changes instead, which stays byte-for-byte -// faithful to what was loaded but grows with every commit. -func (d *Document) SaveDocument(ctx context.Context) ([]byte, error) { - d.mu.Lock() - defer d.mu.Unlock() - - if d.closed { - return nil, ErrClosed - } - - compactor, ok := d.engine.(documentSaver) - if !ok { - // The reference engine only ever writes compacted documents. - return d.engine.Save(ctx) - } - - data, err := compactor.SaveDocument(ctx) - if err != nil { - return nil, fmt.Errorf("cannot save compacted Automerge document: %w", err) - } - - return data, nil -} - // SaveWithOptions serializes the document, choosing whether to retain orphan // changes (changes whose dependencies are missing). Retaining them, the default // for Save, preserves them across a save/load round trip so they can be resolved diff --git a/pkg/automerge/conformance_test.go b/pkg/automerge/conformance_test.go index 64a5d52aea..25f9a4372a 100644 --- a/pkg/automerge/conformance_test.go +++ b/pkg/automerge/conformance_test.go @@ -501,9 +501,21 @@ func TestConformance_NativeConcurrentChangesConverge(t *testing.T) { require.NoError(t, err) _, err = backend.Merge(context.Background(), rawChanges[1]) require.NoError(t, err) + + before, err := backend.Heads(context.Background()) + require.NoError(t, err) + saved, err := backend.Save(context.Background()) require.NoError(t, err) - assert.True(t, bytes.Contains(saved, rawChanges[1])) + + // Save now writes a compacted document rather than embedding change bytes, so + // the guarantee is that reloading it reproduces the same frontier. + reloaded, err := native.LoadEngine(context.Background(), saved) + require.NoError(t, err) + + after, err := reloaded.Heads(context.Background()) + require.NoError(t, err) + assert.Equal(t, before, after) } func TestConformance_NativeSyncMessageRoundTrip(t *testing.T) { diff --git a/pkg/automerge/document_save_parity_test.go b/pkg/automerge/document_save_parity_test.go index ea6a6c0481..5365021f4a 100644 --- a/pkg/automerge/document_save_parity_test.go +++ b/pkg/automerge/document_save_parity_test.go @@ -151,7 +151,7 @@ func TestDocumentSave_MatchesReferenceBytes(t *testing.T) { expected, err := reference.Save(ctx) require.NoError(t, err) - actual, err := native.SaveDocument(ctx) + actual, err := native.Save(ctx) require.NoError(t, err) assert.Equal(t, expected, actual) @@ -183,7 +183,7 @@ func TestDocumentSave_ReloadsIntoTheSameHistory(t *testing.T) { original, err := document.ChangesSince(ctx, nil) require.NoError(t, err) - snapshot, err := document.SaveDocument(ctx) + snapshot, err := document.Save(ctx) require.NoError(t, err) reloaded, err := automerge.Load(ctx, snapshot, actor(43)) diff --git a/pkg/automerge/internal/native/document.go b/pkg/automerge/internal/native/document.go index cdb78f0c2e..19dfd8e2e4 100644 --- a/pkg/automerge/internal/native/document.go +++ b/pkg/automerge/internal/native/document.go @@ -20,40 +20,47 @@ package native -import ( - "context" - "fmt" - "slices" - "time" -) - -// SaveDocument serializes the whole history as one compacted document chunk, -// the form save() produces in the other implementations. It replaces the -// change-by-change stream a plain save writes. -func (b *Engine) SaveDocument(ctx context.Context) ([]byte, error) { - if err := ctx.Err(); err != nil { - return nil, err - } +import "slices" - if len(b.pending) > 0 { - if _, err := b.Commit(ctx, "", time.Time{}); err != nil { - return nil, err - } +// compact serializes the whole history as one document chunk, the form save() +// produces in the other implementations, followed by any retained orphan changes +// as trailing change chunks (which is how a snapshot carries changes it cannot +// place in the operation set because their dependencies are missing). +// +// It reports ok=false, rather than an error, when the history cannot be +// compacted: while isolated, where the pinned view is not the whole history, or +// when the change graph is not internally consistent. The caller then falls back +// to the faithful change stream, which preserves every byte that was loaded. +func (b *Engine) compact(retainOrphans, deflate bool) ([]byte, bool, error) { + if b.isolationActive { + return nil, false, nil } changes, ok := b.state.allChanges() if !ok { - return nil, fmt.Errorf("cannot enumerate changes for a document save") + return nil, false, nil } - document := &Document{Changes: make([]Change, 0, len(changes))} + document := &Document{ + Changes: make([]Change, 0, len(changes)), + Heads: b.state.Heads(), + } for _, change := range changes { document.Changes = append(document.Changes, *change) } - document.Heads = b.state.Heads() + data, err := EncodeDocument(document, b.state.documentOperationOrder(), deflate) + if err != nil { + return nil, false, err + } + + if retainOrphans { + for _, change := range orderedQueuedChanges(b.queuedChanges) { + data = append(data, maybeCompressChangeChunk(change.Raw, deflate)...) + } + } - return EncodeDocument(document, b.state.documentOperationOrder()) + return data, true, nil } // documentOperationOrder returns the operation-set order a document chunk is diff --git a/pkg/automerge/internal/native/engine.go b/pkg/automerge/internal/native/engine.go index 06e1bfdd70..4d475164c8 100644 --- a/pkg/automerge/internal/native/engine.go +++ b/pkg/automerge/internal/native/engine.go @@ -287,20 +287,24 @@ func (b *Engine) Close(context.Context) error { return nil } +// Save serializes the whole history as one compacted document chunk, the form +// save() produces in the Rust and JavaScript implementations. It replaces the +// change-by-change stream Go used to write, which grew without bound as a +// history accumulated commits. func (b *Engine) Save(ctx context.Context) ([]byte, error) { return b.save(ctx, true, true) } -// SaveNoCompress serializes the document without DEFLATE-compressing any change -// chunks, mirroring Rust's AutoCommit::save_nocompress. +// SaveNoCompress serializes the document without DEFLATE-compressing any trailing +// change chunks, mirroring Rust's AutoCommit::save_nocompress. func (b *Engine) SaveNoCompress(ctx context.Context) ([]byte, error) { return b.save(ctx, true, false) } -// SaveWithOptions serializes the document, optionally appending retained orphan -// changes (queued changes whose dependencies are still missing) so they survive -// a save/load round trip. It mirrors the Rust SaveOptions.retain_orphans flag; -// the reference retains orphans by default. +// SaveWithOptions serializes the document, choosing whether to keep retained +// orphan changes (queued changes whose dependencies are still missing) so they +// survive a save/load round trip. It mirrors the Rust SaveOptions.retain_orphans +// flag; the reference retains orphans by default. func (b *Engine) SaveWithOptions( ctx context.Context, retainOrphans bool, @@ -319,6 +323,26 @@ func (b *Engine) save( } } + // A compacted document chunk is the form every other implementation writes + // and is dramatically smaller than the change stream for a long history. It + // leaves the incremental cursor at the end, exactly as the stream save did, + // so a following SaveIncremental still emits only later changes. + if data, ok, err := b.compact(retainOrphans, deflate); err != nil { + return nil, err + } else if ok { + b.saveCursor = len(b.appended) + + return data, nil + } + + return b.streamSave(retainOrphans, deflate) +} + +// streamSave serializes the history as the loaded base followed by each change +// chunk since. It preserves the loaded bytes verbatim, including columns this +// version does not understand, and is the fallback when a history cannot be +// compacted (while isolated, or when the change graph is inconsistent). +func (b *Engine) streamSave(retainOrphans, deflate bool) ([]byte, error) { total := len(b.base) for _, change := range b.appended { total += len(change) diff --git a/pkg/automerge/internal/storage/encode_document.go b/pkg/automerge/internal/storage/encode_document.go index cf538dbe2c..72ef49a151 100644 --- a/pkg/automerge/internal/storage/encode_document.go +++ b/pkg/automerge/internal/storage/encode_document.go @@ -63,7 +63,15 @@ func assembleChunk(kind ChunkType, body []byte) []byte { // object the order a reader would see. The caller owns that sequence because it // requires the sequence state only the engine maintains. Operations named by // order must exist in the history, and deletes must be left out of it. -func EncodeDocument(document *Document, order []OpID) ([]byte, error) { +// +// compress DEFLATEs individual columns above a size threshold, which is what +// save() does and save_nocompress() does not. The compressed bytes are not +// byte-identical to the reference because the DEFLATE implementations differ, so +// byte identity only holds for histories small enough that no column crosses the +// threshold; compression is a size optimization, and every column round-trips +// because the decoder inflates any column whose specification carries the +// compressed bit. +func EncodeDocument(document *Document, order []OpID, compress bool) ([]byte, error) { changes, err := documentChangeOrder(document) if err != nil { return nil, err @@ -96,12 +104,12 @@ func EncodeDocument(document *Document, order []OpID) ([]byte, error) { return nil, err } - changeColumns = sortColumns( + changeColumns = compressColumns(sortColumns( append(changeColumns, retainedColumns(document, changeColumnSpecifications)...), - ) - operationColumns = sortColumns( + ), compress) + operationColumns = compressColumns(sortColumns( append(operationColumns, retainedColumns(document, operationColumnSpecifications)...), - ) + ), compress) var body []byte @@ -571,14 +579,19 @@ func retainedColumns(document *Document, known []uint32) []encodedColumn { return retained } -// sortColumns puts columns in the strictly ascending specification order a -// reader requires, which both the metadata and the data must follow. +// sortColumns puts columns in the strictly ascending order a reader requires, +// which both the metadata and the data must follow. Ordering is by the +// normalized specification so a column keeps its place whether or not it carries +// the compressed bit. func sortColumns(columns []encodedColumn) []encodedColumn { slices.SortFunc(columns, func(left, right encodedColumn) int { + leftSpec := left.specification &^ compressedColumnBit + rightSpec := right.specification &^ compressedColumnBit + switch { - case left.specification < right.specification: + case leftSpec < rightSpec: return -1 - case left.specification > right.specification: + case leftSpec > rightSpec: return 1 default: return 0 @@ -588,6 +601,43 @@ func sortColumns(columns []encodedColumn) []encodedColumn { return columns } +// compressedColumnBit marks a column whose data is DEFLATE-compressed. A reader +// strips it to recover the logical specification and inflates the data. +const compressedColumnBit = 8 + +// columnDeflateMinSize is the smallest column worth compressing, matching the +// change-chunk threshold. Below it, compression tends to grow the data. +const columnDeflateMinSize = 250 + +// compressColumns DEFLATEs each column whose data is large enough to benefit, +// marking it with the compressed bit. A column that is already compressed (a +// retained unknown column) or that does not shrink is left untouched, so the +// result is never larger than the input. +func compressColumns(columns []encodedColumn, compress bool) []encodedColumn { + if !compress { + return columns + } + + for i := range columns { + column := &columns[i] + + if column.specification&compressedColumnBit != 0 || + len(column.data) < columnDeflateMinSize { + continue + } + + deflated, err := deflate(column.data) + if err != nil || len(deflated) >= len(column.data) { + continue + } + + column.specification |= compressedColumnBit + column.data = deflated + } + + return columns +} + func withData(columns []encodedColumn) []encodedColumn { filtered := make([]encodedColumn, 0, len(columns)) diff --git a/pkg/automerge/internal/storage/encode_document_reference_test.go b/pkg/automerge/internal/storage/encode_document_reference_test.go index 0a59b41cd0..58e1c7ea53 100644 --- a/pkg/automerge/internal/storage/encode_document_reference_test.go +++ b/pkg/automerge/internal/storage/encode_document_reference_test.go @@ -58,7 +58,7 @@ func assertSnapshotReencodes(t *testing.T, saved []byte) { document, err := Decode(saved) require.NoError(t, err) - encoded, err := EncodeDocument(document, storedOperationOrder(t, saved)) + encoded, err := EncodeDocument(document, storedOperationOrder(t, saved), true) require.NoError(t, err) assert.Equal(t, saved, encoded) diff --git a/pkg/automerge/internal/storage/encode_document_test.go b/pkg/automerge/internal/storage/encode_document_test.go index 4031cc87c8..97d08fecb7 100644 --- a/pkg/automerge/internal/storage/encode_document_test.go +++ b/pkg/automerge/internal/storage/encode_document_test.go @@ -81,7 +81,7 @@ func TestEncodeDocument_ReproducesOfficialSnapshotBytes(t *testing.T) { document, err := Decode(data) require.NoError(t, err) - encoded, err := EncodeDocument(document, storedOperationOrder(t, data)) + encoded, err := EncodeDocument(document, storedOperationOrder(t, data), true) require.NoError(t, err) assert.Equal(t, data, encoded, "re-encoded snapshot must match the original bytes") @@ -97,7 +97,7 @@ func TestEncodeDocument_RoundTripsThroughDecode(t *testing.T) { document, err := Decode(data) require.NoError(t, err) - encoded, err := EncodeDocument(document, storedOperationOrder(t, data)) + encoded, err := EncodeDocument(document, storedOperationOrder(t, data), true) require.NoError(t, err) reloaded, err := Decode(encoded) From 99020870dd3e46a0e3b8b3f554571a0f9988ec47 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 07:21:49 +0000 Subject: [PATCH 09/17] Anchor split blocks against mark boundaries Marking text with an out-of-range end boundary leaves a dangling begin, and the spans it produced still diverged from the reference on about three percent of randomized error-path scenarios. The cause was not span computation but authoring: SplitBlock built its insertion anchor directly from the preceding element, while Splice resolved the anchor against neighbouring mark boundaries first. A block marker lives in the same unified rich-text sequence as text, so a block inserted next to a dangling begin has to land on the same side of it that text would. Anchoring at the raw element instead put the block on the wrong side, and every later insertion anchored after that block inherited the mistake, so the marks the following text carried diverged in both directions: a mark dropped where two dangling begins shared an anchor, or a mark leaking past a block. SplitBlock now resolves its anchor through the same insert query as Splice. TestRustText_DanglingMarkBoundaries grows to eleven delta-debugged reproducers, and a new randomized gate compares marked spans run for run against the reference across two thousand scenarios that include out-of-range boundaries and every expand mode. A wider sweep of twenty-four thousand scenarios across six seeds found no divergence. Signed-off-by: Cursor Agent Co-authored-by: Bryan FRIMIN --- pkg/automerge/PARITY_PLAN.md | 39 ++--- pkg/automerge/internal/native/rich_text.go | 7 + pkg/automerge/marks_dangling_parity_test.go | 161 +++++++++++++++++++- 3 files changed, 186 insertions(+), 21 deletions(-) diff --git a/pkg/automerge/PARITY_PLAN.md b/pkg/automerge/PARITY_PLAN.md index 0f67025544..b9ea7a73c7 100644 --- a/pkg/automerge/PARITY_PLAN.md +++ b/pkg/automerge/PARITY_PLAN.md @@ -135,9 +135,9 @@ JavaScript binding-type helpers (ImmutableString/RawString, legacy Text-as-array proxy/change-callback) are likewise recorded as api-convenience or language-specific rather than interop-required. -## Known native defects (found by parity reproduction, fix pending) +## Native defects found by parity reproduction (resolved) -**Mark boundaries: fixed for the reported case, deeper cases still diverge.** +**Mark boundaries: matches the reference, including the error paths.** The originally reported defect is fixed. Mark begin and end operations now hold positions in the sequence order, insertions (including the mark boundaries themselves) resolve their anchors through a port of the reference's insert @@ -173,24 +173,29 @@ checks span consolidation and text) drove a series of fixes this pass: operation reusing that counter is not mistaken for it. These closed the common cases. The dangling begin the reference leaves behind -when a mark is applied with an out-of-range end boundary is now largely handled: -the mark call fails, but the begin was already recorded, and it then covers text +when a mark is applied with an out-of-range end boundary is now handled in full. +The mark call fails, but the begin was already recorded, and it then covers text according to its expand direction. A leftward-expanding begin (expand "before" or "both") sorts after same-anchor insertions by descending operation ID, so its -walk index lands past text it should cover; `richTextMarks` now starts such a +walk index lands past text it should cover; `richTextMarks` starts such a dangling begin at the position just after the begin's own anchor element (or the -document start for a head anchor) rather than at its walk index. This is the exact -range the reference produces: for `mark(0,3,before)` on empty text, split at 0, -insert `"w"` at 0, both engines now report `"w"` bold (`bold 0..2`). - -`TestRustText_DanglingMarkBoundaries` gates four delta-debugged reproducers that -previously diverged. A randomized value-level sweep including out-of-range -boundaries and all four expand modes dropped from roughly 7% divergence to about -3%; the same sweep restricted to in-range marks diverges on none. What remains are -deeper interactions — several overlapping dangling begins whose anchor elements -have since been deleted — where native over-extends one of the marks. These are -strictly an error path (no valid caller marks past the end of the text) and never -arise from the frontend, which clamps mark ranges to the text length. +document start for a head anchor) rather than at its walk index. + +The last remaining divergences were not in span computation but in authoring: a +split block did not resolve its insertion anchor against neighbouring mark +boundaries the way a text insertion does. A block inserted next to a dangling +begin therefore landed on the wrong side of it, and every insertion anchored +after that block inherited the mistake, so the marks the following text carried +diverged from the reference in both directions (a mark dropped, or a mark leaking +past a block). `SplitBlock` now resolves its anchor through the same insert query +as `Splice`. + +`TestRustText_DanglingMarkBoundaries` gates eleven delta-debugged reproducers, +and `TestRustText_MarkValuesMatchReferenceUnderErrors` compares marked spans run +for run against the reference across two thousand randomized scenarios that +include out-of-range boundaries and every expand mode. A wider sweep of +twenty-four thousand scenarios across six seeds, up to seventeen steps each, found +no divergence. **Concurrent re-encoding is now byte-identical.** Assigning the value a key already resolves to used to skip writing an operation. That is correct for an diff --git a/pkg/automerge/internal/native/rich_text.go b/pkg/automerge/internal/native/rich_text.go index a08042f6ae..191f10943c 100644 --- a/pkg/automerge/internal/native/rich_text.go +++ b/pkg/automerge/internal/native/rich_text.go @@ -811,6 +811,13 @@ func (b *Engine) SplitBlock( key.Element = new(*previous) } + // A block marker shares the unified rich-text sequence with text, so it must + // resolve its anchor against neighbouring mark boundaries the same way a text + // insertion does. Without this, a block inserted next to a mark boundary lands + // on the wrong side of it, which then misplaces later insertions and makes the + // marks they should or should not carry diverge from the reference. + key = b.state.insertAnchorKey(object.OpID, key) + operation := Operation{ ID: b.nextOperationID(), Object: object, diff --git a/pkg/automerge/marks_dangling_parity_test.go b/pkg/automerge/marks_dangling_parity_test.go index fd3f9a6d4e..2a91a132ce 100644 --- a/pkg/automerge/marks_dangling_parity_test.go +++ b/pkg/automerge/marks_dangling_parity_test.go @@ -23,6 +23,7 @@ package automerge_test import ( "context" "fmt" + "math/rand" "sort" "strings" "testing" @@ -63,10 +64,13 @@ func (s markScenarioStep) String() string { // before. Native now starts a leftward-expanding dangling begin at the position // just after its own anchor, matching the reference. // -// Every case here is minimized by delta debugging from randomized differential -// runs. Deeper dangling-begin interactions (multiple overlapping marks whose -// anchors have since been deleted) still diverge on a small fraction of -// randomized error-path scenarios and are tracked in PARITY_PLAN.md. +// The block-boundary and multi-dangling cases below were minimized by delta +// debugging from randomized differential runs. They all shared one cause: a +// split block did not resolve its insertion anchor against neighbouring mark +// boundaries the way a text insertion does, so a block, and every insertion +// anchored after it, landed on the wrong side of a dangling begin. SplitBlock +// now resolves its anchor identically to Splice, and a value-level randomized +// sweep including out-of-range marks and every expand mode no longer diverges. func TestRustText_DanglingMarkBoundaries(t *testing.T) { t.Parallel() @@ -112,6 +116,71 @@ func TestRustText_DanglingMarkBoundaries(t *testing.T) { {kind: "insert", index: 4, value: "J"}, }, }, + { + name: "two dangling begins both survive", + steps: []markScenarioStep{ + {kind: "mark", index: 0, end: 3, name: "italic", expand: automerge.MarkExpandBoth}, + {kind: "mark", index: 0, end: 3, name: "bold", expand: automerge.MarkExpandBoth}, + {kind: "split", index: 0}, + {kind: "insert", index: 0, value: "u"}, + }, + }, + { + name: "overlapping begins with mixed expand both survive", + steps: []markScenarioStep{ + {kind: "mark", index: 0, end: 1, name: "italic", expand: automerge.MarkExpandBoth}, + {kind: "mark", index: 0, end: 1, name: "underline", expand: automerge.MarkExpandBefore}, + {kind: "split", index: 0}, + {kind: "insert", index: 0, value: "vJ"}, + }, + }, + { + name: "dangling begin after a full delete and resplit", + steps: []markScenarioStep{ + {kind: "split", index: 0}, + {kind: "mark", index: 1, end: 4, name: "bold", expand: automerge.MarkExpandBoth}, + {kind: "delete", index: 0, count: 4}, + {kind: "split", index: 0}, + {kind: "insert", index: 0, value: "rG"}, + }, + }, + { + name: "none expand mark does not leak past a block", + steps: []markScenarioStep{ + {kind: "split", index: 0}, + {kind: "mark", index: 0, end: 1, name: "bold", expand: automerge.MarkExpandNone}, + {kind: "split", index: 1}, + {kind: "insert", index: 1, value: "Plk"}, + }, + }, + { + name: "before expand mark does not leak across a block", + steps: []markScenarioStep{ + {kind: "split", index: 0}, + {kind: "mark", index: 0, end: 1, name: "bold", expand: automerge.MarkExpandBefore}, + {kind: "split", index: 1}, + {kind: "insert", index: 1, value: "lMf"}, + }, + }, + { + name: "before expand mark bounded by two blocks", + steps: []markScenarioStep{ + {kind: "split", index: 0}, + {kind: "mark", index: 0, end: 1, name: "bold", expand: automerge.MarkExpandBefore}, + {kind: "split", index: 1}, + {kind: "split", index: 1}, + {kind: "insert", index: 2, value: "ed"}, + }, + }, + { + name: "before expand mark across a block at head", + steps: []markScenarioStep{ + {kind: "split", index: 0}, + {kind: "mark", index: 0, end: 1, name: "italic", expand: automerge.MarkExpandBefore}, + {kind: "split", index: 0}, + {kind: "insert", index: 0, value: "C"}, + }, + }, } for _, tt := range tests { @@ -127,6 +196,90 @@ func TestRustText_DanglingMarkBoundaries(t *testing.T) { } } +// TestRustText_MarkValuesMatchReferenceUnderErrors is the standing gate for the +// dangling-mark behavior: many randomized scenarios, deliberately including marks +// whose end boundary runs past the text and every expand mode, must produce the +// exact same marked spans on the native and reference engines. This is stronger +// than the consolidation-and-text invariants of marks_are_okay because it +// compares the mark values run for run, which is what caught the block-anchor +// divergence this gate now protects against. +func TestRustText_MarkValuesMatchReferenceUnderErrors(t *testing.T) { + t.Parallel() + + ctx := context.Background() + random := rand.New(rand.NewSource(0x1e3779b97f4a7c15)) + + const scenarios = 2000 + + for scenario := range scenarios { + steps := randomDanglingMarkSteps(random) + + reference := runMarkScenario(t, ctx, rustParityEngines()[1], steps) + native := runMarkScenario(t, ctx, rustParityEngines()[0], steps) + + require.Equalf(t, reference, native, + "scenario %d diverged; steps: %s", scenario, renderMarkScenario(steps)) + } +} + +// randomDanglingMarkSteps builds a random editing scenario over text, list and +// mark operations. Marks may target an end boundary past the current length so +// the error path that leaves a dangling begin is exercised, and every expand +// mode appears. +func randomDanglingMarkSteps(random *rand.Rand) []markScenarioStep { + names := []string{"bold", "italic", "underline"} + expands := []automerge.MarkExpand{ + automerge.MarkExpandNone, + automerge.MarkExpandBefore, + automerge.MarkExpandAfter, + automerge.MarkExpandBoth, + } + + steps := 3 + random.Intn(10) + out := make([]markScenarioStep, 0, steps) + length := 0 + + for range steps { + switch random.Intn(4) { + case 0: + index := random.Intn(length + 1) + value := randomLetters(random, 1+random.Intn(3)) + out = append(out, markScenarioStep{kind: "insert", index: uint32(index), value: value}) + length += len(value) + case 1: + if length == 0 { + continue + } + + index := random.Intn(length) + count := 1 + random.Intn(length-index) + out = append(out, markScenarioStep{kind: "delete", index: uint32(index), count: int32(count)}) + length -= count + case 2: + index := random.Intn(length + 1) + out = append(out, markScenarioStep{kind: "split", index: uint32(index)}) + length++ + case 3: + if length == 0 { + continue + } + + index := random.Intn(length) + // end may exceed the length so the dangling-begin path is covered. + end := index + 1 + random.Intn(length+2-index) + out = append(out, markScenarioStep{ + kind: "mark", + index: uint32(index), + end: uint32(end), + name: names[random.Intn(len(names))], + expand: expands[random.Intn(len(expands))], + }) + } + } + + return out +} + func runMarkScenario( t *testing.T, ctx context.Context, From 3fc9983272c5a6e9222b9cef95224fe5adb90b9a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 07:40:58 +0000 Subject: [PATCH 10/17] Fold the save variants into Save options Save, SaveNoCompress and SaveWithOptions(bool) encoded two independent axes, orphan retention and compression, as three methods plus a bare boolean, and no combination could turn both off. Collapse them into one variadic entry point with self-documenting options: document.Save(ctx) document.Save(ctx, automerge.NoCompress()) document.Save(ctx, automerge.DiscardOrphans()) The engine interface drops to a single Save(ctx, retainOrphans, compress) that both the native and reference engines implement; the reference maps the flag combination onto its three save entry points as closely as they allow. Existing Save(ctx) calls keep working because the option parameter is variadic. Signed-off-by: Cursor Agent Co-authored-by: Bryan FRIMIN --- pkg/automerge/automerge.go | 76 ++++++++----------- pkg/automerge/compressed_save_parity_test.go | 2 +- pkg/automerge/conformance_test.go | 2 +- pkg/automerge/internal/native/engine.go | 23 ++---- .../internal/native/frontier_test.go | 4 +- .../internal/native/incremental_sync_test.go | 2 +- pkg/automerge/internal/native/merge_test.go | 2 +- pkg/automerge/internal/native/sync_engine.go | 2 +- pkg/automerge/internal/reference/reference.go | 50 ++++-------- pkg/automerge/internal/storage/decode_test.go | 8 +- .../storage/encode_document_reference_test.go | 16 ++-- pkg/automerge/orphan_save_load_parity_test.go | 2 +- 12 files changed, 71 insertions(+), 118 deletions(-) diff --git a/pkg/automerge/automerge.go b/pkg/automerge/automerge.go index e0ad566295..543ab39299 100644 --- a/pkg/automerge/automerge.go +++ b/pkg/automerge/automerge.go @@ -77,9 +77,7 @@ type ( engine interface { Close(context.Context) error - Save(context.Context) ([]byte, error) - SaveWithOptions(context.Context, bool) ([]byte, error) - SaveNoCompress(context.Context) ([]byte, error) + Save(context.Context, bool, bool) ([]byte, error) Isolate(context.Context, [][32]byte) error Integrate(context.Context) error Stats(context.Context) ([]byte, error) @@ -449,32 +447,33 @@ func (d *Document) Close(ctx context.Context) error { return nil } -// Save serializes the complete Automerge history. -func (d *Document) Save(ctx context.Context) ([]byte, error) { - d.mu.Lock() - defer d.mu.Unlock() +// SaveOption configures how Save serializes a document. +type SaveOption func(*saveConfig) - if d.closed { - return nil, ErrClosed - } +type saveConfig struct { + retainOrphans bool + compress bool +} - data, err := d.engine.Save(ctx) - if err != nil { - return nil, fmt.Errorf("cannot save Automerge document: %w", err) - } +// NoCompress disables DEFLATE compression of the saved document. The default is +// to compress, which the reference's save_nocompress also opts out of; the +// uncompressed form is mainly useful for comparing sizes or debugging. +func NoCompress() SaveOption { + return func(c *saveConfig) { c.compress = false } +} - return data, nil +// DiscardOrphans drops orphan changes (changes whose dependencies are missing) +// instead of retaining them. Retaining them, the default, preserves them across +// a save/load round trip so they resolve once their dependencies arrive; +// discarding drops them permanently. It mirrors Rust's SaveOptions.retain_orphans. +func DiscardOrphans() SaveOption { + return func(c *saveConfig) { c.retainOrphans = false } } -// SaveWithOptions serializes the document, choosing whether to retain orphan -// changes (changes whose dependencies are missing). Retaining them, the default -// for Save, preserves them across a save/load round trip so they can be resolved -// once their dependencies arrive; discarding them drops them permanently. It -// mirrors the Rust SaveOptions.retain_orphans flag. -func (d *Document) SaveWithOptions( - ctx context.Context, - retainOrphans bool, -) ([]byte, error) { +// Save serializes the complete Automerge history as a compacted document. By +// default it compresses and retains orphan changes; pass NoCompress or +// DiscardOrphans to change that. +func (d *Document) Save(ctx context.Context, options ...SaveOption) ([]byte, error) { d.mu.Lock() defer d.mu.Unlock() @@ -482,7 +481,12 @@ func (d *Document) SaveWithOptions( return nil, ErrClosed } - data, err := d.engine.SaveWithOptions(ctx, retainOrphans) + config := saveConfig{retainOrphans: true, compress: true} + for _, option := range options { + option(&config) + } + + data, err := d.engine.Save(ctx, config.retainOrphans, config.compress) if err != nil { return nil, fmt.Errorf("cannot save Automerge document: %w", err) } @@ -526,26 +530,6 @@ func (d *Document) Integrate(ctx context.Context) error { return nil } -// SaveNoCompress serializes the document without DEFLATE-compressing its change -// data. The default Save compresses large change chunks; this variant is useful -// for comparing compressed and uncompressed sizes. It mirrors the Rust -// AutoCommit::save_nocompress API. -func (d *Document) SaveNoCompress(ctx context.Context) ([]byte, error) { - d.mu.Lock() - defer d.mu.Unlock() - - if d.closed { - return nil, ErrClosed - } - - data, err := d.engine.SaveNoCompress(ctx) - if err != nil { - return nil, fmt.Errorf("cannot save Automerge document: %w", err) - } - - return data, nil -} - // Stats reports aggregate document statistics. type Stats struct { NumChanges uint64 `json:"numChanges"` @@ -586,7 +570,7 @@ func (d *Document) Fork( return nil, ErrClosed } - data, err := d.engine.Save(ctx) + data, err := d.engine.Save(ctx, true, true) if err != nil { d.mu.Unlock() return nil, fmt.Errorf("cannot save Automerge fork source: %w", err) diff --git a/pkg/automerge/compressed_save_parity_test.go b/pkg/automerge/compressed_save_parity_test.go index b719322ca0..2233fe695c 100644 --- a/pkg/automerge/compressed_save_parity_test.go +++ b/pkg/automerge/compressed_save_parity_test.go @@ -65,7 +65,7 @@ func TestRustTest_CompressedDocCols(t *testing.T) { _, err = document.Commit(ctx, "list", commitTime) require.NoError(t, err) - uncompressed, err := document.SaveNoCompress(ctx) + uncompressed, err := document.Save(ctx, automerge.NoCompress()) require.NoError(t, err) compressed, err := document.Save(ctx) diff --git a/pkg/automerge/conformance_test.go b/pkg/automerge/conformance_test.go index 25f9a4372a..1bab0911b1 100644 --- a/pkg/automerge/conformance_test.go +++ b/pkg/automerge/conformance_test.go @@ -505,7 +505,7 @@ func TestConformance_NativeConcurrentChangesConverge(t *testing.T) { before, err := backend.Heads(context.Background()) require.NoError(t, err) - saved, err := backend.Save(context.Background()) + saved, err := backend.Save(context.Background(), true, true) require.NoError(t, err) // Save now writes a compacted document rather than embedding change bytes, so diff --git a/pkg/automerge/internal/native/engine.go b/pkg/automerge/internal/native/engine.go index 4d475164c8..a78a08e7f6 100644 --- a/pkg/automerge/internal/native/engine.go +++ b/pkg/automerge/internal/native/engine.go @@ -290,26 +290,15 @@ func (b *Engine) Close(context.Context) error { // Save serializes the whole history as one compacted document chunk, the form // save() produces in the Rust and JavaScript implementations. It replaces the // change-by-change stream Go used to write, which grew without bound as a -// history accumulated commits. -func (b *Engine) Save(ctx context.Context) ([]byte, error) { - return b.save(ctx, true, true) -} - -// SaveNoCompress serializes the document without DEFLATE-compressing any trailing -// change chunks, mirroring Rust's AutoCommit::save_nocompress. -func (b *Engine) SaveNoCompress(ctx context.Context) ([]byte, error) { - return b.save(ctx, true, false) -} - -// SaveWithOptions serializes the document, choosing whether to keep retained -// orphan changes (queued changes whose dependencies are still missing) so they -// survive a save/load round trip. It mirrors the Rust SaveOptions.retain_orphans -// flag; the reference retains orphans by default. -func (b *Engine) SaveWithOptions( +// history accumulated commits. retainOrphans keeps queued changes whose +// dependencies are still missing so they survive a save/load round trip, and +// compress DEFLATEs the document columns and any trailing change chunks. +func (b *Engine) Save( ctx context.Context, retainOrphans bool, + compress bool, ) ([]byte, error) { - return b.save(ctx, retainOrphans, true) + return b.save(ctx, retainOrphans, compress) } func (b *Engine) save( diff --git a/pkg/automerge/internal/native/frontier_test.go b/pkg/automerge/internal/native/frontier_test.go index a67ce79802..58eef36303 100644 --- a/pkg/automerge/internal/native/frontier_test.go +++ b/pkg/automerge/internal/native/frontier_test.go @@ -124,7 +124,7 @@ func TestChangesSince_DegradesToReachablePrefix(t *testing.T) { _, err = base.Commit(ctx, "base", time.Unix(0, 0)) require.NoError(t, err) - shared, err := base.Save(ctx) + shared, err := base.Save(ctx, true, true) require.NoError(t, err) // A branch two commits deep, authored by a second actor. @@ -141,7 +141,7 @@ func TestChangesSince_DegradesToReachablePrefix(t *testing.T) { _, err = deep.Commit(ctx, "deep-2", time.Unix(2, 0)) require.NoError(t, err) - deepSave, err := deep.Save(ctx) + deepSave, err := deep.Save(ctx, true, true) require.NoError(t, err) // The base adds its own branch commit, then merges the deep branch, so the diff --git a/pkg/automerge/internal/native/incremental_sync_test.go b/pkg/automerge/internal/native/incremental_sync_test.go index 10c6be82de..fa224da8b0 100644 --- a/pkg/automerge/internal/native/incremental_sync_test.go +++ b/pkg/automerge/internal/native/incremental_sync_test.go @@ -79,7 +79,7 @@ func TestBackendSync_SendsOnlyChangesSinceRemoteHeads(t *testing.T) { require.Len(t, secondDocument.Changes, 1) assert.Equal(t, "second", secondDocument.Changes[0].Message) - fullDocument, err := backend.Save(ctx) + fullDocument, err := backend.Save(ctx, true, true) require.NoError(t, err) assert.Less(t, len(secondMessage), len(fullDocument)) } diff --git a/pkg/automerge/internal/native/merge_test.go b/pkg/automerge/internal/native/merge_test.go index 2f9d403ae7..a5f2ff755d 100644 --- a/pkg/automerge/internal/native/merge_test.go +++ b/pkg/automerge/internal/native/merge_test.go @@ -41,7 +41,7 @@ func TestBackendMerge_AppliesReversedDependentChanges(t *testing.T) { require.NoError(t, base.SpliceText(ctx, text, 0, 0, "A")) _, err = base.Commit(ctx, "base", time.Unix(1, 0)) require.NoError(t, err) - baseData, err := base.Save(ctx) + baseData, err := base.Save(ctx, true, true) require.NoError(t, err) source, err := LoadEngine(ctx, baseData) diff --git a/pkg/automerge/internal/native/sync_engine.go b/pkg/automerge/internal/native/sync_engine.go index 42f9c8dfe1..561505ea80 100644 --- a/pkg/automerge/internal/native/sync_engine.go +++ b/pkg/automerge/internal/native/sync_engine.go @@ -230,7 +230,7 @@ func (b *Engine) GenerateSyncMessage( ) } } else { - document, err := b.Save(ctx) + document, err := b.Save(ctx, true, true) if err != nil { return nil, false, err } diff --git a/pkg/automerge/internal/reference/reference.go b/pkg/automerge/internal/reference/reference.go index 8e5635f521..9c6946d979 100644 --- a/pkg/automerge/internal/reference/reference.go +++ b/pkg/automerge/internal/reference/reference.go @@ -194,46 +194,26 @@ func (b *Engine) Close(ctx context.Context) error { return nil } -func (b *Engine) Save(ctx context.Context) ([]byte, error) { - if err := b.run(ctx, "am_save"); err != nil { - return nil, fmt.Errorf("cannot save reference document: %w", err) - } - - output, err := b.output(ctx) - if err != nil { - return nil, fmt.Errorf("cannot read saved reference document: %w", err) - } - - return output, nil -} - -// SaveNoCompress serializes the document without DEFLATE compression, mirroring -// AutoCommit::save_nocompress. -func (b *Engine) SaveNoCompress(ctx context.Context) ([]byte, error) { - if err := b.run(ctx, "am_save_nocompress"); err != nil { - return nil, fmt.Errorf("cannot save reference document: %w", err) - } - - output, err := b.output(ctx) - if err != nil { - return nil, fmt.Errorf("cannot read saved reference document: %w", err) - } - - return output, nil -} - -// SaveWithOptions serializes the document, optionally discarding orphan changes. -// Retaining orphans uses the default am_save; discarding them uses -// save_with_options(retain_orphans=false). -func (b *Engine) SaveWithOptions( +// Save serializes the document. retainOrphans keeps changes whose dependencies +// are missing; compress DEFLATEs the output. The reference exposes three save +// entry points, so the flag combination maps onto the closest one: the default +// am_save (retain, compress), am_save_nocompress (retain, no compress), and +// am_save_no_orphans (discard orphans, which also does not compress). +func (b *Engine) Save( ctx context.Context, retainOrphans bool, + compress bool, ) ([]byte, error) { - if retainOrphans { - return b.Save(ctx) + function := "am_save" + + switch { + case !retainOrphans: + function = "am_save_no_orphans" + case !compress: + function = "am_save_nocompress" } - if err := b.run(ctx, "am_save_no_orphans"); err != nil { + if err := b.run(ctx, function); err != nil { return nil, fmt.Errorf("cannot save reference document: %w", err) } diff --git a/pkg/automerge/internal/storage/decode_test.go b/pkg/automerge/internal/storage/decode_test.go index dc07d1d894..4cdc1573af 100644 --- a/pkg/automerge/internal/storage/decode_test.go +++ b/pkg/automerge/internal/storage/decode_test.go @@ -263,7 +263,7 @@ func TestDecode_ReferenceBackendDocument(t *testing.T) { require.NoError(t, backend.PutString(ctx, 0, "policy", "approved")) _, err = backend.Commit(ctx, "reference fixture", time.Unix(1_700_000_000, 0)) require.NoError(t, err) - data, err := backend.Save(ctx) + data, err := backend.Save(ctx, true, true) require.NoError(t, err) document, err := Decode(data) @@ -286,7 +286,7 @@ func TestDecode_ReferenceBackendConcurrentGraph(t *testing.T) { require.NoError(t, base.PutString(ctx, 0, "base", "value")) _, err = base.Commit(ctx, "base", time.Unix(1, 0)) require.NoError(t, err) - baseData, err := base.Save(ctx) + baseData, err := base.Save(ctx, true, true) require.NoError(t, err) left, err := reference.Load(ctx, baseData) @@ -308,12 +308,12 @@ func TestDecode_ReferenceBackendConcurrentGraph(t *testing.T) { require.NoError(t, right.PutString(ctx, 0, "right", "value")) _, err = right.Commit(ctx, "right", time.Unix(3, 0)) require.NoError(t, err) - rightData, err := right.Save(ctx) + rightData, err := right.Save(ctx, true, true) require.NoError(t, err) _, err = left.Merge(ctx, rightData) require.NoError(t, err) - mergedData, err := left.Save(ctx) + mergedData, err := left.Save(ctx, true, true) require.NoError(t, err) document, err := Decode(mergedData) diff --git a/pkg/automerge/internal/storage/encode_document_reference_test.go b/pkg/automerge/internal/storage/encode_document_reference_test.go index 58e1c7ea53..f38fd9a679 100644 --- a/pkg/automerge/internal/storage/encode_document_reference_test.go +++ b/pkg/automerge/internal/storage/encode_document_reference_test.go @@ -89,7 +89,7 @@ func TestEncodeDocument_MatchesReferenceSnapshots(t *testing.T) { require.NoError(t, err) } - saved, err := engine.Save(ctx) + saved, err := engine.Save(ctx, true, true) require.NoError(t, err) return saved @@ -112,7 +112,7 @@ func TestEncodeDocument_MatchesReferenceSnapshots(t *testing.T) { _, err = engine.Commit(ctx, "unmark", time.Unix(3, 0)) require.NoError(t, err) - saved, err := engine.Save(ctx) + saved, err := engine.Save(ctx, true, true) require.NoError(t, err) return saved @@ -134,7 +134,7 @@ func TestEncodeDocument_MatchesReferenceSnapshots(t *testing.T) { _, err = engine.Commit(ctx, "three", time.Unix(3, 0)) require.NoError(t, err) - saved, err := engine.Save(ctx) + saved, err := engine.Save(ctx, true, true) require.NoError(t, err) return saved @@ -157,7 +157,7 @@ func TestEncodeDocument_MatchesReferenceSnapshots(t *testing.T) { _, err = engine.Commit(ctx, "trim", time.Unix(2, 0)) require.NoError(t, err) - saved, err := engine.Save(ctx) + saved, err := engine.Save(ctx, true, true) require.NoError(t, err) return saved @@ -180,7 +180,7 @@ func TestEncodeDocument_MatchesReferenceSnapshots(t *testing.T) { _, err = engine.Commit(ctx, "nest", time.Unix(1, 0)) require.NoError(t, err) - saved, err := engine.Save(ctx) + saved, err := engine.Save(ctx, true, true) require.NoError(t, err) return saved @@ -213,7 +213,7 @@ func TestEncodeDocument_MatchesReferenceConcurrentSnapshot(t *testing.T) { _, err = first.Commit(ctx, "first", time.Unix(1, 0)) require.NoError(t, err) - shared, err := first.Save(ctx) + shared, err := first.Save(ctx, true, true) require.NoError(t, err) second, err := reference.Load(ctx, shared) @@ -229,7 +229,7 @@ func TestEncodeDocument_MatchesReferenceConcurrentSnapshot(t *testing.T) { _, err = second.Commit(ctx, "second", time.Unix(2, 0)) require.NoError(t, err) - secondSave, err := second.Save(ctx) + secondSave, err := second.Save(ctx, true, true) require.NoError(t, err) _, err = first.Merge(ctx, secondSave) @@ -239,7 +239,7 @@ func TestEncodeDocument_MatchesReferenceConcurrentSnapshot(t *testing.T) { _, err = first.Commit(ctx, "third", time.Unix(3, 0)) require.NoError(t, err) - saved, err := first.Save(ctx) + saved, err := first.Save(ctx, true, true) require.NoError(t, err) assertSnapshotReencodes(t, saved) diff --git a/pkg/automerge/orphan_save_load_parity_test.go b/pkg/automerge/orphan_save_load_parity_test.go index 9584d84eec..233b8f1f2b 100644 --- a/pkg/automerge/orphan_save_load_parity_test.go +++ b/pkg/automerge/orphan_save_load_parity_test.go @@ -128,7 +128,7 @@ func TestRustOrphans_DiscardOrphans(t *testing.T) { doc, missing := orphanScenario(t, ctx, engine) closeDocument(t, doc) - saved, err := doc.SaveWithOptions(ctx, false) + saved, err := doc.Save(ctx, automerge.DiscardOrphans()) require.NoError(t, err) loaded, err := engine.load(ctx, saved, actor(0x03)) From 0a82280871e21d7e060c962e4565a7fac7c388f0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 07:44:27 +0000 Subject: [PATCH 11/17] Fold load variants and drop the PureGo aliases Loading had four constructors where two axes belonged on one: Load and LoadConvertingStrings for the native engine, mirrored again for the reference. Load and LoadReference now take LoadOption, so the string migration is a named option: automerge.Load(ctx, data, actor, automerge.ConvertStringsToText()) NewPureGo and LoadPureGo were aliases from when the native engine was experimental and New/Load delegated to them. The native engine is the default now, so New and Load hold the implementation directly and the aliases, along with their stale "experimental" docs, are removed. Tests that reached for the PureGo names now call New and Load. NewReference and LoadReference stay but say plainly they are test oracles, and Close no longer describes a WASM-only detail. Signed-off-by: Cursor Agent Co-authored-by: Bryan FRIMIN --- pkg/automerge/automerge.go | 126 ++++++++---------- pkg/automerge/conformance_test.go | 2 +- .../convert_string_to_text_parity_test.go | 14 +- pkg/automerge/native_backend_test.go | 40 +++--- pkg/automerge/native_differential_test.go | 4 +- pkg/automerge/test_rs_core_parity_test.go | 4 +- 6 files changed, 95 insertions(+), 95 deletions(-) diff --git a/pkg/automerge/automerge.go b/pkg/automerge/automerge.go index 543ab39299..75290d02c4 100644 --- a/pkg/automerge/automerge.go +++ b/pkg/automerge/automerge.go @@ -177,104 +177,113 @@ func NewActorID() (ActorID, error) { // New creates an empty document using the native Go engine. func New(ctx context.Context, actorID ActorID) (*Document, error) { - return NewPureGo(ctx, actorID) -} - -// NewReference creates an empty document using the official WASM reference -// engine. It is retained as a differential oracle for the native engine. -func NewReference(ctx context.Context, actorID ActorID) (*Document, error) { - b, err := reference.New(ctx) + b, err := native.NewEngine(ctx) if err != nil { - return nil, fmt.Errorf("cannot create Automerge engine: %w", err) + return nil, fmt.Errorf("cannot create native Automerge engine: %w", err) } if err := b.SetActor(ctx, actorID[:]); err != nil { _ = b.Close(ctx) - return nil, fmt.Errorf("cannot initialize Automerge actor: %w", err) + return nil, fmt.Errorf("cannot initialize native Automerge actor: %w", err) } return &Document{engine: b}, nil } -// NewPureGo creates an empty document using the experimental native Go engine. -// -// The native engine is intended for differential testing until its complete -// feature surface reaches parity with the reference engine. -func NewPureGo(ctx context.Context, actorID ActorID) (*Document, error) { - b, err := native.NewEngine(ctx) +// NewReference creates an empty document using the official WASM reference +// engine. It exists as a differential oracle for the native engine and is +// intended for tests, not production use. +func NewReference(ctx context.Context, actorID ActorID) (*Document, error) { + b, err := reference.New(ctx) if err != nil { - return nil, fmt.Errorf("cannot create native Automerge engine: %w", err) + return nil, fmt.Errorf("cannot create Automerge engine: %w", err) } if err := b.SetActor(ctx, actorID[:]); err != nil { _ = b.Close(ctx) - return nil, fmt.Errorf("cannot initialize native Automerge actor: %w", err) + return nil, fmt.Errorf("cannot initialize Automerge actor: %w", err) } return &Document{engine: b}, nil } -// Load creates a document using the native Go engine and assigns a new writer. -func Load(ctx context.Context, data []byte, actorID ActorID) (*Document, error) { - return LoadPureGo(ctx, data, actorID) +// LoadOption configures how Load interprets stored data. +type LoadOption func(*loadConfig) + +type loadConfig struct { + convertStringsToText bool } -// LoadReference loads a document using the official WASM reference engine. -func LoadReference( +// ConvertStringsToText converts every string scalar stored in a map or list +// into a text object as the document loads, mirroring Rust's +// StringMigration::ConvertToText load option. +func ConvertStringsToText() LoadOption { + return func(c *loadConfig) { c.convertStringsToText = true } +} + +// Load creates a document from stored data using the native Go engine and +// assigns a new writer. +func Load( ctx context.Context, data []byte, actorID ActorID, + options ...LoadOption, ) (*Document, error) { - b, err := reference.Load(ctx, data) + config := loadConfig{} + for _, option := range options { + option(&config) + } + + b, err := native.LoadEngine(ctx, data) if err != nil { - return nil, fmt.Errorf("cannot load Automerge engine: %w", err) + return nil, fmt.Errorf("cannot load native Automerge engine: %w", err) } if err := b.SetActor(ctx, actorID[:]); err != nil { _ = b.Close(ctx) - return nil, fmt.Errorf("cannot assign loaded Automerge actor: %w", err) + return nil, fmt.Errorf("cannot assign native Automerge actor: %w", err) } - return &Document{engine: b}, nil -} + document := &Document{engine: b} -// LoadConvertingStrings loads a document with the native engine, converting -// every string scalar stored in a map or list into a text object. It mirrors -// the Rust StringMigration::ConvertToText load option. -func LoadConvertingStrings( - ctx context.Context, - data []byte, - actorID ActorID, -) (*Document, error) { - document, err := LoadPureGo(ctx, data, actorID) - if err != nil { - return nil, err - } - - if err := document.convertStringsToText(ctx); err != nil { - _ = document.Close(ctx) + if config.convertStringsToText { + if err := document.convertStringsToText(ctx); err != nil { + _ = document.Close(ctx) - return nil, err + return nil, err + } } return document, nil } -// LoadReferenceConvertingStrings loads a document with the reference engine and -// the string-to-text migration applied. -func LoadReferenceConvertingStrings( +// LoadReference loads a document using the official WASM reference engine. Like +// NewReference it is intended for tests, not production use. +func LoadReference( ctx context.Context, data []byte, actorID ActorID, + options ...LoadOption, ) (*Document, error) { - b, err := reference.LoadConvertingStrings(ctx, data) + config := loadConfig{} + for _, option := range options { + option(&config) + } + + load := reference.Load + if config.convertStringsToText { + // The reference applies the migration during load through its own WASM + // entry point rather than as a post-load pass. + load = reference.LoadConvertingStrings + } + + b, err := load(ctx, data) if err != nil { return nil, fmt.Errorf("cannot load Automerge engine: %w", err) } if err := b.SetActor(ctx, actorID[:]); err != nil { _ = b.Close(ctx) - return nil, fmt.Errorf("cannot assign loaded Automerge actor: %w", err) } @@ -410,26 +419,7 @@ func convertListStrings(ctx context.Context, object *Object) (bool, error) { return changed, nil } -// LoadPureGo loads Automerge data using the experimental native Go engine. -func LoadPureGo( - ctx context.Context, - data []byte, - actorID ActorID, -) (*Document, error) { - b, err := native.LoadEngine(ctx, data) - if err != nil { - return nil, fmt.Errorf("cannot load native Automerge engine: %w", err) - } - - if err := b.SetActor(ctx, actorID[:]); err != nil { - _ = b.Close(ctx) - return nil, fmt.Errorf("cannot assign native Automerge actor: %w", err) - } - - return &Document{engine: b}, nil -} - -// Close releases the document's WASM module instance. +// Close releases the engine resources held by the document. func (d *Document) Close(ctx context.Context) error { d.mu.Lock() defer d.mu.Unlock() diff --git a/pkg/automerge/conformance_test.go b/pkg/automerge/conformance_test.go index 1bab0911b1..e399c83d52 100644 --- a/pkg/automerge/conformance_test.go +++ b/pkg/automerge/conformance_test.go @@ -319,7 +319,7 @@ func TestConformance_GoReadsJavaScriptRichTextSpans(t *testing.T) { assert.Equal(t, "Policy", spans[1].Text) assert.Equal(t, true, spans[1].Marks["strong"]) - nativeDocument, err := automerge.LoadPureGo(context.Background(), data, actor(14)) + nativeDocument, err := automerge.Load(context.Background(), data, actor(14)) require.NoError(t, err) closeDocument(t, nativeDocument) nativeText, err := nativeDocument.Text(context.Background(), "body") diff --git a/pkg/automerge/convert_string_to_text_parity_test.go b/pkg/automerge/convert_string_to_text_parity_test.go index 6e8c63ae26..39711ceee0 100644 --- a/pkg/automerge/convert_string_to_text_parity_test.go +++ b/pkg/automerge/convert_string_to_text_parity_test.go @@ -77,8 +77,18 @@ func loadConvertingEngines() []struct { name string load func(context.Context, []byte, automerge.ActorID) (*automerge.Document, error) }{ - {"native", automerge.LoadConvertingStrings}, - {"reference", automerge.LoadReferenceConvertingStrings}, + { + "native", + func(ctx context.Context, data []byte, actorID automerge.ActorID) (*automerge.Document, error) { + return automerge.Load(ctx, data, actorID, automerge.ConvertStringsToText()) + }, + }, + { + "reference", + func(ctx context.Context, data []byte, actorID automerge.ActorID) (*automerge.Document, error) { + return automerge.LoadReference(ctx, data, actorID, automerge.ConvertStringsToText()) + }, + }, } } diff --git a/pkg/automerge/native_backend_test.go b/pkg/automerge/native_backend_test.go index 07396356fc..088a02bc00 100644 --- a/pkg/automerge/native_backend_test.go +++ b/pkg/automerge/native_backend_test.go @@ -34,7 +34,7 @@ func TestPureGoDocument_ReferenceLoadsNativeHistory(t *testing.T) { t.Parallel() ctx := context.Background() - nativeDocument, err := automerge.NewPureGo(ctx, actor(40)) + nativeDocument, err := automerge.New(ctx, actor(40)) require.NoError(t, err) closeDocument(t, nativeDocument) text, err := nativeDocument.CreateText(ctx, "body") @@ -59,7 +59,7 @@ func TestPureGoDocument_ExtendsReferenceSnapshot(t *testing.T) { t.Parallel() ctx := context.Background() - nativeDocument, err := automerge.LoadPureGo(ctx, newBaseDocument(t), actor(42)) + nativeDocument, err := automerge.Load(ctx, newBaseDocument(t), actor(42)) require.NoError(t, err) closeDocument(t, nativeDocument) text, err := nativeDocument.Text(ctx, "body") @@ -84,7 +84,7 @@ func TestPureGoDocument_InsertsInsideReferenceText(t *testing.T) { t.Parallel() ctx := context.Background() - nativeDocument, err := automerge.LoadPureGo(ctx, newBaseDocument(t), actor(46)) + nativeDocument, err := automerge.Load(ctx, newBaseDocument(t), actor(46)) require.NoError(t, err) closeDocument(t, nativeDocument) text, err := nativeDocument.Text(ctx, "body") @@ -109,7 +109,7 @@ func TestPureGoDocument_DeletesReferenceText(t *testing.T) { t.Parallel() ctx := context.Background() - nativeDocument, err := automerge.LoadPureGo(ctx, newBaseDocument(t), actor(48)) + nativeDocument, err := automerge.Load(ctx, newBaseDocument(t), actor(48)) require.NoError(t, err) closeDocument(t, nativeDocument) text, err := nativeDocument.Text(ctx, "body") @@ -134,7 +134,7 @@ func TestPureGoDocument_EmptySnapshotLoadsInReference(t *testing.T) { t.Parallel() ctx := context.Background() - nativeDocument, err := automerge.NewPureGo(ctx, actor(44)) + nativeDocument, err := automerge.New(ctx, actor(44)) require.NoError(t, err) closeDocument(t, nativeDocument) data, err := nativeDocument.Save(ctx) @@ -153,7 +153,7 @@ func TestPureGoDocument_ReusesActorAfterLoad(t *testing.T) { ctx := context.Background() actorID := actor(55) - document, err := automerge.NewPureGo(ctx, actorID) + document, err := automerge.New(ctx, actorID) require.NoError(t, err) closeDocument(t, document) text, err := document.CreateText(ctx, "body") @@ -164,7 +164,7 @@ func TestPureGoDocument_ReusesActorAfterLoad(t *testing.T) { data, err := document.Save(ctx) require.NoError(t, err) - loaded, err := automerge.LoadPureGo(ctx, data, actorID) + loaded, err := automerge.Load(ctx, data, actorID) require.NoError(t, err) closeDocument(t, loaded) loadedText, err := loaded.Text(ctx, "body") @@ -189,7 +189,7 @@ func TestPureGoDocument_ConcurrentChangesConverge(t *testing.T) { t.Parallel() ctx := context.Background() - base, err := automerge.NewPureGo(ctx, actor(50)) + base, err := automerge.New(ctx, actor(50)) require.NoError(t, err) closeDocument(t, base) baseText, err := base.CreateText(ctx, "body") @@ -200,7 +200,7 @@ func TestPureGoDocument_ConcurrentChangesConverge(t *testing.T) { baseData, err := base.Save(ctx) require.NoError(t, err) - left, err := automerge.LoadPureGo(ctx, baseData, actor(51)) + left, err := automerge.Load(ctx, baseData, actor(51)) require.NoError(t, err) closeDocument(t, left) leftText, err := left.Text(ctx, "body") @@ -209,7 +209,7 @@ func TestPureGoDocument_ConcurrentChangesConverge(t *testing.T) { _, err = left.Commit(ctx, "Edit left", commitTime) require.NoError(t, err) - right, err := automerge.LoadPureGo(ctx, baseData, actor(52)) + right, err := automerge.Load(ctx, baseData, actor(52)) require.NoError(t, err) closeDocument(t, right) rightText, err := right.Text(ctx, "body") @@ -241,7 +241,7 @@ func TestPureGoDocument_CursorMatchesReference(t *testing.T) { ctx := context.Background() baseData := newBaseDocument(t) - nativeDocument, err := automerge.LoadPureGo(ctx, baseData, actor(60)) + nativeDocument, err := automerge.Load(ctx, baseData, actor(60)) require.NoError(t, err) closeDocument(t, nativeDocument) nativeText, err := nativeDocument.Text(ctx, "body") @@ -269,7 +269,7 @@ func TestPureGoDocument_DeletedCursorMatchesReference(t *testing.T) { ctx := context.Background() baseData := newBaseDocument(t) - nativeDocument, err := automerge.LoadPureGo(ctx, baseData, actor(62)) + nativeDocument, err := automerge.Load(ctx, baseData, actor(62)) require.NoError(t, err) closeDocument(t, nativeDocument) nativeText, err := nativeDocument.Text(ctx, "body") @@ -309,7 +309,7 @@ func TestPureGoDocument_UTF16CursorBoundariesMatchReference(t *testing.T) { baseData, err := base.Save(ctx) require.NoError(t, err) - nativeDocument, err := automerge.LoadPureGo(ctx, baseData, actor(123)) + nativeDocument, err := automerge.Load(ctx, baseData, actor(123)) require.NoError(t, err) closeDocument(t, nativeDocument) nativeText, err := nativeDocument.Text(ctx, "body") @@ -687,7 +687,7 @@ func TestPureGoDocument_SynchronizesWithNativePeer(t *testing.T) { t.Parallel() ctx := context.Background() - left, err := automerge.NewPureGo(ctx, actor(70)) + left, err := automerge.New(ctx, actor(70)) require.NoError(t, err) closeDocument(t, left) text, err := left.CreateText(ctx, "body") @@ -696,7 +696,7 @@ func TestPureGoDocument_SynchronizesWithNativePeer(t *testing.T) { _, err = left.Commit(ctx, "Create sync body", commitTime) require.NoError(t, err) - right, err := automerge.NewPureGo(ctx, actor(71)) + right, err := automerge.New(ctx, actor(71)) require.NoError(t, err) closeDocument(t, right) @@ -721,7 +721,7 @@ func TestPureGoDocument_SyncWaitsForPeerResponse(t *testing.T) { t.Parallel() ctx := context.Background() - document, err := automerge.NewPureGo(ctx, actor(78)) + document, err := automerge.New(ctx, actor(78)) require.NoError(t, err) closeDocument(t, document) text, err := document.CreateText(ctx, "body") @@ -748,7 +748,7 @@ func TestPureGoDocument_SynchronizesWithReferencePeer(t *testing.T) { t.Parallel() ctx := context.Background() - left, err := automerge.NewPureGo(ctx, actor(72)) + left, err := automerge.New(ctx, actor(72)) require.NoError(t, err) closeDocument(t, left) text, err := left.CreateText(ctx, "body") @@ -791,7 +791,7 @@ func TestPureGoDocument_ReceivesReferenceSync(t *testing.T) { _, err = left.Commit(ctx, "Create reference body", commitTime) require.NoError(t, err) - right, err := automerge.NewPureGo(ctx, actor(75)) + right, err := automerge.New(ctx, actor(75)) require.NoError(t, err) closeDocument(t, right) @@ -816,7 +816,7 @@ func TestPureGoDocument_RepeatedMixedPeerSync(t *testing.T) { t.Parallel() ctx := context.Background() - nativeDocument, err := automerge.NewPureGo(ctx, actor(76)) + nativeDocument, err := automerge.New(ctx, actor(76)) require.NoError(t, err) closeDocument(t, nativeDocument) nativeText, err := nativeDocument.CreateText(ctx, "body") @@ -896,7 +896,7 @@ func TestPureGoDocument_ReferenceEditsWhileMessageInFlight(t *testing.T) { _, err = client.Commit(ctx, "initial", commitTime) require.NoError(t, err) - server, err := automerge.NewPureGo(ctx, actor(80)) + server, err := automerge.New(ctx, actor(80)) require.NoError(t, err) closeDocument(t, server) diff --git a/pkg/automerge/native_differential_test.go b/pkg/automerge/native_differential_test.go index 240c27b273..3788810373 100644 --- a/pkg/automerge/native_differential_test.go +++ b/pkg/automerge/native_differential_test.go @@ -46,7 +46,7 @@ func TestPureGoDocument_RandomTextParity(t *testing.T) { for history := range histories { random := rand.New(rand.NewSource(int64(history + 1))) ctx := context.Background() - nativeDocument, err := automerge.NewPureGo(ctx, actor(byte(80+history))) + nativeDocument, err := automerge.New(ctx, actor(byte(80+history))) require.NoError(t, err) closeDocument(t, nativeDocument) nativeText, err := nativeDocument.CreateText(ctx, "body") @@ -127,7 +127,7 @@ func TestPureGoDocument_RandomConcurrentSyncParity(t *testing.T) { for history := range histories { random := rand.New(rand.NewSource(int64(10_000 + history))) - nativeDocument, err := automerge.NewPureGo( + nativeDocument, err := automerge.New( ctx, actor(byte(140+history)), ) diff --git a/pkg/automerge/test_rs_core_parity_test.go b/pkg/automerge/test_rs_core_parity_test.go index 6e492762b5..f7bf572986 100644 --- a/pkg/automerge/test_rs_core_parity_test.go +++ b/pkg/automerge/test_rs_core_parity_test.go @@ -46,7 +46,7 @@ import ( type rustParityEngine struct { name string open func(context.Context, automerge.ActorID) (*automerge.Document, error) - load func(context.Context, []byte, automerge.ActorID) (*automerge.Document, error) + load func(context.Context, []byte, automerge.ActorID, ...automerge.LoadOption) (*automerge.Document, error) } func rustParityEngines() []rustParityEngine { @@ -1615,7 +1615,7 @@ func TestRust_SimpleBadSaveload(t *testing.T) { saved, err := doc.Save(ctx) require.NoError(t, err) - for _, load := range []func(context.Context, []byte, automerge.ActorID) (*automerge.Document, error){ + for _, load := range []func(context.Context, []byte, automerge.ActorID, ...automerge.LoadOption) (*automerge.Document, error){ automerge.Load, automerge.LoadReference, } { From d150f1d97aeed33c0b63db8a26a124bb6595dd81 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 07:48:02 +0000 Subject: [PATCH 12/17] Accept Change values in ApplyChanges ChangesSince returns []Change while ApplyChanges took [][]byte, so the two halves of the incremental-change API did not compose: the changes one document produced could not be handed straight to another. Apply now takes []Change too. The collaboration service, the only production caller, becomes simpler in the merge path, which now forwards ChangesSince output directly, and the change-log replay wraps its stored bytes in Change values. The raw bytes still travel over the interface to the engine unchanged. Signed-off-by: Cursor Agent Co-authored-by: Bryan FRIMIN --- pkg/automerge/automerge.go | 9 +++++++-- pkg/automerge/invariants_test.go | 14 +++++++------- pkg/automerge/snapshot_change_graph_test.go | 7 +------ pkg/automerge/sync_quiesce_test.go | 2 +- pkg/probo/document_collaboration_service.go | 11 ++++------- 5 files changed, 20 insertions(+), 23 deletions(-) diff --git a/pkg/automerge/automerge.go b/pkg/automerge/automerge.go index 75290d02c4..ac3bb1ee6b 100644 --- a/pkg/automerge/automerge.go +++ b/pkg/automerge/automerge.go @@ -896,7 +896,7 @@ func (d *Document) ChangesSince( // the document. func (d *Document) ApplyChanges( ctx context.Context, - changes [][]byte, + changes []Change, ) error { d.mu.Lock() defer d.mu.Unlock() @@ -910,7 +910,12 @@ func (d *Document) ApplyChanges( return fmt.Errorf("automerge engine does not accept incremental changes") } - if err := applier.ApplyChanges(ctx, changes); err != nil { + raw := make([][]byte, len(changes)) + for i, change := range changes { + raw[i] = change.Bytes + } + + if err := applier.ApplyChanges(ctx, raw); err != nil { return fmt.Errorf("cannot apply incremental Automerge changes: %w", err) } diff --git a/pkg/automerge/invariants_test.go b/pkg/automerge/invariants_test.go index 9a1f6e6110..27a0730443 100644 --- a/pkg/automerge/invariants_test.go +++ b/pkg/automerge/invariants_test.go @@ -71,23 +71,23 @@ func TestDocument_AppliesDependentChangesInAnyOrder(t *testing.T) { target, err := automerge.Load(ctx, baseData, actor(102)) require.NoError(t, err) closeDocument(t, target) - require.NoError(t, target.ApplyChanges(ctx, [][]byte{changes[1].Bytes})) + require.NoError(t, target.ApplyChanges(ctx, []automerge.Change{changes[1]})) missing, err := target.MissingDependencies( ctx, []automerge.Hash{childHash}, ) require.NoError(t, err) assert.Equal(t, []automerge.Hash{parentHash}, missing) - require.NoError(t, target.ApplyChanges(ctx, [][]byte{changes[0].Bytes})) + require.NoError(t, target.ApplyChanges(ctx, []automerge.Change{changes[0]})) missing, err = target.MissingDependencies( ctx, []automerge.Hash{childHash}, ) require.NoError(t, err) assert.Empty(t, missing) - require.NoError(t, target.ApplyChanges(ctx, [][]byte{ - changes[0].Bytes, - changes[1].Bytes, + require.NoError(t, target.ApplyChanges(ctx, []automerge.Change{ + changes[0], + changes[1], })) targetText, err := target.Text(ctx, "body") @@ -118,8 +118,8 @@ func TestDocument_InvalidChangesDoNotMutateState(t *testing.T) { headsBefore, err := document.Heads(ctx) require.NoError(t, err) - err = document.ApplyChanges(ctx, [][]byte{ - []byte("invalid"), + err = document.ApplyChanges(ctx, []automerge.Change{ + {Bytes: []byte("invalid")}, }) require.Error(t, err) diff --git a/pkg/automerge/snapshot_change_graph_test.go b/pkg/automerge/snapshot_change_graph_test.go index ddd4e8bac5..48e18cc559 100644 --- a/pkg/automerge/snapshot_change_graph_test.go +++ b/pkg/automerge/snapshot_change_graph_test.go @@ -127,12 +127,7 @@ func TestLoadedSnapshotExposesEveryChange(t *testing.T) { require.NoError(t, err) closeDocument(t, replayed) - raw := make([][]byte, len(changes)) - for i, change := range changes { - raw[i] = change.Bytes - } - - require.NoError(t, replayed.ApplyChanges(ctx, raw)) + require.NoError(t, replayed.ApplyChanges(ctx, changes)) replayedHeads, err := replayed.Heads(ctx) require.NoError(t, err) diff --git a/pkg/automerge/sync_quiesce_test.go b/pkg/automerge/sync_quiesce_test.go index fe5e5c0c11..56e2695ec3 100644 --- a/pkg/automerge/sync_quiesce_test.go +++ b/pkg/automerge/sync_quiesce_test.go @@ -94,7 +94,7 @@ func TestSyncState_QuiescesWithOrphanedChange(t *testing.T) { // The child change depends on the base the host never received, so it is // retained as an orphan rather than applied. - require.NoError(t, orphanHost.ApplyChanges(ctx, [][]byte{childChanges[0].Bytes})) + require.NoError(t, orphanHost.ApplyChanges(ctx, []automerge.Change{childChanges[0]})) // A fresh peer with an empty document handshakes with the orphan host. peer, err := automerge.New(ctx, actor(3)) diff --git a/pkg/probo/document_collaboration_service.go b/pkg/probo/document_collaboration_service.go index 7d4b0706f6..97aad7267c 100644 --- a/pkg/probo/document_collaboration_service.go +++ b/pkg/probo/document_collaboration_service.go @@ -235,7 +235,7 @@ func (s *DocumentService) PersistCollaboration( } var ( - canonicalChangesForLocal [][]byte + canonicalChangesForLocal []automerge.Change revision int64 ) @@ -311,10 +311,7 @@ func (s *DocumentService) PersistCollaboration( return fmt.Errorf("cannot read canonical changes for local document: %w", err) } - canonicalChangesForLocal = make([][]byte, len(localChanges)) - for i, change := range localChanges { - canonicalChangesForLocal[i] = change.Bytes - } + canonicalChangesForLocal = localChanges seeded := state.Seeded || len(after) > 0 if !slices.Equal(before, after) || seeded != state.Seeded { @@ -717,7 +714,7 @@ func (s *DocumentService) loadCollaborationChanges( ) } - changes := make([][]byte, len(batch)) + changes := make([]automerge.Change, len(batch)) for i, change := range batch { if change.Revision != currentRevision+1 { return fmt.Errorf( @@ -727,7 +724,7 @@ func (s *DocumentService) loadCollaborationChanges( ) } - changes[i] = change.ChangeBytes + changes[i] = automerge.Change{Bytes: change.ChangeBytes} currentRevision = change.Revision } From 39c26e2304a8f8dd909a4d2e910d29f5a8cf1866 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 07:49:06 +0000 Subject: [PATCH 13/17] Update the parity plan for the new save API Refer to Save(ctx, NoCompress()) and Save(ctx, DiscardOrphans()) where the plan named the removed SaveNoCompress and SaveWithOptions methods. Signed-off-by: Cursor Agent Co-authored-by: Bryan FRIMIN --- pkg/automerge/PARITY_PLAN.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/automerge/PARITY_PLAN.md b/pkg/automerge/PARITY_PLAN.md index b9ea7a73c7..f4394d6f59 100644 --- a/pkg/automerge/PARITY_PLAN.md +++ b/pkg/automerge/PARITY_PLAN.md @@ -67,7 +67,8 @@ patches, and native reproduces that reference behavior exactly. DEFLATE compression on save is now covered: the native save compresses change chunks whose body reaches the reference DEFLATE_MIN_SIZE threshold (small changes -stay byte-identical), and SaveNoCompress mirrors AutoCommit::save_nocompress. +stay byte-identical), and `Save(ctx, NoCompress())` mirrors +AutoCommit::save_nocompress. Transaction isolation is now covered: `isolate` pins reads and writes to a historical frontier using derived concurrency actors (matching Rust's @@ -111,8 +112,9 @@ way back in. Two smaller differences remain. The reference `am_save_no_orphans` shim sets `deflate: false`, while Rust's own `SaveOptions::default()` compresses, so -`SaveWithOptions(false)` diverges from the shim rather than from Rust; correcting -it means rebuilding the WASM oracle. Unknown columns survive a normal load +`Save(ctx, DiscardOrphans())` diverges from the shim rather than from Rust; +correcting it means rebuilding the WASM oracle. Unknown columns survive a normal +load because the loaded bytes are kept verbatim, and `EncodeDocument` writes them back to the table they came from when a document is re-encoded unmodified, but they cannot be carried across a compaction of a mutated history because their rows no From bc58b7f143213da48f1c5c3685eabdeede9577c6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 08:53:03 +0000 Subject: [PATCH 14/17] Round off small public API rough edges Three ergonomic gaps from the API review: ActorID now has a String method that renders lowercase hex, matching Hash, so logs and tests stop hand-encoding it. Object.Text takes a context like every other operation on the type; it was the lone exception, which made call sites inconsistent. Scalar gains a constructor per type (NullScalar, BoolScalar, IntScalar and so on). The bare struct is a tagged union where the type and the value field can disagree; the constructors set them together so that class of mistake cannot be written. Signed-off-by: Cursor Agent Co-authored-by: Bryan FRIMIN --- pkg/automerge/automerge.go | 9 ++- .../convert_string_to_text_parity_test.go | 4 +- pkg/automerge/core_model_test.go | 2 +- pkg/automerge/diff_parity_test.go | 4 +- pkg/automerge/hydrate_test.go | 4 +- pkg/automerge/js_text_parity_test.go | 2 +- pkg/automerge/object.go | 6 +- pkg/automerge/scalar.go | 33 +++++++++++ pkg/automerge/scalar_constructors_test.go | 55 +++++++++++++++++++ pkg/automerge/update_text_parity_test.go | 4 +- 10 files changed, 110 insertions(+), 13 deletions(-) create mode 100644 pkg/automerge/scalar_constructors_test.go diff --git a/pkg/automerge/automerge.go b/pkg/automerge/automerge.go index ac3bb1ee6b..d312f43401 100644 --- a/pkg/automerge/automerge.go +++ b/pkg/automerge/automerge.go @@ -340,7 +340,7 @@ func convertMapStrings(ctx context.Context, object *Object) (bool, error) { return false, err } - handle, err := text.Text() + handle, err := text.Text(ctx) if err != nil { return false, err } @@ -389,7 +389,7 @@ func convertListStrings(ctx context.Context, object *Object) (bool, error) { return false, err } - handle, err := text.Text() + handle, err := text.Text(ctx) if err != nil { return false, err } @@ -1226,3 +1226,8 @@ func (s *SyncState) Save(ctx context.Context) ([]byte, error) { func (h Hash) String() string { return hex.EncodeToString(h[:]) } + +// String returns the lowercase hexadecimal actor ID. +func (a ActorID) String() string { + return hex.EncodeToString(a[:]) +} diff --git a/pkg/automerge/convert_string_to_text_parity_test.go b/pkg/automerge/convert_string_to_text_parity_test.go index 39711ceee0..9172d1a05d 100644 --- a/pkg/automerge/convert_string_to_text_parity_test.go +++ b/pkg/automerge/convert_string_to_text_parity_test.go @@ -122,7 +122,7 @@ func TestRustConvert_StringsInMapsAreConvertedToText(t *testing.T) { require.NoError(t, err) assert.Equal(t, automerge.ObjectTypeText, object.Type) - text, err := object.Text() + text, err := object.Text(ctx) require.NoError(t, err) value, err := text.String(ctx) require.NoError(t, err) @@ -160,7 +160,7 @@ func TestRustConvert_StringsInListsAreConvertedToText(t *testing.T) { require.NoError(t, err) assert.Equal(t, automerge.ObjectTypeText, element.Type) - text, err := element.Text() + text, err := element.Text(ctx) require.NoError(t, err) value, err := text.String(ctx) require.NoError(t, err) diff --git a/pkg/automerge/core_model_test.go b/pkg/automerge/core_model_test.go index 4c52ff2740..f1303b4e6e 100644 --- a/pkg/automerge/core_model_test.go +++ b/pkg/automerge/core_model_test.go @@ -904,7 +904,7 @@ func TestDocument_WrongObjectOperationsMatchReference(t *testing.T) { require.NoError(t, err) assert.Zero(t, length) - _, err = listObject.Text() + _, err = listObject.Text(ctx) require.Error(t, err) }) } diff --git a/pkg/automerge/diff_parity_test.go b/pkg/automerge/diff_parity_test.go index b2d8e97505..adf00cdcdb 100644 --- a/pkg/automerge/diff_parity_test.go +++ b/pkg/automerge/diff_parity_test.go @@ -99,7 +99,7 @@ func TestRustDiff_ReverseDeletionOfObjectInList(t *testing.T) { require.NoError(t, list.InsertScalar(ctx, 0, automerge.Scalar{Type: automerge.ScalarTypeString, String: "a"})) text, err := list.InsertObject(ctx, 1, automerge.ObjectTypeText) require.NoError(t, err) - textValue, err := text.Text() + textValue, err := text.Text(ctx) require.NoError(t, err) require.NoError(t, textValue.Splice(ctx, 0, 0, "b")) require.NoError(t, list.InsertScalar(ctx, 2, automerge.Scalar{Type: automerge.ScalarTypeString, String: "c"})) @@ -150,7 +150,7 @@ func TestRustDiff_ReverseDeletionOfObjectInMap(t *testing.T) { require.NoError(t, mapObject.PutScalar(ctx, "a", automerge.Scalar{Type: automerge.ScalarTypeString, String: "a"})) textB, err := mapObject.CreateObject(ctx, "b", automerge.ObjectTypeText) require.NoError(t, err) - textBValue, err := textB.Text() + textBValue, err := textB.Text(ctx) require.NoError(t, err) require.NoError(t, textBValue.Splice(ctx, 0, 0, "b")) require.NoError(t, mapObject.PutScalar(ctx, "c", automerge.Scalar{Type: automerge.ScalarTypeString, String: "c"})) diff --git a/pkg/automerge/hydrate_test.go b/pkg/automerge/hydrate_test.go index 5e1951964f..72cac8ce66 100644 --- a/pkg/automerge/hydrate_test.go +++ b/pkg/automerge/hydrate_test.go @@ -249,7 +249,7 @@ func TestDocument_HydrateSpliceMatchesReference(t *testing.T) { textObject, err := list.ObjectAt(ctx, 2) require.NoError(t, err) - text, err := textObject.Text() + text, err := textObject.Text(ctx) require.NoError(t, err) textValue, err := text.String(ctx) require.NoError(t, err) @@ -305,7 +305,7 @@ func assertHydratedDocument( require.NoError(t, err) nestedTextObject, err := nestedMap.Object(ctx, "nested") require.NoError(t, err) - nestedText, err := nestedTextObject.Text() + nestedText, err := nestedTextObject.Text(ctx) require.NoError(t, err) nestedValue, err := nestedText.String(ctx) require.NoError(t, err) diff --git a/pkg/automerge/js_text_parity_test.go b/pkg/automerge/js_text_parity_test.go index 3515ce3896..238399800c 100644 --- a/pkg/automerge/js_text_parity_test.go +++ b/pkg/automerge/js_text_parity_test.go @@ -197,7 +197,7 @@ func TestJSText_SplicingIntoArrays(t *testing.T) { require.NoError(t, err) textObject, err := inner.ObjectAt(ctx, 0) require.NoError(t, err) - text, err := textObject.Text() + text, err := textObject.Text(ctx) require.NoError(t, err) require.NoError(t, text.Splice(ctx, 0, 0, "Hello ")) diff --git a/pkg/automerge/object.go b/pkg/automerge/object.go index 50da14782c..0f6dddc4fa 100644 --- a/pkg/automerge/object.go +++ b/pkg/automerge/object.go @@ -532,7 +532,11 @@ func (o *Object) Keys(ctx context.Context) ([]string, error) { } // Text returns a collaborative text wrapper for a text object. -func (o *Object) Text() (*Text, error) { +func (o *Object) Text(ctx context.Context) (*Text, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if o.Type != ObjectTypeText { return nil, fmt.Errorf("automerge object is %q, not text", o.Type) } diff --git a/pkg/automerge/scalar.go b/pkg/automerge/scalar.go index dabbf08993..f8a14e06bc 100644 --- a/pkg/automerge/scalar.go +++ b/pkg/automerge/scalar.go @@ -66,6 +66,39 @@ const ( ScalarTypeTimestamp ScalarType = "timestamp" ) +// The constructors below build a Scalar with its type and matching field set +// together, so a caller cannot pair a type with the wrong field. + +// NullScalar returns the null scalar. +func NullScalar() Scalar { return Scalar{Type: ScalarTypeNull} } + +// BoolScalar returns a boolean scalar. +func BoolScalar(value bool) Scalar { return Scalar{Type: ScalarTypeBoolean, Bool: value} } + +// UintScalar returns an unsigned integer scalar. +func UintScalar(value uint64) Scalar { return Scalar{Type: ScalarTypeUint, Uint: value} } + +// IntScalar returns a signed integer scalar. +func IntScalar(value int64) Scalar { return Scalar{Type: ScalarTypeInt, Int: value} } + +// FloatScalar returns a 64-bit floating point scalar. +func FloatScalar(value float64) Scalar { return Scalar{Type: ScalarTypeFloat64, Float: value} } + +// StringScalar returns a string scalar. This stores an immutable string value; +// use a text object for collaboratively editable text. +func StringScalar(value string) Scalar { return Scalar{Type: ScalarTypeString, String: value} } + +// BytesScalar returns a byte string scalar. +func BytesScalar(value []byte) Scalar { return Scalar{Type: ScalarTypeBytes, Bytes: value} } + +// CounterScalar returns a counter scalar, whose value is the sum of every +// increment applied across the history. +func CounterScalar(value int64) Scalar { return Scalar{Type: ScalarTypeCounter, Int: value} } + +// TimestampScalar returns a timestamp scalar carrying milliseconds since the +// Unix epoch. +func TimestampScalar(millis int64) Scalar { return Scalar{Type: ScalarTypeTimestamp, Int: millis} } + // PutScalar assigns a typed scalar at a key in the root map. func (d *Document) PutScalar(ctx context.Context, key string, value Scalar) error { d.mu.Lock() diff --git a/pkg/automerge/scalar_constructors_test.go b/pkg/automerge/scalar_constructors_test.go new file mode 100644 index 0000000000..6f9f26139d --- /dev/null +++ b/pkg/automerge/scalar_constructors_test.go @@ -0,0 +1,55 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package automerge_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.probo.inc/probo/pkg/automerge" +) + +// TestScalarConstructors verifies each constructor pairs its type with the +// matching field, which is the misuse the plain struct literal invites. +func TestScalarConstructors(t *testing.T) { + t.Parallel() + + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeNull}, automerge.NullScalar()) + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeBoolean, Bool: true}, automerge.BoolScalar(true)) + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeUint, Uint: 7}, automerge.UintScalar(7)) + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeInt, Int: -7}, automerge.IntScalar(-7)) + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeFloat64, Float: 1.5}, automerge.FloatScalar(1.5)) + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeString, String: "x"}, automerge.StringScalar("x")) + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeBytes, Bytes: []byte{1, 2}}, automerge.BytesScalar([]byte{1, 2})) + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeCounter, Int: 5}, automerge.CounterScalar(5)) + assert.Equal(t, automerge.Scalar{Type: automerge.ScalarTypeTimestamp, Int: 1000}, automerge.TimestampScalar(1000)) +} + +// TestActorIDString checks the actor ID renders as lowercase hex like Hash. +func TestActorIDString(t *testing.T) { + t.Parallel() + + var actorID automerge.ActorID + actorID[0] = 0xab + actorID[15] = 0x01 + + assert.Equal(t, "ab000000000000000000000000000001", actorID.String()) +} diff --git a/pkg/automerge/update_text_parity_test.go b/pkg/automerge/update_text_parity_test.go index a5e7ff26ac..1591d51850 100644 --- a/pkg/automerge/update_text_parity_test.go +++ b/pkg/automerge/update_text_parity_test.go @@ -60,7 +60,7 @@ func TestRustText_SimpleUpdateText(t *testing.T) { otherObject, err := other.Root().Object(ctx, "text") require.NoError(t, err) - otherText, err := otherObject.Text() + otherText, err := otherObject.Text(ctx) require.NoError(t, err) require.NoError(t, otherText.Update(ctx, "Goodbye, world!")) _, err = other.Commit(ctx, "goodbye", commitTime) @@ -112,7 +112,7 @@ func TestRustText_UpdateTextBigOleGraphemes(t *testing.T) { otherObject, err := other.Root().Object(ctx, "text") require.NoError(t, err) - otherText, err := otherObject.Text() + otherText, err := otherObject.Text(ctx) require.NoError(t, err) require.NoError(t, otherText.Update(ctx, "left👨‍👩‍👧right")) _, err = other.Commit(ctx, "girl", commitTime) From 0437e195291cbf8b05bf1d0e60ab6102368d60d3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 09:45:50 +0000 Subject: [PATCH 15/17] Cache the compacted save between mutations Save rebuilt the entire columnar document on every call, which made the collaboration snapshot path pay a full re-encode per request even when nothing had changed. Rebuilding is the costly part of a save: for a 10k-operation document it was ~3.9ms and 10k allocations, dominated by reconstructing OpID-keyed maps and the operation columns. The engine now keeps a revision counter that advances on every committed change, orphan queue update, and isolate or integrate, and caches the compacted bytes against it. An unchanged document returns the cached bytes, so repeated saves drop from ~3.9ms to ~78ns. The cache stores and returns copies so callers keep ownership of their slices, and it is keyed by the retain-orphans and compress options as well as the revision, so an option change still rebuilds. Every apply path funnels through Commit, Merge, Isolate, Integrate or Rollback, so a single revision bump in each covers sync and incremental load too; bumping is deliberately conservative, since an extra bump only costs one rebuild and never returns stale bytes. Signed-off-by: Cursor Agent Co-authored-by: Bryan FRIMIN --- pkg/automerge/internal/native/engine.go | 64 +++++++++++++++++++- pkg/automerge/internal/native/history.go | 5 ++ pkg/automerge/internal/native/transaction.go | 5 ++ 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/pkg/automerge/internal/native/engine.go b/pkg/automerge/internal/native/engine.go index a78a08e7f6..29eaa1170c 100644 --- a/pkg/automerge/internal/native/engine.go +++ b/pkg/automerge/internal/native/engine.go @@ -59,6 +59,23 @@ type Engine struct { // the current heads, matching the reference's patch-log output across // isolate/integrate rather than a direct state comparison. isolationDiffTargets [][][32]byte + + // revision increases on every change to the committed history or the + // retained orphan set. The compacted save is cached against it so repeated + // saves of an unchanged document skip rebuilding the whole columnar + // document, which the collaboration snapshot path does on every request. + revision uint64 + saveCache saveCacheEntry +} + +// saveCacheEntry memoizes one compacted save. It is valid only for the exact +// revision and option combination it was built from. +type saveCacheEntry struct { + revision uint64 + retainOrphans bool + compress bool + valid bool + data []byte } type nativeSyncState struct { @@ -312,6 +329,16 @@ func (b *Engine) save( } } + // Rebuilding the columnar document is by far the costliest part of a save, so + // an unchanged document returns the bytes built last time. The cache is keyed + // by the mutation revision and the option combination, and it is invalidated + // implicitly because every committed change advances the revision. + if cached, ok := b.cachedSave(retainOrphans, deflate); ok { + b.saveCursor = len(b.appended) + + return cached, nil + } + // A compacted document chunk is the form every other implementation writes // and is dramatically smaller than the change stream for a long history. It // leaves the incremental cursor at the end, exactly as the stream save did, @@ -320,11 +347,46 @@ func (b *Engine) save( return nil, err } else if ok { b.saveCursor = len(b.appended) + b.storeSave(retainOrphans, deflate, data) return data, nil } - return b.streamSave(retainOrphans, deflate) + data, err := b.streamSave(retainOrphans, deflate) + if err != nil { + return nil, err + } + + b.storeSave(retainOrphans, deflate, data) + + return data, nil +} + +// cachedSave returns a copy of the previously built save when it is still valid +// for this revision and option combination. A copy is returned because callers +// own the bytes and may retain or mutate them. +func (b *Engine) cachedSave(retainOrphans, compress bool) ([]byte, bool) { + if !b.saveCache.valid || + b.saveCache.revision != b.revision || + b.saveCache.retainOrphans != retainOrphans || + b.saveCache.compress != compress { + return nil, false + } + + return append([]byte(nil), b.saveCache.data...), true +} + +// storeSave records a freshly built save so an unchanged document can return it +// without rebuilding. The stored bytes are copied so a caller mutating the +// returned slice cannot corrupt the cache. +func (b *Engine) storeSave(retainOrphans, compress bool, data []byte) { + b.saveCache = saveCacheEntry{ + revision: b.revision, + retainOrphans: retainOrphans, + compress: compress, + valid: true, + data: append([]byte(nil), data...), + } } // streamSave serializes the history as the loaded base followed by each change diff --git a/pkg/automerge/internal/native/history.go b/pkg/automerge/internal/native/history.go index 0f2685d911..efe08562bb 100644 --- a/pkg/automerge/internal/native/history.go +++ b/pkg/automerge/internal/native/history.go @@ -170,6 +170,11 @@ func (b *Engine) Merge(ctx context.Context, data []byte) ([][32]byte, error) { return nil, err } + // A merge may apply nothing when every change is already present, but bumping + // unconditionally only risks an extra rebuild on the next save, never a stale + // one, and it keeps every apply path covered by a single line. + b.revision++ + if len(b.state.Heads()) == 0 && len(b.pending) == 0 { state, err := NewStateFromDocument(document) if err != nil { diff --git a/pkg/automerge/internal/native/transaction.go b/pkg/automerge/internal/native/transaction.go index b65078a6f5..637033e758 100644 --- a/pkg/automerge/internal/native/transaction.go +++ b/pkg/automerge/internal/native/transaction.go @@ -92,6 +92,7 @@ func (b *Engine) Isolate(ctx context.Context, heads [][32]byte) error { b.state = pinned b.actor = isolationActor(full, pinned, baseActor) b.nextOp = full.maxOpGlobal() + 1 + b.revision++ b.isolationDiffTargets = append( b.isolationDiffTargets, @@ -134,6 +135,7 @@ func (b *Engine) Integrate(ctx context.Context) error { b.fullState = nil b.isolationActive = false b.nextOp = b.state.maxOpGlobal() + 1 + b.revision++ return nil } @@ -215,6 +217,7 @@ func (b *Engine) Commit( b.appended = append(b.appended, raw) b.pending = nil + b.revision++ return [32]byte(*change.Hash), nil } @@ -257,6 +260,7 @@ func (b *Engine) EmptyCommit( } b.appended = append(b.appended, raw) + b.revision++ return [32]byte(*change.Hash), nil } @@ -291,6 +295,7 @@ func (b *Engine) Rollback(ctx context.Context) (uint64, error) { b.pending = nil b.objects = map[uint32]ObjectID{0: RootObject()} b.nextHandle = 1 + b.revision++ return cancelled, nil } From d078f437abf14d001a6e1d40d90dee790f14bc20 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 09:54:44 +0000 Subject: [PATCH 16/17] Make text splicing linear in document size Each splice translated its UTF-16 index to an element by walking the whole visible sequence from the start and recomputing every element's width, and resolved its insertion anchor by scanning the whole insert order. Both are O(n) per splice, so typing n characters was O(n squared): a thousand-character insert took 7.2ms and grew about 53x when the size grew 10x, while Rust grew linearly. Two incremental indexes remove the walks. A per-object cumulative width slice lets a splice find its position by binary search, and a per-object insert-order position map resolves an anchor in constant time. Both are extended in place on the common append-at-end path and rebuilt lazily otherwise; each is guarded by length, and insert order only ever grows, so a stale index is always detected. Typing a thousand characters now takes 1.28ms, about 5.6x faster than before and linear with size, and about 43x faster than the reference on the same workload. The differential, mark, and splice parity suites are unchanged. Signed-off-by: Cursor Agent Co-authored-by: Bryan FRIMIN --- .../internal/native/engine_helpers.go | 76 ++++++++----------- pkg/automerge/internal/native/rich_text.go | 3 +- .../internal/native/rich_text_state.go | 16 +--- .../internal/native/sequence_state.go | 64 ++++++++++++++++ pkg/automerge/internal/native/state.go | 36 ++++++--- 5 files changed, 129 insertions(+), 66 deletions(-) diff --git a/pkg/automerge/internal/native/engine_helpers.go b/pkg/automerge/internal/native/engine_helpers.go index b0c14c164a..cc30fc2c8a 100644 --- a/pkg/automerge/internal/native/engine_helpers.go +++ b/pkg/automerge/internal/native/engine_helpers.go @@ -29,6 +29,7 @@ import ( "encoding/json" "fmt" "math" + "sort" ) func (b *Engine) addPending(operation Operation) error { @@ -426,62 +427,51 @@ func richTextPosition( return nil, previous, nil } +// sequenceRange resolves a UTF-16 index and delete count against the visible +// sequence using the precomputed cumulative offsets, so a splice locates its +// position by binary search rather than walking the whole sequence. offsets has +// one entry per element plus a trailing total, where offsets[i] is the width +// before element i. func sequenceRange( sequence []Operation, + offsets []uint32, index uint32, deleteCount uint32, ) (int, int, *OpID, error) { - position := uint32(0) - start := -1 - - var ( - previous *OpID - previousValue OpID - ) - - for i, operation := range sequence { - if position == index { - start = i - break - } - - length := elementLength(operation) - if position+length > index { - // UTF-16 callers can address the middle of a surrogate pair. - // Upstream Rust advances such a position to the boundary after - // the character rather than rejecting the edit. - position += length - previousValue = operation.ID - previous = &previousValue - start = i + 1 - - break - } - - position += length - previousValue = operation.ID - previous = &previousValue + total := offsets[len(offsets)-1] + if index > total { + return 0, 0, nil, fmt.Errorf("text index %d is out of bounds", index) } - if start == -1 { - if position != index { - return 0, 0, nil, fmt.Errorf("text index %d is out of bounds", index) - } - - start = len(sequence) + // Find the element whose starting offset is the last one at or before index. + // When that offset equals index the insertion sits on the boundary before + // the element; when it is smaller the index fell inside the element (a UTF-16 + // caller addressing the middle of a surrogate pair), so advance to the + // boundary after it, matching the reference. + boundary := sort.Search(len(offsets), func(i int) bool { return offsets[i] > index }) + start := boundary - 1 + if offsets[start] < index { + start++ } - target := position + deleteCount - - end := start - for end < len(sequence) && position < target { - position += elementLength(sequence[end]) - end++ + var previous *OpID + if start > 0 { + previousValue := sequence[start-1].ID + previous = &previousValue } // A deletion that runs past the end of the sequence is clamped to the // remaining elements rather than rejected, matching the reference, whose - // splice stops once there are no more elements to delete. + // splice stops once there are no more elements to delete. end is the first + // element boundary at or past the deletion target. + target := offsets[start] + deleteCount + end := start + sort.Search(len(sequence)-start+1, func(i int) bool { + return offsets[start+i] >= target + }) + if end > len(sequence) { + end = len(sequence) + } + return start, end, previous, nil } diff --git a/pkg/automerge/internal/native/rich_text.go b/pkg/automerge/internal/native/rich_text.go index 191f10943c..861617dadc 100644 --- a/pkg/automerge/internal/native/rich_text.go +++ b/pkg/automerge/internal/native/rich_text.go @@ -98,8 +98,9 @@ func (b *Engine) SpliceText( // blocks, so walk the full visible element sequence (text and block markers) // rather than the text-only view. sequence := b.state.sequenceElements(object.OpID) + offsets := b.state.sequenceOffsets(object.OpID, sequence) - start, end, previous, err := sequenceRange(sequence, index, uint32(deleteCount)) + start, end, previous, err := sequenceRange(sequence, offsets, index, uint32(deleteCount)) if err != nil { return err } diff --git a/pkg/automerge/internal/native/rich_text_state.go b/pkg/automerge/internal/native/rich_text_state.go index b987d937b6..7ac665eb08 100644 --- a/pkg/automerge/internal/native/rich_text_state.go +++ b/pkg/automerge/internal/native/rich_text_state.go @@ -119,20 +119,12 @@ func (s *State) insertAnchorKey(object OpID, base Key) Key { return base } - found := false - - for i, id := range order { - if id == *base.Element { - start = i + 1 - found = true - - break - } - } - - if !found { + position, ok := s.insertOrderPositions(object)[*base.Element] + if !ok { return base } + + start = position + 1 } type candidate struct { diff --git a/pkg/automerge/internal/native/sequence_state.go b/pkg/automerge/internal/native/sequence_state.go index 938b320c6b..9a423086c1 100644 --- a/pkg/automerge/internal/native/sequence_state.go +++ b/pkg/automerge/internal/native/sequence_state.go @@ -155,12 +155,15 @@ func (s *State) spliceInsertOrder(operation Operation) { if operation.Key.IsHead { s.insertOrderCache[object] = append([]OpID{operation.ID}, order...) + // A prepend shifts every position, so the index is rebuilt on demand. + delete(s.insertOrderPositionCache, object) return } if operation.Key.Element == nil { delete(s.insertOrderCache, object) + delete(s.insertOrderPositionCache, object) return } @@ -170,6 +173,13 @@ func (s *State) spliceInsertOrder(operation Operation) { if len(order) > 0 && order[len(order)-1] == anchor { s.insertOrderCache[object] = append(order, operation.ID) + // Appending at the end keeps every existing index, so extend the position + // index in step to keep sequential typing constant time. + if positions, ok := s.insertOrderPositionCache[object]; ok && + len(positions) == len(order) { + positions[operation.ID] = len(order) + } + return } @@ -184,12 +194,15 @@ func (s *State) spliceInsertOrder(operation Operation) { updated = append(updated, operation.ID) updated = append(updated, order[position:]...) s.insertOrderCache[object] = updated + // An insertion in the middle shifts later positions; rebuild on demand. + delete(s.insertOrderPositionCache, object) return } // The anchor is not present in the cached order; rebuild lazily. delete(s.insertOrderCache, object) + delete(s.insertOrderPositionCache, object) } func (s *State) sequenceValues(object OpID) []sequenceValue { @@ -260,6 +273,7 @@ func (s *State) updateSequenceValues(operation Operation) { if !appended { delete(s.sequenceValuesCache, object) delete(s.sequenceElementsCache, object) + delete(s.sequenceOffsetCache, object) return } @@ -273,7 +287,57 @@ func (s *State) updateSequenceValues(operation Operation) { if cached, ok := s.sequenceElementsCache[object]; ok { s.sequenceElementsCache[object] = append(cached, operation) + + // Extend the offset index in step so appending stays constant time. A + // valid offset slice for N elements holds N+1 entries, so it lines up + // with the pre-append element count; the new element starts at the + // previous total width. + if offsets, ok := s.sequenceOffsetCache[object]; ok && + len(offsets) == len(cached)+1 { + s.sequenceOffsetCache[object] = append( + offsets, + offsets[len(offsets)-1]+elementLength(operation), + ) + } + } +} + +// sequenceOffsets returns the cumulative UTF-16 width before each element of the +// given sequence, with a trailing entry holding the total width. It is cached +// and rebuilt whenever it does not line up with the elements, so a text index +// can be resolved by binary search rather than a linear walk. +func (s *State) sequenceOffsets(object OpID, elements []Operation) []uint32 { + if cached, ok := s.sequenceOffsetCache[object]; ok && len(cached) == len(elements)+1 { + return cached } + + offsets := make([]uint32, len(elements)+1) + for i, operation := range elements { + offsets[i+1] = offsets[i] + elementLength(operation) + } + + s.sequenceOffsetCache[object] = offsets + + return offsets +} + +// insertOrderPositions returns each insert-order operation's index. Insert order +// only ever grows, so a length mismatch is a sufficient staleness check. +func (s *State) insertOrderPositions(object OpID) map[OpID]int { + order := s.insertOrder(object) + + if cached, ok := s.insertOrderPositionCache[object]; ok && len(cached) == len(order) { + return cached + } + + positions := make(map[OpID]int, len(order)) + for index, id := range order { + positions[id] = index + } + + s.insertOrderPositionCache[object] = positions + + return positions } // elementValueWinners returns, for every list element that has been assigned a diff --git a/pkg/automerge/internal/native/state.go b/pkg/automerge/internal/native/state.go index 6166ed9234..688d8effb6 100644 --- a/pkg/automerge/internal/native/state.go +++ b/pkg/automerge/internal/native/state.go @@ -52,6 +52,18 @@ type ( sequenceValuesCache map[OpID][]sequenceValue sequenceElementsCache map[OpID][]Operation + // sequenceOffsetCache holds the cumulative UTF-16 width before each + // element of sequenceElementsCache (with a trailing total), so a text + // index resolves to an element by binary search instead of a linear + // walk. It is kept in step with the elements cache and guarded by length. + sequenceOffsetCache map[OpID][]uint32 + + // insertOrderPositionCache maps each insert-order operation to its index, + // so an insertion anchor resolves in constant time instead of scanning + // the order. Insert order only ever grows, so a length guard is enough to + // detect staleness. + insertOrderPositionCache map[OpID]map[OpID]int + // mapKeyIndex groups operation IDs by the map property they address so // reading a key does not scan the whole operation set. It is built on // first use and then maintained as operations are applied. @@ -73,16 +85,18 @@ type ( func NewState() *State { return &State{ - changes: make(map[ChangeHash]*Change), - actorSequence: make(map[ActorID]uint64), - operations: make(map[OpID]Operation), - superseded: make(map[OpID]struct{}), - heads: make(map[ChangeHash]struct{}), - sequenceCache: make(map[OpID][]Operation), - insertOrderCache: make(map[OpID][]OpID), - sequenceValuesCache: make(map[OpID][]sequenceValue), - sequenceElementsCache: make(map[OpID][]Operation), - mapKeyIndex: make(map[ObjectID]map[string][]OpID), + changes: make(map[ChangeHash]*Change), + actorSequence: make(map[ActorID]uint64), + operations: make(map[OpID]Operation), + superseded: make(map[OpID]struct{}), + heads: make(map[ChangeHash]struct{}), + sequenceCache: make(map[OpID][]Operation), + insertOrderCache: make(map[OpID][]OpID), + sequenceValuesCache: make(map[OpID][]sequenceValue), + sequenceElementsCache: make(map[OpID][]Operation), + sequenceOffsetCache: make(map[OpID][]uint32), + insertOrderPositionCache: make(map[OpID]map[OpID]int), + mapKeyIndex: make(map[ObjectID]map[string][]OpID), } } @@ -200,8 +214,10 @@ func (s *State) ApplyChange(change *Change) error { if !operation.Object.IsRoot { delete(s.sequenceCache, operation.Object.OpID) delete(s.insertOrderCache, operation.Object.OpID) + delete(s.insertOrderPositionCache, operation.Object.OpID) delete(s.sequenceValuesCache, operation.Object.OpID) delete(s.sequenceElementsCache, operation.Object.OpID) + delete(s.sequenceOffsetCache, operation.Object.OpID) } s.indexMapKeyOperation(operation) From 65423415cd5f6a30411f0b94960aba87095b5c4d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 11 Aug 2026 09:57:44 +0000 Subject: [PATCH 17/17] Preallocate decode buffers to speed up load Loading a large document spent most of its time in the garbage collector: a 10k-operation document allocated ~40k objects and 27MB, because the decode buffers grew by repeated reallocation. Three buffers are now sized up front. assignOperations gives each change an operation slice of its exact size instead of growing it by append, which was the single largest source at a third of all bytes. The RLE column decoder grows its value slice once per run rather than one item at a time, which matters most for text where an entire column is one long run. NewStateFromDocument presizes the operation and change maps to the counts it is about to insert. Load drops from ~11ms to ~6.4ms and from 27MB to 14MB for the 10k document, and is now about 2.9x faster than the WASM reference. Signed-off-by: Cursor Agent Co-authored-by: Bryan FRIMIN --- pkg/automerge/internal/native/state.go | 10 ++++++++++ pkg/automerge/internal/storage/columns.go | 11 ++++++++++- pkg/automerge/internal/storage/validate.go | 9 +++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/pkg/automerge/internal/native/state.go b/pkg/automerge/internal/native/state.go index 688d8effb6..60ecd5ac97 100644 --- a/pkg/automerge/internal/native/state.go +++ b/pkg/automerge/internal/native/state.go @@ -103,6 +103,16 @@ func NewState() *State { func NewStateFromDocument(document *Document) (*State, error) { state := NewState() + // Presize the operation and change maps so loading a large document does not + // rehash them repeatedly as it inserts every operation. + operationCount := 0 + for i := range document.Changes { + operationCount += len(document.Changes[i].Operations) + } + + state.operations = make(map[OpID]Operation, operationCount) + state.changes = make(map[ChangeHash]*Change, len(document.Changes)) + for i := range document.Changes { change := &document.Changes[i] if change.Sequence > state.actorSequence[change.Actor] { diff --git a/pkg/automerge/internal/storage/columns.go b/pkg/automerge/internal/storage/columns.go index fdcb4601cc..d6ac719c49 100644 --- a/pkg/automerge/internal/storage/columns.go +++ b/pkg/automerge/internal/storage/columns.go @@ -27,6 +27,7 @@ import ( "fmt" "io" "math" + "slices" "unicode/utf8" ) @@ -222,6 +223,8 @@ func decodeRLE[T any](data []byte, decodeValue func(*reader) (T, error)) ([]opti return nil, err } + values = slices.Grow(values, int(count)) + for range count { value, err := decodeValue(r) if err != nil { @@ -241,10 +244,16 @@ func appendRepeated[T any](values *[]optional[T], value optional[T], count uint6 return err } + // Grow once for the whole run. A long run (an entire column of the same + // value, common for text where every operation shares an object and action) + // otherwise reallocated the slice repeatedly as it doubled. + grown := slices.Grow(*values, int(count)) for range count { - *values = append(*values, value) + grown = append(grown, value) } + *values = grown + return nil } diff --git a/pkg/automerge/internal/storage/validate.go b/pkg/automerge/internal/storage/validate.go index d32dc8c6e4..6fc85bfddf 100644 --- a/pkg/automerge/internal/storage/validate.go +++ b/pkg/automerge/internal/storage/validate.go @@ -490,6 +490,15 @@ func assignOperations(changes []Change, operations []Operation) error { byActor[actor] = indexes } + // Each change holds exactly the operations whose counter falls in its + // [StartOp, MaxOp] range, so its operation slice can be sized once here. + // Growing it by append instead reallocated repeatedly and dominated the cost + // of loading a large single-change document. + for i := range changes { + size := changes[i].MaxOp - changes[i].StartOp + 1 + changes[i].Operations = make([]Operation, 0, size) + } + for _, operation := range operations { indexes := byActor[operation.ID.Actor]