diff --git a/README.md b/README.md index 1d0aa21e..819c901c 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,8 @@ store: merkle_service: url: "https://merkle-service-us-1.bsvb.tech" + # optional extra instances for redundancy; see docs/getting-started.md + # urls: ["https://merkle-service-us-2.bsvb.tech"] p2p: datahub_discovery: true diff --git a/app/app.go b/app/app.go index 986a14ec..3719055f 100644 --- a/app/app.go +++ b/app/app.go @@ -57,7 +57,7 @@ type Deps struct { Leaser store.Leaser TxTracker *store.TxTracker TeranodeClient *teranode.Client - MerkleClient *merkleservice.Client // nil when MerkleService.URL is unset + MerkleClient merkleservice.Service // nil when no merkle_service endpoint is configured Validator *validator.Validator // Chaintracks is the shared in-process header tracker. nil when // chaintracks_server is disabled (regtest, or explicit opt-out) — services @@ -184,10 +184,22 @@ func Bootstrap(ctx context.Context, cfg *config.Config, logger *zap.Logger) (*De seedCancel() } - var merkleClient *merkleservice.Client - if cfg.MerkleService.URL != "" { - merkleClient = merkleservice.NewClient(cfg.MerkleService.URL, cfg.MerkleService.AuthToken, 0) - merkleClient.SetLogger(logger.Named("merkle-client")) + // The interface stays nil when the integration is disabled; a typed-nil + // *Pool assigned here would make every consumer's nil guard pass and + // dereference it. + var merkleClient merkleservice.Service + if cfg.MerkleService.Enabled() { + endpoints := cfg.MerkleService.Endpoints() + // One Pool for every endpoint, single or many: each /watch and + // /reprocess fans out to all of them, a tx counts as registered once + // any endpoint accepts it, and a per-endpoint breaker keeps a dead + // merkle-service from adding its timeout to every batch. Start runs + // the /health probe that closes breakers again. + pool := merkleservice.NewPool(endpoints, cfg.MerkleService.AuthToken, 0) + pool.SetLogger(logger.Named("merkle-client")) + pool.Start(ctx) + merkleClient = pool + logger.Info("merkle-service endpoints configured", zap.Strings("endpoints", endpoints)) if cfg.MerkleService.AuthToken == "" { // Not fatal: many merkle-service deployments don't enforce auth, and // the standalone/no-auth profile is legitimate. But if the target @@ -196,8 +208,8 @@ func Bootstrap(ctx context.Context, cfg *config.Config, logger *zap.Logger) (*De // this loudly (issue #269) rather than misreporting it as a // consensus problem. Warn at startup so the cause is obvious. logger.Warn( - "merkle_service.url is set but merkle_service.auth_token is empty; " + - "if the target merkle-service enforces auth, tx registration (/watch) " + + "merkle_service endpoints are set but merkle_service.auth_token is empty; " + + "if a target merkle-service enforces auth, tx registration (/watch) " + "and block-processed recovery (/reprocess) will fail with 401 at runtime — " + "set merkle_service.auth_token to authenticate", ) @@ -632,7 +644,7 @@ func BuildServices(d *Deps) []services.Service { if wd := watchdog.NewService(cfg, d.Logger, d.Store, d.Leaser, d.MerkleClient); wd != nil { svcs = append(svcs, wd) } else { - d.Logger.Info("watchdog skipped: merkle_service.url or leaser not configured") + d.Logger.Info("watchdog skipped: merkle_service endpoints or leaser not configured") } } if shouldRun("sse") { diff --git a/bump/bump.go b/bump/bump.go index 2d26e066..cee66dd1 100644 --- a/bump/bump.go +++ b/bump/bump.go @@ -648,24 +648,29 @@ func BuildCompoundBUMP(stumps []*models.Stump, subtreeHashes []chainhash.Hash, c numSubtrees := len(subtreeHashes) + // Several merkle-services may each deliver a STUMP for the same subtree, + // and a service that missed registrations delivers one that lacks those + // txs' paths. Same-index variants are merged into one path per subtree + // so the compound covers the union of every tracked tx any service knew. + // Single-subtree edge case (and pathological zero-subtree case): the // STUMP IS the full BUMP. Reuse assembleFullPath which handles the // coinbase placeholder swap and odd-leaf padding for us. if numSubtrees <= 1 { - full, _, err := assembleFullPath(stumps[0].StumpData, stumps[0].SubtreeIndex, subtreeHashes, coinbaseBUMP) + merged, err := mergeStumpVariants(stumps) if err != nil { - return nil, nil, fmt.Errorf("failed to assemble single-subtree BUMP: %w", err) + return nil, nil, err } - txids := make([]string, 0) - for _, h := range ExtractLevel0Hashes(stumps[0].StumpData) { - txids = append(txids, h.String()) + full, _, err := assembleFullPath(merged[0].data, merged[0].index, subtreeHashes, coinbaseBUMP) + if err != nil { + return nil, nil, fmt.Errorf("failed to assemble single-subtree BUMP: %w", err) } - return full, txids, nil + return full, merged[0].txids, nil } firstPath, err := transaction.NewMerklePathFromBinary(stumps[0].StumpData) if err != nil { - return nil, nil, fmt.Errorf("failed to parse first STUMP: %w", err) + return nil, nil, fmt.Errorf("parse STUMP for subtree %d: %w", stumps[0].SubtreeIndex, err) } blockHeight := firstPath.BlockHeight @@ -695,6 +700,22 @@ func BuildCompoundBUMP(stumps []*models.Stump, subtreeHashes []chainhash.Hash, c finalIdx := numSubtrees - 1 finalNaturalHeight := internalHeight - liftLevels + // The lifted final subtree's STUMP is legitimately shorter: its natural + // height is internalHeight - liftLevels. Every other subtree is complete + // and must cover internalHeight levels. + requiredHeightFor := func(index int) int { + if index == finalIdx { + return finalNaturalHeight + } + return internalHeight + } + + stumps = dropVariantsOffSubtreeRoot(stumps, subtreeHashes, requiredHeightFor, coinbaseTxID, coinbaseBUMP) + merged, err := mergeStumpVariants(stumps) + if err != nil { + return nil, nil, err + } + compound := &transaction.MerklePath{ BlockHeight: blockHeight, Path: make([][]*transaction.PathElement, totalHeight), @@ -704,49 +725,34 @@ func BuildCompoundBUMP(stumps []*models.Stump, subtreeHashes []chainhash.Hash, c // of its levels' offsets from local-to-subtree to global-to-block, and // place the elements directly into the compound. Tracked-leaf Txid // markers carried on the parsed PathElements are preserved by reference. - haveSTUMP := make(map[int]bool, len(stumps)) + haveSTUMP := make(map[int]bool, len(merged)) var txids []string - for _, stump := range stumps { - if haveSTUMP[stump.SubtreeIndex] { - continue // duplicate STUMP for the same subtree — keep the first - } - haveSTUMP[stump.SubtreeIndex] = true - - path, err := transaction.NewMerklePathFromBinary(stump.StumpData) - if err != nil { - return nil, nil, fmt.Errorf("parse STUMP for subtree %d: %w", stump.SubtreeIndex, err) - } - // The lifted final subtree's STUMP is legitimately shorter: its - // natural height is internalHeight - liftLevels. Every other subtree - // is complete and must cover internalHeight levels. - requiredHeight := internalHeight - if stump.SubtreeIndex == finalIdx { - requiredHeight = finalNaturalHeight - } + for _, stump := range merged { + haveSTUMP[stump.index] = true + path := stump.path + requiredHeight := requiredHeightFor(stump.index) // A STUMP taller than requiredHeight is tolerated: merkle-service // sometimes delivers a STUMP at full block height, and only its first // requiredHeight levels are the subtree-internal path — the placement // loop below reads exactly those. A STUMP SHORTER than requiredHeight // cannot cover its subtree and is rejected. if len(path.Path) < requiredHeight { - return nil, nil, fmt.Errorf("subtree %d STUMP has internal height %d, expected at least %d", stump.SubtreeIndex, len(path.Path), requiredHeight) + return nil, nil, fmt.Errorf("subtree %d STUMP has internal height %d, expected at least %d", stump.index, len(path.Path), requiredHeight) } - if stump.SubtreeIndex == 0 && coinbaseTxID != nil { + if stump.index == 0 && coinbaseTxID != nil { applyCoinbaseToSTUMP(path, coinbaseTxID, coinbaseBUMP) } for level := 0; level < requiredHeight; level++ { - shift := uint64(stump.SubtreeIndex) << uint(internalHeight-level) //nolint:gosec // subtreeIndex is bounded by numSubtrees; height is small + shift := uint64(stump.index) << uint(internalHeight-level) //nolint:gosec // subtreeIndex is bounded by numSubtrees; height is small for _, elem := range path.Path[level] { elem.Offset += shift addLeaf(compound, level, elem) } } - for _, h := range ExtractLevel0Hashes(stump.StumpData) { - txids = append(txids, h.String()) - } + txids = append(txids, stump.txids...) } // Represent the final subtree's lift levels: from its natural height up @@ -791,6 +797,195 @@ func BuildCompoundBUMP(stumps []*models.Stump, subtreeHashes []chainhash.Hash, c return compound, txids, nil } +// dropVariantsOffSubtreeRoot filters the STUMP variants of every subtree that +// has more than one. mergeStumpVariants only rejects variants that disagree +// on a slot they share; a variant whose slots are disjoint from the others' +// is merged as-is, so one bad copy (a misbehaving merkle-service) would put +// leaves the subtree does not contain into the compound, or fail the whole +// block's validation and hold back the good variants with it. Each variant's +// tracked leaves are folded up the subtree's own height and compared with the +// block's subtree root; when at least one variant matches, the ones that do +// not are dropped. When none matches, all are kept: the check cannot tell +// which is wrong, and the compound-root validation still guards what is +// persisted. A subtree with a single STUMP is left alone. +func dropVariantsOffSubtreeRoot(stumps []*models.Stump, subtreeHashes []chainhash.Hash, requiredHeight func(index int) int, coinbaseTxID *chainhash.Hash, coinbaseBUMP []byte) []*models.Stump { + count := make(map[int]int, len(stumps)) + for _, st := range stumps { + count[st.SubtreeIndex]++ + } + matches := make(map[*models.Stump]bool) + anyMatch := make(map[int]bool) + for _, st := range stumps { + idx := st.SubtreeIndex + if count[idx] < 2 || idx < 0 || idx >= len(subtreeHashes) { + continue + } + if variantFoldsToRoot(st, &subtreeHashes[idx], requiredHeight(idx), coinbaseTxID, coinbaseBUMP) { + matches[st] = true + anyMatch[idx] = true + } + } + out := stumps[:0:0] + for _, st := range stumps { + idx := st.SubtreeIndex + if count[idx] >= 2 && anyMatch[idx] && !matches[st] { + continue + } + out = append(out, st) + } + return out +} + +// variantFoldsToRoot reports whether every tracked level-0 leaf of st (every +// leaf when none is marked) folds up height levels to root. It works on a +// fresh parse so the coinbase swap for subtree 0 does not touch the copy the +// build places. +func variantFoldsToRoot(st *models.Stump, root *chainhash.Hash, height int, coinbaseTxID *chainhash.Hash, coinbaseBUMP []byte) bool { + path, err := transaction.NewMerklePathFromBinary(st.StumpData) + if err != nil || height <= 0 || len(path.Path) < height { + return false + } + if st.SubtreeIndex == 0 && coinbaseTxID != nil { + applyCoinbaseToSTUMP(path, coinbaseTxID, coinbaseBUMP) + } + subtree := &transaction.MerklePath{BlockHeight: path.BlockHeight, Path: path.Path[:height]} + var leaves []*chainhash.Hash + for _, leaf := range subtree.Path[0] { + if leaf.Hash != nil && leaf.Txid != nil && *leaf.Txid { + leaves = append(leaves, leaf.Hash) + } + } + if len(leaves) == 0 { + for _, leaf := range subtree.Path[0] { + if leaf.Hash != nil { + leaves = append(leaves, leaf.Hash) + } + } + } + if len(leaves) == 0 { + return false + } + for _, leaf := range leaves { + got, err := subtree.ComputeRoot(leaf) + if err != nil || !got.IsEqual(root) { + return false + } + } + return true +} + +// mergedStump is one subtree's STUMP after same-index variants were merged. +type mergedStump struct { + index int + path *transaction.MerklePath + // data is the BRC-74 encoding of path: the original bytes when a single + // STUMP was delivered for the subtree, a re-encoding of the union + // otherwise. Captured before placement shifts the element offsets. + data []byte + // txids is every level-0 hash of the merged path, deduplicated, in + // first-seen order. + txids []string +} + +// mergeStumpVariants parses every STUMP and merges those that share a +// subtree index into one path by (level, offset) union. Variants come from +// different merkle-services: each builds its STUMP from the txids IT has +// registrations for, so a service that was down while txs were registered +// delivers a path that lacks them. Two variants describe the same subtree of +// the same block, so a slot they both carry must hold the same hash; a +// disagreement is a merkle-service bug and is reported rather than built. +// The Txid marker is OR-ed so a leaf tracked by any service stays tracked. +// Output preserves first-seen subtree order. +func mergeStumpVariants(stumps []*models.Stump) ([]mergedStump, error) { + byIndex := make(map[int]*mergedStump, len(stumps)) + variants := make(map[int]int, len(stumps)) + order := make([]int, 0, len(stumps)) + for _, st := range stumps { + path, err := transaction.NewMerklePathFromBinary(st.StumpData) + if err != nil { + return nil, fmt.Errorf("parse STUMP for subtree %d: %w", st.SubtreeIndex, err) + } + variants[st.SubtreeIndex]++ + g, ok := byIndex[st.SubtreeIndex] + if !ok { + byIndex[st.SubtreeIndex] = &mergedStump{index: st.SubtreeIndex, path: path, data: st.StumpData} + order = append(order, st.SubtreeIndex) + continue + } + if err := unionMerklePath(g.path, path); err != nil { + return nil, fmt.Errorf("subtree %d: %w", st.SubtreeIndex, err) + } + } + out := make([]mergedStump, 0, len(order)) + for _, idx := range order { + g := byIndex[idx] + if variants[idx] > 1 { + g.data = g.path.Bytes() + } + g.txids = level0TxIDs(g.path) + out = append(out, *g) + } + return out, nil +} + +// unionMerklePath adds every element of src that dst lacks, keyed by +// (level, offset), and reconciles shared slots: hashes must agree, a hash +// beats a duplicate marker, and the Txid flag is OR-ed. Levels are kept +// sorted by offset so the re-encoded path is canonical. +func unionMerklePath(dst, src *transaction.MerklePath) error { + for level := range src.Path { + for level >= len(dst.Path) { + dst.Path = append(dst.Path, nil) + } + existing := make(map[uint64]*transaction.PathElement, len(dst.Path[level])) + for _, e := range dst.Path[level] { + existing[e.Offset] = e + } + for _, e := range src.Path[level] { + have, ok := existing[e.Offset] + if !ok { + dst.Path[level] = append(dst.Path[level], e) + existing[e.Offset] = e + continue + } + switch { + case have.Hash != nil && e.Hash != nil && !have.Hash.IsEqual(e.Hash): + return fmt.Errorf("STUMP variants disagree at level %d offset %d: %s vs %s", level, e.Offset, have.Hash, e.Hash) + case have.Hash == nil && e.Hash != nil: + have.Hash = e.Hash + have.Duplicate = nil + } + if e.Txid != nil && *e.Txid && (have.Txid == nil || !*have.Txid) { + tracked := true + have.Txid = &tracked + } + } + slices.SortFunc(dst.Path[level], func(a, b *transaction.PathElement) int { return cmp.Compare(a.Offset, b.Offset) }) + } + return nil +} + +// level0TxIDs lists the level-0 hashes of a path, deduplicated, in order. +func level0TxIDs(path *transaction.MerklePath) []string { + if len(path.Path) == 0 { + return nil + } + seen := make(map[string]struct{}, len(path.Path[0])) + out := make([]string, 0, len(path.Path[0])) + for _, leaf := range path.Path[0] { + if leaf.Hash == nil { + continue + } + id := leaf.Hash.String() + if _, dup := seen[id]; dup { + continue + } + seen[id] = struct{}{} + out = append(out, id) + } + return out +} + // topTreeRoot folds subtree-root-layer hashes to the block merkle root using // Bitcoin-canonical duplicate-last-on-odd padding — the same rule // computeAndPadCompound applies at and above the subtree-root layer. diff --git a/bump/variants_test.go b/bump/variants_test.go new file mode 100644 index 00000000..ea69164a --- /dev/null +++ b/bump/variants_test.go @@ -0,0 +1,253 @@ +package bump + +import ( + "strings" + "testing" + + "github.com/bsv-blockchain/go-sdk/chainhash" + "github.com/bsv-blockchain/go-sdk/transaction" + + "github.com/bsv-blockchain/arcade/models" +) + +// Two merkle-services deliver the same subtree, each tracking a different tx +// (the second service missed the first tx's registration during an outage). +// The compound must cover both tracked txs, carry the Txid marker on each, +// and validate against the block root. +func TestBuildCompoundBUMP_MergesSameSubtreeVariants(t *testing.T) { + allLeaves, subtreeHashes, blockRoot := multiSubtreeTestSetup(2, 4) + + // Subtree 1 arrives as two variants: one tracking leaf 0, one leaf 2. + // Subtree 0 arrives once, tracking leaf 1. + stumps := []*models.Stump{ + {BlockHash: "blockhash", SubtreeIndex: 0, StumpData: buildSTUMP(allLeaves[0], 1, 990000)}, + {BlockHash: "blockhash", SubtreeIndex: 1, StumpData: buildSTUMP(allLeaves[1], 0, 990000)}, + {BlockHash: "blockhash", SubtreeIndex: 1, StumpData: buildSTUMP(allLeaves[1], 2, 990000)}, + } + + compound, txids, err := BuildCompoundBUMP(stumps, subtreeHashes, nil, &blockRoot) + if err != nil { + t.Fatalf("BuildCompoundBUMP: %v", err) + } + if vErr := ValidateCompoundRoot(compound, &blockRoot); vErr != nil { + t.Fatalf("merged compound does not validate: %v", vErr) + } + + // Level-0 of subtree 1 now carries all four leaves (each variant brings + // its tracked leaf plus sibling), and txids lists every level-0 hash once. + wantTracked := map[string]bool{allLeaves[1][0].String(): true, allLeaves[1][2].String(): true, allLeaves[0][1].String(): true} + seen := map[string]int{} + for _, id := range txids { + seen[id]++ + } + for id, n := range seen { + if n != 1 { + t.Fatalf("txid %s listed %d times", id, n) + } + } + for id := range wantTracked { + if seen[id] != 1 { + t.Fatalf("tracked tx %s missing from txids %v", id, txids) + } + } + + parsed, err := transaction.NewMerklePathFromBinary(compound.Bytes()) + if err != nil { + t.Fatal(err) + } + tracked := 0 + for _, leaf := range parsed.Path[0] { + if leaf.Hash == nil { + continue + } + if leaf.Txid != nil && *leaf.Txid { + tracked++ + if !wantTracked[leaf.Hash.String()] { + t.Fatalf("unexpected Txid marker on %s", leaf.Hash) + } + } + minimal := ExtractMinimalPath(parsed, leaf.Offset) + root, err := minimal.ComputeRoot(leaf.Hash) + if err != nil { + t.Fatalf("ComputeRoot(%s): %v", leaf.Hash, err) + } + if *root != blockRoot { + t.Fatalf("root for %s = %s, want %s", leaf.Hash, root, blockRoot) + } + } + if tracked != len(wantTracked) { + t.Fatalf("tracked markers = %d, want %d", tracked, len(wantTracked)) + } +} + +// Identical STUMPs from several services (the common case) merge to exactly +// the single-STUMP result. +func TestBuildCompoundBUMP_IdenticalVariantsAreOneSTUMP(t *testing.T) { + allLeaves, subtreeHashes, blockRoot := multiSubtreeTestSetup(2, 4) + data := buildSTUMP(allLeaves[1], 3, 990000) + single, singleTxIDs, err := BuildCompoundBUMP( + []*models.Stump{{BlockHash: "b", SubtreeIndex: 1, StumpData: data}}, subtreeHashes, nil, &blockRoot) + if err != nil { + t.Fatal(err) + } + dup, dupTxIDs, err := BuildCompoundBUMP([]*models.Stump{ + {BlockHash: "b", SubtreeIndex: 1, StumpData: data}, + {BlockHash: "b", SubtreeIndex: 1, StumpData: data}, + {BlockHash: "b", SubtreeIndex: 1, StumpData: data}, + }, subtreeHashes, nil, &blockRoot) + if err != nil { + t.Fatal(err) + } + if string(single.Bytes()) != string(dup.Bytes()) { + t.Fatal("three identical deliveries must build the same compound as one") + } + if strings.Join(singleTxIDs, ",") != strings.Join(dupTxIDs, ",") { + t.Fatalf("txids differ: %v vs %v", singleTxIDs, dupTxIDs) + } +} + +// Variants that disagree on a hash at the same slot cannot both be right. +// When one of them folds to the subtree root, the other is dropped before +// the merge and the block builds from the good one. +func TestBuildCompoundBUMP_ConflictingVariantOffRootIsDropped(t *testing.T) { + allLeaves, subtreeHashes, blockRoot := multiSubtreeTestSetup(2, 4) + // A STUMP built from subtree 0's leaves but labelled subtree 1 carries + // different hashes at the same (level, offset) slots. + stumps := []*models.Stump{ + {BlockHash: "b", SubtreeIndex: 1, StumpData: buildSTUMP(allLeaves[1], 0, 990000)}, + {BlockHash: "b", SubtreeIndex: 1, StumpData: buildSTUMP(allLeaves[0], 0, 990000)}, + } + compound, txids, err := BuildCompoundBUMP(stumps, subtreeHashes, nil, &blockRoot) + if err != nil { + t.Fatalf("BuildCompoundBUMP: %v", err) + } + if vErr := ValidateCompoundRoot(compound, &blockRoot); vErr != nil { + t.Fatalf("compound does not validate: %v", vErr) + } + for _, id := range txids { + if id == allLeaves[0][0].String() || id == allLeaves[0][1].String() { + t.Fatalf("leaf %s of the off-root variant made it into txids %v", id, txids) + } + } +} + +// When neither conflicting variant can be told apart by the subtree root, +// the merge still refuses to build rather than produce a compound that +// validates for one leaf set and silently drops the other. +func TestBuildCompoundBUMP_ConflictingVariantsRejected(t *testing.T) { + allLeaves, subtreeHashes, blockRoot := multiSubtreeTestSetup(2, 4) + subtreeHashes[1] = chainhash.DoubleHashH([]byte("not the subtree root")) + stumps := []*models.Stump{ + {BlockHash: "b", SubtreeIndex: 1, StumpData: buildSTUMP(allLeaves[1], 0, 990000)}, + {BlockHash: "b", SubtreeIndex: 1, StumpData: buildSTUMP(allLeaves[0], 0, 990000)}, + } + if _, _, err := BuildCompoundBUMP(stumps, subtreeHashes, nil, &blockRoot); err == nil || !strings.Contains(err.Error(), "disagree") { + t.Fatalf("conflicting variants must be rejected, got err=%v", err) + } +} + +// A variant that shares no slot with the good one passes the per-slot +// disagreement check, so a misbehaving merkle-service could slip leaves the +// subtree does not contain into the compound. Each variant of a multi-variant +// subtree is checked against the block's subtree root; one that does not +// fold to it is dropped and the build proceeds from the variants that do. +func TestBuildCompoundBUMP_DisjointBadVariantIsDropped(t *testing.T) { + allLeaves, subtreeHashes, blockRoot := multiSubtreeTestSetup(2, 4) + // Same shape as subtree 1, different hashes: its STUMP for offset 2 + // occupies slots (L0: 2,3; L1: 0) that the good variant for offset 0 + // (L0: 0,1; L1: 1) does not. + fake := make([]chainhash.Hash, 4) + for i := range fake { + fake[i] = chainhash.DoubleHashH([]byte{0xfa, byte(i)}) + } + stumps := []*models.Stump{ + {BlockHash: "b", SubtreeIndex: 0, StumpData: buildSTUMP(allLeaves[0], 1, 990000)}, + {BlockHash: "b", SubtreeIndex: 1, StumpData: buildSTUMP(allLeaves[1], 0, 990000)}, + {BlockHash: "b", SubtreeIndex: 1, StumpData: buildSTUMP(fake, 2, 990000)}, + } + compound, txids, err := BuildCompoundBUMP(stumps, subtreeHashes, nil, &blockRoot) + if err != nil { + t.Fatalf("BuildCompoundBUMP: %v", err) + } + if vErr := ValidateCompoundRoot(compound, &blockRoot); vErr != nil { + t.Fatalf("compound does not validate: %v", vErr) + } + for _, id := range txids { + for _, f := range fake { + if id == f.String() { + t.Fatalf("leaf %s from the bad variant made it into txids %v", id, txids) + } + } + } + parsed, err := transaction.NewMerklePathFromBinary(compound.Bytes()) + if err != nil { + t.Fatal(err) + } + for _, leaf := range parsed.Path[0] { + if leaf.Hash == nil { + continue + } + root, err := ExtractMinimalPath(parsed, leaf.Offset).ComputeRoot(leaf.Hash) + if err != nil || *root != blockRoot { + t.Fatalf("proof for %s does not hold: root=%v err=%v", leaf.Hash, root, err) + } + } +} + +// When no variant of a subtree folds to its root, nothing is dropped: the +// check cannot tell which one is wrong, and the compound-root validation +// still guards what gets persisted. +func TestBuildCompoundBUMP_NoVariantMatchingKeepsAll(t *testing.T) { + allLeaves, subtreeHashes, blockRoot := multiSubtreeTestSetup(2, 4) + subtreeHashes[1] = chainhash.DoubleHashH([]byte("not the subtree root")) + stumps := []*models.Stump{ + {BlockHash: "b", SubtreeIndex: 1, StumpData: buildSTUMP(allLeaves[1], 0, 990000)}, + {BlockHash: "b", SubtreeIndex: 1, StumpData: buildSTUMP(allLeaves[1], 2, 990000)}, + } + _, txids, err := BuildCompoundBUMP(stumps, subtreeHashes, nil, &blockRoot) + if err != nil { + t.Fatalf("BuildCompoundBUMP: %v", err) + } + if len(txids) != 4 { + t.Fatalf("both variants must be kept when none matches, txids = %v", txids) + } +} + +// Subtree 0's variants fold to the coinbase-corrected root only after the +// placeholder swap; both legitimate shapes (one carrying the coinbase slot, +// one minimal) must survive the root check and be merged. +func TestBuildCompoundBUMP_Subtree0VariantsSurviveRootCheckWithCoinbase(t *testing.T) { + allLeaves, trueAllLeaves, subtreeHashes, coinbaseTxID, trueBlockRoot := setupCoinbaseBlock(2, 4) + cbBUMP := buildCoinbaseBUMP(trueAllLeaves[0], coinbaseTxID, 990000, subtreeHashes) + stumps := []*models.Stump{ + {BlockHash: "b", SubtreeIndex: 0, StumpData: buildSTUMP(allLeaves[0], 1, 990000)}, // carries offset 0 + {BlockHash: "b", SubtreeIndex: 0, StumpData: buildSTUMP(allLeaves[0], 3, 990000)}, // minimal + } + // Both must match on their own: if neither did, nothing would be + // dropped and the build below would pass without proving the check. + corrected := correctedSubtree0Root(cbBUMP, len(subtreeHashes)) + if corrected == nil { + t.Fatal("corrected subtree-0 root unavailable") + } + for i, st := range stumps { + if !variantFoldsToRoot(st, corrected, 2, &coinbaseTxID, cbBUMP) { + t.Fatalf("variant %d does not fold to the corrected subtree-0 root", i) + } + } + compound, txids, err := BuildCompoundBUMP(stumps, subtreeHashes, cbBUMP, &trueBlockRoot) + if err != nil { + t.Fatalf("BuildCompoundBUMP: %v", err) + } + if vErr := ValidateCompoundRoot(compound, &trueBlockRoot); vErr != nil { + t.Fatalf("compound does not validate: %v", vErr) + } + have := map[string]bool{} + for _, id := range txids { + have[id] = true + } + for _, want := range []chainhash.Hash{allLeaves[0][1], allLeaves[0][3]} { + if !have[want.String()] { + t.Fatalf("tracked leaf %s dropped; txids %v", want, txids) + } + } +} diff --git a/compose/arcade-config.yaml b/compose/arcade-config.yaml index f6859f7b..c211bcb0 100644 --- a/compose/arcade-config.yaml +++ b/compose/arcade-config.yaml @@ -52,6 +52,10 @@ validator: merkle_service: url: http://merkle-service:8080 + # Add further merkle-service instances for redundancy; every tx is + # registered with all of them and duplicate callbacks are no-ops. + # urls: + # - http://merkle-service-2:8080 # Every chaintracks consumer (bump-builder header lookups, the validator # finality gate in api-server) reads headers from the chaintracks service diff --git a/config.example.standalone.yaml b/config.example.standalone.yaml index a268a932..56e0bc0c 100644 --- a/config.example.standalone.yaml +++ b/config.example.standalone.yaml @@ -98,6 +98,7 @@ teranode: merkle_service: url: "" + urls: [] auth_token: "" # P2P discovery via go-teranode-p2p-client. When enabled, arcade joins the diff --git a/config.example.yaml b/config.example.yaml index 4891ee9c..89a9c525 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -121,11 +121,30 @@ teranode: auth_token: "" merkle_service: - # Merkle Service base URL. Leaving this empty disables the Merkle integration - # (standalone profiles). When set, the top-level `callback_token` becomes - # REQUIRED — arcade fails to start otherwise so that the inbound callback - # endpoint never runs unauthenticated. See issue #76 / finding F-018. + # Merkle Service base URL(s). Leaving both `url` and `urls` empty disables + # the Merkle integration (standalone profiles). When at least one endpoint + # is set, the top-level `callback_token` becomes REQUIRED — arcade fails to + # start otherwise so that the inbound callback endpoint never runs + # unauthenticated. See issue #76 / finding F-018. + # + # `url` is the single-endpoint form; `urls` adds more. Every tx is + # registered with every endpoint and each endpoint delivers its own + # callbacks, so one merkle-service going down neither blocks registration + # (a tx is registered once any endpoint accepts it) nor stops SEEN/STUMP/ + # BLOCK_PROCESSED delivery. Duplicate callbacks are no-ops. A registration + # one endpoint accepted but another missed is re-sent to it in the + # background. An endpoint that fails three consecutive requests is skipped + # until GET /health on it answers. + # The breaker applies to a single `url` as well: after three consecutive + # transport/5xx failures, registrations are refused locally (propagation + # requeues them) until the next probe (every 5s) finds the service healthy, + # instead of every batch waiting out the 30s HTTP timeout. + # All endpoints share `auth_token`. Env form: ARCADE_MERKLE_SERVICE_URLS as + # a comma-separated list. url: "" + # urls: + # - "https://merkle-service-us-1.bsvb.tech" + # - "https://merkle-service-us-2.bsvb.tech" auth_token: "" p2p: @@ -344,6 +363,12 @@ bump_builder: # Mitigates F-007. # datahub_max_block_bytes: 1073741824 # + # How long STUMP rows stay after the block's BUMP is built. With several + # merkle-services each delivers its own STUMP set; a later BLOCK_PROCESSED + # re-reads the retained set and rebuilds only if it adds leaves the stored + # BUMP lacks. The janitor deletes rows older than this. + # stump_retention_minutes: 60 + # # Write-time anchor guard (issue #279): refuse to mark a block's txs MINED # when chaintracks' active chain provably has a DIFFERENT block at its # height (same-height competition loser). Fails open on any uncertainty. diff --git a/config/config.go b/config/config.go index f555cf51..48dc215b 100644 --- a/config/config.go +++ b/config/config.go @@ -3,8 +3,10 @@ package config import ( "errors" "fmt" + "net/url" "os" "path/filepath" + "strconv" "strings" chaintracksconfig "github.com/bsv-blockchain/go-chaintracks/config" @@ -310,9 +312,52 @@ type TeranodeConfig struct { AuthToken string `mapstructure:"auth_token"` } +// MerkleServiceConfig selects the merkle-service endpoints arcade registers +// transactions with. URL is the original single-endpoint form; URLs is the +// multi-endpoint form. Both feed Endpoints(), so an existing config with only +// `url` keeps working and `urls` may be added alongside it. +// +// With several endpoints every tx is registered with all of them and every +// endpoint delivers callbacks; arcade treats a callback that carries nothing +// new as a no-op. A tx counts as registered once at least one endpoint +// accepted it, so a single unreachable merkle-service no longer blocks +// broadcast (F-024 still holds: zero acceptances means no broadcast). type MerkleServiceConfig struct { - URL string `mapstructure:"url"` - AuthToken string `mapstructure:"auth_token"` + URL string `mapstructure:"url"` + URLs []string `mapstructure:"urls"` + AuthToken string `mapstructure:"auth_token"` +} + +// Endpoints returns every configured merkle-service base URL, in config +// order (URL first, then URLs), trimmed, without a trailing slash, and with +// duplicates removed. Empty entries are dropped here; validate rejects them so +// a typo in `urls` fails loudly at startup instead of silently shrinking the +// redundancy set. +func (c MerkleServiceConfig) Endpoints() []string { + raw := make([]string, 0, 1+len(c.URLs)) + raw = append(raw, c.URL) + raw = append(raw, c.URLs...) + seen := make(map[string]struct{}, len(raw)) + out := make([]string, 0, len(raw)) + for _, u := range raw { + u = strings.TrimSuffix(strings.TrimSpace(u), "/") + if u == "" { + continue + } + if _, dup := seen[u]; dup { + continue + } + seen[u] = struct{}{} + out = append(out, u) + } + return out +} + +// Enabled reports whether the Merkle integration is configured at all. It is +// the single toggle every consumer checks: no endpoints means no /watch +// registration, no /reprocess recovery, and no watchdog. +func (c MerkleServiceConfig) Enabled() bool { + return len(c.Endpoints()) > 0 } // P2PConfig controls the libp2p-based peer discovery service. Seeds is the @@ -548,10 +593,13 @@ type PropagationConfig struct { // MerkleRegisteredAt is within this window. /watch is INSERT ... ON // CONFLICT DO NOTHING on the merkle-service side and does not refresh // expires_at, so re-registering a tx merkle-service already knows about - // is wasted work. Default 30 (matches merkle-service postMineTTLSec). - // Set to 0 to disable the skip and re-register every non-terminal tx — - // useful for forcing a full re-sync after a known merkle-service wipe. - // Issue #145. + // is wasted work. Default -1 (auto): 30 (matches merkle-service + // postMineTTLSec) with a single merkle-service, 0 with several, because + // the stamp is set once any one of them accepted the tx and the pool's + // queue of registrations owed to a lagging endpoint is in memory, so a + // restart must re-send them. Set to 0 to disable the skip and + // re-register every non-terminal tx — useful for forcing a full re-sync + // after a known merkle-service wipe. Issue #145. MerkleReplaySkipRecentMinutes int `mapstructure:"merkle_replay_skip_recent_minutes"` // MerkleReplayRPS caps the average requests-per-second the startup // replay issues against merkle-service. Implemented as an inter-batch @@ -664,6 +712,13 @@ type BumpBuilderConfig struct { // open on any uncertainty. Default true (viper); false restores the // legacy last-writer-wins behavior as an instant rollback lever. AnchorGuardEnabled bool `mapstructure:"anchor_guard_enabled"` + // StumpRetentionMinutes is how long STUMP rows are kept after the + // block's BUMP was built before the janitor deletes them. STUMPs are no + // longer pruned right after a build: with several merkle-services each + // one delivers its own STUMP set, and a later BLOCK_PROCESSED must be + // able to re-read the full set to decide whether it adds leaves the + // stored BUMP lacks. <= 0 selects the default (60). + StumpRetentionMinutes int `mapstructure:"stump_retention_minutes"` // Reconciler tunes the anchor reconciler that heals txs anchored to // orphaned blocks (issue #279). Reconciler ReconcilerConfig `mapstructure:"reconciler"` @@ -751,8 +806,8 @@ type ReconcilerConfig struct { // WatchdogConfig tunes the stale-block recovery watchdog. Defaults are // applied in setDefaults; zero values for everything except Enabled fall -// back to those defaults. The watchdog requires merkle_service.url to be -// set (no /reprocess target otherwise) and runs as a standalone arcade +// back to those defaults. The watchdog requires at least one merkle_service +// endpoint (no /reprocess target otherwise) and runs as a standalone arcade // service (mode=watchdog) or alongside other services when mode=all. At // most one replica fires per tick — coordination via the // `block-processing-watchdog` lease. @@ -1190,6 +1245,17 @@ func setDefaults() { viper.SetDefault("api.host", "0.0.0.0") viper.SetDefault("api.port", 8080) viper.SetDefault("kafka.backend", "sarama") + // callback_* and merkle_service.* defaults exist so viper.AutomaticEnv + // knows the keys: without a registered key an ARCADE_MERKLE_SERVICE_URLS + // or ARCADE_CALLBACK_TOKEN override is silently ignored (see + // TestPropagationBatchCapsBind for the same trap). Empty values keep the + // integration disabled. + viper.SetDefault("callback_url", "") + viper.SetDefault("callback_token", "") + viper.SetDefault("merkle_service.url", "") + viper.SetDefault("merkle_service.urls", []string{}) + viper.SetDefault("merkle_service.auth_token", "") + viper.SetDefault("kafka.brokers", []string{"localhost:9092"}) viper.SetDefault("kafka.consumer_group", "arcade") viper.SetDefault("kafka.max_retries", 5) @@ -1276,7 +1342,7 @@ func setDefaults() { // callbacks for everything in-flight. viper.SetDefault("propagation.register_replay_on_start", true) viper.SetDefault("propagation.register_replay_lookback_hours", 24) - viper.SetDefault("propagation.merkle_replay_skip_recent_minutes", 30) + viper.SetDefault("propagation.merkle_replay_skip_recent_minutes", -1) viper.SetDefault("propagation.merkle_replay_rps", 50) viper.SetDefault("network", NetworkMainnet) @@ -1330,6 +1396,7 @@ func setDefaults() { // while still bounding memory against a hostile DataHub. See F-007. viper.SetDefault("bump_builder.datahub_max_block_bytes", int64(1*1024*1024*1024)) viper.SetDefault("bump_builder.anchor_guard_enabled", true) + viper.SetDefault("bump_builder.stump_retention_minutes", 60) viper.SetDefault("bump_builder.reconciler.enabled", true) viper.SetDefault("bump_builder.reconciler.interval_ms", 30000) viper.SetDefault("bump_builder.reconciler.batch_size", 5000) @@ -1345,7 +1412,7 @@ func setDefaults() { // Block-processing watchdog (standalone arcade service — mode=watchdog // in production, in-process under mode=all): on by default. The runtime // nil-guards the merkle-service client; an unconfigured deployment - // (merkle_service.url unset) skips the watchdog regardless of this flag. + // (no merkle_service.url/urls) skips the watchdog regardless of this flag. viper.SetDefault("watchdog.enabled", true) viper.SetDefault("watchdog.interval_ms", 30000) viper.SetDefault("watchdog.stale_threshold_ms", 120000) @@ -1401,6 +1468,75 @@ func setDefaults() { viper.SetDefault("validator.observed_fee_refresh_ms", DefaultValidatorObservedFeeRefreshMs) } +// validateMerkleService checks the merkle_service block. Every entry of +// `url`/`urls` must be an absolute http(s) URL; a blank entry in `urls` is a +// misconfiguration rather than "disabled" (only an entirely empty block +// disables the integration). When at least one endpoint is set the top-level +// callback_token is mandatory — see the comment in validate. +func validateMerkleService(cfg *Config) error { + for i, u := range cfg.MerkleService.URLs { + if strings.TrimSpace(u) == "" { + return fmt.Errorf("merkle_service.urls[%d] is empty; remove the entry or set a URL", i) + } + } + // Every error below names the URL through redactURL: the URL may carry + // credentials (user info, a token in the query, or a scheme-less + // "user:pass@host" that parses as an opaque URL), and the startup error + // is logged. + for _, u := range cfg.MerkleService.Endpoints() { + parsed, err := url.Parse(u) + if err != nil { + return fmt.Errorf("merkle_service url %s does not parse", redactURL(u)) + } + // The URL is shown in /health, metric labels and startup logs, so + // credentials in it would leak there; auth_token is the supported + // way to authenticate. + if parsed.User != nil { + return fmt.Errorf("merkle_service url %s must not contain user info; use merkle_service.auth_token for authentication", redactURL(u)) + } + if (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + return fmt.Errorf("merkle_service url %s must be an absolute http(s) URL", redactURL(u)) + } + // The client appends /watch, /reprocess and /health to the base URL + // as a string, so a query or fragment would swallow the path. + if parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" { + return fmt.Errorf("merkle_service url %s must not contain a query or fragment", redactURL(u)) + } + } + if cfg.MerkleService.Enabled() && cfg.CallbackToken == "" { + return fmt.Errorf("callback_token is required when merkle_service.url/urls is set " + + "(unauthenticated /api/v1/merkle-service/callback would accept forged callbacks; see issue #76)") + } + return nil +} + +// redactURL renders a configured URL for an error message with everything +// that may be a secret replaced: user info, an opaque part (a scheme-less +// "user:pass@host:port" parses with its first segment as the scheme and the +// rest opaque), the query and the fragment. A URL that does not parse is not +// echoed at all. +func redactURL(raw string) string { + u, err := url.Parse(raw) + if err != nil { + return "(unparseable)" + } + const redacted = "REDACTED" + c := *u + if c.User != nil { + c.User = url.User(redacted) + } + if c.Opaque != "" { + c.Opaque = redacted + } + if c.RawQuery != "" { + c.RawQuery = redacted + } + if c.Fragment != "" { + c.Fragment, c.RawFragment = redacted, "" + } + return strconv.Quote(c.String()) +} + func validate(cfg *Config) error { switch cfg.Kafka.Backend { case "", "sarama": @@ -1456,15 +1592,16 @@ func validate(cfg *Config) error { default: return fmt.Errorf("unknown store.backend %q (expected aerospike, pebble, postgres, or mongodb)", cfg.Store.Backend) } - // merkle_service.url is intentionally optional: an empty value means the - // Merkle integration is disabled. The runtime treats URL-presence as the - // toggle — cmd/arcade/main.go only constructs a merkleservice.Client when - // the URL is set, and propagation.Propagator nil-guards every dereference - // of the client. The documented standalone and zero-dependency profiles + // merkle_service.url / urls are intentionally optional: no endpoints means + // the Merkle integration is disabled. The runtime treats + // MerkleServiceConfig.Enabled() as the toggle — app.Bootstrap only + // constructs a merkleservice.Pool when at least one endpoint is set, and + // propagation.Propagator nil-guards every dereference of the client. The + // documented standalone and zero-dependency profiles // (config.example.standalone.yaml) ship with merkle_service.url: "" for // exactly this reason. See issue #59 / finding F-001. // - // When the Merkle integration IS enabled (URL set), callback_token is + // When the Merkle integration IS enabled, callback_token is // mandatory. The /api/v1/merkle-service/callback endpoint accepts forged // status updates for any txid in the system if it runs without bearer-token // auth, so we fail-closed here at config load rather than silently exposing @@ -1476,9 +1613,8 @@ func validate(cfg *Config) error { // callbacks. Without a configured token there's nothing to forward AND the // inbound receiver would 401 anyway — the same fail-closed posture covers // both ends, so a duplicate "outbound token required" check is unnecessary. - if cfg.MerkleService.URL != "" && cfg.CallbackToken == "" { - return fmt.Errorf("callback_token is required when merkle_service.url is set " + - "(unauthenticated /api/v1/merkle-service/callback would accept forged callbacks; see issue #76)") + if err := validateMerkleService(cfg); err != nil { + return err } if cfg.Network == "" { cfg.Network = NetworkMainnet diff --git a/config/config_test.go b/config/config_test.go index 78ee0f8e..71e79cd3 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -521,3 +521,117 @@ func TestValidate_MongoDBBounds(t *testing.T) { t.Fatalf("store.mongodb.index_timeout_ms default = %d, want 300000 (matches postgres schema_apply_timeout_ms)", got) } } + +// Multi-endpoint merkle_service: `urls` extends `url`, and Endpoints() is +// the single normalized view every consumer uses. +func TestMerkleServiceEndpoints_MergesURLAndURLs(t *testing.T) { + cfg := MerkleServiceConfig{ + URL: "http://merkle-a.local/", + URLs: []string{" http://merkle-b.local ", "http://merkle-a.local", "http://merkle-b.local/"}, + } + got := cfg.Endpoints() + want := []string{"http://merkle-a.local", "http://merkle-b.local"} + if len(got) != len(want) { + t.Fatalf("Endpoints() = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("Endpoints()[%d] = %q, want %q (full: %v)", i, got[i], want[i], got) + } + } + if !cfg.Enabled() { + t.Fatal("Enabled() must be true with endpoints") + } + if (MerkleServiceConfig{}).Enabled() { + t.Fatal("Enabled() must be false with no endpoints") + } +} + +func TestValidate_MerkleServiceURLsOnlyRequiresCallbackToken(t *testing.T) { + cfg := baseValidConfig() + cfg.MerkleService.URL = "" + cfg.MerkleService.URLs = []string{"http://merkle-a.local", "http://merkle-b.local"} + if err := validate(cfg); err != nil { + t.Fatalf("urls-only config should validate, got: %v", err) + } + cfg.CallbackToken = "" + if err := validate(cfg); err == nil { + t.Fatal("callback_token must be required when merkle_service.urls is set") + } +} + +func TestValidate_MerkleServiceRejectsEmptyAndMalformedEntries(t *testing.T) { + cfg := baseValidConfig() + cfg.MerkleService.URLs = []string{"http://merkle-b.local", " "} + if err := validate(cfg); err == nil { + t.Fatal("a blank merkle_service.urls entry must be rejected") + } + + for _, bad := range []string{ + "merkle.local", "ftp://merkle.local", "http://", + // The client string-appends /watch, so these would lose the path. + "http://merkle.local?tenant=a", "http://merkle.local?", "http://merkle.local#frag", + } { + cfg := baseValidConfig() + cfg.MerkleService.URL = bad + if err := validate(cfg); err == nil { + t.Fatalf("merkle_service.url %q must be rejected", bad) + } + } +} + +// ARCADE_MERKLE_SERVICE_URLS is the env form of the list; viper splits it +// on commas the same way it does ARCADE_KAFKA_BROKERS. This only works +// because setDefaults registers the key. +func TestMerkleServiceURLsBindFromEnv(t *testing.T) { + t.Setenv("ARCADE_MERKLE_SERVICE_URLS", "http://merkle-a.local,http://merkle-b.local/") + t.Setenv("ARCADE_CALLBACK_TOKEN", "tok") + cfg, err := Load(&cobra.Command{}) + if err != nil { + t.Fatalf("load: %v", err) + } + got := cfg.MerkleService.Endpoints() + if len(got) != 2 || got[0] != "http://merkle-a.local" || got[1] != "http://merkle-b.local" { + t.Fatalf("Endpoints() from env = %v", got) + } +} + +// Credentials in a merkle_service URL would surface in /health, metric +// labels and startup logs; auth_token is the supported mechanism. The error +// must not echo the password either. +func TestValidate_MerkleServiceRejectsUserinfo(t *testing.T) { + for _, bad := range []string{"https://user:s3cret@merkle.local", "http://user@merkle.local"} { + cfg := baseValidConfig() + cfg.MerkleService.URLs = []string{"http://merkle-a.local", bad} + err := validate(cfg) + if err == nil { + t.Fatalf("merkle_service url %q must be rejected", bad) + } + if strings.Contains(err.Error(), "s3cret") { + t.Fatalf("validation error leaks the password: %v", err) + } + } +} + +// No merkle_service URL validation error echoes a secret: not for a URL that +// does not parse, a scheme-less user:pass@host (which parses as opaque), a +// token in the query, or a fragment. +func TestValidate_MerkleServiceErrorsRedactSecrets(t *testing.T) { + for _, bad := range []string{ + "http://merkle.local/%zz-s3cret", // does not parse + "svc:s3cret@merkle.local:8080", // scheme-less user info + "https://merkle.local/?token=s3cret", // query + "https://merkle.local/#s3cret", // fragment + "ftp://user:s3cret@merkle.local", // user info with a bad scheme + } { + cfg := baseValidConfig() + cfg.MerkleService.URLs = []string{bad} + err := validate(cfg) + if err == nil { + t.Fatalf("merkle_service url %q must be rejected", bad) + } + if strings.Contains(err.Error(), "s3cret") { + t.Fatalf("validation error for %q leaks the secret: %v", bad, err) + } + } +} diff --git a/doc.md b/doc.md index 8c584efa..51f1d590 100644 --- a/doc.md +++ b/doc.md @@ -145,14 +145,20 @@ The `api_server` exposes: ### Merkle-service callback states -The `/api/v1/merkle-service/callback` endpoint is a thin Kafka publisher. All real work happens downstream so the API server stays cheap and horizontally scalable. Recognized states: +The `/api/v1/merkle-service/callback` endpoint applies SEEN transitions and +stores STUMPs synchronously (a 2xx from arcade implies durability) and hands +`BLOCK_PROCESSED` to Kafka so BUMP construction runs in the bump-builder. +Recognized callback types: -- `REJECTED` - `SEEN_ON_NETWORK` - `SEEN_MULTIPLE_NODES` -- `STUMP` — STUMP is stored synchronously, so a 2xx from arcade implies durability +- `STUMP` — stored keyed by block hash and subtree index - `BLOCK_PROCESSED` — triggers BUMP construction after `grace_window_ms` +With several merkle-services configured every callback arrives once per +service; a delivery that carries nothing new is a no-op (see +`docs/merkle-lifecycle-durability.md`). + ## Observability Prometheus metrics are exposed at `/metrics` on both the API server (port `api.port`, default 8080) and the per-pod health server (port `health.port`, default 8081). All metric names are namespaced under `arcade_`. diff --git a/docs/getting-started.md b/docs/getting-started.md index efb4f812..af7b4627 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -133,10 +133,44 @@ The service watches the network for arcade's registered txids and drives the transaction lifecycle forward via callbacks (`SEEN_ON_NETWORK` → `MINED` → `IMMUTABLE`). -**If `merkle_service.url` is left empty, arcade still accepts and broadcasts -transactions, but every row stays at `RECEIVED` forever** — no callback source -means nothing advances the state machine. Configure Merkle Service whenever -you want real status progression. +**If `merkle_service.url` and `merkle_service.urls` are both left empty, arcade +still accepts and broadcasts transactions, but every row stays at `RECEIVED` +forever** — no callback source means nothing advances the state machine. +Configure Merkle Service whenever you want real status progression. + +### Running against several Merkle Services + +List additional instances under `merkle_service.urls` (or set +`ARCADE_MERKLE_SERVICE_URLS` to a comma-separated list). Arcade registers every +transaction with every endpoint and each endpoint delivers its own callbacks: + +- A transaction is registered, and therefore broadcast, as soon as **one** + endpoint accepts it. Only when every endpoint refuses is the tx requeued. +- Duplicate `SEEN_ON_NETWORK` / `SEEN_MULTIPLE_NODES` callbacks are no-ops; + each transition is published to SSE/webhook subscribers once. A duplicate + `BLOCK_PROCESSED` re-uses the stored BUMP and publishes `MINED` only for + transactions that were not already mined; if the later service's STUMPs + cover transactions the first build lacked, the BUMP is rebuilt and only + those are published. +- A registration one endpoint accepted but another refused, timed out on, + or was skipped for is queued and re-sent to that endpoint in the + background, so every endpoint ends up watching the same set. An endpoint + that fails three consecutive requests is skipped until its `GET /health` + answers again. `/health` lists each endpoint under `merkle_endpoints` + (breaker state and queued registrations) as the answering process sees + it; every process runs its own pool, and the propagation pods carry the + registrations, so their `arcade_merkle_endpoint_healthy` / + `arcade_merkle_endpoint_catchup_pending` series are the authoritative + view. +- The breaker applies to a single `url` too. After three consecutive + transport/5xx failures, registrations are refused locally (propagation + requeues them) until the next probe, every 5s, finds the service healthy. + Before, each batch waited out the 30s HTTP timeout instead. +- `/reprocess` (watchdog and `POST /api/v1/blocks/:hash/reprocess`) is sent + to every endpoint and succeeds if any accepts. + +All endpoints share `merkle_service.auth_token` and the top-level +`callback_url` / `callback_token`. ### Public Merkle Service endpoints diff --git a/docs/merkle-lifecycle-durability.md b/docs/merkle-lifecycle-durability.md index 7b85aac5..e2fd4174 100644 --- a/docs/merkle-lifecycle-durability.md +++ b/docs/merkle-lifecycle-durability.md @@ -76,3 +76,112 @@ tracked subtree. If that field is omitted, a dropped STUMP set is indistinguishable from an empty block and `finalizeEmptyBlock` stamps `processed_at`. Confirm this for a deployment's Merkle before enabling callbacks. + +## Multiple merkle-services + +`merkle_service.urls` lists additional endpoints. The `merkleservice.Pool` +fans every `/watch` and `/reprocess` out to all of them, so each service +watches the same txids and each delivers its own callbacks. + +Registration succeeds when **any** endpoint accepts; only a tx refused or +unreachable on every endpoint is requeued (F-024 still gates broadcast). When +every endpoint failed, the surfaced error is the most retryable one (context +error, then network/5xx, then 401/403, then other 4xx). So `auth_error` on the +registration metrics means no endpoint accepted the tx, none failed with a +network error or 5xx, and at least one rejected the token; the others may have +answered with another 4xx. A token problem stays visible even when it is not +the only failure. + +A tx the pool reported registered must still reach every endpoint, or the +endpoints' watch sets diverge and their STUMPs disagree. Every endpoint- +specific failure behind a pool-level success — a refused or timed-out +`/watch`, or an endpoint skipped because its breaker was open — is kept in +that endpoint's in-memory catch-up queue (`arcade_merkle_endpoint_catchup_pending`) +and re-sent from the pool's background loop once the endpoint answers again, +500 per 5s tick with backoff; what fails again goes back to the queue, except +a registration the endpoint rejected outright (a 4xx other than 401, 403, 408 +or 429), which is dropped with a warning. A 401/403 is kept and logged, since +the breaker does not count it. The +queue is bounded (200k entries per endpoint, about half an hour at 100 TPS); +past that the oldest entries are dropped and, once the queue drains, the +pool asks propagation to resync that endpoint with a full lookback replay +(`register_replay_lookback_hours`, ignoring `merkle_registered_at`, since +the stamp is pool-wide). A `/watch` that fails during the resync goes back +to the queue. `/watch` is idempotent on merkle-service. + +The queue lives in memory, so a restart drops it. With several endpoints the +startup replay therefore re-registers every non-terminal tx in the lookback +window with the whole pool: `merkle_replay_skip_recent_minutes` defaults to +`-1`, which means no recent-row skip with several merkle-services (30 minutes +with one), because `merkle_registered_at` is stamped once any one endpoint +accepted. Setting it to a positive value with several endpoints brings the +skip back, and with it the risk that a lagging endpoint never receives the +txs registered in that window before the restart. + +An endpoint that fails three consecutive transport/5xx requests has its +breaker opened and is skipped; 4xx answers never trip it. Results are recorded +per request, and nothing more is sent to an endpoint once its breaker opens, +so a black-holed endpoint delays the batch that trips it by at most the HTTP +timeout of the requests already in flight; later batches skip it at once. +Each endpoint's catch-up drain runs on its own goroutine, so a hanging +endpoint does not hold up the probes or another endpoint's catch-up. The pool probes +`GET /health` on open endpoints every 5s (concurrently) and closes the breaker +on any non-5xx answer, provided one owed registration re-sent as a trial +`/watch` (bounded by the 5s probe timeout) also answers below 500, so an +endpoint whose `/health` is up while `/watch` hangs stays open; the catch-up +queue then drains. + +Duplicate callbacks: + +- `SEEN_ON_NETWORK` / `SEEN_MULTIPLE_NODES` from a second service hit the + tracker prefilter or the lattice re-assert path and are not published. A + forward move (`SEEN_ON_NETWORK` → `SEEN_MULTIPLE_NODES`) from whichever + service reports it first is applied and published once; a late + `SEEN_ON_NETWORK` after that is a lattice skip. +- `STUMP` rows are content-addressed: the key is + `(block_hash, subtree_index, sha256(stump_data))`. Identical bytes from a + second service are a no-op; a different STUMP for the same subtree is a + second row. The difference is real: each merkle-service builds its STUMP + from the txids *it* has registrations for, so a service that was down while + txs were registered delivers a STUMP that lacks those txs' paths. + `bump.BuildCompoundBUMP` merges same-subtree variants by `(level, offset)` + union (hashes must agree; the tracked marker is OR-ed). +- `BLOCK_PROCESSED` is processed every time. When a compound BUMP already + exists, the builder reads the retained STUMP set and diffs its level-0 + hashes against the stored BUMP: + - nothing new ⇒ short-circuit: re-mine the stored level-0 set with + `onlyChanged=true` (rows already `MINED` against this block are not + re-published; a tx registered after the first build still publishes), + re-stamp `processed_at`. Outcome `short_circuited`, once per extra + service. + - new leaves ⇒ rebuild from the full STUMP set, overwrite the BUMP, mine the + union with `onlyChanged=true` so only the newly covered txs are + published. Outcome `rebuilt`. A rebuild always waits the grace window + first: the expected set names subtree indices, and an index covered by + the earlier service's variant says nothing about whether the later + service's own STUMP for it has landed. The rebuilt compound must cover + every leaf of the stored one; if it would not (only possible if the + janitor pruned the earlier STUMPs), the stored BUMP is kept and the block + is deferred. + + Residual window: a later service's STUMP that straggles past the grace + window for an index the earlier service already covered is not detected + (the index looks satisfied and no further `BLOCK_PROCESSED` follows). It + needs both a partial registration and a STUMP retry slower than + `grace_window_ms`; `/reprocess` on the block recovers it. + +## STUMP retention and deferral with several sources + +STUMPs are no longer pruned right after a build. The janitor +(`pruneOrphanStumps`, run at startup and every quarter of the window) deletes +a block's STUMP rows once its `bump_built_at` is older than +`bump_builder.stump_retention_minutes` (default 60). Anchor-denied and +reconciler cleanups still delete immediately. + +`processed_at` is still the watchdog's signal. The first service's build +stamps it. If a later service's `BLOCK_PROCESSED` then defers the block (its +expected STUMP set is still missing after the grace window, or a rebuild would +drop leaves), `deferStampedBlock` clears the stamp with the new store method +`ClearBlockProcessed`, so `ListStaleBlockProcessingStatus` surfaces the block +and the watchdog's `/reprocess` (fanned out to every merkle-service) re-drives +it. `ClearBlockProcessed` only unsets `processed_at` and never creates a row. diff --git a/docs/overview.md b/docs/overview.md index aa0d349e..f7d6e902 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -125,7 +125,7 @@ A transaction's journey through Arcade v2 has five phases. ### Phase 1 — Submission -Client calls `SubmitTransaction()`. Arcade parses the raw tx (BEEF or raw bytes), deduplicates via `store.GetOrInsertStatus()`, validates against policy (fee rates, script rules, size limits), tracks it in TxTracker (an in-memory O(1) hash map), registers it with Merkle Service via `POST /watch {txid, callbackUrl}` (best-effort, non-blocking — registration failure does not block broadcast), and broadcasts concurrently to Teranode endpoints, returning on first success. +Client calls `SubmitTransaction()`. Arcade parses the raw tx (BEEF or raw bytes), deduplicates via `store.GetOrInsertStatus()`, validates against policy (fee rates, script rules, size limits), tracks it in TxTracker (an in-memory O(1) hash map), registers it with every configured Merkle Service via `POST /watch {txid, callbackUrl}` (a tx is registered once any endpoint accepts it; a tx no endpoint accepted is requeued and never broadcast, so a proof can always be delivered for a broadcast tx), and broadcasts concurrently to Teranode endpoints, returning on first success. Status: `RECEIVED` → `SENT_TO_NETWORK` → `ACCEPTED_BY_NETWORK`. @@ -257,7 +257,7 @@ Operators can detect the stale-chain-view condition before it causes non-final r **TxTracker as source of truth for tracked transactions.** Both STUMP processing and BUMP construction use `TxTracker.FilterTrackedHashes()` to discover which level-0 hashes correspond to tracked transactions. The tracker is an in-memory concurrent hash map loaded from the store at startup. -**Best-effort registration.** Merkle Service registration failures don't block broadcast. Transactions can be re-registered later. +**Registration before broadcast, with redundancy.** A tx is broadcast only after at least one Merkle Service accepted its `/watch`; several Merkle Services can be configured and each one delivers callbacks, with duplicates handled idempotently. In-flight transactions are re-registered on startup and whenever an endpoint comes back from an outage. **Backward compatibility.** STUMP callback handling works against both old per-tx and new per-subtree Merkle Service implementations. diff --git a/merkleservice/pool.go b/merkleservice/pool.go new file mode 100644 index 00000000..39bed775 --- /dev/null +++ b/merkleservice/pool.go @@ -0,0 +1,1144 @@ +package merkleservice + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "sync" + "sync/atomic" + "time" + + "go.uber.org/zap" + + "github.com/bsv-blockchain/arcade/metrics" +) + +// ErrNoHealthyEndpoints is returned when every endpoint's breaker is open, +// so nothing was sent. It is a transport-class error: the propagator +// classifies it as register_error and requeues the tx, and the watchdog treats +// it as a transient /reprocess failure. +var ErrNoHealthyEndpoints = errors.New("merkle service: no healthy endpoints") + +const ( + // defaultFailureThreshold is how many consecutive transport/5xx failures + // open an endpoint's breaker. 4xx responses never count: the service is + // reachable and the loud auth warning in propagation must keep firing. + defaultFailureThreshold = 3 + // defaultProbeInterval is how often open endpoints are probed with + // GET /health and how often closed endpoints drain their catch-up queue. + defaultProbeInterval = 5 * time.Second + defaultProbeTimeout = 5 * time.Second + // defaultCatchupQueueSize bounds the per-endpoint catch-up queue. At + // 100 TPS it holds roughly half an hour of registrations an endpoint + // missed; past that the oldest entries are dropped and a full resync is + // requested once the queue drains (see Recoverable.OnResyncNeeded). + defaultCatchupQueueSize = 200_000 + // catchupBatchSize is how many queued registrations one drain tick sends + // to an endpoint; with the default probe interval that is ~100/s. + catchupBatchSize = 500 + // catchupConcurrency bounds the in-flight /watch calls of one drain. + catchupConcurrency = 10 + // defaultStragglerGrace is how long a /watch batch keeps waiting for the + // remaining endpoints once every tx in it has been accepted by some + // endpoint. Past it they are canceled and owe the txs via catch-up. Same + // bar as the trial /watch before a breaker closes. + defaultStragglerGrace = defaultProbeTimeout + // catchupMaxBackoffTicks caps how many ticks a drain waits after a batch + // in which nothing landed; the breaker handles a fully-down endpoint. + catchupMaxBackoffTicks = 12 + + opWatch = "watch" + opCatchup = "catchup" + opReprocess = "reprocess" + opProbe = "probe" + + outcomeOK = "ok" + outcomeAuth = "err_auth" + outcome4xx = "err_4xx" + outcome5xx = "err_5xx" + outcomeNetwork = "err_network" + outcomeCanceled = "canceled" + outcomeSkippedOpen = "skipped_open" +) + +// endpoint is one merkle-service plus its circuit-breaker state and the +// catch-up queue of registrations it has not acknowledged yet. +type endpoint struct { + url string + client *Client + + mu sync.Mutex + consecutiveFailures int + open bool + openedAt time.Time + lastFailureAt time.Time + + pending *catchupQueue + // draining is set while a catch-up drain for this endpoint is running, so + // a slow endpoint never has two drains in flight. Guarded by mu. + draining bool + // backoffTicks is how many drain ticks to skip after a drain batch in + // which nothing landed; doubles up to catchupMaxBackoffTicks. Guarded by + // mu, as is skipTicks. + backoffTicks int + skipTicks int +} + +func (ep *endpoint) isOpen() bool { + ep.mu.Lock() + defer ep.mu.Unlock() + return ep.open +} + +// syncPendingGauge publishes the catch-up queue length. Called after every +// change to the queue so the gauge agrees with /health, including while the +// breaker is open. +func (ep *endpoint) syncPendingGauge() { + metrics.MerkleEndpointCatchupPending.WithLabelValues(ep.url).Set(float64(ep.pending.len())) +} + +// catchupQueue is the bounded, deduplicated set of registrations one +// endpoint still owes. A tx the pool reported registered (some endpoint +// accepted it) but this endpoint refused, timed out on, or was skipped for +// (breaker open) lands here and is re-sent once the endpoint answers again. +// It is in-memory on purpose: with several endpoints the startup replay +// re-registers every in-flight tx in the lookback window with the whole pool +// (the merkle_registered_at skip is off by default, since that stamp is set +// when any one endpoint accepted), so what a restart drops is re-sent then. +type catchupQueue struct { + mu sync.Mutex + max int + order []string + byTxID map[string]Registration + // inflight counts entries taken for a catch-up send and not yet settled; + // they still count as owed until the endpoint has answered. + inflight int + dropped int + // overflowed is set when entries were dropped and cleared when the + // queue next drains empty; that transition requests a full resync. + overflowed bool +} + +func newCatchupQueue(capacity int) *catchupQueue { + return &catchupQueue{max: capacity, byTxID: make(map[string]Registration)} +} + +// add queues reg (replacing an older entry for the same txid). When the +// queue is full the oldest entry is dropped and the overflow flag set. +func (q *catchupQueue) add(reg Registration) { + q.mu.Lock() + defer q.mu.Unlock() + if _, dup := q.byTxID[reg.TxID]; dup { + q.byTxID[reg.TxID] = reg + return + } + for len(q.order) >= q.max && len(q.order) > 0 { + oldest := q.order[0] + q.order = q.order[1:] + delete(q.byTxID, oldest) + q.dropped++ + q.overflowed = true + } + q.order = append(q.order, reg.TxID) + q.byTxID[reg.TxID] = reg +} + +// take removes and returns up to n entries from the front. They stay counted +// by len until settle is called for them. +func (q *catchupQueue) take(n int) []Registration { + q.mu.Lock() + defer q.mu.Unlock() + if n > len(q.order) { + n = len(q.order) + } + out := make([]Registration, 0, n) + for _, txid := range q.order[:n] { + out = append(out, q.byTxID[txid]) + delete(q.byTxID, txid) + } + q.order = q.order[n:] + q.inflight += len(out) + return out +} + +// settle marks n taken entries as answered; any that failed have been added +// back by then. +func (q *catchupQueue) settle(n int) { + q.mu.Lock() + defer q.mu.Unlock() + q.inflight -= n +} + +// len is how many registrations are owed: queued plus taken but unsettled. +func (q *catchupQueue) len() int { + q.mu.Lock() + defer q.mu.Unlock() + return len(q.order) + q.inflight +} + +// drainedAfterOverflow reports and clears the overflow flag once the queue +// is empty, returning how many entries were dropped since the last resync. +func (q *catchupQueue) drainedAfterOverflow() (int, bool) { + q.mu.Lock() + defer q.mu.Unlock() + if !q.overflowed || len(q.order)+q.inflight > 0 { + return 0, false + } + n := q.dropped + q.overflowed, q.dropped = false, 0 + return n, true +} + +// PoolOption tunes a Pool. The defaults suit production; tests shrink the +// probe interval and queue size. +type PoolOption func(*Pool) + +// WithFailureThreshold sets how many consecutive transport/5xx failures open +// an endpoint's breaker. Values below 1 keep the default. +func WithFailureThreshold(n int) PoolOption { + return func(p *Pool) { + if n > 0 { + p.threshold = n + } + } +} + +// WithProbeInterval sets how often open endpoints are probed and closed +// endpoints drain their catch-up queue. +func WithProbeInterval(d time.Duration) PoolOption { + return func(p *Pool) { + if d > 0 { + p.probeInterval = d + } + } +} + +// WithCatchupQueueSize bounds each endpoint's catch-up queue. +func WithCatchupQueueSize(n int) PoolOption { + return func(p *Pool) { + if n > 0 { + p.queueSize = n + } + } +} + +// WithStragglerGrace sets how long a /watch batch waits for the remaining +// endpoints once every tx has been accepted somewhere. +func WithStragglerGrace(d time.Duration) PoolOption { + return func(p *Pool) { + if d > 0 { + p.stragglerGrace = d + } + } +} + +// WithClock overrides the pool's clock (tests). +func WithClock(now func() time.Time) PoolOption { + return func(p *Pool) { + if now != nil { + p.now = now + } + } +} + +// Pool fans every merkle-service call out to all configured endpoints so a +// tx is watched by each of them and any one of them delivering callbacks is +// enough. Per call the pool reports success when at least one endpoint +// accepted; otherwise it surfaces the most retryable failure so the existing +// error classification in propagation and the watchdog keeps its meaning +// (context error > network/5xx > 401/403 > other 4xx). +// +// A tx the pool reported registered must still reach every endpoint, or the +// endpoints' watch sets diverge and their STUMPs disagree. So every +// endpoint-specific failure behind a pool-level success — a refused or timed +// out /watch, or an endpoint skipped because its breaker was open — is kept +// in that endpoint's catch-up queue and re-sent from the background loop +// once the endpoint answers again, with backoff. The queue is bounded; past +// its size the oldest entries are dropped and, once it drains, the +// OnResyncNeeded hooks fire so propagation can run a full lookback replay +// against that endpoint alone. +// +// Each endpoint has a circuit breaker: after threshold consecutive +// transport/5xx failures it is skipped on the hot path so a black-holed +// service cannot add its full HTTP timeout to every registration batch. The +// background loop (Start) GETs /health on open endpoints and closes the +// breaker on any non-5xx answer, after one owed registration (if any) also +// went through as a trial /watch. +// +// A Pool with a single endpoint behaves like the Client it wraps, plus the +// breaker, queue and per-endpoint metrics. +type Pool struct { + endpoints []*endpoint + threshold int + probeInterval time.Duration + queueSize int + // stragglerGrace bounds the wait for slower endpoints once every tx of + // a /watch batch is accepted (see RegisterBatchWithResults). + stragglerGrace time.Duration + probeClient *http.Client + now func() time.Time + logger *zap.Logger + + hooksMu sync.Mutex + resyncHooks []func(endpoint string) +} + +// NewPool builds one Client per base URL (all sharing authToken and the +// per-request timeout, 0 = Client default) and wraps them in a Pool. Duplicate +// and empty URLs are dropped; config.MerkleServiceConfig.Endpoints already +// normalizes the list, but NewPool is defensive so tests can pass raw slices. +func NewPool(baseURLs []string, authToken string, timeout time.Duration, opts ...PoolOption) *Pool { + p := &Pool{ + threshold: defaultFailureThreshold, + probeInterval: defaultProbeInterval, + queueSize: defaultCatchupQueueSize, + stragglerGrace: defaultStragglerGrace, + probeClient: &http.Client{Timeout: defaultProbeTimeout}, + now: time.Now, + logger: zap.NewNop(), + } + for _, o := range opts { + o(p) + } + seen := make(map[string]struct{}, len(baseURLs)) + for _, u := range baseURLs { + u = strings.TrimSuffix(strings.TrimSpace(u), "/") + if u == "" { + continue + } + if _, dup := seen[u]; dup { + continue + } + seen[u] = struct{}{} + p.endpoints = append(p.endpoints, &endpoint{ + url: u, + client: NewClient(u, authToken, timeout), + pending: newCatchupQueue(p.queueSize), + }) + metrics.MerkleEndpointHealthy.WithLabelValues(u).Set(1) + metrics.MerkleEndpointCatchupPending.WithLabelValues(u).Set(0) + } + return p +} + +// SetLogger sets the pool's logger and gives every endpoint client a child +// logger tagged with its base URL. Mirrors Client.SetLogger. +func (p *Pool) SetLogger(logger *zap.Logger) { + if logger == nil { + return + } + p.logger = logger + for _, ep := range p.endpoints { + ep.client.SetLogger(logger.With(zap.String("endpoint", ep.url))) + } +} + +// Endpoints lists the configured base URLs in config order. +func (p *Pool) Endpoints() []string { + out := make([]string, 0, len(p.endpoints)) + for _, ep := range p.endpoints { + out = append(out, ep.url) + } + return out +} + +// Endpoint returns a Service bound to exactly one endpoint. Calls through it +// still honor that endpoint's breaker and feed its metrics, and a failed +// registration lands in the endpoint's catch-up queue rather than being +// lost. nil when baseURL is unknown. +func (p *Pool) Endpoint(baseURL string) Service { + baseURL = strings.TrimSuffix(strings.TrimSpace(baseURL), "/") + for _, ep := range p.endpoints { + if ep.url == baseURL { + return &endpointService{pool: p, ep: ep} + } + } + return nil +} + +// OnResyncNeeded registers fn to run each time an endpoint's catch-up queue +// has overflowed and then drained: the registrations it dropped are unknown, +// so the endpoint needs a full lookback replay. fn runs on the pool's +// background goroutine and must return quickly. +func (p *Pool) OnResyncNeeded(fn func(endpoint string)) { + if fn == nil { + return + } + p.hooksMu.Lock() + defer p.hooksMu.Unlock() + p.resyncHooks = append(p.resyncHooks, fn) +} + +// Retain implements Recoverable.Retain. +func (p *Pool) Retain(registrations []Registration, errs []error) { + for _, ep := range p.endpoints { + for i, reg := range registrations { + if i >= len(errs) || errs[i] == nil || permanentRejection(errs[i]) { + continue + } + ep.pending.add(reg) + } + ep.syncPendingGauge() + } +} + +// EndpointStatuses snapshots every endpoint's breaker state. +func (p *Pool) EndpointStatuses() []EndpointStatus { + out := make([]EndpointStatus, 0, len(p.endpoints)) + for _, ep := range p.endpoints { + ep.mu.Lock() + st := EndpointStatus{ + URL: ep.url, + Healthy: !ep.open, + ConsecutiveFailures: ep.consecutiveFailures, + } + if !ep.lastFailureAt.IsZero() { + t := ep.lastFailureAt + st.LastFailureAt = &t + } + if ep.open { + t := ep.openedAt + st.OpenSince = &t + } + ep.mu.Unlock() + st.CatchupPending = ep.pending.len() + out = append(out, st) + } + return out +} + +// Start launches the background loop that probes open endpoints and drains +// closed endpoints' catch-up queues. It returns immediately; the loop exits +// when ctx is canceled. Without Start an open breaker never closes and a +// queued registration is never re-sent, so production always calls it. +func (p *Pool) Start(ctx context.Context) { + if len(p.endpoints) == 0 { + return + } + go p.loop(ctx) +} + +// Register fans one /watch out to every closed endpoint. +func (p *Pool) Register(ctx context.Context, txid, callbackURL, callbackToken string) error { + errs := p.RegisterBatchWithResults(ctx, []Registration{{TxID: txid, CallbackURL: callbackURL, CallbackToken: callbackToken}}, 1) + return errs[0] +} + +// Reprocess fans one /reprocess out to every closed endpoint. nil when any +// endpoint accepted: every endpoint re-emits its own STUMP set, and the +// bump-builder is idempotent across sources. +func (p *Pool) Reprocess(ctx context.Context, blockHash, callbackURL, callbackToken string) error { + targets := p.closedEndpoints(opReprocess, 1) + if len(targets) == 0 { + return ErrNoHealthyEndpoints + } + // One slot per configured endpoint: an endpoint skipped for an open + // breaker answers ErrNoHealthyEndpoints, so a terminal answer from the + // others (a 404, say) cannot outrank the possibility that the skipped + // one is merely down and would have accepted. + errs := make([]error, len(p.endpoints)) + for i := len(targets); i < len(errs); i++ { + errs[i] = ErrNoHealthyEndpoints + } + var wg sync.WaitGroup + for i, ep := range targets { + wg.Add(1) + go func() { + defer wg.Done() + err := ep.client.Reprocess(ctx, blockHash, callbackURL, callbackToken) + p.record(ctx, ep, opReprocess, err) + errs[i] = err + }() + } + wg.Wait() + return mergeErrors(ctx, errs) +} + +// RegisterBatch keeps Client.RegisterBatch's contract (one error, nil only +// when every tx registered) on top of the per-tx fan-out. It runs the whole +// batch rather than failing fast: a fail-fast across endpoints would cancel +// in-flight /watch calls on healthy endpoints because one endpoint hiccuped. +func (p *Pool) RegisterBatch(ctx context.Context, registrations []Registration, maxConcurrency int) error { + for _, err := range p.RegisterBatchWithResults(ctx, registrations, maxConcurrency) { + if err != nil { + return err + } + } + return nil +} + +// RegisterBatchWithResults runs the batch against every closed endpoint +// concurrently (each endpoint bounded by maxConcurrency, as in +// Client.RegisterBatchWithResults) and merges per input index: nil when any +// endpoint accepted the tx, otherwise the most retryable of that tx's errors. +// Every tx the pool reports registered that some endpoint did not accept +// (refused, timed out, or skipped for an open breaker) is queued for that +// endpoint. A batch that every tx has cleared waits at most stragglerGrace +// for the remaining endpoints (see collectEndpointBatches), so one hung +// endpoint does not hold registration — and therefore broadcast — for its +// full HTTP timeout. Output length always equals len(registrations). +func (p *Pool) RegisterBatchWithResults(ctx context.Context, registrations []Registration, maxConcurrency int) []error { + if len(registrations) == 0 { + return nil + } + out := make([]error, len(registrations)) + targets := p.closedEndpoints(opWatch, len(registrations)) + if len(targets) == 0 { + for i := range out { + out[i] = ErrNoHealthyEndpoints + } + return out + } + + perEndpoint := p.collectEndpointBatches(ctx, targets, registrations, maxConcurrency) + + // Endpoints skipped up front for an open breaker take part in the merge + // as ErrNoHealthyEndpoints (transport-class), as mid-batch skips already + // do: a 401 or 4xx from the endpoints that answered must not read as the + // pool's verdict while another endpoint is merely down. + skipped := len(p.endpoints) - len(targets) + candidates := make([]error, 0, len(p.endpoints)) + for j := range registrations { + candidates = candidates[:0] + for _, ep := range targets { + candidates = append(candidates, perEndpoint[ep][j]) + } + for range skipped { + candidates = append(candidates, ErrNoHealthyEndpoints) + } + out[j] = mergeErrors(ctx, candidates) + } + // Behind every pool-level success, queue the endpoints that did not get + // the tx: refused/timed out, or not even asked because the breaker was + // open. Without this a single transport blip (below the breaker + // threshold) would leave the endpoint permanently missing the txid. + for _, ep := range p.endpoints { + res := perEndpoint[ep] // nil for endpoints that were skipped + for j := range registrations { + if out[j] != nil { + continue + } + if res == nil || res[j] != nil { + ep.pending.add(registrations[j]) + } + } + ep.syncPendingGauge() + } + return out +} + +// collectEndpointBatches runs the batch against every target concurrently +// and returns each endpoint's per-tx results. It returns as soon as every +// endpoint has answered, or stragglerGrace after every tx has been accepted +// by some endpoint, whichever is first: the stragglers' sends are then +// canceled. Acceptance is tracked per tx as each /watch lands, not per +// finished endpoint batch, so complementary coverage counts too: A accepting +// tx1 while hanging on tx2 and B accepting tx2 while hanging on tx1 still +// starts the grace timer. A straggler that acknowledged nothing in the batch +// has the cancellations recorded against its breaker (see runEndpointBatch), +// so a black-holed endpoint trips on its first batch instead of holding every +// batch for a full HTTP timeout; one that acknowledged something is only +// slower and keeps its health. What the +// stragglers did not acknowledge lands in their catch-up queue through the +// caller's usual path. +func (p *Pool) collectEndpointBatches(ctx context.Context, targets []*endpoint, registrations []Registration, maxConcurrency int) map[*endpoint][]error { + sendCtx, cancelSends := context.WithCancel(ctx) + defer cancelSends() + type endpointResult struct { + ep *endpoint + res []error + } + results := make(chan endpointResult, len(targets)) + // Sized for every possible acceptance so a sender never blocks, even + // after the collector has returned. + acks := make(chan int, len(targets)*len(registrations)) + onAccepted := func(j int) { acks <- j } + for _, ep := range targets { + go func() { + results <- endpointResult{ep, p.runEndpointBatch(ctx, sendCtx, ep, opWatch, registrations, maxConcurrency, onAccepted)} + }() + } + + perEndpoint := make(map[*endpoint][]error, len(targets)) + accepted := make([]bool, len(registrations)) + unaccepted := len(registrations) + accept := func(j int) { + if !accepted[j] { + accepted[j] = true + unaccepted-- + } + } + var grace <-chan time.Time + for len(perEndpoint) < len(targets) { + select { + case j := <-acks: + accept(j) + case r := <-results: + perEndpoint[r.ep] = r.res + for j, err := range r.res { + if err == nil { + accept(j) + } + } + case <-grace: + // Every tx is registered somewhere; stop waiting on the rest. + // Their sends return promptly and are still collected above. + cancelSends() + grace = nil + continue + } + if unaccepted == 0 && grace == nil && len(perEndpoint) < len(targets) && sendCtx.Err() == nil { + t := time.NewTimer(p.stragglerGrace) + defer t.Stop() + grace = t.C + } + } + return perEndpoint +} + +// runEndpointBatch sends registrations to ep alone, at most maxConcurrency at +// a time, and records each result as it lands so the breaker can open in the +// middle of a batch. Once it is open nothing more is sent: the remaining +// entries come back as ErrNoHealthyEndpoints right away. Without that a +// black-holed endpoint would hold the caller for one HTTP timeout per +// maxConcurrency entries, even though another endpoint accepted them all +// long ago; with it the first batch waits for at most the requests already +// in flight when the breaker trips. res[i] corresponds to registrations[i]. +// +// sendCtx bounds the requests; ctx is the caller's and decides whether a +// failure was the caller's own cancellation (not the endpoint's fault) when it +// is recorded. They differ only when the pool cuts a straggler short. +// +// onAccepted, when non-nil, is called with the index of each registration the +// endpoint acknowledges, as it lands; it must not block. +func (p *Pool) runEndpointBatch(ctx, sendCtx context.Context, ep *endpoint, op string, registrations []Registration, maxConcurrency int, onAccepted func(int)) []error { + if maxConcurrency <= 0 { + maxConcurrency = 10 + } + res := make([]error, len(registrations)) + sem := make(chan struct{}, maxConcurrency) + var wg sync.WaitGroup + skipped := 0 + // acked counts this batch's acknowledgements. A request the pool cut + // short (sendCtx canceled after the straggler grace) is held against the + // endpoint only when the endpoint has acknowledged nothing in the batch: + // a black-holed endpoint acknowledges nothing and still trips on its + // first batch, while one that is merely slower than the others keeps + // its health and just owes the rest through catch-up. + var acked atomic.Int64 + for i, reg := range registrations { + select { + case sem <- struct{}{}: + case <-sendCtx.Done(): + res[i] = sendCtx.Err() + continue + } + // Checked after taking a slot: a slot frees only when an earlier + // request has finished and been recorded, so a trip it caused is + // already visible here. + if ep.isOpen() { + <-sem + res[i] = ErrNoHealthyEndpoints + skipped++ + continue + } + wg.Add(1) + go func() { + defer wg.Done() + defer func() { <-sem }() + err := ep.client.Register(sendCtx, reg.TxID, reg.CallbackURL, reg.CallbackToken) + classifyCtx := ctx + if err == nil { + acked.Add(1) + } else if ctx.Err() == nil && sendCtx.Err() != nil && acked.Load() > 0 { + classifyCtx = sendCtx // recorded as canceled, not a failure + } + p.record(classifyCtx, ep, op, err) + res[i] = err + if err == nil && onAccepted != nil { + onAccepted(i) + } + }() + } + wg.Wait() + if skipped > 0 { + metrics.MerkleEndpointRequestsTotal.WithLabelValues(ep.url, op, outcomeSkippedOpen).Add(float64(skipped)) + } + return res +} + +// closedEndpoints returns the endpoints whose breaker is closed and counts +// requests skipped_open samples for each one that is open: the number of +// calls the endpoint would otherwise have received (one /watch per +// registration, one /reprocess), so the counter matches mid-batch skips. +func (p *Pool) closedEndpoints(op string, requests int) []*endpoint { + out := make([]*endpoint, 0, len(p.endpoints)) + for _, ep := range p.endpoints { + if ep.isOpen() { + metrics.MerkleEndpointRequestsTotal.WithLabelValues(ep.url, op, outcomeSkippedOpen).Add(float64(requests)) + continue + } + out = append(out, ep) + } + return out +} + +// classify maps one call's error to its metric outcome and whether it counts +// toward the breaker. ctx is consulted because an http.Client timeout +// satisfies errors.Is(err, context.DeadlineExceeded) on recent Go versions, +// and that timeout IS the black-holed-endpoint signal the breaker exists for; +// only the caller's own cancellation is exempt. +func classify(ctx context.Context, err error) (outcome string, failure bool) { + if err == nil { + return outcomeOK, false + } + if ctx.Err() != nil { + return outcomeCanceled, false + } + var regErr *RegisterError + var repErr *ReprocessError + var prbErr *probeError + var code int + switch { + case errors.As(err, ®Err): + code = regErr.StatusCode + case errors.As(err, &repErr): + code = repErr.StatusCode + case errors.As(err, &prbErr): + code = prbErr.StatusCode + default: + return outcomeNetwork, true + } + switch { + case code >= http.StatusInternalServerError: + return outcome5xx, true + case code == http.StatusUnauthorized || code == http.StatusForbidden: + return outcomeAuth, false + default: + return outcome4xx, false + } +} + +// errRank orders failures by how much a retry is worth: a lower rank is +// surfaced in preference to a higher one when every endpoint failed. +func errRank(ctx context.Context, err error) int { + switch outcome, _ := classify(ctx, err); outcome { + case outcomeCanceled: + return 0 + case outcomeNetwork: + return 1 + case outcome5xx: + return 2 + case outcomeAuth: + return 3 + default: + return 4 + } +} + +// mergeErrors reduces one call's per-endpoint results to the pool's answer. +func mergeErrors(ctx context.Context, errs []error) error { + var best error + bestRank := int(^uint(0) >> 1) + for _, err := range errs { + if err == nil { + return nil + } + if r := errRank(ctx, err); r < bestRank { + best, bestRank = err, r + } + } + return best +} + +// record updates the endpoint's breaker and metrics after one call. +func (p *Pool) record(ctx context.Context, ep *endpoint, op string, err error) { + outcome, failure := classify(ctx, err) + metrics.MerkleEndpointRequestsTotal.WithLabelValues(ep.url, op, outcome).Inc() + if outcome == outcomeCanceled { + return // the caller's context, not the endpoint's fault + } + if !failure { + // Any HTTP answer below 500 proves reachability. A success while the + // breaker is open can only come from the probe path, but if it + // happens the endpoint is evidently back. + ep.mu.Lock() + ep.consecutiveFailures = 0 + wasOpen := ep.open + ep.mu.Unlock() + if wasOpen && err == nil { + p.closeBreaker(ep, op) + } + return + } + ep.mu.Lock() + ep.consecutiveFailures++ + ep.lastFailureAt = p.now() + trip := !ep.open && ep.consecutiveFailures >= p.threshold + if trip { + ep.open = true + ep.openedAt = ep.lastFailureAt + } + failures := ep.consecutiveFailures + ep.mu.Unlock() + if trip { + metrics.MerkleEndpointHealthy.WithLabelValues(ep.url).Set(0) + p.logger.Warn( + "merkle-service endpoint unhealthy; skipping it until /health answers", + zap.String("endpoint", ep.url), + zap.String("op", op), + zap.Int("consecutive_failures", failures), + zap.Error(err), + ) + } +} + +// closeBreaker marks ep healthy. No-op when the breaker is already closed. +// The catch-up queue drains on the next loop tick, so nothing else needs to +// happen here. +func (p *Pool) closeBreaker(ep *endpoint, via string) { + ep.mu.Lock() + if !ep.open { + ep.mu.Unlock() + return + } + openedAt := ep.openedAt + ep.open = false + ep.openedAt = time.Time{} + ep.consecutiveFailures = 0 + ep.mu.Unlock() + + metrics.MerkleEndpointHealthy.WithLabelValues(ep.url).Set(1) + p.logger.Info( + "merkle-service endpoint healthy again", + zap.String("endpoint", ep.url), + zap.String("via", via), + zap.Duration("open_for", p.now().Sub(openedAt)), + zap.Int("catchup_pending", ep.pending.len()), + ) +} + +func (p *Pool) loop(ctx context.Context) { + ticker := time.NewTicker(p.probeInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + p.probeOpenEndpoints(ctx) + p.drainCatchup(ctx) + } + } +} + +// probeOpenEndpoints probes every open endpoint concurrently, so several +// black-holed endpoints cost one probe timeout per tick rather than one each. +func (p *Pool) probeOpenEndpoints(ctx context.Context) { + var wg sync.WaitGroup + for _, ep := range p.endpoints { + if !ep.isOpen() { + continue + } + wg.Add(1) + go func() { + defer wg.Done() + p.probeEndpoint(ctx, ep) + }() + } + wg.Wait() +} + +// probeEndpoint GETs /health on one open endpoint and closes its breaker when +// it answers with anything but a 5xx and a trial /watch (see trialWatch) also +// answers. A 4xx (e.g. 404 from a build without /health) still proves the +// process is reachable; a 5xx means the service itself reports unhealthy, so +// it stays open. +func (p *Pool) probeEndpoint(ctx context.Context, ep *endpoint) { + err := p.probe(ctx, ep) + outcome, _ := classify(ctx, err) + metrics.MerkleEndpointRequestsTotal.WithLabelValues(ep.url, opProbe, outcome).Inc() + if err != nil { + p.logger.Debug("merkle-service probe failed", zap.String("endpoint", ep.url), zap.String("outcome", outcome), zap.Error(err)) + return + } + if !p.trialWatch(ctx, ep) { + return + } + p.closeBreaker(ep, opProbe) +} + +// trialWatch re-sends one registration the endpoint owes, bounded by the +// probe timeout, before its breaker closes. /health is shallow: an endpoint +// whose /health answers while /watch hangs would otherwise be closed every +// probe tick and hold the next hot-path batch for a full HTTP timeout before +// re-tripping. It reports whether /watch answered below 500. With nothing +// owed there is nothing to try, and /health alone decides. +func (p *Pool) trialWatch(ctx context.Context, ep *endpoint) bool { + batch := ep.pending.take(1) + if len(batch) == 0 { + return true + } + reg := batch[0] + trialCtx, cancel := context.WithTimeout(ctx, defaultProbeTimeout) + err := ep.client.Register(trialCtx, reg.TxID, reg.CallbackURL, reg.CallbackToken) + cancel() + // Classified against the parent ctx: the trial's own deadline is the + // hanging-/watch signal and must count as a failure. + outcome, failure := classify(ctx, err) + p.record(ctx, ep, opCatchup, err) + if err != nil && !permanentRejection(err) { + ep.pending.add(reg) + } + ep.pending.settle(1) + ep.syncPendingGauge() + if outcome == outcomeCanceled || failure { + p.logger.Debug("merkle-service trial /watch failed; keeping breaker open", zap.String("endpoint", ep.url), zap.String("outcome", outcome), zap.Error(err)) + return false + } + return true +} + +// drainCatchup starts, for every closed endpoint that owes registrations and +// is not backing off, a drain of one batch. Each endpoint drains on its own +// goroutine, at most one at a time, so a hanging endpoint cannot hold up the +// probes or another endpoint's catch-up. +func (p *Pool) drainCatchup(ctx context.Context) { + for _, ep := range p.endpoints { + if ctx.Err() != nil { + return + } + ep.syncPendingGauge() + ep.mu.Lock() + if ep.open || ep.draining { + ep.mu.Unlock() + continue + } + if ep.skipTicks > 0 { + ep.skipTicks-- + ep.mu.Unlock() + continue + } + ep.draining = true + ep.mu.Unlock() + go func() { + defer func() { + ep.mu.Lock() + ep.draining = false + ep.mu.Unlock() + }() + p.drainEndpoint(ctx, ep) + }() + } +} + +// drainEndpoint re-sends one batch of ep's queued registrations. Failures go +// back to the queue (and count toward the breaker) unless the endpoint +// rejected the registration outright (see permanentRejection); a batch that +// made no progress backs the endpoint off for a growing number of ticks. +// When a queue that had overflowed drains empty, the resync hooks fire. +func (p *Pool) drainEndpoint(ctx context.Context, ep *endpoint) { + if batch := ep.pending.take(catchupBatchSize); len(batch) > 0 { + p.catchupBatch(ctx, ep, batch) + } + ep.syncPendingGauge() + if dropped, ok := ep.pending.drainedAfterOverflow(); ok { + p.logger.Warn( + "merkle-service catch-up queue had overflowed; requesting a full resync of this endpoint", + zap.String("endpoint", ep.url), + zap.Int("dropped", dropped), + ) + p.fireResync(ep.url) + } +} + +// catchupBatch sends one taken batch, settles it, adjusts the endpoint's +// backoff and logs the outcome. +func (p *Pool) catchupBatch(ctx context.Context, ep *endpoint, batch []Registration) { + r := p.sendCatchup(ctx, ep, batch) + ep.pending.settle(len(batch)) + progress := r.landed + r.dropped + ep.mu.Lock() + switch { + case progress == 0 && ctx.Err() == nil: + if ep.backoffTicks == 0 { + ep.backoffTicks = 1 + } else if ep.backoffTicks < catchupMaxBackoffTicks { + ep.backoffTicks *= 2 + } + ep.skipTicks = ep.backoffTicks + case progress > 0: + ep.backoffTicks = 0 + } + ep.mu.Unlock() + if r.landed > 0 { + p.logger.Info( + "merkle-service catch-up delivered missed registrations", + zap.String("endpoint", ep.url), + zap.Int("landed", r.landed), + zap.Int("failed", len(batch)-progress), + zap.Int("still_pending", ep.pending.len()), + ) + } + if r.dropped > 0 { + p.logger.Warn( + "merkle-service endpoint rejected catch-up registrations; dropped them", + zap.String("endpoint", ep.url), + zap.Int("dropped", r.dropped), + zap.Error(r.sampleDropped), + ) + } + if r.auth > 0 { + // 401/403 do not count toward the breaker, so this log is the only + // signal besides the gauge. They are kept: a token fixed on the + // merkle-service side should not need an arcade restart. + p.logger.Warn( + "merkle-service endpoint rejected the auth token on catch-up; check merkle_service.auth_token", + zap.String("endpoint", ep.url), + zap.Int("rejected", r.auth), + zap.Int("still_pending", ep.pending.len()), + ) + } +} + +// catchupResult summarizes one catch-up batch. +type catchupResult struct { + landed int // acknowledged by the endpoint + dropped int // rejected outright; not re-queued + auth int // refused with 401/403; re-queued + // sampleDropped is one of the rejections behind dropped, for the log. + sampleDropped error +} + +// sendCatchup registers batch with ep alone and re-queues every failure +// except an outright rejection. +func (p *Pool) sendCatchup(ctx context.Context, ep *endpoint, batch []Registration) catchupResult { + var r catchupResult + for i, err := range p.runEndpointBatch(ctx, ctx, ep, opCatchup, batch, catchupConcurrency, nil) { + switch outcome, _ := classify(ctx, err); { + case err == nil: + r.landed++ + case permanentRejection(err): + r.dropped++ + if r.sampleDropped == nil { + r.sampleDropped = err + } + default: + if outcome == outcomeAuth { + r.auth++ + } + ep.pending.add(batch[i]) + } + } + return r +} + +// permanentRejection reports whether err is a /watch refusal that sending the +// same registration again cannot fix: a 4xx other than 401/403 (a token +// problem, fixable on either side), 408 or 429 (the endpoint asked us to +// slow down or try again). +func permanentRejection(err error) bool { + var regErr *RegisterError + if !errors.As(err, ®Err) { + return false + } + switch code := regErr.StatusCode; { + case code < http.StatusBadRequest || code >= http.StatusInternalServerError: + return false + case code == http.StatusUnauthorized, code == http.StatusForbidden, + code == http.StatusRequestTimeout, code == http.StatusTooManyRequests: + return false + default: + return true + } +} + +func (p *Pool) fireResync(url string) { + p.hooksMu.Lock() + hooks := make([]func(string), len(p.resyncHooks)) + copy(hooks, p.resyncHooks) + p.hooksMu.Unlock() + for _, fn := range hooks { + fn(url) + } +} + +func (p *Pool) probe(ctx context.Context, ep *endpoint) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, ep.url+"/health", nil) + if err != nil { + return err + } + resp, err := p.probeClient.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 512)) + if resp.StatusCode >= http.StatusInternalServerError { + return &probeError{StatusCode: resp.StatusCode} + } + return nil +} + +// probeError is a 5xx answer to GET /health. +type probeError struct { + StatusCode int +} + +func (e *probeError) Error() string { + return "merkle service /health returned status " + http.StatusText(e.StatusCode) +} + +// endpointService is the Service returned by Pool.Endpoint: one endpoint, +// honoring its breaker (an open endpoint refuses fast with +// ErrNoHealthyEndpoints instead of waiting out the HTTP timeout) and +// keeping every failed registration in the catch-up queue. +type endpointService struct { + pool *Pool + ep *endpoint +} + +func (s *endpointService) Register(ctx context.Context, txid, callbackURL, callbackToken string) error { + return s.RegisterBatchWithResults(ctx, []Registration{{TxID: txid, CallbackURL: callbackURL, CallbackToken: callbackToken}}, 1)[0] +} + +func (s *endpointService) RegisterBatch(ctx context.Context, registrations []Registration, maxConcurrency int) error { + for _, err := range s.RegisterBatchWithResults(ctx, registrations, maxConcurrency) { + if err != nil { + return err + } + } + return nil +} + +func (s *endpointService) RegisterBatchWithResults(ctx context.Context, registrations []Registration, maxConcurrency int) []error { + if len(registrations) == 0 { + return nil + } + // runEndpointBatch refuses fast (ErrNoHealthyEndpoints, nothing sent) + // for every entry once the breaker is open, including from the start. + out := s.pool.runEndpointBatch(ctx, ctx, s.ep, opWatch, registrations, maxConcurrency, nil) + // An outright rejection is not queued: re-sending it cannot succeed + // (same rule as sendCatchup and Retain). + for i, reg := range registrations { + if out[i] != nil && !permanentRejection(out[i]) { + s.ep.pending.add(reg) + } + } + s.ep.syncPendingGauge() + return out +} + +func (s *endpointService) Reprocess(ctx context.Context, blockHash, callbackURL, callbackToken string) error { + if s.ep.isOpen() { + metrics.MerkleEndpointRequestsTotal.WithLabelValues(s.ep.url, opReprocess, outcomeSkippedOpen).Inc() + return ErrNoHealthyEndpoints + } + err := s.ep.client.Reprocess(ctx, blockHash, callbackURL, callbackToken) + s.pool.record(ctx, s.ep, opReprocess, err) + return err +} + +var _ Service = (*endpointService)(nil) diff --git a/merkleservice/pool_test.go b/merkleservice/pool_test.go new file mode 100644 index 00000000..cb2c742e --- /dev/null +++ b/merkleservice/pool_test.go @@ -0,0 +1,1003 @@ +package merkleservice + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + + "github.com/bsv-blockchain/arcade/metrics" +) + +// fakeMerkle is one merkle-service endpoint whose /watch, /reprocess and +// /health answers can be flipped at runtime. +type fakeMerkle struct { + srv *httptest.Server + + mu sync.Mutex + watched []string // txids received on /watch + // watchCode is the status for /watch (and /reprocess); healthCode for + // /health. Both default to 200. + watchCode atomic.Int32 + healthCode atomic.Int32 + // failTxIDs are refused with the mapped status, regardless of + // watchCode; setFail uses 500. + failTxIDs map[string]int +} + +func newFakeMerkle(t *testing.T) *fakeMerkle { + t.Helper() + f := &fakeMerkle{failTxIDs: map[string]int{}} + f.watchCode.Store(http.StatusOK) + f.healthCode.Store(http.StatusOK) + f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/health": + w.WriteHeader(int(f.healthCode.Load())) + case "/watch": + var req struct { + TxID string `json:"txid"` + } + decodeJSON(r, &req) + f.mu.Lock() + f.watched = append(f.watched, req.TxID) + failCode := f.failTxIDs[req.TxID] + f.mu.Unlock() + if failCode != 0 { + w.WriteHeader(failCode) + return + } + w.WriteHeader(int(f.watchCode.Load())) + case "/reprocess": + w.WriteHeader(int(f.watchCode.Load())) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(f.srv.Close) + return f +} + +func (f *fakeMerkle) watchedCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.watched) +} + +func (f *fakeMerkle) watchedSet() map[string]int { + f.mu.Lock() + defer f.mu.Unlock() + out := make(map[string]int, len(f.watched)) + for _, id := range f.watched { + out[id]++ + } + return out +} + +func (f *fakeMerkle) setFail(txid string, fail bool) { + if fail { + f.setFailCode(txid, http.StatusInternalServerError) + } else { + f.setFailCode(txid, 0) + } +} + +// setFailCode refuses txid's /watch with code; 0 clears it. +func (f *fakeMerkle) setFailCode(txid string, code int) { + f.mu.Lock() + defer f.mu.Unlock() + if code == 0 { + delete(f.failTxIDs, txid) + } else { + f.failTxIDs[txid] = code + } +} + +// pendingFor reads one endpoint's catch-up queue length from the pool's +// status surface. +func pendingFor(p *Pool, url string) int { + for _, st := range p.EndpointStatuses() { + if st.URL == url { + return st.CatchupPending + } + } + return -1 +} + +// waitFor polls cond for up to 3s. +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + +func (f *fakeMerkle) url() string { return f.srv.URL } + +func decodeJSON(r *http.Request, v any) { + _ = json.NewDecoder(r.Body).Decode(v) +} + +func TestPool_RegisterSucceedsWhenAnyEndpointAccepts(t *testing.T) { + a, b := newFakeMerkle(t), newFakeMerkle(t) + b.watchCode.Store(http.StatusInternalServerError) + p := NewPool([]string{a.url(), b.url()}, "", time.Second) + + if err := p.Register(context.Background(), "tx1", testCallbackURL, ""); err != nil { + t.Fatalf("one healthy endpoint is enough, got %v", err) + } + if a.watchedCount() != 1 || b.watchedCount() != 1 { + t.Fatalf("both endpoints must be called: a=%d b=%d", a.watchedCount(), b.watchedCount()) + } +} + +func TestPool_RegisterBatchWithResults_MergesPerTx(t *testing.T) { + a, b := newFakeMerkle(t), newFakeMerkle(t) + a.failTxIDs["tx2"] = http.StatusInternalServerError + b.failTxIDs["tx2"] = http.StatusInternalServerError + b.failTxIDs["tx3"] = http.StatusInternalServerError // tx3 still lands on a + p := NewPool([]string{a.url(), b.url()}, "", time.Second) + + regs := []Registration{{TxID: "tx1", CallbackURL: testCallbackURL}, {TxID: "tx2", CallbackURL: testCallbackURL}, {TxID: "tx3", CallbackURL: testCallbackURL}} + errs := p.RegisterBatchWithResults(context.Background(), regs, 2) + if len(errs) != 3 { + t.Fatalf("len(errs)=%d want 3", len(errs)) + } + if errs[0] != nil || errs[2] != nil { + t.Fatalf("tx1/tx3 accepted by at least one endpoint, got %v / %v", errs[0], errs[2]) + } + var regErr *RegisterError + if !errors.As(errs[1], ®Err) || regErr.StatusCode != http.StatusInternalServerError { + t.Fatalf("tx2 refused everywhere must surface the 5xx, got %v", errs[1]) + } + if a.watchedCount() != 3 || b.watchedCount() != 3 { + t.Fatalf("every tx goes to every endpoint: a=%d b=%d", a.watchedCount(), b.watchedCount()) + } + // tx3 was accepted by a but refused by b: b owes it. tx2 failed + // everywhere (the caller requeues it), so nobody owes it. + if got := pendingFor(p, b.url()); got != 1 { + t.Fatalf("b catch-up pending = %d, want 1 (tx3)", got) + } + if got := pendingFor(p, a.url()); got != 0 { + t.Fatalf("a catch-up pending = %d, want 0", got) + } +} + +// An isolated failure on one endpoint (below the breaker threshold) behind +// a pool-level success is not forgotten: the background loop re-sends it +// until the endpoint acknowledges, so both watch sets converge. +func TestPool_IsolatedFailureIsQueuedAndRedelivered(t *testing.T) { + a, b := newFakeMerkle(t), newFakeMerkle(t) + b.setFail("tx2", true) + p := NewPool([]string{a.url(), b.url()}, "", time.Second, WithProbeInterval(10*time.Millisecond)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + + regs := []Registration{{TxID: "tx1", CallbackURL: testCallbackURL}, {TxID: "tx2", CallbackURL: testCallbackURL}} + for i, err := range p.RegisterBatchWithResults(ctx, regs, 2) { + if err != nil { + t.Fatalf("errs[%d]=%v, a accepted both", i, err) + } + } + if st := p.EndpointStatuses()[1]; !st.Healthy || st.CatchupPending != 1 { + t.Fatalf("b must stay healthy with tx2 owed: %+v", st) + } + + b.setFail("tx2", false) + waitFor(t, "b to acknowledge tx2", func() bool { return b.watchedSet()["tx2"] >= 2 }) + waitFor(t, "b's queue to drain", func() bool { return pendingFor(p, b.url()) == 0 }) + if got := testutil.ToFloat64(metrics.MerkleEndpointCatchupPending.WithLabelValues(b.url())); got != 0 { + t.Fatalf("catchup_pending gauge = %v, want 0", got) + } +} + +// Registrations skipped while an endpoint's breaker was open are owed to it +// and delivered once the probe closes the breaker. +func TestPool_SkippedWhileOpenIsDeliveredAfterRecovery(t *testing.T) { + a, b := newFakeMerkle(t), newFakeMerkle(t) + b.watchCode.Store(http.StatusServiceUnavailable) + b.healthCode.Store(http.StatusServiceUnavailable) + p := NewPool([]string{a.url(), b.url()}, "", time.Second, WithFailureThreshold(2), WithProbeInterval(10*time.Millisecond)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + + // Two refusals open b; both txs are owed to it. + for _, id := range []string{"tx1", "tx2", "tx3"} { + if err := p.Register(ctx, id, testCallbackURL, ""); err != nil { + t.Fatalf("register %s: %v (a accepts)", id, err) + } + } + if st := p.EndpointStatuses()[1]; st.Healthy || st.CatchupPending != 3 { + t.Fatalf("b must be open and owe all three: %+v", st) + } + // A 5xx /health keeps it open and nothing drains. + time.Sleep(50 * time.Millisecond) + if st := p.EndpointStatuses()[1]; st.Healthy || st.CatchupPending != 3 { + t.Fatalf("b must stay open while /health is 503: %+v", st) + } + if got := testutil.ToFloat64(metrics.MerkleEndpointRequestsTotal.WithLabelValues(b.url(), opProbe, outcome5xx)); got == 0 { + t.Fatal("a 503 /health must be counted as err_5xx on the probe op") + } + + b.healthCode.Store(http.StatusOK) + b.watchCode.Store(http.StatusOK) + waitFor(t, "b to recover and drain", func() bool { + st := p.EndpointStatuses()[1] + return st.Healthy && st.CatchupPending == 0 + }) + got := b.watchedSet() + for _, id := range []string{"tx1", "tx2", "tx3"} { + if got[id] == 0 { + t.Fatalf("b never received %s after recovery: %v", id, got) + } + } +} + +// Past the queue bound the oldest entries are dropped; once the queue +// drains, the pool asks for a full resync of that endpoint exactly once. +func TestPool_CatchupOverflowRequestsResync(t *testing.T) { + a, b := newFakeMerkle(t), newFakeMerkle(t) + b.watchCode.Store(http.StatusServiceUnavailable) + b.healthCode.Store(http.StatusServiceUnavailable) + p := NewPool([]string{a.url(), b.url()}, "", time.Second, + WithFailureThreshold(1), WithProbeInterval(10*time.Millisecond), WithCatchupQueueSize(2)) + resyncs := make(chan string, 4) + p.OnResyncNeeded(func(endpoint string) { resyncs <- endpoint }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + + for _, id := range []string{"tx1", "tx2", "tx3", "tx4", "tx5"} { + if err := p.Register(ctx, id, testCallbackURL, ""); err != nil { + t.Fatalf("register %s: %v", id, err) + } + } + if got := pendingFor(p, b.url()); got != 2 { + t.Fatalf("queue must be capped at 2, got %d", got) + } + select { + case ep := <-resyncs: + t.Fatalf("resync must not fire before the queue drains, got %s", ep) + default: + } + + b.healthCode.Store(http.StatusOK) + b.watchCode.Store(http.StatusOK) + select { + case ep := <-resyncs: + if ep != b.url() { + t.Fatalf("resync for %s, want %s", ep, b.url()) + } + case <-time.After(3 * time.Second): + t.Fatal("resync hook never fired after the overflowed queue drained") + } + if got := pendingFor(p, b.url()); got != 0 { + t.Fatalf("queue must be empty after drain, got %d", got) + } + time.Sleep(50 * time.Millisecond) + select { + case ep := <-resyncs: + t.Fatalf("resync must fire once per overflow, got a second one for %s", ep) + default: + } +} + +// The Service from Endpoint() targets one endpoint, keeps its failures in +// the catch-up queue, and refuses fast while the breaker is open. +func TestPool_EndpointServiceQueuesFailures(t *testing.T) { + a, b := newFakeMerkle(t), newFakeMerkle(t) + p := NewPool([]string{a.url(), b.url()}, "", time.Second, WithFailureThreshold(2)) + ctx := context.Background() + single := p.Endpoint(b.url()) + if single == nil { + t.Fatal("known endpoint must be returned") + } + + b.setFail("tx2", true) + errs := single.RegisterBatchWithResults(ctx, []Registration{{TxID: "tx1"}, {TxID: "tx2"}}, 2) + if errs[0] != nil || errs[1] == nil { + t.Fatalf("errs = %v, want tx1 ok / tx2 refused", errs) + } + if a.watchedCount() != 0 { + t.Fatal("Endpoint() must not fan out to other endpoints") + } + if got := pendingFor(p, b.url()); got != 1 { + t.Fatalf("refused tx must be queued for b, pending = %d", got) + } + + b.watchCode.Store(http.StatusServiceUnavailable) + _ = single.Register(ctx, "tx3", testCallbackURL, "") + _ = single.Register(ctx, "tx4", testCallbackURL, "") + if st := p.EndpointStatuses()[1]; st.Healthy { + t.Fatalf("two 5xx through Endpoint() must open the breaker: %+v", st) + } + before := b.watchedCount() + if err := single.Register(ctx, "tx5", testCallbackURL, ""); !errors.Is(err, ErrNoHealthyEndpoints) { + t.Fatalf("open endpoint must refuse fast, got %v", err) + } + if b.watchedCount() != before { + t.Fatal("open endpoint must not be called") + } + if got := pendingFor(p, b.url()); got != 4 { + t.Fatalf("tx2..tx5 must all be owed to b, pending = %d", got) + } + if got := testutil.ToFloat64(metrics.MerkleEndpointCatchupPending.WithLabelValues(b.url())); got != 4 { + t.Fatalf("catchup_pending gauge = %v, want 4 (refused while open must update it)", got) + } +} + +// An outright 4xx rejection through Endpoint() is not queued: re-sending it +// cannot succeed. +func TestPool_EndpointServiceDoesNotQueuePermanentRejections(t *testing.T) { + b := newFakeMerkle(t) + p := NewPool([]string{b.url()}, "", time.Second) + b.setFailCode("tx1", http.StatusBadRequest) + errs := p.Endpoint(b.url()).RegisterBatchWithResults(context.Background(), []Registration{{TxID: "tx1"}}, 1) + if errs[0] == nil { + t.Fatal("tx1 must be refused") + } + if got := pendingFor(p, b.url()); got != 0 { + t.Fatalf("a 400 must not be queued, pending = %d", got) + } +} + +// A catch-up entry the endpoint refuses again goes back to the queue: it is +// still owed, and dropping it would leave the endpoint missing the txid. +func TestPool_FailedCatchupIsRequeued(t *testing.T) { + a, b := newFakeMerkle(t), newFakeMerkle(t) + b.setFail("tx2", true) + // A high threshold keeps the breaker out of it: this is about the queue. + p := NewPool([]string{a.url(), b.url()}, "", time.Second, + WithFailureThreshold(1000), WithProbeInterval(10*time.Millisecond)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + + if err := p.Register(ctx, "tx2", testCallbackURL, ""); err != nil { + t.Fatalf("register: %v (a accepts)", err) + } + // A second catch-up attempt can only come from the entry having been + // put back after the first one failed. + refused := metrics.MerkleEndpointRequestsTotal.WithLabelValues(b.url(), opCatchup, outcome5xx) + waitFor(t, "a second catch-up attempt on b", func() bool { return testutil.ToFloat64(refused) >= 2 }) + if got := pendingFor(p, b.url()); got != 1 { + t.Fatalf("tx2 must still be owed to b, pending = %d", got) + } +} + +// A registration the endpoint rejects outright (400) is dropped from the +// catch-up queue; a throttled one (429) is kept and retried. +func TestPool_CatchupDropsPermanentRejectionsOnly(t *testing.T) { + a, b := newFakeMerkle(t), newFakeMerkle(t) + b.setFailCode("bad", http.StatusBadRequest) + b.setFailCode("slow", http.StatusTooManyRequests) + p := NewPool([]string{a.url(), b.url()}, "", time.Second, WithProbeInterval(10*time.Millisecond)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + + regs := []Registration{{TxID: "bad", CallbackURL: testCallbackURL}, {TxID: "slow", CallbackURL: testCallbackURL}} + for i, err := range p.RegisterBatchWithResults(ctx, regs, 2) { + if err != nil { + t.Fatalf("errs[%d]=%v, a accepted both", i, err) + } + } + if got := pendingFor(p, b.url()); got != 2 { + t.Fatalf("both refusals must be owed to b first, pending = %d", got) + } + waitFor(t, "a catch-up attempt for both", func() bool { + got := b.watchedSet() + return got["bad"] >= 2 && got["slow"] >= 2 + }) + waitFor(t, "only the 429 to stay owed", func() bool { return pendingFor(p, b.url()) == 1 }) + time.Sleep(100 * time.Millisecond) + if got := b.watchedSet()["bad"]; got != 2 { + t.Fatalf("a 400 must not be re-sent after the catch-up attempt, sent %d times", got) + } + + b.setFailCode("slow", 0) + waitFor(t, "the 429 entry to land", func() bool { return pendingFor(p, b.url()) == 0 }) +} + +// An endpoint that hangs (no answer until the HTTP timeout) must cost the +// batch that trips its breaker no more than the requests already in flight, +// not one timeout per maxConcurrency entries; everything it did not get is +// owed to it. +func TestPool_HangingEndpointDoesNotHoldTheBatch(t *testing.T) { + a := newFakeMerkle(t) + release := make(chan struct{}) + hang := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + select { + case <-r.Context().Done(): + case <-release: + } + })) + t.Cleanup(hang.Close) + t.Cleanup(func() { close(release) }) // runs first: unblock handlers so Close returns + + const timeout = 200 * time.Millisecond + p := NewPool([]string{a.url(), hang.URL}, "", timeout, WithFailureThreshold(3)) + regs := make([]Registration, 12) + for i := range regs { + regs[i] = Registration{TxID: "tx" + strconv.Itoa(i), CallbackURL: testCallbackURL} + } + + start := time.Now() + errs := p.RegisterBatchWithResults(context.Background(), regs, 2) + elapsed := time.Since(start) + for i, err := range errs { + if err != nil { + t.Fatalf("errs[%d]=%v, a accepted every tx", i, err) + } + } + // Concurrency 2, threshold 3: two rounds of timeouts trip the breaker, + // then the other eight are skipped. Sending all twelve would take six. + if elapsed > 4*timeout { + t.Fatalf("first batch took %v; the breaker must stop the remaining sends (6 rounds = %v)", elapsed, 6*timeout) + } + if st := p.EndpointStatuses()[1]; st.Healthy || st.CatchupPending != len(regs) { + t.Fatalf("hanging endpoint must be open and owed all %d txs: %+v", len(regs), st) + } + + start = time.Now() + _ = p.RegisterBatchWithResults(context.Background(), regs, 2) + if elapsed := time.Since(start); elapsed > timeout { + t.Fatalf("with the breaker open the batch must not wait on the endpoint, took %v", elapsed) + } +} + +func TestPool_RegisterBatch_ReturnsFirstFailureWithoutFailFast(t *testing.T) { + a, b := newFakeMerkle(t), newFakeMerkle(t) + a.failTxIDs["tx1"] = http.StatusInternalServerError + b.failTxIDs["tx1"] = http.StatusInternalServerError + p := NewPool([]string{a.url(), b.url()}, "", time.Second) + + regs := []Registration{{TxID: "tx1", CallbackURL: testCallbackURL}, {TxID: "tx2", CallbackURL: testCallbackURL}} + if err := p.RegisterBatch(context.Background(), regs, 4); err == nil { + t.Fatal("a tx refused everywhere must fail the batch") + } + // Not fail-fast: tx2 was still sent to both. + if a.watchedCount() != 2 || b.watchedCount() != 2 { + t.Fatalf("remaining txs must still be registered: a=%d b=%d", a.watchedCount(), b.watchedCount()) + } +} + +// The merged error is the most retryable one so propagation's classification +// (claim_revoked > register_error > auth_error) keeps its meaning. +func TestPool_MergePrefersMostRetryableError(t *testing.T) { + ctx := context.Background() + net := errors.New("dial tcp: connection refused") + auth := &RegisterError{StatusCode: http.StatusUnauthorized} + srvErr := &RegisterError{StatusCode: http.StatusBadGateway} + bad := &RegisterError{StatusCode: http.StatusBadRequest} + + cases := []struct { + name string + errs []error + want error + }{ + {"any success wins", []error{auth, nil, net}, nil}, + {"network over 5xx", []error{srvErr, net}, net}, + {"5xx over auth", []error{auth, srvErr}, srvErr}, + {"auth over other 4xx", []error{bad, auth}, auth}, + {"all auth", []error{auth, auth}, auth}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := mergeErrors(ctx, tc.errs); !errors.Is(got, tc.want) { + t.Fatalf("got %v want %v", got, tc.want) + } + }) + } +} + +func TestPool_AllAuthFailuresSurfaceAsAuth(t *testing.T) { + a, b := newFakeMerkle(t), newFakeMerkle(t) + a.watchCode.Store(http.StatusUnauthorized) + b.watchCode.Store(http.StatusForbidden) + p := NewPool([]string{a.url(), b.url()}, "", time.Second) + + err := p.Register(context.Background(), "tx1", testCallbackURL, "") + var regErr *RegisterError + if !errors.As(err, ®Err) || (regErr.StatusCode != http.StatusUnauthorized && regErr.StatusCode != http.StatusForbidden) { + t.Fatalf("want a 401/403 RegisterError, got %v", err) + } + // 4xx never trips the breaker: the service is reachable. + for _, st := range p.EndpointStatuses() { + if !st.Healthy || st.ConsecutiveFailures != 0 { + t.Fatalf("auth rejection must not count toward the breaker: %+v", st) + } + } +} + +func TestPool_CanceledContextIsNotAnEndpointFailure(t *testing.T) { + a := newFakeMerkle(t) + p := NewPool([]string{a.url()}, "", time.Second) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := p.Register(ctx, "tx1", testCallbackURL, "") + if err == nil { + t.Fatal("expected an error from a canceled context") + } + if st := p.EndpointStatuses()[0]; !st.Healthy || st.ConsecutiveFailures != 0 { + t.Fatalf("caller cancellation must not move the breaker: %+v", st) + } +} + +func TestPool_BreakerOpensAfterThresholdAndSkipsEndpoint(t *testing.T) { + a := newFakeMerkle(t) + dead := "http://127.0.0.1:1" // nothing listens: connection refused + p := NewPool([]string{a.url(), dead}, "", time.Second, WithFailureThreshold(3)) + + ctx := context.Background() + for i := 0; i < 3; i++ { + if err := p.Register(ctx, "tx", testCallbackURL, ""); err != nil { + t.Fatalf("call %d: healthy endpoint must carry the call, got %v", i, err) + } + } + statuses := p.EndpointStatuses() + if len(statuses) != 2 { + t.Fatalf("statuses=%d", len(statuses)) + } + if !statuses[0].Healthy { + t.Fatalf("healthy endpoint reported unhealthy: %+v", statuses[0]) + } + if statuses[1].Healthy || statuses[1].ConsecutiveFailures != 3 || statuses[1].OpenSince == nil { + t.Fatalf("dead endpoint must be open after 3 failures: %+v", statuses[1]) + } + if got := testutil.ToFloat64(metrics.MerkleEndpointHealthy.WithLabelValues(dead)); got != 0 { + t.Fatalf("healthy gauge for open endpoint = %v want 0", got) + } + + skippedBefore := testutil.ToFloat64(metrics.MerkleEndpointRequestsTotal.WithLabelValues(dead, opWatch, outcomeSkippedOpen)) + if err := p.Register(ctx, "tx", testCallbackURL, ""); err != nil { + t.Fatalf("register with one open endpoint: %v", err) + } + if got := testutil.ToFloat64(metrics.MerkleEndpointRequestsTotal.WithLabelValues(dead, opWatch, outcomeSkippedOpen)) - skippedBefore; got != 1 { + t.Fatalf("open endpoint must be skipped (and counted), delta=%v", got) + } + if a.watchedCount() != 4 { + t.Fatalf("healthy endpoint calls=%d want 4", a.watchedCount()) + } +} + +func TestPool_AllBreakersOpenFailsFast(t *testing.T) { + p := NewPool([]string{"http://127.0.0.1:1"}, "", time.Second, WithFailureThreshold(1)) + ctx := context.Background() + _ = p.Register(ctx, "tx", testCallbackURL, "") + + errs := p.RegisterBatchWithResults(ctx, []Registration{{TxID: "a"}, {TxID: "b"}}, 1) + for i, err := range errs { + if !errors.Is(err, ErrNoHealthyEndpoints) { + t.Fatalf("errs[%d]=%v want ErrNoHealthyEndpoints", i, err) + } + } + if err := p.Reprocess(ctx, "hash", testCallbackURL, ""); !errors.Is(err, ErrNoHealthyEndpoints) { + t.Fatalf("reprocess=%v want ErrNoHealthyEndpoints", err) + } +} + +// A canceled context during a probe is counted as canceled, not as an +// endpoint failure. +func TestPool_ProbeCancellationIsNotNetworkFailure(t *testing.T) { + a := newFakeMerkle(t) + a.watchCode.Store(http.StatusServiceUnavailable) + p := NewPool([]string{a.url()}, "", time.Second, WithFailureThreshold(1)) + _ = p.Register(context.Background(), "tx", testCallbackURL, "") + if st := p.EndpointStatuses()[0]; st.Healthy { + t.Fatalf("breaker should be open: %+v", st) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + before := testutil.ToFloat64(metrics.MerkleEndpointRequestsTotal.WithLabelValues(a.url(), opProbe, outcomeCanceled)) + p.probeOpenEndpoints(ctx) + if got := testutil.ToFloat64(metrics.MerkleEndpointRequestsTotal.WithLabelValues(a.url(), opProbe, outcomeCanceled)) - before; got != 1 { + t.Fatalf("probe under a canceled context must count as canceled, delta=%v", got) + } + if st := p.EndpointStatuses()[0]; st.Healthy { + t.Fatalf("a canceled probe must not close the breaker: %+v", st) + } +} + +func TestPool_ReprocessFansOutAndSucceedsOnAny(t *testing.T) { + a, b := newFakeMerkle(t), newFakeMerkle(t) + b.watchCode.Store(http.StatusNotFound) + p := NewPool([]string{a.url(), b.url()}, "", time.Second) + if err := p.Reprocess(context.Background(), "hash", testCallbackURL, "tok"); err != nil { + t.Fatalf("one 202 is enough, got %v", err) + } + + a.watchCode.Store(http.StatusNotFound) + err := p.Reprocess(context.Background(), "hash", testCallbackURL, "tok") + var repErr *ReprocessError + if !errors.As(err, &repErr) || repErr.StatusCode != http.StatusNotFound { + t.Fatalf("all-4xx must surface a ReprocessError 404, got %v", err) + } +} + +func TestPool_EndpointsAndEndpoint(t *testing.T) { + a := newFakeMerkle(t) + p := NewPool([]string{a.url() + "/", a.url(), "", " "}, "tok", time.Second) + if got := p.Endpoints(); len(got) != 1 || got[0] != a.url() { + t.Fatalf("Endpoints()=%v want [%s] (deduped, trimmed)", got, a.url()) + } + if p.Endpoint("http://nope.invalid") != nil { + t.Fatal("unknown endpoint must be nil") + } + single := p.Endpoint(a.url() + "/") + if single == nil { + t.Fatal("known endpoint must be returned") + } + if err := single.Register(context.Background(), "tx", testCallbackURL, ""); err != nil { + t.Fatalf("direct endpoint call: %v", err) + } + if a.watchedCount() != 1 { + t.Fatalf("direct call reached endpoint %d times, want 1", a.watchedCount()) + } +} + +// /health is shallow: an endpoint whose /health answers while /watch hangs +// must stay open (a trial /watch of one owed registration fails) instead of +// closing every probe tick and holding the next hot-path batch. +func TestPool_ProbeKeepsBreakerOpenWhileWatchHangs(t *testing.T) { + a := newFakeMerkle(t) + var watchOK atomic.Bool + release := make(chan struct{}) + b := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/health" || watchOK.Load() { + w.WriteHeader(http.StatusOK) + return + } + select { + case <-r.Context().Done(): + case <-release: + } + })) + t.Cleanup(b.Close) + t.Cleanup(func() { close(release) }) + + p := NewPool([]string{a.url(), b.URL}, "", 100*time.Millisecond, WithFailureThreshold(1)) + ctx := context.Background() + if err := p.Register(ctx, "tx1", testCallbackURL, ""); err != nil { + t.Fatalf("register: %v (a accepts)", err) + } + if st := p.EndpointStatuses()[1]; st.Healthy || st.CatchupPending != 1 { + t.Fatalf("b must be open and owe tx1: %+v", st) + } + + p.probeOpenEndpoints(ctx) + if st := p.EndpointStatuses()[1]; st.Healthy || st.CatchupPending != 1 { + t.Fatalf("/health up but /watch hanging must keep b open and owing tx1: %+v", st) + } + + watchOK.Store(true) + p.probeOpenEndpoints(ctx) + if st := p.EndpointStatuses()[1]; !st.Healthy || st.CatchupPending != 0 { + t.Fatalf("b must close once the trial /watch lands: %+v", st) + } +} + +// hangingServer answers nothing until the request is canceled or release is +// closed: a black-holed merkle-service (paused container, dead IP). +func hangingServer(t *testing.T) *httptest.Server { + t.Helper() + release := make(chan struct{}) + hang := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + select { + case <-r.Context().Done(): + case <-release: + } + })) + t.Cleanup(hang.Close) + t.Cleanup(func() { close(release) }) // runs first: unblock handlers so Close returns + return hang +} + +func regsN(n int) []Registration { + regs := make([]Registration, n) + for i := range regs { + regs[i] = Registration{TxID: "tx" + strconv.Itoa(i), CallbackURL: testCallbackURL} + } + return regs +} + +// Found by chaos testing: with one of three merkle-services black-holed, every +// batch waited out the full HTTP timeout (30s in production) on that endpoint +// although the other two had accepted every tx in milliseconds, so broadcast +// stalled for the whole window before the breaker tripped. Once every tx is +// accepted somewhere, the stragglers get a short grace and are then canceled; +// the cancellation counts against their breaker and the txs they owe go to +// the catch-up queue. +func TestPool_StragglerDoesNotHoldTheBatchOnceEveryTxIsAccepted(t *testing.T) { + a := newFakeMerkle(t) + hang := hangingServer(t) + const timeout, grace = 10 * time.Second, 100 * time.Millisecond + p := NewPool([]string{a.url(), hang.URL}, "", timeout, WithFailureThreshold(3), WithStragglerGrace(grace)) + regs := regsN(12) + + start := time.Now() + errs := p.RegisterBatchWithResults(context.Background(), regs, 4) + elapsed := time.Since(start) + for i, err := range errs { + if err != nil { + t.Fatalf("errs[%d]=%v, a accepted every tx", i, err) + } + } + if elapsed > 10*grace { + t.Fatalf("batch took %v; it must return about one grace (%v) after a accepted everything, not wait for the %v timeout", elapsed, grace, timeout) + } + st := p.EndpointStatuses()[1] + if st.Healthy { + t.Fatalf("the canceled in-flight /watch calls must count against the hanging endpoint's breaker: %+v", st) + } + if st.CatchupPending != len(regs) { + t.Fatalf("hanging endpoint must owe all %d txs, owes %d", len(regs), st.CatchupPending) + } +} + +// With no other endpoint to accept, nothing is covered and nothing is cut +// short: the batch waits for the endpoint's own answer (here its timeout). +func TestPool_SingleHangingEndpointStillWaitsForItsTimeout(t *testing.T) { + hang := hangingServer(t) + const timeout, grace = 300 * time.Millisecond, 20 * time.Millisecond + p := NewPool([]string{hang.URL}, "", timeout, WithStragglerGrace(grace)) + start := time.Now() + errs := p.RegisterBatchWithResults(context.Background(), regsN(2), 2) + if elapsed := time.Since(start); elapsed < timeout { + t.Fatalf("returned after %v, before the endpoint's %v timeout", elapsed, timeout) + } + for i, err := range errs { + if err == nil { + t.Fatalf("errs[%d] = nil, nothing accepted it", i) + } + } +} + +// A slower endpoint that answers within the grace keeps its registrations and +// its health. +func TestPool_SlowEndpointWithinGraceIsNotPenalized(t *testing.T) { + a := newFakeMerkle(t) + b := newFakeMerkle(t) + slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(50 * time.Millisecond) + b.srv.Config.Handler.ServeHTTP(w, r) + })) + t.Cleanup(slow.Close) + p := NewPool([]string{a.url(), slow.URL}, "", 5*time.Second, WithStragglerGrace(2*time.Second)) + regs := regsN(6) + for i, err := range p.RegisterBatchWithResults(context.Background(), regs, 6) { + if err != nil { + t.Fatalf("errs[%d]=%v", i, err) + } + } + st := p.EndpointStatuses()[1] + if !st.Healthy || st.CatchupPending != 0 || st.ConsecutiveFailures != 0 { + t.Fatalf("slow-but-answering endpoint must be healthy and owe nothing: %+v", st) + } + if b.watchedCount() != len(regs) { + t.Fatalf("slow endpoint got %d of %d registrations", b.watchedCount(), len(regs)) + } +} + +// What a canceled straggler owes is not lost: once the endpoint answers again +// the probe closes its breaker and the catch-up drain delivers every tx. +func TestPool_CanceledStragglerOwedTxsDrainAfterRecovery(t *testing.T) { + a := newFakeMerkle(t) + b := newFakeMerkle(t) + var hung atomic.Bool + hung.Store(true) + release := make(chan struct{}) + flaky := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hung.Load() { + select { + case <-r.Context().Done(): + case <-release: + } + return + } + b.srv.Config.Handler.ServeHTTP(w, r) + })) + t.Cleanup(flaky.Close) + t.Cleanup(func() { close(release) }) + + p := NewPool([]string{a.url(), flaky.URL}, "", 10*time.Second, + WithFailureThreshold(3), WithStragglerGrace(50*time.Millisecond), WithProbeInterval(20*time.Millisecond)) + regs := regsN(8) + for i, err := range p.RegisterBatchWithResults(context.Background(), regs, 4) { + if err != nil { + t.Fatalf("errs[%d]=%v", i, err) + } + } + if st := p.EndpointStatuses()[1]; st.Healthy || st.CatchupPending != len(regs) { + t.Fatalf("straggler must be open and owe all %d: %+v", len(regs), st) + } + + hung.Store(false) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Start(ctx) + waitFor(t, "straggler's owed registrations to drain", func() bool { + st := p.EndpointStatuses()[1] + return st.Healthy && st.CatchupPending == 0 + }) + got := b.watchedSet() + for _, r := range regs { + if got[r.TxID] == 0 { + t.Fatalf("%s never reached the recovered endpoint: %v", r.TxID, got) + } + } +} + +// selectiveHangServer accepts /watch for the txids in accept and hangs on +// every other one until the request is canceled. +func selectiveHangServer(t *testing.T, accept ...string) *httptest.Server { + t.Helper() + ok := make(map[string]bool, len(accept)) + for _, id := range accept { + ok[id] = true + } + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + TxID string `json:"txid"` + } + decodeJSON(r, &req) + if ok[req.TxID] { + w.WriteHeader(http.StatusOK) + return + } + select { + case <-r.Context().Done(): + case <-release: + } + })) + t.Cleanup(srv.Close) + t.Cleanup(func() { close(release) }) + return srv +} + +// Neither endpoint finishes its own batch, but between them every tx is +// accepted: A takes tx0 and hangs on tx1, B the reverse. The grace timer must +// start from the per-tx acceptances, not wait for a whole endpoint batch. +func TestPool_ComplementaryAcceptanceStartsTheGrace(t *testing.T) { + a := selectiveHangServer(t, "tx0") + b := selectiveHangServer(t, "tx1") + const timeout, grace = 10 * time.Second, 100 * time.Millisecond + p := NewPool([]string{a.URL, b.URL}, "", timeout, WithFailureThreshold(3), WithStragglerGrace(grace)) + regs := regsN(2) + + start := time.Now() + errs := p.RegisterBatchWithResults(context.Background(), regs, 2) + elapsed := time.Since(start) + for i, err := range errs { + if err != nil { + t.Fatalf("errs[%d]=%v, one endpoint accepted each tx", i, err) + } + } + if elapsed > 10*grace { + t.Fatalf("batch took %v; complementary coverage must return about one grace (%v) after the last acceptance, not wait for the %v timeout", elapsed, grace, timeout) + } + if got := pendingFor(p, a.URL); got != 1 { + t.Fatalf("a must owe tx1, owes %d", got) + } + if got := pendingFor(p, b.URL); got != 1 { + t.Fatalf("b must owe tx0, owes %d", got) + } +} + +// An endpoint skipped because its breaker is already open counts one +// skipped_open per registration it would have received, matching the +// per-request help text and mid-batch skips; /reprocess counts one. +func TestPool_SkippedOpenCountsEveryRegistration(t *testing.T) { + a := newFakeMerkle(t) + b := newFakeMerkle(t) + b.watchCode.Store(http.StatusInternalServerError) + p := NewPool([]string{a.url(), b.url()}, "", time.Second, WithFailureThreshold(1)) + if err := p.Register(context.Background(), "trip", testCallbackURL, ""); err != nil { + t.Fatalf("Register: %v", err) + } + if p.EndpointStatuses()[1].Healthy { + t.Fatal("b's breaker must be open after one failure") + } + watch := metrics.MerkleEndpointRequestsTotal.WithLabelValues(b.url(), opWatch, outcomeSkippedOpen) + before := testutil.ToFloat64(watch) + p.RegisterBatchWithResults(context.Background(), regsN(25), 5) + if got := testutil.ToFloat64(watch) - before; got != 25 { + t.Fatalf("skipped_open grew by %v for a 25-entry batch, want 25", got) + } + reprocess := metrics.MerkleEndpointRequestsTotal.WithLabelValues(b.url(), opReprocess, outcomeSkippedOpen) + before = testutil.ToFloat64(reprocess) + _ = p.Reprocess(context.Background(), "blockhash", testCallbackURL, "") + if got := testutil.ToFloat64(reprocess) - before; got != 1 { + t.Fatalf("skipped_open grew by %v for one /reprocess, want 1", got) + } +} + +// Retain queues only retryable failures, and queues them for every endpoint. +func TestPool_RetainSkipsSuccessesAndPermanentRejections(t *testing.T) { + a := newFakeMerkle(t) + b := newFakeMerkle(t) + p := NewPool([]string{a.url(), b.url()}, "", time.Second) + regs := regsN(3) + p.Retain(regs, []error{ + nil, + &RegisterError{StatusCode: http.StatusBadRequest}, + &RegisterError{StatusCode: http.StatusServiceUnavailable}, + }) + for _, st := range p.EndpointStatuses() { + if st.CatchupPending != 1 { + t.Fatalf("endpoint %s must owe only the 503'd tx, got %+v", st.URL, st) + } + } +} + +// An endpoint skipped for an open breaker takes part in the merge as +// ErrNoHealthyEndpoints, so a terminal answer from the others (404 on +// /reprocess, 401 on /watch) does not become the pool's verdict while the +// skipped endpoint may merely be down. +func TestPool_SkippedOpenEndpointOutranksTerminalAnswers(t *testing.T) { + a := newFakeMerkle(t) + b := newFakeMerkle(t) + p := NewPool([]string{a.url(), b.url()}, "", time.Second, WithFailureThreshold(1)) + a.watchCode.Store(http.StatusInternalServerError) + _ = p.Register(context.Background(), "trip", testCallbackURL, "") + if p.EndpointStatuses()[0].Healthy { + t.Fatal("a's breaker must be open") + } + + b.watchCode.Store(http.StatusNotFound) + if err := p.Reprocess(context.Background(), "blockhash", testCallbackURL, ""); !errors.Is(err, ErrNoHealthyEndpoints) { + t.Fatalf("Reprocess with a open and b 404 = %v, want ErrNoHealthyEndpoints", err) + } + b.watchCode.Store(http.StatusUnauthorized) + errs := p.RegisterBatchWithResults(context.Background(), regsN(2), 2) + for i, err := range errs { + if !errors.Is(err, ErrNoHealthyEndpoints) { + t.Fatalf("errs[%d] with a open and b 401 = %v, want ErrNoHealthyEndpoints", i, err) + } + } +} + +// An endpoint that is only slower than the others, and acknowledged requests +// in this batch before the straggler grace cut it short, keeps its health: +// the cancellations are the pool's doing, not evidence the endpoint is down. +// It still owes what it did not acknowledge. +func TestPool_SlowerEndpointCutByGraceKeepsItsHealth(t *testing.T) { + a := newFakeMerkle(t) + b := newFakeMerkle(t) + slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(60 * time.Millisecond) + b.srv.Config.Handler.ServeHTTP(w, r) + })) + t.Cleanup(slow.Close) + p := NewPool([]string{a.url(), slow.URL}, "", 5*time.Second, WithFailureThreshold(3), WithStragglerGrace(100*time.Millisecond)) + regs := regsN(40) + for i, err := range p.RegisterBatchWithResults(context.Background(), regs, 2) { + if err != nil { + t.Fatalf("errs[%d]=%v", i, err) + } + } + st := p.EndpointStatuses()[1] + if !st.Healthy || st.ConsecutiveFailures != 0 { + t.Fatalf("a slower endpoint that acknowledged requests must keep its health: %+v", st) + } + if st.CatchupPending == 0 || st.CatchupPending >= len(regs) { + t.Fatalf("the slower endpoint must owe only what it did not acknowledge, owes %d of %d", st.CatchupPending, len(regs)) + } +} diff --git a/merkleservice/service.go b/merkleservice/service.go new file mode 100644 index 00000000..2ef462d9 --- /dev/null +++ b/merkleservice/service.go @@ -0,0 +1,77 @@ +package merkleservice + +import ( + "context" + "time" +) + +// Service is the merkle-service surface arcade's services depend on. The +// single-endpoint *Client and the multi-endpoint *Pool both satisfy it, so a +// consumer never needs to know how many merkle-services are configured: it +// registers a tx or asks for a block reprocess, and the implementation decides +// which endpoints that reaches. +type Service interface { + // Register registers one txid for watching. See Client.Register. + Register(ctx context.Context, txid, callbackURL, callbackToken string) error + // RegisterBatch registers many txids and fails fast on the first error. + // See Client.RegisterBatch. + RegisterBatch(ctx context.Context, registrations []Registration, maxConcurrency int) error + // RegisterBatchWithResults registers many txids and reports one error per + // input index (nil = registered). See Client.RegisterBatchWithResults. + RegisterBatchWithResults(ctx context.Context, registrations []Registration, maxConcurrency int) []error + // Reprocess asks for a block's STUMP + BLOCK_PROCESSED callbacks to be + // re-emitted. See Client.Reprocess. + Reprocess(ctx context.Context, blockHash, callbackURL, callbackToken string) error +} + +// Recoverable is the optional surface a multi-endpoint implementation exposes +// so a consumer can resync one endpoint. *Pool implements it; *Client does +// not. The pool itself re-sends every registration an endpoint missed from a +// bounded per-endpoint catch-up queue; propagation type-asserts for this +// interface to run a full lookback replay against one endpoint when that +// queue overflowed and the dropped registrations are unknown. +type Recoverable interface { + // Endpoints lists the configured base URLs, in config order. + Endpoints() []string + // Endpoint returns a Service that talks to exactly one endpoint (still + // honoring its breaker and queueing its failures), or nil when baseURL + // is not part of the pool. + Endpoint(baseURL string) Service + // OnResyncNeeded registers fn to run (synchronously, on the pool's + // background goroutine) when an endpoint's catch-up queue overflowed and + // has since drained. fn must return quickly; spawn work if needed. + OnResyncNeeded(fn func(endpoint string)) + // Retain queues each registration whose /watch failed on every endpoint + // into every endpoint's catch-up queue, so the background loop re-sends + // it once an endpoint answers. errs[i] is registrations[i]'s pool-level + // result; nil entries and outright rejections (a 4xx that a resend cannot + // fix) are skipped. Used by the startup replay, which has no other retry + // for the entries it could not land. + Retain(registrations []Registration, errs []error) +} + +// HealthReporter is the optional surface the api-server's /health uses to +// show per-endpoint breaker state. *Pool implements it. +type HealthReporter interface { + EndpointStatuses() []EndpointStatus +} + +// EndpointStatus is one endpoint's breaker and catch-up view for /health +// and tests. +type EndpointStatus struct { + URL string `json:"url"` + Healthy bool `json:"healthy"` + ConsecutiveFailures int `json:"consecutiveFailures"` + LastFailureAt *time.Time `json:"lastFailureAt,omitempty"` + OpenSince *time.Time `json:"openSince,omitempty"` + // CatchupPending is how many registrations the pool still owes this + // endpoint (accepted elsewhere, not yet acknowledged here). + CatchupPending int `json:"catchupPending"` +} + +var ( + _ Service = (*Client)(nil) + _ Service = (*Pool)(nil) + _ Recoverable = (*Pool)(nil) + _ HealthReporter = (*Pool)(nil) +) diff --git a/metrics/README.md b/metrics/README.md index b65d1292..93131544 100644 --- a/metrics/README.md +++ b/metrics/README.md @@ -46,7 +46,7 @@ rate(arcade_bump_builder_build_duration_seconds_count{ ``` **Do not** negate the successful-build labels (e.g. -`{outcome!~"finalized_complete_no_grace|grace_waited"}`) — five outcomes are +`{outcome!~"finalized_complete_no_grace|grace_waited"}`) — six outcomes are benign and would fire it: - `finalized_complete_no_grace` — BUMP built; merkle's expected-STUMP set was @@ -54,8 +54,14 @@ benign and would fire it: case on a healthy deployment. - `grace_waited` — BUMP built via the grace-window path (completeness could not be verified up-front). -- `short_circuited` — a BUMP already existed, so a redelivered `BLOCK_PROCESSED` - skipped the rebuild. Expected whenever `/reprocess` re-drives a block. +- `short_circuited` — a BUMP already existed and the redelivered + `BLOCK_PROCESSED` added no leaves, so the rebuild was skipped. Expected + whenever `/reprocess` re-drives a block and once per extra merkle-service. +- `rebuilt` — a redelivered `BLOCK_PROCESSED` (another merkle-service) carried + STUMPs with leaves the stored BUMP lacked, so the compound was rebuilt and + only the newly covered txs were published. Expected after a merkle-service + outage; a steady rate means one service is consistently missing + registrations. - `no_stumps` — the block contained no tracked txs. Expected on most blocks. - `context_canceled` — shutdown. @@ -82,6 +88,29 @@ landing. - `histogram_quantile(0.95, ...broadcast_duration_seconds_bucket{path="batch"})` — p95 batch broadcast - `histogram_quantile(0.95, ...merkle_register_duration_seconds_bucket)` — merkle is often the bottleneck +### Merkle-service endpoint board + +- Per-endpoint `arcade_merkle_endpoint_healthy` (one row per `endpoint` per + pod). Every arcade process runs its own pool with its own breakers. + Propagation sends `/watch`; api-server, watchdog and bump-builder send + `/reprocess`, and a transport/5xx failure there trips the same breaker, so + they can report 0 too. Only a process that has made neither call stays at + 1, which is why a `sum()` across pods says little. The propagation pods + carry every registration and are the authoritative view: alert on + `min by (endpoint) (arcade_merkle_endpoint_healthy{}) == 0`, + with the selector your scrape config uses for the propagation pods. +- `sum by (endpoint, outcome) (rate(arcade_merkle_endpoint_requests_total{op="watch",outcome!="ok"}[5m]))` + — which merkle-service is failing and how (`err_network`, `err_5xx`, + `err_auth`, `err_4xx`, `skipped_open`). +- `arcade_merkle_endpoint_catchup_pending` (one row per `endpoint`) — + registrations another endpoint accepted that this one has not acknowledged + yet; the pool re-sends them in the background (`op="catchup"` on the + requests counter). Alert when it stays above 0 for longer than a few + minutes while the endpoint is healthy. +- `arcade_propagation_merkle_register_batch_outcome_total` stays the + pool-level view: `fully_ok` while any endpoint accepts, `all_failed` only + when every endpoint refused. + ### Datahub health board - Per-endpoint `arcade_teranode_endpoint_healthy` (one row per `endpoint`) diff --git a/metrics/metrics.go b/metrics/metrics.go index 8f286c93..0cd1ea0e 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -204,6 +204,38 @@ var PropagationMerkleRegisterBatchOutcomeTotal = promauto.NewCounterVec(promethe Help: "Per-batch merkle-service registration outcome.", }, []string{labelOutcome}) // fully_ok, partial, all_failed +// MerkleEndpointRequestsTotal counts every request the merkle-service pool +// issued to (or skipped for) one endpoint, by operation and outcome. The +// existing pool-level merkle metrics answer "did the tx get registered"; +// this one answers "which endpoint is failing". op is watch, catchup +// (re-sending a registration the endpoint missed), reprocess or probe; outcome is ok, err_auth, err_4xx, err_5xx, err_network, canceled or +// skipped_open (breaker open, request not sent). The endpoint label is the +// configured base URL, so cardinality is bounded by the config. +var MerkleEndpointRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "arcade_merkle_endpoint_requests_total", + Help: "Per-endpoint merkle-service request outcomes by operation.", +}, []string{"endpoint", "op", labelOutcome}) + +// MerkleEndpointCatchupPending is how many registrations the merkle-service +// pool still owes an endpoint: txs some other endpoint accepted that this +// one refused, timed out on, or was skipped for while its breaker was open. +// The pool re-sends them in the background; a value that never returns to 0 +// means the endpoint keeps refusing /watch. +var MerkleEndpointCatchupPending = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "arcade_merkle_endpoint_catchup_pending", + Help: "Registrations queued for re-delivery to a merkle-service endpoint that missed them.", +}, []string{"endpoint"}) + +// MerkleEndpointHealthy is 1 while the endpoint's circuit breaker is closed +// (requests are sent to it) and 0 while it is open (skipped until the +// background /health probe answers). Each process runs its own pool, and +// ones that never send /watch stay at 1, so alert on min by (endpoint) == 0 +// rather than on a sum across pods. +var MerkleEndpointHealthy = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "arcade_merkle_endpoint_healthy", + Help: "1 while the merkle-service endpoint's circuit breaker is closed, 0 while open.", +}, []string{"endpoint"}) + // PropagationReaperLease is 1 when this pod holds the reaper lease, 0 otherwise. // In K8s, sum across pods should always equal 1 (or 0 during failover). var PropagationReaperLease = promauto.NewGauge(prometheus.GaugeOpts{ @@ -460,9 +492,11 @@ var StuckTransientTxsByCallback = promauto.NewGaugeVec(prometheus.GaugeOpts{ // (expected-STUMP set complete on the first read — no grace wait) and // `grace_waited` (built via the grace-window path; also stamped when the // window is configured to 0). Three more are benign non-failures: -// `short_circuited` (a BUMP already existed, so a redelivery was skipped), -// `no_stumps` (block contains no tracked txs) and `context_canceled` -// (shutdown). The rest are failures: +// `short_circuited` (a BUMP already existed and the redelivery added no +// leaves, so the rebuild was skipped), `rebuilt` (a redelivery from another +// merkle-service carried leaves the stored BUMP lacked, so the compound was +// rebuilt), `no_stumps` (block contains no tracked txs) and +// `context_canceled` (shutdown). The rest are failures: // // parse_failed, deferred_incomplete, fetch_failed, no_subtrees, // build_failed, validation_failed, store_failed diff --git a/metrics/preregister.go b/metrics/preregister.go index c35d0f1d..35503401 100644 --- a/metrics/preregister.go +++ b/metrics/preregister.go @@ -49,6 +49,9 @@ var bumpBuildOutcomes = []string{ // window was skipped, grace_waited otherwise) plus the non-build ones "finalized_complete_no_grace", "grace_waited", "short_circuited", "no_stumps", "context_canceled", + // a redelivered BLOCK_PROCESSED (another merkle-service) carried STUMPs + // with leaves the stored BUMP lacked, so the compound was rebuilt + "rebuilt", // the anchor guard refused to mark the block's txs MINED because the // active chain has a different block at its height (issue #279) "skipped_inactive_chain", diff --git a/models/callback.go b/models/callback.go index ad22a13f..9b065675 100644 --- a/models/callback.go +++ b/models/callback.go @@ -1,5 +1,10 @@ package models +import ( + "crypto/sha256" + "encoding/hex" +) + // CallbackType represents the type of callback message from Merkle Service type CallbackType string @@ -74,9 +79,44 @@ func (msg *CallbackMessage) ResolveSeenTxIDs() []string { return nil } -// Stump represents a stored STUMP (Subtree Unified Merkle Path), keyed by subtree. +// Stump represents a stored STUMP (Subtree Unified Merkle Path). Rows are +// keyed by (block_hash, subtree_index, content_hash): a STUMP's bytes depend +// on which txids the emitting merkle-service had registrations for, so two +// merkle-services can legitimately deliver different STUMPs for the same +// subtree (one of them missed registrations during an outage). Content +// addressing lets identical deliveries collapse to one row while variants +// coexist for the bump-builder to merge. type Stump struct { BlockHash string `json:"block_hash"` SubtreeIndex int `json:"subtree_index"` StumpData []byte `json:"stump_data"` + // ContentHash is hex(sha256(StumpData)). Stores compute it when empty; + // NewStump fills it in. + ContentHash string `json:"content_hash,omitempty"` +} + +// NewStump builds a Stump with its ContentHash set. +func NewStump(blockHash string, subtreeIndex int, data []byte) *Stump { + return &Stump{ + BlockHash: blockHash, + SubtreeIndex: subtreeIndex, + StumpData: data, + ContentHash: StumpContentHash(data), + } +} + +// StumpContentHash returns the content address of STUMP bytes. +func StumpContentHash(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +// EnsureContentHash fills ContentHash from StumpData when it is empty and +// returns it. Store backends call this so a caller that built the struct by +// hand still lands on the content-addressed key. +func (s *Stump) EnsureContentHash() string { + if s.ContentHash == "" { + s.ContentHash = StumpContentHash(s.StumpData) + } + return s.ContentHash } diff --git a/openspec/specs/api-server/spec.md b/openspec/specs/api-server/spec.md index f717a073..a3be36fe 100644 --- a/openspec/specs/api-server/spec.md +++ b/openspec/specs/api-server/spec.md @@ -11,8 +11,12 @@ The API server SHALL expose a POST `/callback` endpoint that accepts merkle-serv - **THEN** the server SHALL publish a message to the `transaction` Kafka topic with the TXID and new state, and return HTTP 200 #### Scenario: Receive STUMP callback -- **WHEN** merkle-service sends a POST to `/callback` with state `STUMP` including a TXID, block hash, and STUMP data -- **THEN** the server SHALL publish a message to the `stump` Kafka topic with the full STUMP payload, and return HTTP 200 +- **WHEN** merkle-service sends a POST to `/callback` with state `STUMP` including a block hash, subtree index, and STUMP data +- **THEN** the server SHALL store the STUMP synchronously keyed by block hash, subtree index and the SHA-256 of the STUMP bytes, and return HTTP 200 only once the write is durable + +#### Scenario: Duplicate deliveries from several merkle-services +- **WHEN** more than one merkle-service delivers the same `SEEN_*`, `STUMP` or `BLOCK_PROCESSED` callback +- **THEN** the server SHALL accept each delivery with HTTP 200; a `SEEN_*` that does not advance the status lattice SHALL NOT be published, an identical `STUMP` SHALL NOT create a second row, and every `BLOCK_PROCESSED` SHALL be forwarded for the bump-builder to process idempotently #### Scenario: Receive BLOCK_PROCESSED callback - **WHEN** merkle-service sends a POST to `/callback` with state `BLOCK_PROCESSED` and a block hash diff --git a/openspec/specs/bump-builder/spec.md b/openspec/specs/bump-builder/spec.md index 86fa37ea..cbe5a568 100644 --- a/openspec/specs/bump-builder/spec.md +++ b/openspec/specs/bump-builder/spec.md @@ -25,14 +25,29 @@ The BUMP builder SHALL store the computed merkle proof for each registered trans - **WHEN** BUMPs are constructed for 10,000 transactions in a block - **THEN** the builder SHALL batch-write all merkle proofs to Aerospike and update each transaction's state to include its proof -### Requirement: Prune STUMPs after BUMP construction -The BUMP builder SHALL delete STUMPs from Aerospike after the full BUMP has been successfully constructed and all merkle proofs stored. +### Requirement: Prune STUMPs after a retention window +The BUMP builder SHALL retain a block's STUMP rows for `bump_builder.stump_retention_minutes` after the compound BUMP was built, and SHALL delete them afterwards. STUMP rows are content-addressed (`block_hash`, `subtree_index`, `sha256(stump_data)`) so several merkle-services can deliver the same subtree: identical bytes are one row, divergent variants coexist. -#### Scenario: Successful BUMP construction -- **WHEN** the BUMP for block `abc123` is fully constructed and all proofs are stored -- **THEN** the builder SHALL delete all STUMP records for that block from Aerospike +#### Scenario: Retention window elapsed +- **WHEN** the BUMP for block `abc123` was built more than the retention window ago +- **THEN** the janitor SHALL delete all STUMP records for that block -#### Scenario: Partial BUMP construction failure -- **WHEN** BUMP construction fails for some transactions in a block -- **THEN** the builder SHALL retain the STUMPs for failed transactions and route a retry message to Kafka +#### Scenario: Within the retention window +- **WHEN** the BUMP for block `abc123` was built less than the retention window ago +- **THEN** the builder SHALL keep the STUMP records so a later `block_processed` delivery can be compared against the stored BUMP + +### Requirement: Idempotent processing of repeated block_processed messages +With several merkle-services configured every block's `block_processed` arrives once per service. The BUMP builder SHALL process every delivery and SHALL rebuild the compound BUMP only when the retained STUMP set contains level-0 hashes the stored BUMP lacks. + +#### Scenario: Redelivery adds nothing +- **WHEN** a `block_processed` message arrives for a block whose stored BUMP already covers every level-0 hash in the retained STUMPs +- **THEN** the builder SHALL re-mine the stored BUMP's transactions without publishing `MINED` for rows already mined against that block, re-stamp `processed_at`, and record outcome `short_circuited` + +#### Scenario: Redelivery adds leaves +- **WHEN** a `block_processed` message arrives and the retained STUMPs (merged per subtree) cover transactions the stored BUMP does not +- **THEN** the builder SHALL rebuild the compound from the full STUMP set, overwrite the stored BUMP, publish `MINED` only for the newly covered transactions, and record outcome `rebuilt` + +#### Scenario: Later delivery is incomplete for an already-finalized block +- **WHEN** a `block_processed` message for a block with `processed_at` set is deferred (expected STUMPs missing after the grace window, or a rebuild would drop leaves of the stored BUMP) +- **THEN** the builder SHALL clear `processed_at` so the watchdog re-drives the block via `/reprocess` to every merkle-service diff --git a/openspec/specs/propagation/spec.md b/openspec/specs/propagation/spec.md index 5af4a800..816ded45 100644 --- a/openspec/specs/propagation/spec.md +++ b/openspec/specs/propagation/spec.md @@ -19,15 +19,19 @@ The propagation service SHALL consume validated transactions and broadcast them - **THEN** the service SHALL log the rejection, retry with backoff, and after max retries route to a dead-letter topic ### Requirement: Register transaction with merkle-service -After successful propagation, the propagation service SHALL register the transaction with the merkle-service server, providing the TXID and the arcade callback URL. +Before broadcasting, the propagation service SHALL register the transaction with every configured merkle-service endpoint, providing the TXID and the arcade callback URL. A transaction counts as registered once at least one endpoint accepted it. #### Scenario: Successful registration -- **WHEN** a transaction has been propagated to at least one datahub successfully -- **THEN** the service SHALL register the TXID and callback URL with the merkle-service +- **WHEN** a transaction is ready for broadcast and at least one merkle-service endpoint accepts its `/watch` +- **THEN** the service SHALL broadcast the transaction and stamp `merkle_registered_at` -#### Scenario: Registration failure -- **WHEN** merkle-service registration fails -- **THEN** the service SHALL retry with exponential backoff and log the failure +#### Scenario: Registration failure on every endpoint +- **WHEN** every merkle-service endpoint refuses or cannot be reached +- **THEN** the service SHALL NOT broadcast the transaction, SHALL requeue it, and SHALL log the failure + +#### Scenario: Endpoint outage +- **WHEN** one endpoint fails three consecutive requests while others remain healthy +- **THEN** the service SHALL skip that endpoint until its `/health` answers, keep registering with the healthy endpoints, and on recovery re-register the non-terminal transactions the endpoint missed ### Requirement: Concurrent datahub broadcasting The propagation service SHALL broadcast to multiple datahub URLs concurrently, not sequentially, to minimize latency. diff --git a/services/api_server/callback_multi_source_test.go b/services/api_server/callback_multi_source_test.go new file mode 100644 index 00000000..ea617a86 --- /dev/null +++ b/services/api_server/callback_multi_source_test.go @@ -0,0 +1,158 @@ +package api_server + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" + + "github.com/bsv-blockchain/arcade/config" + "github.com/bsv-blockchain/arcade/kafka" + "github.com/bsv-blockchain/arcade/merkleservice" + "github.com/bsv-blockchain/arcade/models" + "github.com/bsv-blockchain/arcade/store" + "github.com/bsv-blockchain/arcade/store/pebble" +) + +// With merkle-services B, C and D all watching the same tx, each SEEN state +// arrives three times. The first SEEN_ON_NETWORK transitions and publishes; +// the other two are no-ops. The first SEEN_MULTIPLE_NODES (from whichever +// service gets there first) transitions and publishes; the other two are +// no-ops. A late SEEN_ON_NETWORK after that neither regresses the row nor +// publishes. +func TestHandleCallback_MultiSourceSeen_PublishesEachTransitionOnce(t *testing.T) { + st, err := pebble.New(config.Pebble{Path: t.TempDir()}) + if err != nil { + t.Fatalf("pebble.New: %v", err) + } + t.Cleanup(func() { _ = st.Close() }) + + const txid = "abcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcd" + ctx := context.Background() + if _, _, err := st.GetOrInsertStatus(ctx, &models.TransactionStatus{ + TxID: txid, Status: models.StatusAcceptedByNetwork, Timestamp: time.Now().Add(-time.Second), + }); err != nil { + t.Fatal(err) + } + tracker := store.NewTxTracker() + tracker.Add(txid, models.StatusAcceptedByNetwork) + pub := &recordingCallbackPub{} + gin.SetMode(gin.TestMode) + srv := &Server{ + cfg: &config.Config{CallbackToken: testCallbackToken}, + logger: zap.NewNop(), + producer: kafka.NewProducer(&kafka.RecordingBroker{}), + store: st, + publisher: pub, + txTracker: tracker, + submissionCh: make(chan submissionRecord, submissionRecorderBuffer), + submissionStop: make(chan struct{}), + } + router := gin.New() + srv.registerRoutes(router) + + deliver := func(cb models.CallbackType) { + t.Helper() + req := authedCallbackRequest(t, mustMarshalJSON(t, models.CallbackMessage{Type: cb, TxIDs: []string{txid}})) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("%s: status %d, want 200: %s", cb, w.Code, w.Body.String()) + } + } + bulkCount := func() int { + pub.mu.Lock() + defer pub.mu.Unlock() + return len(pub.bulkPublishes) + } + status := func() models.Status { + t.Helper() + got, err := st.GetStatus(ctx, txid) + if err != nil || got == nil { + t.Fatalf("GetStatus: %v %v", got, err) + } + return got.Status + } + + // B, C, D: SEEN_ON_NETWORK. + for i := 0; i < 3; i++ { + deliver(models.CallbackSeenOnNetwork) + } + if got := bulkCount(); got != 1 { + t.Fatalf("after 3x SEEN_ON_NETWORK publishes=%d want 1", got) + } + if got := status(); got != models.StatusSeenOnNetwork { + t.Fatalf("status=%s want SEEN_ON_NETWORK", got) + } + + // C first, then D and B: SEEN_MULTIPLE_NODES. + for i := 0; i < 3; i++ { + deliver(models.CallbackSeenMultipleNodes) + } + if got := bulkCount(); got != 2 { + t.Fatalf("after 3x SEEN_MULTIPLE_NODES publishes=%d want 2", got) + } + if got := status(); got != models.StatusSeenMultipleNodes { + t.Fatalf("status=%s want SEEN_MULTIPLE_NODES", got) + } + pub.mu.Lock() + if pub.bulkPublishes[0].Status != models.StatusSeenOnNetwork || pub.bulkPublishes[1].Status != models.StatusSeenMultipleNodes { + t.Fatalf("publish order = %s, %s", pub.bulkPublishes[0].Status, pub.bulkPublishes[1].Status) + } + pub.mu.Unlock() + + // A straggling SEEN_ON_NETWORK from a slow service. + deliver(models.CallbackSeenOnNetwork) + if got := bulkCount(); got != 2 { + t.Fatalf("late SEEN_ON_NETWORK must not publish, publishes=%d", got) + } + if got := status(); got != models.StatusSeenMultipleNodes { + t.Fatalf("late SEEN_ON_NETWORK regressed the row to %s", got) + } + if tracked, ok := tracker.GetStatus(txid); !ok || tracked != models.StatusSeenMultipleNodes { + t.Fatalf("tracker = %s ok=%v, want SEEN_MULTIPLE_NODES", tracked, ok) + } +} + +// /health lists every configured merkle-service endpoint with its breaker +// state, and omits the key entirely when the integration is disabled. +func TestHandleHealth_MerkleEndpoints(t *testing.T) { + pool := merkleservice.NewPool([]string{"http://merkle-a.example", "http://merkle-b.example"}, "", time.Second) + srv := &Server{ + cfg: &config.Config{}, + logger: zap.NewNop(), + merkleClient: pool, + } + code, _, raw := doHealth(t, srv) + if code != http.StatusOK { + t.Fatalf("status=%d", code) + } + var body struct { + MerkleEndpoints []merkleservice.EndpointStatus `json:"merkle_endpoints"` + } + if err := json.Unmarshal(raw, &body); err != nil { + t.Fatal(err) + } + if len(body.MerkleEndpoints) != 2 { + t.Fatalf("merkle_endpoints = %+v, want 2 entries", body.MerkleEndpoints) + } + for _, ep := range body.MerkleEndpoints { + if !ep.Healthy { + t.Fatalf("fresh pool endpoint must be healthy: %+v", ep) + } + } + + var keys map[string]json.RawMessage + _, _, raw = doHealth(t, &Server{cfg: &config.Config{}, logger: zap.NewNop()}) + if err := json.Unmarshal(raw, &keys); err != nil { + t.Fatal(err) + } + if _, present := keys["merkle_endpoints"]; present { + t.Fatal("merkle_endpoints must be omitted when merkle-service is not configured") + } +} diff --git a/services/api_server/handlers.go b/services/api_server/handlers.go index 4be752c1..227b670a 100644 --- a/services/api_server/handlers.go +++ b/services/api_server/handlers.go @@ -22,6 +22,7 @@ import ( arcerrors "github.com/bsv-blockchain/arcade/errors" "github.com/bsv-blockchain/arcade/kafka" "github.com/bsv-blockchain/arcade/logfields" + "github.com/bsv-blockchain/arcade/merkleservice" "github.com/bsv-blockchain/arcade/metrics" "github.com/bsv-blockchain/arcade/models" "github.com/bsv-blockchain/arcade/store" @@ -205,6 +206,10 @@ type healthResponse struct { Status string `json:"status"` BlockHeight uint64 `json:"blockHeight,omitempty"` DatahubURLs []datahubStatus `json:"datahub_urls"` + // MerkleEndpoints is the per-endpoint breaker view of the configured + // merkle-services (url, healthy, consecutive failures, open since). + // Omitted when the Merkle integration is disabled. + MerkleEndpoints []merkleservice.EndpointStatus `json:"merkle_endpoints,omitempty"` } // datahubStatus is one entry of health.datahub_urls: the circuit-breaker view @@ -334,6 +339,9 @@ func (s *Server) handleHealth(c *gin.Context) { BlockHeight: s.activeTipHeight(c.Request.Context()), DatahubURLs: []datahubStatus{}, } + if hr, ok := s.merkleClient.(merkleservice.HealthReporter); ok { + resp.MerkleEndpoints = hr.EndpointStatuses() + } if s.teranode != nil { statuses := s.teranode.GetEndpointStatuses() policies := s.endpointPolicies(c.Request.Context()) @@ -637,15 +645,28 @@ func (s *Server) handleStump(c *gin.Context, msg models.CallbackMessage, logger return } - // Store STUMP keyed by (blockHash, subtreeIndex). Synchronous write so that - // a 200 to merkle-service is a durability guarantee — merkle-service only - // fires BLOCK_PROCESSED after all STUMPs succeed, so the bump builder can - // then rely on finding them all in Aerospike. - stump := &models.Stump{ - BlockHash: msg.BlockHash, - SubtreeIndex: msg.SubtreeIndex, - StumpData: msg.Stump, + // Refuse a STUMP that does not parse. Rows are content-addressed, so a + // bad copy is never overwritten by a good resend; it would sit beside + // the good one and fail every build and redelivery of the block until + // the Kafka message dead-letters. Rejecting it here surfaces the problem + // at the sending merkle-service instead. + if mp, err := sdkTx.NewMerklePathFromBinary(msg.Stump); err != nil || len(mp.Path) == 0 { + logger.Warn("malformed STUMP callback", + logfields.BlockHash(msg.BlockHash), + zap.Int("subtree_index", msg.SubtreeIndex), + zap.Error(err), + ) + c.JSON(http.StatusBadRequest, gin.H{jsonKeyError: "stump is not a valid BRC-74 merkle path"}) + return } + + // Store STUMP keyed by (blockHash, subtreeIndex, contentHash). Synchronous + // write so that a 200 to merkle-service is a durability guarantee — + // merkle-service only fires BLOCK_PROCESSED after all STUMPs succeed, so + // the bump builder can then rely on finding them all. Content addressing + // makes the same STUMP from a second merkle-service a no-op and keeps a + // divergent one (a service that missed registrations) as its own row. + stump := models.NewStump(msg.BlockHash, msg.SubtreeIndex, msg.Stump) if err := s.store.InsertStump(c.Request.Context(), stump); err != nil { logger.Error("failed to store STUMP", zap.Error(err)) c.JSON(http.StatusInternalServerError, gin.H{jsonKeyError: "failed to store stump"}) diff --git a/services/api_server/handlers_test.go b/services/api_server/handlers_test.go index 99cc0785..57ccf81e 100644 --- a/services/api_server/handlers_test.go +++ b/services/api_server/handlers_test.go @@ -1600,7 +1600,7 @@ func TestHandleCallback_Stump_StorageError_Returns500(t *testing.T) { payload := models.CallbackMessage{ Type: models.CallbackStump, BlockHash: "abc123", - Stump: []byte{0x01, 0x02}, + Stump: validStumpBytes(0, 0x01), } body := mustMarshalJSON(t, payload) @@ -1617,6 +1617,27 @@ func TestHandleCallback_Stump_StorageError_Returns500(t *testing.T) { } } +// A STUMP that does not parse is refused with 400 and never stored: rows are +// content-addressed, so a stored bad copy would sit beside every good resend +// and fail each build of the block. +func TestHandleCallback_Stump_Malformed_Returns400(t *testing.T) { + var stored int + ms := &mockStore{insertStumpFn: func(*models.Stump) error { stored++; return nil }} + _, router := setupServerWithStore(&kafka.RecordingBroker{}, ms) + + for _, stump := range [][]byte{{0x01, 0x02}, make([]byte, 64)} { + payload := models.CallbackMessage{Type: models.CallbackStump, BlockHash: "abc123", Stump: stump} + w := httptest.NewRecorder() + router.ServeHTTP(w, authedCallbackRequest(t, mustMarshalJSON(t, payload))) + if w.Code != http.StatusBadRequest { + t.Fatalf("stump %x: expected 400, got %d: %s", stump, w.Code, w.Body.String()) + } + } + if stored != 0 { + t.Fatalf("a malformed STUMP must not be stored, InsertStump calls = %d", stored) + } +} + func TestHandleCallback_SeenMultipleNodes_EmptyTxIDs(t *testing.T) { ms := &mockStore{} _, router := setupServerWithStore(&kafka.RecordingBroker{}, ms) @@ -1676,8 +1697,8 @@ func TestHandleCallback_FullBlockFlow_20Subtrees(t *testing.T) { // Deterministic per-subtree payload so byte-level assertions are stable. stumpPayloads := make([][]byte, numSubtrees) for i := range stumpPayloads { - buf := make([]byte, stumpSize) - for j := range buf { + buf := validStumpBytes(stumpSize, byte(i)) + for j := minimalStumpLen; j < len(buf); j++ { buf[j] = byte((i*31 + j) & 0xFF) } stumpPayloads[i] = buf @@ -1863,8 +1884,8 @@ func TestHandleCallback_FullBlockFlow_PartialStumpFailure(t *testing.T) { stumpPayloads := make([][]byte, numSubtrees) for i := range stumpPayloads { - buf := make([]byte, stumpSize) - for j := range buf { + buf := validStumpBytes(stumpSize, byte(i)) + for j := minimalStumpLen; j < len(buf); j++ { buf[j] = byte((i*31 + j) & 0xFF) } stumpPayloads[i] = buf @@ -2255,10 +2276,7 @@ func TestHandleCallback_BodySizeLimit_UnderLimitSucceeds(t *testing.T) { // hex-encodes the bytes via models.HexBytes — is just under the cap. Hex // roughly doubles the byte count, so half the limit (less envelope // overhead) leaves headroom for the JSON wrapper. - stumpBytes := make([]byte, limit/2-1024) - for i := range stumpBytes { - stumpBytes[i] = byte(i & 0xFF) - } + stumpBytes := validStumpBytes(limit/2-1024, 0x01) payload := models.CallbackMessage{ Type: models.CallbackStump, BlockHash: "0000000000000000000000000000000000000000000000000000000000000001", @@ -2515,3 +2533,24 @@ func TestSubmitCounters_SingleDedupResults(t *testing.T) { t.Errorf("duplicate resubmit must not count as new (delta %v)", got) } } + +// minimalStumpLen is the size of the smallest parseable BRC-74 STUMP: block +// height, tree height, leaf count, offset and flags (one byte each) plus one +// 32-byte hash. +const minimalStumpLen = 37 + +// validStumpBytes returns a STUMP the callback handler accepts, size bytes +// long (at least minimalStumpLen): a one-leaf merkle path whose hash is seed +// repeated, padded with zeros. The parser ignores trailing bytes, which lets +// size-sensitive tests keep their payload sizes. +func validStumpBytes(size int, seed byte) []byte { + if size < minimalStumpLen { + size = minimalStumpLen + } + buf := make([]byte, size) + copy(buf, []byte{0x01, 0x01, 0x01, 0x00, 0x02}) + for i := 5; i < minimalStumpLen; i++ { + buf[i] = seed + } + return buf +} diff --git a/services/api_server/lifecycle_log_test.go b/services/api_server/lifecycle_log_test.go index 12b35613..4c4a15af 100644 --- a/services/api_server/lifecycle_log_test.go +++ b/services/api_server/lifecycle_log_test.go @@ -250,7 +250,7 @@ func TestHandleStump_LogsStumpStored(t *testing.T) { Type: models.CallbackStump, BlockHash: "blockhash-stump-test", SubtreeIndex: 3, - Stump: []byte{0x01, 0x02}, + Stump: validStumpBytes(0, 0x03), } req := authedCallbackRequest(t, mustMarshalJSON(t, payload)) w := httptest.NewRecorder() diff --git a/services/api_server/server.go b/services/api_server/server.go index 3a736762..33d491dd 100644 --- a/services/api_server/server.go +++ b/services/api_server/server.go @@ -50,7 +50,7 @@ type Server struct { store store.Store txTracker *store.TxTracker teranode *teranode.Client // used by /health for datahub URL inventory; nil in tests - merkleClient *merkleservice.Client // nil when merkle_service.url is unset; gates POST /api/v1/blocks/:blockHash/reprocess + merkleClient merkleservice.Service // nil when no merkle_service endpoint is configured; gates POST /api/v1/blocks/:blockHash/reprocess // validator runs synchronous policy validation in the submit handler. // Nil-safe: tests that use struct-literal construction may leave it // unset, in which case the handler skips validation. Production @@ -97,7 +97,7 @@ type submissionRecord struct { sub *models.Submission } -func New(cfg *config.Config, logger *zap.Logger, producer *kafka.Producer, publisher events.Publisher, st store.Store, tracker *store.TxTracker, tc *teranode.Client, mc *merkleservice.Client, val *validator.Validator, fin *finality.Checker) *Server { +func New(cfg *config.Config, logger *zap.Logger, producer *kafka.Producer, publisher events.Publisher, st store.Store, tracker *store.TxTracker, tc *teranode.Client, mc merkleservice.Service, val *validator.Validator, fin *finality.Checker) *Server { // Export every series this service can emit at 0 from the first scrape — // a series born mid-burst is invisible to increase() until its second // sample, which undercounted submissions after every rollout. diff --git a/services/bump_builder/builder.go b/services/bump_builder/builder.go index 7b40af65..a2f19676 100644 --- a/services/bump_builder/builder.go +++ b/services/bump_builder/builder.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "math" + "slices" "strings" "time" @@ -318,9 +319,59 @@ func (b *Builder) handleAnchorDenied(ctx context.Context, logger *zap.Logger, bl // It reports whether the store write completed in full: false means some of // the txids are still SEEN_* and the block must stay un-stamped so the // watchdog re-drives it. -func (b *Builder) markMinedAndPublish(ctx context.Context, logger *zap.Logger, blockHash string, blockHeight uint64, txids []string) bool { - _, err := setMinedAndPublish(ctx, logger, b.store, b.publisher, blockHash, blockHeight, txids, "", false) - return err == nil +// +// onlyChanged suppresses MINED events for rows already MINED against this +// block. With several merkle-services every block's BLOCK_PROCESSED arrives +// once per service, and each redelivery re-mines the stored BUMP's level-0 +// set; those rows must not fan out a second MINED event to SSE/webhook +// subscribers. But "already MINED" only proves the store write landed, not +// that the event went out: a crash between SetMinedByTxIDs and PublishBulk +// leaves the rows MINED and unpublished. The build path therefore passes the +// result of blockFinalized, read before it touches the block: processed_at +// is stamped only after the publish, so a stamped block's MINED rows were +// published, and an unstamped one is republished in full (a duplicate event +// beats a lost one). The short-circuit passes it too, which is safe because +// it un-stamps the block when its mine or publish fails (see shortCircuit). +// +// That only holds if a failed publish also withholds the stamp, so a +// PublishBulk failure reports false like a partial store write: the block +// stays in the watchdog's stale scan and its re-drive republishes the set. +func (b *Builder) markMinedAndPublish(ctx context.Context, logger *zap.Logger, blockHash string, blockHeight uint64, txids []string, onlyChanged bool) bool { + var publisher events.Publisher + var recorder *publishFailureRecorder + if b.publisher != nil { + recorder = &publishFailureRecorder{Publisher: b.publisher} + publisher = recorder + } + _, err := setMinedAndPublish(ctx, logger, b.store, publisher, blockHash, blockHeight, txids, "", onlyChanged) + return err == nil && (recorder == nil || !recorder.failed) +} + +// publishFailureRecorder notes whether any PublishBulk call failed. +// setMinedAndPublish logs and carries on past a failed chunk (the rows are +// MINED in the store either way); markMinedAndPublish needs to know so it can +// keep the block un-stamped. Calls are sequential. +type publishFailureRecorder struct { + events.Publisher + + failed bool +} + +func (r *publishFailureRecorder) PublishBulk(ctx context.Context, template *models.TransactionStatus) error { + err := r.Publisher.PublishBulk(ctx, template) + if err != nil { + r.failed = true + } + return err +} + +// blockFinalized reports whether the block's processed_at is stamped, which +// markMinedAndPublish uses as proof that every row an earlier delivery mined +// against it was also published. Any read failure answers false: the caller +// then republishes rather than risk withholding an event that never went out. +func (b *Builder) blockFinalized(ctx context.Context, blockHash string) bool { + row, err := b.store.GetBlockProcessingStatus(ctx, blockHash) + return err == nil && row != nil && row.ProcessedAt != nil } // setMinedAndPublish is the shared "mark MINED + fan out" core behind the @@ -528,12 +579,13 @@ func (b *Builder) Start(ctx context.Context) error { zap.Int("grace_window_ms", b.cfg.BumpBuilder.GraceWindowMs), ) - // One-shot startup janitor: drop orphan STUMPs left behind by blocks - // whose BUMP build already succeeded (bump_built_at IS NOT NULL). The - // happy-path DeleteStumpsByBlockHash at the end of handleMessage is - // best-effort — any transient store error there leaves orphans that - // never get cleaned up by the normal flow. - go b.pruneOrphanStumps(ctx) + // STUMP janitor: drops STUMP rows of blocks whose BUMP was built more + // than bump_builder.stump_retention_minutes ago. The build path no + // longer prunes right away — every merkle-service delivers its own + // STUMP set and a later BLOCK_PROCESSED re-reads the full set to decide + // whether it adds leaves — so retention is what bounds the stumps table. + // Runs once at startup and then periodically. + go b.runStumpJanitor(ctx) // Readiness: the consumer group is constructed (brokers reachable); Run // blocks until shutdown, so the signal precedes the actual group join @@ -543,10 +595,51 @@ func (b *Builder) Start(ctx context.Context) error { return consumer.Run(ctx) } +// defaultStumpRetention is used when bump_builder.stump_retention_minutes +// is unset. +const defaultStumpRetention = 60 * time.Minute + +// stumpRetention is how long a block's STUMPs outlive its BUMP build. +func (b *Builder) stumpRetention() time.Duration { + if m := b.cfg.BumpBuilder.StumpRetentionMinutes; m > 0 { + return time.Duration(m) * time.Minute + } + return defaultStumpRetention +} + +// runStumpJanitor runs pruneOrphanStumps at startup and then every quarter +// of the retention window (at least once a minute) until ctx is canceled. +func (b *Builder) runStumpJanitor(ctx context.Context) { + b.pruneOrphanStumps(ctx) + interval := b.stumpRetention() / 4 + if interval < time.Minute { + interval = time.Minute + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + b.pruneOrphanStumps(ctx) + } + } +} + // pruneOrphanStumps walks the most recent block_processing rows and deletes -// STUMP rows for any block that already has bump_built_at set. Bounded by -// the watchdog's recency window so a long-running deployment doesn't -// repeatedly scan ancient history. Runs once at startup and exits. +// STUMP rows for any block whose bump_built_at is older than the retention +// window. +// +// Eligibility is by bump_built_at age while the walk is by height, so the +// horizon has to cover every block that may still be inside the retention +// window: the walk continues past the watchdog's recency depth while the +// page just read holds a built row younger than retention or a row whose +// header was seen within retention plus an hour of slack. Only once a page +// below the depth has neither does the walk stop, which keeps a long-running +// deployment from scanning ancient history every tick. A block built by a +// late /reprocess long after its header was seen and already below that +// horizon is the residual case and leaks a few STUMP rows at most. // // This catches the "happy-path delete failed transiently" orphan case. The // "perma-failure" case (BUMP build never validates, stumps stuck forever @@ -575,9 +668,14 @@ func (b *Builder) pruneOrphanStumps(ctx context.Context) { const pageSize = 200 var ( - cursor = tip + 1 // exclusive upper bound; first page starts strictly below tip+1 - scanned int - pruned int + cursor = tip + 1 // exclusive upper bound; first page starts strictly below tip+1 + scanned int + pruned int + now = time.Now() + retention = b.stumpRetention() + // timeHorizon is the oldest header_seen_at a page may hold before + // the walk is allowed to stop below the recency depth. + timeHorizon = now.Add(-retention - time.Hour) ) for { if err := ctx.Err(); err != nil { @@ -595,11 +693,26 @@ func (b *Builder) pruneOrphanStumps(ctx context.Context) { if len(rows) == 0 { break } + pageMayHoldRetained := false for _, r := range rows { scanned++ + if r.HeaderSeenAt.After(timeHorizon) { + pageMayHoldRetained = true + } if r.BUMPBuiltAt == nil { continue } + // An un-stamped block is still being driven: deferred by a later + // delivery, or mid re-drive by the watchdog, whose /reprocess + // re-sends STUMPs the next build needs. bump_built_at can be old + // by then, so its age alone is no reason to prune. + if r.ProcessedAt == nil { + continue + } + if now.Sub(*r.BUMPBuiltAt) < retention { + pageMayHoldRetained = true + continue + } // Idempotent: no-op when there are no stumps for the block. if err := b.store.DeleteStumpsByBlockHash(ctx, r.BlockHash); err != nil { b.logger.Warn( @@ -611,9 +724,10 @@ func (b *Builder) pruneOrphanStumps(ctx context.Context) { } pruned++ } - // Stop when we've stepped outside the recency window. + // Stop once we are below the recency window AND this page holds + // nothing that may still be inside the retention window. oldest := rows[len(rows)-1] - if tip > depth && oldest.BlockHeight <= tip-depth { + if tip > depth && oldest.BlockHeight <= tip-depth && !pageMayHoldRetained { break } // Advance the keyset cursor strictly below the oldest row we saw. @@ -639,6 +753,10 @@ func (b *Builder) Stop() error { return nil } +// buildOutcomeStoreFailed is the BumpBuilderBuildDuration outcome for a +// BLOCK_PROCESSED that could not finish because a store read or write failed. +const buildOutcomeStoreFailed = "store_failed" + func (b *Builder) handleMessage(ctx context.Context, msg *kafka.Message) error { overallStart := time.Now() metrics.BumpBuilderBlocksProcessedTotal.Inc() @@ -668,28 +786,50 @@ func (b *Builder) handleMessage(ctx context.Context, msg *kafka.Message) error { logger := telemetry.LoggerWith(ctx, b.logger.With(logfields.BlockHash(blockHash))) - // Short-circuit: if a compound BUMP already exists for this block, skip - // the datahub fetch + recompute path entirely. See tryShortCircuit for - // the full contract. - if handled, mineFailed := b.tryShortCircuit(ctx, logger, blockHash); handled { - // A stored BUMP means the rebuild was skipped. A failed mine on - // that path is still a persistence failure: record store_failed, - // not the benign short_circuited label. processed_at stays unset - // inside tryShortCircuit so the watchdog can re-drive. - if mineFailed { - outcome = "store_failed" - } else { - outcome = "short_circuited" - } + // A stored compound BUMP means this BLOCK_PROCESSED is a redelivery: + // another merkle-service's copy, a watchdog /reprocess, or a DLQ + // redrive. Every merkle-service builds its STUMPs from the txids IT has + // registrations for, so a redelivery may carry leaves the stored BUMP + // lacks (a service that was down while those txs were registered has a + // smaller set; the one that was up has the larger one). Compare the + // retained STUMP set against the stored BUMP: nothing new ⇒ short-circuit + // (re-mine idempotently, re-stamp, publish nothing already MINED); + // new leaves ⇒ fall through and rebuild from the full set. + // Each merkle-service announces only the subtrees IT tracks, so judge + // completeness against every index an earlier, deferred delivery + // announced too. Otherwise a later delivery with a smaller set (another + // service's duplicate, or its own watchdog re-drive) could stamp a block + // that is still owed a STUMP. + if err := b.widenExpectedSubtrees(ctx, &callback); err != nil { + outcome = buildOutcomeStoreFailed + return err + } + + storedLeaves, handledOutcome, err := b.handleRedelivery(ctx, logger, &callback) + if err != nil { + // A redelivery whose stored BUMP or STUMP set cannot be read must + // not short-circuit or rebuild: re-mining the stored BUMP and + // re-stamping processed_at would retire a block this delivery may + // have added leaves to, and rebuilding without the stored leaves + // skips the no-drop check. Fail the message so Kafka redelivers it + // once the store answers. + outcome = buildOutcomeStoreFailed + return fmt.Errorf("reading stored BUMP/STUMPs on redelivery: %w", err) + } + if handledOutcome != "" { + outcome = handledOutcome return nil } + // Read before anything below writes the block: see markMinedAndPublish. + finalized := b.blockFinalized(ctx, blockHash) + // Resolve the STUMP set with completeness-first ordering: STUMPs are read // BEFORE any grace-window wait, so a block whose expected-STUMP set is // already complete (or provably empty) finalizes immediately instead of // burning the window. Only an unverifiable set still waits + re-reads. // See resolveStumps for the full contract. - stumps, disposition, done, err := b.resolveStumps(ctx, logger, &callback) + stumps, disposition, done, err := b.resolveStumps(ctx, logger, &callback, storedLeaves != nil) if err != nil { outcome = disposition return err @@ -773,16 +913,57 @@ func (b *Builder) handleMessage(ctx context.Context, msg *kafka.Message) error { return fmt.Errorf("compound BUMP root mismatch for block %s: %w", blockHash, err) } - // 5. Store compound BUMP as binary - if err := b.store.InsertBUMP(ctx, blockHash, blockHeight, bumpBytes); err != nil { - outcome = "store_failed" - return fmt.Errorf("storing BUMP: %w", err) + // 5. Store compound BUMP as binary. On a rebuild the new compound must + // cover every leaf of the stored one: overwriting with a smaller set + // would silently drop proofs. That is only reachable if the janitor + // pruned the first service's STUMPs before this redelivery arrived, and + // the fix is the same as for an incomplete expected set — hand the block + // back to the watchdog, whose /reprocess fans out to every merkle-service. + if storedLeaves != nil { + // Compound against compound: txids comes from the STUMPs and still + // carries the coinbase placeholder at subtree 0 offset 0, while the + // stored BUMP (like this compound) has the real coinbase txid there. + if dropped := leavesNotIn(storedLeaves, compoundLeaves(compound)); dropped > 0 { + logger.Error( + "rebuilt compound would drop leaves of the stored BUMP; keeping the stored BUMP and deferring to the watchdog", + zap.Int("dropped_leaves", dropped), + zap.Int("rebuilt_leaves", len(txids)), + ) + if err := b.deferIncompleteBlock(ctx, logger, blockHash, callback.ExpectedSubtreeIndices); err != nil { + outcome = buildOutcomeStoreFailed + return err + } + outcome = "deferred_incomplete" + return nil + } + } + // A rebuild of a block an earlier build finalized takes the old stamp + // off before touching the BUMP. Until the expanded set is mined in full, + // the block must stay visible to the watchdog's stale scan: if the mine + // below fails or partially lands, or the process dies between the + // overwrite and the mine, the newly covered txs have no other way back. + // The stamp goes back on below, only once the mine landed. + if storedLeaves != nil && finalized { + if err := b.store.ClearBlockProcessed(ctx, blockHash); err != nil { + outcome = buildOutcomeStoreFailed + return fmt.Errorf("clearing processed_at before rebuild: %w", err) + } } - // Observability-only: record bump_built_at on the block-processing row. - // Failure here must not block the MINED status updates downstream. + // bump_built_at is the STUMP janitor's retention basis, so it is written + // before the BUMP and a failure fails the message. Written after, a + // failed first write would leave the block's STUMPs ineligible forever, + // and on a rebuild the Kafka retry would short-circuit (the expanded + // BUMP already covers every retained leaf) and never refresh the old + // timestamp, making the STUMPs eligible for pruning straight away. If + // InsertBUMP fails after this, the retry rewrites both. if err := b.store.MarkBlockBUMPBuilt(ctx, blockHash, blockHeight, time.Now()); err != nil { - logger.Warn("failed to record bump_built status", zap.Error(err)) + outcome = buildOutcomeStoreFailed + return fmt.Errorf("recording bump_built_at: %w", err) + } + if err := b.store.InsertBUMP(ctx, blockHash, blockHeight, bumpBytes); err != nil { + outcome = buildOutcomeStoreFailed + return fmt.Errorf("storing BUMP: %w", err) } // 6. Set tracked transactions to MINED — unless the anchor guard has @@ -810,7 +991,7 @@ func (b *Builder) handleMessage(ctx context.Context, msg *kafka.Message) error { } minedComplete := true if len(txids) > 0 { - minedComplete = b.markMinedAndPublish(ctx, logger, blockHash, blockHeight, txids) + minedComplete = b.markMinedAndPublish(ctx, logger, blockHash, blockHeight, txids, finalized) } // 7. Finalize: the BUMP is built and validated, every expected STUMP was @@ -828,9 +1009,13 @@ func (b *Builder) handleMessage(ctx context.Context, msg *kafka.Message) error { // txids. Withholding the stamp is the retry signal — the watchdog re-fires // BLOCK_PROCESSED, tryShortCircuit re-mines the stored BUMP's full level-0 // set (idempotent: rows already MINED here are a no-op), and stamps. - if minedComplete { + switch { + case minedComplete && storedLeaves != nil: b.markBlockProcessed(ctx, logger, blockHash, blockHeight) - } else { + outcome = "rebuilt" + case minedComplete: + b.markBlockProcessed(ctx, logger, blockHash, blockHeight) + default: // Not a terminal lifecycle success. The histogram must not keep // the benign build disposition (finalized_complete_no_grace or // grace_waited): those labels mean the block finished. store_failed @@ -838,7 +1023,7 @@ func (b *Builder) handleMessage(ctx context.Context, msg *kafka.Message) error { // ListStaleBlockProcessingStatus (processed_at IS NULL, status // active, header already seen) still returns this block and the // watchdog re-drives it via /reprocess. The stored BUMP is what - // that re-drive mines; tryShortCircuit is idempotent for rows + // that re-drive mines; shortCircuit is idempotent for rows // already MINED. // // Kafka redelivery is not that recovery path. This handler returns @@ -846,7 +1031,7 @@ func (b *Builder) handleMessage(ctx context.Context, msg *kafka.Message) error { // handler returns nil. Returning the SetMined error would retry // and then dead-letter a message that cannot finish until the // watchdog re-drives the block. - outcome = "store_failed" + outcome = buildOutcomeStoreFailed logger.Warn( "SetMinedByTxIDs did not complete; leaving processed_at unset so the watchdog can re-drive this block", logfields.BlockHash(blockHash), @@ -855,19 +1040,219 @@ func (b *Builder) handleMessage(ctx context.Context, msg *kafka.Message) error { ) } - // 8. Prune STUMPs - if err := b.store.DeleteStumpsByBlockHash(ctx, blockHash); err != nil { - logger.Warn("failed to clean up STUMPs", zap.Error(err)) - } - + // STUMPs are NOT pruned here. Every configured merkle-service delivers + // its own STUMP set for the block, and the next BLOCK_PROCESSED decides + // whether it adds leaves by re-reading the full retained set (see the + // top of this function). The janitor deletes them once the BUMP is older + // than bump_builder.stump_retention_minutes. logger.Info( "BUMP built successfully", zap.Int("level0_count", len(txids)), - zap.Int("stumps_pruned", len(stumps)), + zap.Int("stump_count", len(stumps)), + zap.Bool("rebuilt", storedLeaves != nil), ) return nil } +// handleRedelivery decides what a BLOCK_PROCESSED for a block that already +// has a compound BUMP means. It returns a non-empty outcome when the +// delivery was fully handled by the short-circuit (nothing new; the stored +// level-0 set was re-mined and the block re-stamped), or the stored BUMP's +// leaf set when the caller must go through resolveStumps as a rebuild: the +// retained STUMPs add leaves, or the callback expects a subtree index with +// no retained STUMP yet. Both are nil/"" when no BUMP exists yet. A read +// failure is returned as an error: without the stored BUMP and the STUMP set +// there is no telling whether this delivery adds leaves, so neither path may +// run. +func (b *Builder) handleRedelivery(ctx context.Context, logger *zap.Logger, callback *models.CallbackMessage) (storedLeaves map[string]struct{}, outcome string, err error) { + blockHash := callback.BlockHash + height, storedTxIDs, ok, err := b.loadStoredBUMP(ctx, logger, blockHash) + if err != nil || !ok { + return nil, "", err + } + stumps, err := b.store.GetStumpsByBlockHash(ctx, blockHash) + if err != nil { + return nil, "", err + } + newLeaves, err := leavesBeyondStoredBUMP(stumps, storedTxIDs) + if err != nil { + return nil, "", err + } + // Nothing new retained is not the same as nothing new coming: this + // service may expect a subtree whose STUMP is still retrying. The + // short-circuit would re-stamp the block and retire it from the + // watchdog, so such a delivery takes the rebuild path instead, whose + // grace window and completeness gate defer the block if the STUMP does + // not show up. + // + // Only while STUMPs are retained, though: once the janitor has pruned + // them (the block was finalized long enough ago), every index would read + // as missing and a plain redelivery (Kafka retry, DLQ redrive) would + // un-stamp a complete block. + var missing []int + if len(stumps) > 0 { + missing = missingStumpIndices(callback.ExpectedSubtreeIndices, stumps) + } + if len(newLeaves) == 0 && len(missing) == 0 { + // A failed mine on this path is still a persistence failure: + // record store_failed, not the benign short_circuited label. + // shortCircuit leaves the block un-stamped (clearing an earlier + // stamp) so the watchdog can re-drive; a failure to clear is + // returned and fails the message. + mineFailed, err := b.shortCircuit(ctx, logger, blockHash, height, storedTxIDs) + if err != nil { + return nil, buildOutcomeStoreFailed, err + } + if mineFailed { + return nil, buildOutcomeStoreFailed, nil + } + return nil, "short_circuited", nil + } + if len(newLeaves) > 0 { + logger.Info( + "redelivered STUMP set carries leaves the stored BUMP lacks; rebuilding the compound", + zap.Int("new_leaves", len(newLeaves)), + zap.Int("stored_leaves", len(storedTxIDs)), + ) + } else { + logger.Info( + "redelivery expects STUMPs that are not retained yet; checking completeness before re-stamping", + zap.Ints("missing_subtree_indices", missing), + zap.Int("stored_leaves", len(storedTxIDs)), + ) + } + storedLeaves = make(map[string]struct{}, len(storedTxIDs)) + for _, txid := range storedTxIDs { + storedLeaves[txid] = struct{}{} + } + return storedLeaves, "", nil +} + +// compoundLeaves lists the level-0 hashes of a built compound BUMP, the same +// set levelZeroTxidsFromBUMP reads back from a stored one. +func compoundLeaves(compound *transaction.MerklePath) []string { + if len(compound.Path) == 0 { + return nil + } + out := make([]string, 0, len(compound.Path[0])) + for _, elem := range compound.Path[0] { + if elem.Hash != nil { + out = append(out, elem.Hash.String()) + } + } + return out +} + +// leavesNotIn counts the stored leaves that txids does not cover. +func leavesNotIn(stored map[string]struct{}, txids []string) int { + have := make(map[string]struct{}, len(txids)) + for _, txid := range txids { + have[txid] = struct{}{} + } + missing := 0 + for txid := range stored { + if _, ok := have[txid]; !ok { + missing++ + } + } + return missing +} + +// leavesBeyondStoredBUMP returns the level-0 hashes present in the block's +// retained STUMPs that the stored BUMP does not cover. Empty means the +// retained set carries nothing new. +func leavesBeyondStoredBUMP(stumps []*models.Stump, storedTxIDs []string) ([]string, error) { + have := make(map[string]struct{}, len(storedTxIDs)) + for _, txid := range storedTxIDs { + have[txid] = struct{}{} + } + seen := make(map[string]struct{}) + var missing []string + for _, st := range stumps { + mp, perr := transaction.NewMerklePathFromBinary(st.StumpData) + if perr != nil || len(mp.Path) == 0 { + // Reading an unparseable STUMP as "adds nothing" would let the + // short-circuit stamp a block it never built from it. Fail the + // message instead, as the build path does for the same STUMP. + return nil, fmt.Errorf("stored STUMP for subtree %d does not parse: %w", st.SubtreeIndex, errors.Join(perr, errEmptyMerklePath(mp))) + } + for _, leaf := range mp.Path[0] { + // Subtree 0 offset 0 is the coinbase slot: every STUMP carries + // the placeholder there while the stored compound carries the + // real coinbase txid, and arcade never tracks a coinbase. Counting + // it would turn every identical redelivery into a rebuild. + if leaf.Hash == nil || (st.SubtreeIndex == 0 && leaf.Offset == 0) { + continue + } + txid := leaf.Hash.String() + if _, ok := have[txid]; ok { + continue + } + if _, dup := seen[txid]; dup { + continue + } + seen[txid] = struct{}{} + missing = append(missing, txid) + } + } + return missing, nil +} + +// errEmptyMerklePath reports a STUMP that parsed but has no levels. +func errEmptyMerklePath(mp *transaction.MerklePath) error { + if mp != nil && len(mp.Path) == 0 { + return errors.New("empty merkle path") + } + return nil +} + +// deferIncompleteBlock leaves a block un-finalized for the watchdog. It +// retains this delivery's expected-subtree set first, so whichever delivery +// comes next (in any order) is held to it, then takes off any stamp an +// earlier build left. A failed write is returned and callers fail the Kafka +// message: committing it could let the block be finalized while it is still +// owed a STUMP. +func (b *Builder) deferIncompleteBlock(ctx context.Context, logger *zap.Logger, blockHash string, expected []int) error { + if err := b.store.AddBlockExpectedSubtrees(ctx, blockHash, expected); err != nil { + return fmt.Errorf("retaining expected subtrees for deferred block: %w", err) + } + return b.deferStampedBlock(ctx, logger, blockHash, "an earlier build finalized this block but a later STUMP set is incomplete") +} + +// deferStampedBlock hands a block back to the watchdog after an earlier +// build already finalized it. With several merkle-services the first +// service's BLOCK_PROCESSED builds and stamps processed_at; if a later +// service's delivery then proves incomplete (its expected STUMP set is not +// satisfied, or a rebuild would drop leaves), or a short-circuit's re-mine or +// MINED publish does not land, the stamp must come off, or +// ListStaleBlockProcessingStatus never surfaces the block and the txs only +// that service tracked stay SEEN_* forever. The watchdog's /reprocess fans +// out to every merkle-service, so the re-drive carries every variant. +// +// A store failure is returned, and callers fail the Kafka message on it: +// committing the message with the old stamp still set would leave the block +// known-incomplete and invisible to the watchdog, the exact gap this helper +// closes. A missing row is a no-op (nothing is stamped). +// +// reason completes the log line: why the stamp is coming off. +func (b *Builder) deferStampedBlock(ctx context.Context, logger *zap.Logger, blockHash, reason string) error { + row, err := b.store.GetBlockProcessingStatus(ctx, blockHash) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + return nil + } + return fmt.Errorf("reading block_processing for deferral: %w", err) + } + if row.ProcessedAt == nil { + return nil + } + if err := b.store.ClearBlockProcessed(ctx, blockHash); err != nil { + return fmt.Errorf("clearing processed_at on deferred block: %w", err) + } + logger.Warn("cleared processed_at; the watchdog will re-drive this block via /reprocess", zap.String("reason", reason)) + return nil +} + // resolveStumps decides which STUMP set handleMessage builds from and whether // the grace window has to be waited at all. Extracted from handleMessage to // keep its nesting under the lint bar (same motivation as tryShortCircuit). @@ -896,16 +1281,25 @@ func (b *Builder) handleMessage(ctx context.Context, msg *kafka.Message) error { // return nil; otherwise it is the benign label the successful build lands // under. BumpBuilderStumpCount is observed exactly once per message, on the // STUMP set that reaches the decision. -func (b *Builder) resolveStumps(ctx context.Context, logger *zap.Logger, callback *models.CallbackMessage) (stumps []*models.Stump, disposition string, done bool, err error) { +// +// rebuild is set when a compound BUMP already exists and the redelivery adds +// leaves. The expected set then names subtree indices, not which +// merkle-service's variant satisfies them: an index can be covered by the +// earlier service's STUMP while the later service's own variant for it is +// still retrying. So a rebuild always waits the grace window before reading +// the set it builds from, even when the index set already looks complete. +func (b *Builder) resolveStumps(ctx context.Context, logger *zap.Logger, callback *models.CallbackMessage, rebuild bool) (stumps []*models.Stump, disposition string, done bool, err error) { blockHash := callback.BlockHash expected := callback.ExpectedSubtreeIndices stumps, err = b.store.GetStumpsByBlockHash(ctx, blockHash) if err != nil { - return nil, "store_failed", false, fmt.Errorf("getting STUMPs for block: %w", err) + return nil, buildOutcomeStoreFailed, false, fmt.Errorf("getting STUMPs for block: %w", err) } switch { + case rebuild: + // Fall through to the grace window below. case len(expected) > 0 && len(missingStumpIndices(expected, stumps)) == 0: // Complete: every subtree index merkle told us to expect already has // a stored STUMP — there is nothing left to wait for. @@ -929,7 +1323,7 @@ func (b *Builder) resolveStumps(ctx context.Context, logger *zap.Logger, callbac // set to verify against): completeness cannot be established from the // first read, so the grace window is still the only defense against late // STUMP retries. - if len(expected) == 0 { + if len(expected) == 0 && !rebuild { logger.Warn( "BLOCK_PROCESSED lacks expected-STUMP set but STUMPs exist — cannot verify completeness; waiting grace window", zap.Int("received_stumps", len(stumps)), @@ -946,7 +1340,7 @@ func (b *Builder) resolveStumps(ctx context.Context, logger *zap.Logger, callbac // Re-read: catching late arrivals is the whole point of the wait. stumps, err = b.store.GetStumpsByBlockHash(ctx, blockHash) if err != nil { - return nil, "store_failed", false, fmt.Errorf("getting STUMPs for block: %w", err) + return nil, buildOutcomeStoreFailed, false, fmt.Errorf("getting STUMPs for block: %w", err) } } @@ -967,6 +1361,9 @@ func (b *Builder) resolveStumps(ctx context.Context, logger *zap.Logger, callbac zap.Int("expected_stumps", len(expected)), zap.Int("received_stumps", len(stumps)), ) + if derr := b.deferIncompleteBlock(ctx, logger, blockHash, expected); derr != nil { + return nil, buildOutcomeStoreFailed, false, derr + } return nil, "deferred_incomplete", true, nil } @@ -1110,35 +1507,31 @@ func callbackBlockData(callback *models.CallbackMessage, minSubtrees int, logger return hashes, []byte(callback.CoinbaseBUMP), root, true } -// tryShortCircuit attempts the BUMP-already-exists redelivery path. -// handled is true when the short-circuit consumed the message and the -// caller must not rebuild. mineFailed is true only when handled is true -// and SetMinedByTxIDs did not complete: the caller records store_failed, -// and processed_at is left unset so the watchdog can re-drive. A denied -// anchor is handled but not a mine failure. -// -// Returns handled=false when no usable BUMP exists and the caller should -// fall through to the normal rebuild. -// -// Errors at the store-read step (not-found, etc.) are intentionally -// swallowed — they're indistinguishable from "no BUMP" and we want the -// rebuild path to handle both. A parseErr on a corrupt stored BUMP is -// logged so operators can see the divergence; the rebuild still proceeds. -// -// Extracted from processBlockProcessed to reduce nesting depth (nestif). -// The short-circuit is exercised on /reprocess redeliveries of -// BLOCK_PROCESSED — typical case is the watchdog re-firing after a missed -// callback. The SetMinedByTxIDs UPDATE is idempotent so repeated calls are -// safe; the value is letting a tx registered after the original build still -// get marked MINED on the redelivery. -func (b *Builder) tryShortCircuit(ctx context.Context, logger *zap.Logger, blockHash string) (handled bool, mineFailed bool) { +// Redelivery path: a BLOCK_PROCESSED for a block whose compound BUMP is +// already stored (another merkle-service's copy, a watchdog /reprocess, a +// DLQ redrive). handleRedelivery decides between shortCircuit (nothing new) +// and a rebuild (the retained STUMPs add leaves). shortCircuit's +// SetMinedByTxIDs is idempotent, so repeated calls are safe; its value is +// letting a tx registered after the original build still get marked MINED, +// and re-stamping processed_at on a block whose first stamp never landed. +// loadStoredBUMP reads and parses the block's stored compound BUMP. ok is +// false when there is none, or when the stored bytes are empty or do not +// parse (a corrupt BUMP is rebuilt rather than pinning the block); the caller +// then takes the full build path. Any other read failure is returned: an +// unanswered read is not proof of absence, and treating it as such would +// rebuild from whatever STUMPs are still retained and overwrite a complete +// BUMP without the no-drop check. +func (b *Builder) loadStoredBUMP(ctx context.Context, logger *zap.Logger, blockHash string) (height uint64, txids []string, ok bool, err error) { existingHeight, bumpBytes, getErr := b.store.GetBUMP(ctx, blockHash) + if errors.Is(getErr, store.ErrNotFound) { + return 0, nil, false, nil + } if getErr != nil { - // not-found / transient store error — fall through to rebuild - return false, false + return 0, nil, false, fmt.Errorf("reading stored BUMP: %w", getErr) } if len(bumpBytes) == 0 { - return false, false + logger.Warn("stored BUMP is empty on redelivery — rebuilding") + return 0, nil, false, nil } txids, parseErr := levelZeroTxidsFromBUMP(bumpBytes) if parseErr != nil { @@ -1146,8 +1539,14 @@ func (b *Builder) tryShortCircuit(ctx context.Context, logger *zap.Logger, block // path so an upstream corruption doesn't pin a block in a broken // state forever. logger.Warn("stored BUMP failed to parse on redelivery — rebuilding", zap.Error(parseErr)) - return false, false + return 0, nil, false, nil } + return existingHeight, txids, true, nil +} + +// shortCircuit re-mines the stored BUMP's level-0 set and re-stamps +// processed_at. It reports whether the mine failed to land in full. +func (b *Builder) shortCircuit(ctx context.Context, logger *zap.Logger, blockHash string, existingHeight uint64, txids []string) (mineFailed bool, err error) { metrics.BumpBuilderShortCircuitTotal.Inc() logger.Info( "BUMP already built — skipping datahub fetch on redelivery", @@ -1160,11 +1559,16 @@ func (b *Builder) tryShortCircuit(ctx context.Context, logger *zap.Logger, block // block is finalized + orphan-marked exactly like the fresh-build path. if b.anchorDecision(ctx, logger, blockHash, existingHeight) == anchorDeny { b.handleAnchorDenied(ctx, logger, blockHash, existingHeight, "short_circuit") - return true, false + return false, nil } + // Rows already MINED against a stamped block are not republished: the + // stamp proves their events went out, because a mine or publish failure + // below takes it off again (deferStampedBlock) before the next delivery + // can read the block as finalized. minedComplete := true if len(txids) > 0 { - minedComplete = b.markMinedAndPublish(ctx, logger, blockHash, existingHeight, txids) + finalized := b.blockFinalized(ctx, blockHash) + minedComplete = b.markMinedAndPublish(ctx, logger, blockHash, existingHeight, txids, finalized) } // Stamp processed_at on the redelivery too. The typical short-circuit // trigger IS the watchdog re-firing BLOCK_PROCESSED for a block whose BUMP @@ -1174,29 +1578,34 @@ func (b *Builder) tryShortCircuit(ctx context.Context, logger *zap.Logger, block // since the stale query keys on processed_at IS NULL and a built block can // never be re-detected as incomplete. // - // A partial mine withholds the stamp for the same reason the build path - // does: this IS the recovery path, so retiring the block from the stale - // scan while some of its txids are still SEEN_* would strand them. + // A partial mine or a failed publish withholds the stamp for the same + // reason the build path does: this IS the recovery path, so retiring the + // block from the stale scan while some of its txids are still SEEN_*, or + // their MINED events unsent, would strand them. + // + // Withholding is not enough when an earlier delivery already stamped the + // block (every extra merkle-service's BLOCK_PROCESSED lands here): the + // stamp would stay, the watchdog would never re-drive the block, and the + // next delivery would read it as finalized and filter the now-MINED rows + // out of the publish. So take the stamp off; a failure to do that fails + // the message so Kafka retries. if minedComplete { b.markBlockProcessed(ctx, logger, blockHash, existingHeight) } else { - // Same contract as the build path: Kafka redelivery is not the - // recovery path (the caller returns nil and the offset is - // committed). processed_at stays unset so the watchdog re-drives. logger.Warn( - "SetMinedByTxIDs did not complete; leaving processed_at unset so the watchdog can re-drive this block", + "SetMinedByTxIDs or the MINED publish did not complete; leaving the block un-stamped so the watchdog can re-drive it", logfields.BlockHash(blockHash), logfields.BlockHeight(existingHeight), zap.Int("requested", len(txids)), ) + if err := b.deferStampedBlock(ctx, logger, blockHash, "short-circuit mine or publish did not complete"); err != nil { + return true, err + } } - // STUMP rows for this block should already have been pruned at the end - // of the original build; ensure stragglers are cleared in case a STUMP - // arrived after pruning ran. - if delErr := b.store.DeleteStumpsByBlockHash(ctx, blockHash); delErr != nil { - logger.Warn("failed to clean up STUMPs on short-circuit", zap.Error(delErr)) - } - return true, !minedComplete + // STUMP rows are left for the janitor (bump_builder.stump_retention_minutes): + // a later merkle-service's BLOCK_PROCESSED compares its STUMPs against + // the stored BUMP, and that needs the full retained set. + return !minedComplete, nil } // markBlockProcessed stamps processed_at on the block-processing row — the @@ -1212,7 +1621,8 @@ func (b *Builder) tryShortCircuit(ctx context.Context, logger *zap.Logger, block // MarkBlockProcessed upsert preserves any chaintracks-supplied height on // conflict and only writes processed_at. func (b *Builder) markBlockProcessed(ctx context.Context, logger *zap.Logger, blockHash string, blockHeight uint64) { - if err := b.store.MarkBlockProcessed(ctx, blockHash, blockHeight, time.Now()); err != nil { + err := b.store.MarkBlockProcessed(ctx, blockHash, blockHeight, time.Now()) + if err != nil { // Include the block identity: a sustained stream of these is the signal // that finalized blocks are looping through the watchdog (processed_at // never lands), which an aggregator can only group by block hash. @@ -1222,9 +1632,35 @@ func (b *Builder) markBlockProcessed(ctx context.Context, logger *zap.Logger, bl logfields.BlockHeight(blockHeight), zap.Error(err), ) + return + } + // A stamped block owes nothing, so drop the retained expected set (see + // widenExpectedSubtrees). Soft: a leftover set only costs a later + // redelivery a deferral round-trip through the watchdog. + if err := b.store.ClearBlockExpectedSubtrees(ctx, blockHash); err != nil { + logger.Warn("failed to clear retained expected subtrees", logfields.BlockHash(blockHash), zap.Error(err)) } } +// widenExpectedSubtrees merges the block's retained expected-subtree set +// (written by an earlier delivery that deferred the block) into the +// callback's own, ascending and deduplicated. A read failure is returned: +// judging completeness against a set that may be missing entries could +// finalize an incomplete block. +func (b *Builder) widenExpectedSubtrees(ctx context.Context, callback *models.CallbackMessage) error { + retained, err := b.store.GetBlockExpectedSubtrees(ctx, callback.BlockHash) + if err != nil { + return fmt.Errorf("reading retained expected subtrees: %w", err) + } + if len(retained) == 0 { + return nil + } + merged := append(slices.Clone(callback.ExpectedSubtreeIndices), retained...) + slices.Sort(merged) + callback.ExpectedSubtreeIndices = slices.Compact(merged) + return nil +} + // missingStumpIndices returns the subtree indices merkle-service expected to // produce a STUMP for this block (callback.ExpectedSubtreeIndices) that are NOT // present among the STUMPs arcade actually stored, preserving the ascending diff --git a/services/bump_builder/builder_bulk_mined_test.go b/services/bump_builder/builder_bulk_mined_test.go index 266bf0bd..434795aa 100644 --- a/services/bump_builder/builder_bulk_mined_test.go +++ b/services/bump_builder/builder_bulk_mined_test.go @@ -54,7 +54,7 @@ func TestBuilder_MarkMinedAndPublish_ChunksLargeTxidList(t *testing.T) { txids[i] = fmt.Sprintf("tx%064d", i) } - b.markMinedAndPublish(context.Background(), zap.NewNop(), "blkD", 4242, txids) + b.markMinedAndPublish(context.Background(), zap.NewNop(), "blkD", 4242, txids, false) wantChunks := (n + maxTxIDsPerBulkEvent - 1) / maxTxIDsPerBulkEvent if len(pub.chunks) != wantChunks { diff --git a/services/bump_builder/builder_callback_test.go b/services/bump_builder/builder_callback_test.go index b3896ca9..ead8b078 100644 --- a/services/bump_builder/builder_callback_test.go +++ b/services/bump_builder/builder_callback_test.go @@ -141,8 +141,11 @@ func TestBuilder_HandleMessage_CallbackPath_NoDatahubCall(t *testing.T) { if len(ms.minedCalls) != 1 { t.Errorf("expected 1 SetMinedByTxIDs call, got %d", len(ms.minedCalls)) } - if len(ms.deletedBlocks) != 1 || ms.deletedBlocks[0] != blockHash { - t.Errorf("expected STUMPs for %s pruned, got %v", blockHash, ms.deletedBlocks) + // STUMPs are retained after the build (the janitor prunes them after + // bump_builder.stump_retention_minutes) so a later merkle-service's + // BLOCK_PROCESSED can be compared against the stored BUMP. + if len(ms.deletedBlocks) != 0 { + t.Errorf("STUMPs must be retained after the build, got deletes %v", ms.deletedBlocks) } if after := testutil.ToFloat64(metrics.BumpBuilderBlockDataSourceTotal.WithLabelValues("callback")); after != before+1 { t.Errorf("block_data_source{source=callback} = %v, want %v", after, before+1) diff --git a/services/bump_builder/builder_test.go b/services/bump_builder/builder_test.go index f1761ad2..bf68bc2b 100644 --- a/services/bump_builder/builder_test.go +++ b/services/bump_builder/builder_test.go @@ -8,6 +8,7 @@ import ( "errors" "net/http" "net/http/httptest" + "slices" "strings" "sync" "testing" @@ -49,6 +50,7 @@ type mockStore struct { bumpHeights map[string]uint64 // blockHash → blockHeight minedCalls []minedCall deletedBlocks []string + clearedBlocks []string bumpBuiltCalls []bumpBuiltCall processedCalls []bumpBuiltCall getStumpsErr error @@ -56,15 +58,23 @@ type mockStore struct { insertStumpErr error setMinedErr error setMinedAttempts int - deleteStumpsErr error - markBumpBuiltErr error - markProcessedErr error + // alreadyMined maps txid → block hash for rows the store already holds + // at MINED; SetMinedByTxIDs reports them with a MINED previous status so + // tests can drive the onlyChanged filter. + alreadyMined map[string]string + deleteStumpsErr error + markBumpBuiltErr error + markProcessedErr error + clearProcessedErr error // Janitor fixtures: tipHeight feeds GetActiveTipBlockHeight; blockProc // feeds ListBlockProcessingStatus. Both default empty so existing tests // (which never set them) see the "fresh deployment" early-exit path. tipHeight uint64 blockProc []*models.BlockProcessingStatus + + // expectedSubtrees backs the retained expected-subtree set ops. + expectedSubtrees map[string][]int } type bumpBuiltCall struct { @@ -143,13 +153,83 @@ func (m *mockStore) ListBlockProcessingStatus(_ context.Context, beforeHeight ui func (m *mockStore) EnsureIndexes(context.Context) error { return nil } +// InsertStump mirrors the store contract: rows are content-addressed, so +// the same bytes for the same subtree are one row and a variant is another. func (m *mockStore) InsertStump(_ context.Context, stump *models.Stump) error { m.mu.Lock() defer m.mu.Unlock() if m.insertStumpErr != nil { return m.insertStumpErr } + m.addStumpLocked(stump) + return nil +} + +func (m *mockStore) addStumpLocked(stump *models.Stump) { + stump.EnsureContentHash() + for _, have := range m.stumps[stump.BlockHash] { + if have.SubtreeIndex == stump.SubtreeIndex && have.ContentHash == stump.ContentHash { + return + } + } m.stumps[stump.BlockHash] = append(m.stumps[stump.BlockHash], stump) +} + +// AddBlockExpectedSubtrees unions indices into the retained set. Unlike the +// real backends it does not require a block_processing row, so tests need no +// fixture row to exercise the union. +func (m *mockStore) AddBlockExpectedSubtrees(_ context.Context, blockHash string, indices []int) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.expectedSubtrees == nil { + m.expectedSubtrees = map[string][]int{} + } + merged := append(slices.Clone(m.expectedSubtrees[blockHash]), indices...) + slices.Sort(merged) + m.expectedSubtrees[blockHash] = slices.Compact(merged) + return nil +} + +func (m *mockStore) GetBlockExpectedSubtrees(_ context.Context, blockHash string) ([]int, error) { + m.mu.Lock() + defer m.mu.Unlock() + return slices.Clone(m.expectedSubtrees[blockHash]), nil +} + +func (m *mockStore) ClearBlockExpectedSubtrees(_ context.Context, blockHash string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.expectedSubtrees, blockHash) + return nil +} + +// GetBlockProcessingStatus serves rows from blockProc by hash. +func (m *mockStore) GetBlockProcessingStatus(_ context.Context, blockHash string) (*models.BlockProcessingStatus, error) { + m.mu.Lock() + defer m.mu.Unlock() + for _, r := range m.blockProc { + if r.BlockHash == blockHash { + cp := *r + return &cp, nil + } + } + return nil, store.ErrNotFound +} + +// ClearBlockProcessed records the call and unsets processed_at on the +// blockProc fixture row, if any. +func (m *mockStore) ClearBlockProcessed(_ context.Context, blockHash string) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.clearProcessedErr != nil { + return m.clearProcessedErr + } + m.clearedBlocks = append(m.clearedBlocks, blockHash) + for _, r := range m.blockProc { + if r.BlockHash == blockHash { + r.ProcessedAt = nil + } + } return nil } @@ -182,11 +262,17 @@ func (m *mockStore) SetMinedByTxIDs(_ context.Context, blockHash string, blockHe m.minedCalls = append(m.minedCalls, minedCall{blockHash, blockHeight, txids}) var prevs, statuses []*models.TransactionStatus for _, txid := range txids { - prevs = append(prevs, &models.TransactionStatus{ + prev := &models.TransactionStatus{ TxID: txid, Status: models.StatusSeenOnNetwork, Timestamp: time.Now(), - }) + } + if bh, ok := m.alreadyMined[txid]; ok { + prev.Status = models.StatusMined + prev.BlockHash = bh + prev.BlockHeight = blockHeight + } + prevs = append(prevs, prev) statuses = append(statuses, &models.TransactionStatus{ TxID: txid, Status: models.StatusMined, @@ -233,17 +319,23 @@ func (m *mockStore) MarkBlockProcessed(_ context.Context, blockHash string, bloc return m.markProcessedErr } m.processedCalls = append(m.processedCalls, bumpBuiltCall{blockHash: blockHash, blockHeight: blockHeight}) + // Upsert processed_at on the fixture row like the real backends, so a + // later delivery's blockFinalized sees the stamp. + now := time.Now() + for _, r := range m.blockProc { + if r.BlockHash == blockHash { + r.ProcessedAt = &now + return nil + } + } + m.blockProc = append(m.blockProc, &models.BlockProcessingStatus{BlockHash: blockHash, BlockHeight: blockHeight, ProcessedAt: &now}) return nil } func (m *mockStore) addStump(blockHash string, subtreeIndex int, stumpData []byte) { m.mu.Lock() defer m.mu.Unlock() - m.stumps[blockHash] = append(m.stumps[blockHash], &models.Stump{ - BlockHash: blockHash, - SubtreeIndex: subtreeIndex, - StumpData: stumpData, - }) + m.addStumpLocked(models.NewStump(blockHash, subtreeIndex, stumpData)) } // --- Helpers --- @@ -643,16 +735,24 @@ func TestBuilder_HandleMessage_ShortCircuit_BUMPAlreadyExists(t *testing.T) { func TestBuilder_PruneOrphanStumps_DeletesAfterSuccess(t *testing.T) { ms := newMockStore() ms.tipHeight = 1000 - built := time.Now() + // Older than the default retention: STUMPs now outlive the build by + // bump_builder.stump_retention_minutes so a later merkle-service's + // BLOCK_PROCESSED can re-read the full set. + built := time.Now().Add(-2 * time.Hour) + recent := time.Now().Add(-time.Minute) ms.blockProc = []*models.BlockProcessingStatus{ - {BlockHash: "done-1", BlockHeight: 999, BUMPBuiltAt: &built}, // BUMP built — prune - {BlockHash: "done-2", BlockHeight: 998, BUMPBuiltAt: &built}, // BUMP built — prune - {BlockHash: "in-flight", BlockHeight: 997, BUMPBuiltAt: nil}, // still pending — keep + {BlockHash: "done-1", BlockHeight: 999, BUMPBuiltAt: &built, ProcessedAt: &built}, // BUMP built, aged out — prune + {BlockHash: "done-2", BlockHeight: 998, BUMPBuiltAt: &built, ProcessedAt: &built}, // BUMP built, aged out — prune + {BlockHash: "in-flight", BlockHeight: 997, BUMPBuiltAt: nil}, // still pending — keep + {BlockHash: "fresh", BlockHeight: 996, BUMPBuiltAt: &recent, ProcessedAt: &recent}, // built, within retention — keep + {BlockHash: "deferred", BlockHeight: 995, BUMPBuiltAt: &built}, // built long ago, un-stamped (deferred / re-driven) — keep } - // Seed stumps for all three so we can verify the in-flight one survives. + // Seed stumps for every row so we can verify the kept ones survive. ms.addStump("done-1", 0, []byte("orphan-1")) ms.addStump("done-2", 0, []byte("orphan-2")) ms.addStump("in-flight", 0, []byte("pending")) + ms.addStump("fresh", 0, []byte("fresh")) + ms.addStump("deferred", 0, []byte("deferred")) b := &Builder{ cfg: &config.Config{}, @@ -675,6 +775,12 @@ func TestBuilder_PruneOrphanStumps_DeletesAfterSuccess(t *testing.T) { if len(ms.stumps["in-flight"]) != 1 { t.Errorf("in-flight STUMPs should have been preserved, got %d", len(ms.stumps["in-flight"])) } + if len(ms.stumps["deferred"]) != 1 { + t.Errorf("an un-stamped block's STUMPs must be kept whatever bump_built_at says, got %d", len(ms.stumps["deferred"])) + } + if len(ms.stumps["fresh"]) != 1 { + t.Errorf("STUMPs of a block built within the retention window must be kept, got %d", len(ms.stumps["fresh"])) + } } // TestBuilder_PruneOrphanStumps_EmptyStore exits early when there's no tip. @@ -718,9 +824,11 @@ func TestBuilder_HandleMessage_HappyPath_SingleSubtree(t *testing.T) { t.Error("expected BUMP to be stored") } - // Verify STUMPs were pruned - if len(ms.deletedBlocks) != 1 || ms.deletedBlocks[0] != blockHash { - t.Errorf("expected STUMPs for %s to be deleted, got: %v", blockHash, ms.deletedBlocks) + // STUMPs are retained after the build for the janitor's retention + // window, so a redelivered BLOCK_PROCESSED from another merkle-service + // can be compared against the stored BUMP. + if len(ms.deletedBlocks) != 0 || len(ms.stumps[blockHash]) == 0 { + t.Errorf("STUMPs must be retained after the build, deletes=%v stumps=%d", ms.deletedBlocks, len(ms.stumps[blockHash])) } // Verify the block-processing observability row was updated. @@ -963,7 +1071,10 @@ func TestBuilder_HandleMessage_FieldAbsentFromWire_BuildsNormally(t *testing.T) } } -func TestBuilder_HandleMessage_BumpBuiltStoreError_StillSucceeds(t *testing.T) { +// bump_built_at is the STUMP janitor's retention basis, so a failure to +// record it fails the message before the BUMP is written: a stored BUMP +// without the timestamp would leave its STUMPs ineligible for pruning forever. +func TestBuilder_HandleMessage_BumpBuiltStoreError_FailsBeforeInsert(t *testing.T) { ms := newMockStore() ms.markBumpBuiltErr = errors.New("store down") blockHash := testBlockHash @@ -981,14 +1092,17 @@ func TestBuilder_HandleMessage_BumpBuiltStoreError_StillSucceeds(t *testing.T) { b := newTestBuilder(ms, datahub.URL) - if err := b.handleMessage(context.Background(), makeBlockProcessedMsg(blockHash)); err != nil { - t.Fatalf("MarkBlockBUMPBuilt error must not fail handler, got: %v", err) + if err := b.handleMessage(context.Background(), makeBlockProcessedMsg(blockHash)); err == nil { + t.Fatal("a MarkBlockBUMPBuilt failure must fail the message so Kafka retries it") } ms.mu.Lock() defer ms.mu.Unlock() - if _, ok := ms.bumps[blockHash]; !ok { - t.Error("BUMP should still be stored despite the observability write failure") + if _, ok := ms.bumps[blockHash]; ok { + t.Error("the BUMP must not be stored without its bump_built_at") + } + if len(ms.minedCalls) != 0 || len(ms.processedCalls) != 0 { + t.Errorf("nothing may be mined or stamped: mined=%d stamped=%d", len(ms.minedCalls), len(ms.processedCalls)) } } @@ -1142,9 +1256,9 @@ func TestBuilder_E2E_InsertStump_GetStumps_BuildBUMP(t *testing.T) { t.Error("expected non-empty BUMP data") } - // Verify STUMPs were cleaned up - if len(ms.deletedBlocks) != 1 || ms.deletedBlocks[0] != blockHash { - t.Errorf("expected STUMPs for %s to be deleted, got: %v", blockHash, ms.deletedBlocks) + // STUMPs are retained after the build (janitor-pruned later). + if len(ms.deletedBlocks) != 0 { + t.Errorf("STUMPs must be retained after the build, got deletes: %v", ms.deletedBlocks) } } @@ -1406,7 +1520,7 @@ func TestBuilder_MarkMinedAndPublish_UsesBulkEvent(t *testing.T) { b := newTestBuilder(ms, "http://unused.example/api/v1") b.publisher = pub - b.markMinedAndPublish(context.Background(), zap.NewNop(), blockHash, blockHeight, txids) + b.markMinedAndPublish(context.Background(), zap.NewNop(), blockHash, blockHeight, txids, false) // recordingPublisher.PublishBulk unfans into one entry per TxID for // assertion convenience, so snapshot() shows N entries — but the diff --git a/services/bump_builder/lifecycle_log_test.go b/services/bump_builder/lifecycle_log_test.go index b29a3dfe..68d51f3b 100644 --- a/services/bump_builder/lifecycle_log_test.go +++ b/services/bump_builder/lifecycle_log_test.go @@ -35,7 +35,7 @@ func TestMarkMinedAndPublish_LogsTransactionsMinedLevelSplit(t *testing.T) { txids[i] = fmt.Sprintf("mined-tx-%05d", i) } - b.markMinedAndPublish(context.Background(), logger, "block-hash-mined-test", 777, txids) + b.markMinedAndPublish(context.Background(), logger, "block-hash-mined-test", 777, txids, false) // Info: exactly one bounded line, TRUE total in txid_count, list capped. infoEntries := recorded.FilterMessage("transactions mined"). @@ -118,7 +118,7 @@ func TestMarkMinedAndPublish_LogsTransactionsMinedSingleChunk(t *testing.T) { logger := zap.New(core) txids := []string{"tx-a", "tx-b", "tx-c"} - b.markMinedAndPublish(context.Background(), logger, "block-hash-small", 42, txids) + b.markMinedAndPublish(context.Background(), logger, "block-hash-small", 42, txids, false) infoEntries := recorded.FilterMessage("transactions mined"). FilterLevelExact(zapcore.InfoLevel).All() diff --git a/services/bump_builder/multi_source_test.go b/services/bump_builder/multi_source_test.go new file mode 100644 index 00000000..f140b050 --- /dev/null +++ b/services/bump_builder/multi_source_test.go @@ -0,0 +1,967 @@ +package bump_builder + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/bsv-blockchain/go-sdk/chainhash" + "github.com/bsv-blockchain/go-sdk/transaction" + "github.com/prometheus/client_golang/prometheus/testutil" + "go.uber.org/zap" + + "github.com/bsv-blockchain/arcade/bump" + "github.com/bsv-blockchain/arcade/config" + "github.com/bsv-blockchain/arcade/kafka" + "github.com/bsv-blockchain/arcade/metrics" + "github.com/bsv-blockchain/arcade/models" +) + +// With several merkle-services every block's BLOCK_PROCESSED arrives once +// per service. The redelivery takes the short-circuit path, which re-mines +// the stored BUMP's level-0 set; rows already MINED against this block must +// not fan out a second MINED event, while a tx that was still SEEN_* (e.g. +// registered after the first build) must still publish. +func TestBuilder_HandleMessage_DuplicateBlockProcessed_PublishesOnlyNewlyMined(t *testing.T) { + const alreadyMinedTx = testTxidHex + const lateTx = "2222222222222222222222222222222222222222222222222222222222222222" + ms := newMockStore() + blockHash := testBlockHash + + stumpA := makeMinimalSTUMP(alreadyMinedTx) + stumpB := makeMinimalSTUMP(lateTx) + compound, _, err := bump.BuildCompoundBUMP( + []*models.Stump{ + {BlockHash: blockHash, SubtreeIndex: 0, StumpData: stumpA}, + {BlockHash: blockHash, SubtreeIndex: 1, StumpData: stumpB}, + }, + []chainhash.Hash{mustHash(t, alreadyMinedTx), mustHash(t, lateTx)}, nil, nil, + ) + if err != nil { + t.Fatalf("BuildCompoundBUMP: %v", err) + } + ms.bumps[blockHash] = compound.Bytes() + ms.bumpHeights[blockHash] = uint64(compound.BlockHeight) + ms.alreadyMined = map[string]string{alreadyMinedTx: blockHash} + stampedBlock(ms, blockHash) + + pub := &recordingPublisher{} + b := &Builder{ + cfg: &config.Config{}, + logger: zap.NewNop().Named("bump-builder"), + store: ms, + publisher: pub, + } + + before := bumpOutcomeSampleCount(t, "short_circuited") + if err := b.handleMessage(context.Background(), makeBlockProcessedMsg(blockHash)); err != nil { + t.Fatalf("duplicate BLOCK_PROCESSED returned error: %v", err) + } + if got := bumpOutcomeSampleCount(t, "short_circuited"); got != before+1 { + t.Fatalf("short_circuited samples = %d, want %d", got, before+1) + } + + ms.mu.Lock() + minedCalls := len(ms.minedCalls) + processed := len(ms.processedCalls) + ms.mu.Unlock() + if minedCalls != 1 { + t.Fatalf("SetMinedByTxIDs calls = %d, want 1 (idempotent re-mine of the stored set)", minedCalls) + } + if processed != 1 { + t.Fatalf("processed_at must be (re)stamped on the redelivery, got %d calls", processed) + } + + emitted := pub.snapshot() + if len(emitted) != 1 { + t.Fatalf("expected exactly one MINED event (the late tx), got %d: %+v", len(emitted), emitted) + } + if emitted[0].TxID != lateTx || emitted[0].Status != models.StatusMined || emitted[0].BlockHash != blockHash { + t.Fatalf("published = %+v, want MINED %s@%s", emitted[0], lateTx, blockHash) + } +} + +// A redelivery where every row is already MINED against this block is a pure +// no-op for subscribers: no MINED event at all. +func TestBuilder_HandleMessage_DuplicateBlockProcessed_AllMined_PublishesNothing(t *testing.T) { + ms := newMockStore() + blockHash := testBlockHash + compound, _, err := bump.BuildCompoundBUMP( + []*models.Stump{{BlockHash: blockHash, SubtreeIndex: 0, StumpData: makeMinimalSTUMP(testTxidHex)}}, + []chainhash.Hash{mustHash(t, testTxidHex)}, nil, nil, + ) + if err != nil { + t.Fatalf("BuildCompoundBUMP: %v", err) + } + ms.bumps[blockHash] = compound.Bytes() + ms.bumpHeights[blockHash] = uint64(compound.BlockHeight) + ms.alreadyMined = map[string]string{testTxidHex: blockHash} + stampedBlock(ms, blockHash) + + pub := &recordingPublisher{} + b := &Builder{cfg: &config.Config{}, logger: zap.NewNop(), store: ms, publisher: pub} + if err := b.handleMessage(context.Background(), makeBlockProcessedMsg(blockHash)); err != nil { + t.Fatalf("handleMessage: %v", err) + } + if emitted := pub.snapshot(); len(emitted) != 0 { + t.Fatalf("already-MINED rows must not be re-published, got %+v", emitted) + } +} + +// fourLeafVariant encodes a BRC-74 STUMP for a 4-leaf subtree that tracks +// leaves[tracked]: level 0 holds the tracked leaf (Txid marker) and its +// sibling, level 1 the other pair's parent. It is what one merkle-service +// emits when it only has a registration for that one tx. +func fourLeafVariant(leaves [4]chainhash.Hash, tracked int) []byte { + sibling := tracked ^ 1 + otherPair := (tracked / 2) ^ 1 + parent := transaction.MerkleTreeParent(&leaves[otherPair*2], &leaves[otherPair*2+1]) + isTx := true + tl, sl := leaves[tracked], leaves[sibling] + mp := &transaction.MerklePath{ + BlockHeight: 1, + Path: [][]*transaction.PathElement{ + { + {Offset: uint64(tracked), Hash: &tl, Txid: &isTx}, //nolint:gosec // tiny test index + {Offset: uint64(sibling), Hash: &sl}, //nolint:gosec // tiny test index + }, + {{Offset: uint64(otherPair), Hash: parent}}, //nolint:gosec // tiny test index + }, + } + return mp.Bytes() +} + +func fourLeafRoot(leaves [4]chainhash.Hash) chainhash.Hash { + p01 := transaction.MerkleTreeParent(&leaves[0], &leaves[1]) + p23 := transaction.MerkleTreeParent(&leaves[2], &leaves[3]) + return *transaction.MerkleTreeParent(p01, p23) +} + +// multiSourceFixture is a single-subtree block of four txs. Service A had +// registrations for L0 only; service B (down while L2 was registered, so it +// never learned about L0) tracks L2. The datahub serves the block so the +// build path has subtree hashes and a header root to validate against. +type multiSourceFixture struct { + ms *mockStore + b *Builder + pub *recordingPublisher + blockHash string + leaves [4]chainhash.Hash + variantA []byte + variantB []byte +} + +func newMultiSourceFixture(t *testing.T) *multiSourceFixture { + t.Helper() + f := &multiSourceFixture{ms: newMockStore(), pub: &recordingPublisher{}, blockHash: testBlockHash} + hexes := [4]string{ + "1111111111111111111111111111111111111111111111111111111111111111", + "2222222222222222222222222222222222222222222222222222222222222222", + "3333333333333333333333333333333333333333333333333333333333333333", + "4444444444444444444444444444444444444444444444444444444444444444", + } + for i, h := range hexes { + f.leaves[i] = mustHash(t, h) + } + f.variantA = fourLeafVariant(f.leaves, 0) + f.variantB = fourLeafVariant(f.leaves, 2) + root := fourLeafRoot(f.leaves) + datahub := newDatahubServer(root.CloneBytes(), []chainhash.Hash{root}) + t.Cleanup(datahub.Close) + f.b = newTestBuilder(f.ms, datahub.URL) + f.b.publisher = f.pub + return f +} + +func (f *multiSourceFixture) storedLeafCount(t *testing.T) int { + t.Helper() + f.ms.mu.Lock() + data := f.ms.bumps[f.blockHash] + f.ms.mu.Unlock() + txids, err := levelZeroTxidsFromBUMP(data) + if err != nil { + t.Fatalf("stored BUMP: %v", err) + } + return len(txids) +} + +// Service A's BLOCK_PROCESSED builds from its STUMP alone. Service B's +// delivery then carries leaves the stored BUMP lacks: the compound is rebuilt +// from both variants, the BUMP is overwritten, and only the newly covered txs +// are published. A third delivery (identical set) short-circuits silently. +func TestBuilder_HandleMessage_RedeliveryWithNewLeaves_RebuildsAndPublishesOnlyNew(t *testing.T) { + f := newMultiSourceFixture(t) + ctx := context.Background() + + // 1. Service A: STUMP tracking L0, then BLOCK_PROCESSED. + f.ms.addStump(f.blockHash, 0, f.variantA) + if err := f.b.handleMessage(ctx, makeBlockProcessedMsg(f.blockHash)); err != nil { + t.Fatalf("first build: %v", err) + } + if got := f.storedLeafCount(t); got != 2 { + t.Fatalf("first build leaves = %d, want 2 (L0 + sibling)", got) + } + if got := len(f.pub.snapshot()); got != 2 { + t.Fatalf("first build published %d, want 2", got) + } + f.ms.mu.Lock() + if len(f.ms.stumps[f.blockHash]) != 1 || len(f.ms.deletedBlocks) != 0 { + t.Fatalf("STUMPs must be retained after the build: stumps=%d deletes=%v", len(f.ms.stumps[f.blockHash]), f.ms.deletedBlocks) + } + // The rows the first build mined are now MINED in the store. + f.ms.alreadyMined = map[string]string{f.leaves[0].String(): f.blockHash, f.leaves[1].String(): f.blockHash} + f.ms.mu.Unlock() + + // 2. Service B: STUMP tracking L2 (a different variant), then BLOCK_PROCESSED. + f.ms.addStump(f.blockHash, 0, f.variantB) + beforeRebuilt := bumpOutcomeSampleCount(t, "rebuilt") + if err := f.b.handleMessage(ctx, makeBlockProcessedMsg(f.blockHash)); err != nil { + t.Fatalf("rebuild: %v", err) + } + if got := bumpOutcomeSampleCount(t, "rebuilt"); got != beforeRebuilt+1 { + t.Fatalf("rebuilt samples = %d, want %d", got, beforeRebuilt+1) + } + if got := f.storedLeafCount(t); got != 4 { + t.Fatalf("rebuilt BUMP leaves = %d, want 4", got) + } + emitted := f.pub.snapshot() + if len(emitted) != 4 { + t.Fatalf("after rebuild published total = %d, want 4 (2 from the first build + L2, L3)", len(emitted)) + } + newly := map[string]bool{} + for _, st := range emitted[2:] { + newly[st.TxID] = true + } + if !newly[f.leaves[2].String()] || !newly[f.leaves[3].String()] { + t.Fatalf("rebuild must publish exactly the newly covered txs, got %v", newly) + } + f.ms.mu.Lock() + if len(f.ms.processedCalls) != 2 { + t.Fatalf("processed_at must be stamped on the rebuild too, got %d stamps", len(f.ms.processedCalls)) + } + f.ms.alreadyMined[f.leaves[2].String()] = f.blockHash + f.ms.alreadyMined[f.leaves[3].String()] = f.blockHash + f.ms.mu.Unlock() + + // 3. Service C delivers the same STUMP as B: nothing new, short-circuit. + f.ms.addStump(f.blockHash, 0, f.variantB) + beforeShort := bumpOutcomeSampleCount(t, "short_circuited") + if err := f.b.handleMessage(ctx, makeBlockProcessedMsg(f.blockHash)); err != nil { + t.Fatalf("third delivery: %v", err) + } + if got := bumpOutcomeSampleCount(t, "short_circuited"); got != beforeShort+1 { + t.Fatalf("short_circuited samples = %d, want %d", got, beforeShort+1) + } + if got := len(f.pub.snapshot()); got != 4 { + t.Fatalf("short-circuit must publish nothing, total = %d", got) + } + if got := f.storedLeafCount(t); got != 4 { + t.Fatalf("short-circuit must not rewrite the BUMP, leaves = %d", got) + } +} + +// Both variants are present before the first build (the common ordering when +// services run in lockstep): one build, one BUMP covering the union. +func TestBuilder_HandleMessage_VariantsBeforeFirstBuild_SingleBuildCoversUnion(t *testing.T) { + f := newMultiSourceFixture(t) + f.ms.addStump(f.blockHash, 0, f.variantA) + f.ms.addStump(f.blockHash, 0, f.variantB) + if err := f.b.handleMessage(context.Background(), makeBlockProcessedMsg(f.blockHash)); err != nil { + t.Fatalf("build: %v", err) + } + if got := f.storedLeafCount(t); got != 4 { + t.Fatalf("leaves = %d, want 4", got) + } + if got := len(f.pub.snapshot()); got != 4 { + t.Fatalf("published = %d, want 4", got) + } +} + +// Service A's build stamped processed_at. Service B's STUMP for a new subtree +// arrived, but B's expected set names another subtree whose STUMP is still +// missing after the grace window: the block is deferred AND the stamp is +// cleared so the watchdog re-drives it. The stored BUMP is untouched. +func TestBuilder_HandleMessage_DeferredAfterStamp_ClearsProcessedAt(t *testing.T) { + f := newMultiSourceFixture(t) + ctx := context.Background() + f.ms.addStump(f.blockHash, 0, f.variantA) + if err := f.b.handleMessage(ctx, makeBlockProcessedMsg(f.blockHash)); err != nil { + t.Fatalf("first build: %v", err) + } + stamped := time.Now() + f.ms.mu.Lock() + f.ms.blockProc = []*models.BlockProcessingStatus{{BlockHash: f.blockHash, BlockHeight: 1, ProcessedAt: &stamped}} + f.ms.mu.Unlock() + f.b.cfg.BumpBuilder.GraceWindowMs = 0 + + const lateTx = "5555555555555555555555555555555555555555555555555555555555555555" + f.ms.addStump(f.blockHash, 1, makeMinimalSTUMP(lateTx)) // new leaf ⇒ rebuild path + beforeDeferred := bumpOutcomeSampleCount(t, "deferred_incomplete") + if err := f.b.handleMessage(ctx, makeBlockProcessedMsgWithExpected(f.blockHash, []int{1, 2})); err != nil { + t.Fatalf("deferred delivery must return nil (watchdog recovers), got %v", err) + } + if got := bumpOutcomeSampleCount(t, "deferred_incomplete"); got != beforeDeferred+1 { + t.Fatalf("deferred_incomplete samples = %d, want %d", got, beforeDeferred+1) + } + f.ms.mu.Lock() + defer f.ms.mu.Unlock() + if len(f.ms.clearedBlocks) != 1 || f.ms.clearedBlocks[0] != f.blockHash { + t.Fatalf("ClearBlockProcessed calls = %v, want [%s]", f.ms.clearedBlocks, f.blockHash) + } + if f.ms.blockProc[0].ProcessedAt != nil { + t.Fatal("processed_at must be cleared on the fixture row") + } + if got := len(f.pub.snapshot()); got != 2 { + t.Fatalf("deferral must publish nothing new, total = %d", got) + } +} + +// If the janitor already pruned service A's STUMP, a rebuild from B alone +// would drop L0/L1 from the stored BUMP. The builder keeps the stored BUMP, +// publishes nothing, and hands the block back to the watchdog. +func TestBuilder_HandleMessage_RebuildWouldDropLeaves_KeepsBUMPAndDefers(t *testing.T) { + f := newMultiSourceFixture(t) + ctx := context.Background() + f.ms.addStump(f.blockHash, 0, f.variantA) + if err := f.b.handleMessage(ctx, makeBlockProcessedMsg(f.blockHash)); err != nil { + t.Fatalf("first build: %v", err) + } + stamped := time.Now() + f.ms.mu.Lock() + f.ms.blockProc = []*models.BlockProcessingStatus{{BlockHash: f.blockHash, BlockHeight: 1, ProcessedAt: &stamped}} + delete(f.ms.stumps, f.blockHash) // janitor pruned A's STUMP + f.ms.alreadyMined = map[string]string{f.leaves[0].String(): f.blockHash, f.leaves[1].String(): f.blockHash} + before := f.ms.bumps[f.blockHash] + f.ms.mu.Unlock() + + f.ms.addStump(f.blockHash, 0, f.variantB) + if err := f.b.handleMessage(ctx, makeBlockProcessedMsg(f.blockHash)); err != nil { + t.Fatalf("delivery: %v", err) + } + f.ms.mu.Lock() + defer f.ms.mu.Unlock() + if string(f.ms.bumps[f.blockHash]) != string(before) { + t.Fatal("stored BUMP must not be overwritten by a compound that drops leaves") + } + if len(f.ms.clearedBlocks) != 1 { + t.Fatalf("block must be handed back to the watchdog, cleared=%v", f.ms.clearedBlocks) + } + if got := len(f.pub.snapshot()); got != 2 { + t.Fatalf("nothing new may be published, total = %d", got) + } +} + +// A redelivery whose STUMP set cannot be read must not take the +// short-circuit: that would re-stamp processed_at and retire a block this +// delivery may have added leaves to. The message fails so Kafka retries it. +func TestBuilder_HandleMessage_RedeliveryStumpReadError_FailsMessage(t *testing.T) { + f := newMultiSourceFixture(t) + ctx := context.Background() + f.ms.addStump(f.blockHash, 0, f.variantA) + if err := f.b.handleMessage(ctx, makeBlockProcessedMsg(f.blockHash)); err != nil { + t.Fatalf("first build: %v", err) + } + f.ms.mu.Lock() + f.ms.getStumpsErr = errors.New("store: read timeout") + minedBefore, stampedBefore := len(f.ms.minedCalls), len(f.ms.processedCalls) + f.ms.mu.Unlock() + + before := bumpOutcomeSampleCount(t, "store_failed") + err := f.b.handleMessage(ctx, makeBlockProcessedMsg(f.blockHash)) + if err == nil { + t.Fatal("a STUMP read failure on redelivery must fail the message") + } + if got := bumpOutcomeSampleCount(t, "store_failed"); got != before+1 { + t.Fatalf("store_failed samples = %d, want %d", got, before+1) + } + f.ms.mu.Lock() + defer f.ms.mu.Unlock() + if len(f.ms.minedCalls) != minedBefore || len(f.ms.processedCalls) != stampedBefore { + t.Fatalf("nothing may be mined or stamped on a failed read: mined %d->%d stamped %d->%d", + minedBefore, len(f.ms.minedCalls), stampedBefore, len(f.ms.processedCalls)) + } +} + +// If clearing processed_at fails, the deferral must fail the message too: +// committing it with the old stamp still set would leave the block +// known-incomplete and invisible to the watchdog. +func TestBuilder_HandleMessage_DeferralClearFails_FailsMessage(t *testing.T) { + f := newMultiSourceFixture(t) + ctx := context.Background() + f.ms.addStump(f.blockHash, 0, f.variantA) + if err := f.b.handleMessage(ctx, makeBlockProcessedMsg(f.blockHash)); err != nil { + t.Fatalf("first build: %v", err) + } + stamped := time.Now() + f.ms.mu.Lock() + f.ms.blockProc = []*models.BlockProcessingStatus{{BlockHash: f.blockHash, BlockHeight: 1, ProcessedAt: &stamped}} + f.ms.clearProcessedErr = errors.New("store: write timeout") + f.ms.mu.Unlock() + f.b.cfg.BumpBuilder.GraceWindowMs = 0 + + const lateTx = "5555555555555555555555555555555555555555555555555555555555555555" + f.ms.addStump(f.blockHash, 1, makeMinimalSTUMP(lateTx)) + beforeDeferred := bumpOutcomeSampleCount(t, "deferred_incomplete") + beforeFailed := bumpOutcomeSampleCount(t, "store_failed") + err := f.b.handleMessage(ctx, makeBlockProcessedMsgWithExpected(f.blockHash, []int{1, 2})) + if err == nil { + t.Fatal("a failed ClearBlockProcessed must fail the message") + } + if got := bumpOutcomeSampleCount(t, "deferred_incomplete"); got != beforeDeferred { + t.Fatalf("deferred_incomplete must not be recorded when the clear failed, got %d want %d", got, beforeDeferred) + } + if got := bumpOutcomeSampleCount(t, "store_failed"); got != beforeFailed+1 { + t.Fatalf("store_failed samples = %d, want %d", got, beforeFailed+1) + } + + // The rebuild-would-drop path fails the same way. + f.ms.mu.Lock() + delete(f.ms.stumps, f.blockHash) + f.ms.alreadyMined = map[string]string{f.leaves[0].String(): f.blockHash, f.leaves[1].String(): f.blockHash} + f.ms.mu.Unlock() + f.ms.addStump(f.blockHash, 0, f.variantB) + if err := f.b.handleMessage(ctx, makeBlockProcessedMsg(f.blockHash)); err == nil { + t.Fatal("a failed ClearBlockProcessed on the rebuild-would-drop path must fail the message") + } +} + +// The janitor walks by height but prunes by bump_built_at age, so it must +// keep walking below the watchdog's recency depth while a page may still +// hold retained rows, or a block that ages out after dropping below the +// depth is never visited again. +func TestBuilder_PruneOrphanStumps_WalksPastRecencyDepthWhileRowsRetained(t *testing.T) { + ms := newMockStore() + ms.tipHeight = 1000 + old := time.Now().Add(-3 * time.Hour) + recent := time.Now().Add(-time.Minute) + // 200 in-flight rows fill the first page down to height 801, below the + // 144-block recency depth (horizon 856). Their headers are recent, so + // the page may still hold retained rows and the walk continues to the + // deep built rows. + for h := uint64(1000); h >= 801; h-- { + ms.blockProc = append(ms.blockProc, &models.BlockProcessingStatus{BlockHash: fmt.Sprintf("inflight-%d", h), BlockHeight: h, HeaderSeenAt: recent}) + } + ms.blockProc = append(ms.blockProc, + &models.BlockProcessingStatus{BlockHash: "deep-fresh", BlockHeight: 700, HeaderSeenAt: old, BUMPBuiltAt: &recent, ProcessedAt: &recent}, // late build, keep + &models.BlockProcessingStatus{BlockHash: "deep-old", BlockHeight: 690, HeaderSeenAt: old, BUMPBuiltAt: &old, ProcessedAt: &old}, // aged out, prune + ) + ms.addStump("deep-fresh", 0, []byte("fresh")) + ms.addStump("deep-old", 0, []byte("old")) + + b := &Builder{cfg: &config.Config{}, logger: zap.NewNop(), store: ms} + b.cfg.Watchdog.RecencyDepth = 144 + b.pruneOrphanStumps(context.Background()) + + ms.mu.Lock() + defer ms.mu.Unlock() + if len(ms.stumps["deep-old"]) != 0 { + t.Fatalf("deep-old aged out and must be pruned even below the recency depth, still has %d", len(ms.stumps["deep-old"])) + } + if len(ms.stumps["deep-fresh"]) != 1 { + t.Fatalf("deep-fresh is inside retention and must be kept, got %d", len(ms.stumps["deep-fresh"])) + } +} + +// coinbaseSlotFixture is the shape every small block has: one subtree whose +// offset 0 is the coinbase placeholder and whose tracked tx sits at offset 1, +// so each service's STUMP carries the placeholder at level 0 while the stored +// compound carries the real coinbase txid there (the build swaps it in from +// the coinbase BUMP). Delivered through the enriched BLOCK_PROCESSED path, +// as merkle-service sends it. +func coinbaseSlotFixture(t *testing.T) (f *multiSourceFixture, msg func() *kafka.Message) { + t.Helper() + f = &multiSourceFixture{ms: newMockStore(), pub: &recordingPublisher{}, blockHash: testBlockHash} + placeholder := mustHash(t, "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + tracked := mustHash(t, "1111111111111111111111111111111111111111111111111111111111111111") + coinbase := mustHash(t, "4444444444444444444444444444444444444444444444444444444444444444") + isTx := true + stump := (&transaction.MerklePath{BlockHeight: 1, Path: [][]*transaction.PathElement{{ + {Offset: 0, Hash: &placeholder}, + {Offset: 1, Hash: &tracked, Txid: &isTx}, + }}}).Bytes() + cbBUMP := (&transaction.MerklePath{BlockHeight: 1, Path: [][]*transaction.PathElement{{ + {Offset: 0, Hash: &coinbase, Txid: &isTx}, + {Offset: 1, Hash: &tracked}, + }}}).Bytes() + subtreeHashes := []chainhash.Hash{*transaction.MerkleTreeParent(&placeholder, &tracked)} + root := *transaction.MerkleTreeParent(&coinbase, &tracked) + f.variantA = stump + datahub := newRecordingDatahub(t) + f.b = newTestBuilder(f.ms, datahub.server.URL) + f.b.publisher = f.pub + return f, func() *kafka.Message { + return makeBlockProcessedMsgWithEnrichment(t, f.blockHash, root, subtreeHashes, cbBUMP) + } +} + +// With several merkle-services every one of them delivers an identical +// STUMP for the block, placeholder included. The redelivery must be +// recognised as carrying nothing new: it must not rebuild, must not flag the +// real coinbase leaf as dropped, and must not clear processed_at (which +// hands a finished block back to the watchdog on every block). +func TestBuilder_HandleMessage_IdenticalRedelivery_CoinbaseSlot_ShortCircuits(t *testing.T) { + f, msg := coinbaseSlotFixture(t) + ctx := context.Background() + + f.ms.addStump(f.blockHash, 0, f.variantA) + if err := f.b.handleMessage(ctx, msg()); err != nil { + t.Fatalf("first build: %v", err) + } + stamped := time.Now() + f.ms.mu.Lock() + if _, ok := f.ms.bumps[f.blockHash]; !ok { + f.ms.mu.Unlock() + t.Fatal("first delivery must build the compound") + } + f.ms.blockProc = []*models.BlockProcessingStatus{{BlockHash: f.blockHash, BlockHeight: 1, ProcessedAt: &stamped}} + before := string(f.ms.bumps[f.blockHash]) + f.ms.mu.Unlock() + + // Services B and C deliver the same STUMP. + for svc := 0; svc < 2; svc++ { + f.ms.addStump(f.blockHash, 0, f.variantA) + beforeShort := bumpOutcomeSampleCount(t, "short_circuited") + if err := f.b.handleMessage(ctx, msg()); err != nil { + t.Fatalf("redelivery %d: %v", svc, err) + } + if got := bumpOutcomeSampleCount(t, "short_circuited"); got != beforeShort+1 { + t.Fatalf("redelivery %d: short_circuited samples = %d, want %d", svc, got, beforeShort+1) + } + } + f.ms.mu.Lock() + defer f.ms.mu.Unlock() + if len(f.ms.clearedBlocks) != 0 { + t.Fatalf("an identical redelivery must not hand the block back to the watchdog, cleared=%v", f.ms.clearedBlocks) + } + if string(f.ms.bumps[f.blockHash]) != before { + t.Fatal("an identical redelivery must not rewrite the stored BUMP") + } +} + +// Service B tracks a different pair of the same subtree. Its redelivery adds +// leaves, so the compound is rebuilt; the rebuild still carries the real +// coinbase at offset 0 and must not be judged as dropping it (the STUMP-side +// txids hold the placeholder there). +func TestBuilder_HandleMessage_RedeliveryNewLeaves_CoinbaseSlot_Rebuilds(t *testing.T) { + ms := newMockStore() + pub := &recordingPublisher{} + blockHash := testBlockHash + ph := mustHash(t, "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + t1 := mustHash(t, "1111111111111111111111111111111111111111111111111111111111111111") + l2 := mustHash(t, "2222222222222222222222222222222222222222222222222222222222222222") + l3 := mustHash(t, "3333333333333333333333333333333333333333333333333333333333333333") + cb := mustHash(t, "4444444444444444444444444444444444444444444444444444444444444444") + isTx := true + p01 := transaction.MerkleTreeParent(&ph, &t1) + p23 := transaction.MerkleTreeParent(&l2, &l3) + stumpA := (&transaction.MerklePath{BlockHeight: 1, Path: [][]*transaction.PathElement{ + {{Offset: 0, Hash: &ph}, {Offset: 1, Hash: &t1, Txid: &isTx}}, + {{Offset: 1, Hash: p23}}, + }}).Bytes() + stumpB := (&transaction.MerklePath{BlockHeight: 1, Path: [][]*transaction.PathElement{ + {{Offset: 2, Hash: &l2, Txid: &isTx}, {Offset: 3, Hash: &l3}}, + {{Offset: 0, Hash: p01}}, + }}).Bytes() + cbBUMP := (&transaction.MerklePath{BlockHeight: 1, Path: [][]*transaction.PathElement{ + {{Offset: 0, Hash: &cb, Txid: &isTx}, {Offset: 1, Hash: &t1}}, + {{Offset: 1, Hash: p23}}, + }}).Bytes() + subtreeHashes := []chainhash.Hash{*transaction.MerkleTreeParent(p01, p23)} + root := *transaction.MerkleTreeParent(transaction.MerkleTreeParent(&cb, &t1), p23) + datahub := newRecordingDatahub(t) + b := newTestBuilder(ms, datahub.server.URL) + b.publisher = pub + msg := func() *kafka.Message { + return makeBlockProcessedMsgWithEnrichment(t, blockHash, root, subtreeHashes, cbBUMP) + } + ctx := context.Background() + + ms.addStump(blockHash, 0, stumpA) + if err := b.handleMessage(ctx, msg()); err != nil { + t.Fatalf("first build: %v", err) + } + stamped := time.Now() + ms.mu.Lock() + ms.blockProc = []*models.BlockProcessingStatus{{BlockHash: blockHash, BlockHeight: 1, ProcessedAt: &stamped}} + ms.mu.Unlock() + + ms.addStump(blockHash, 0, stumpB) + beforeRebuilt := bumpOutcomeSampleCount(t, "rebuilt") + if err := b.handleMessage(ctx, msg()); err != nil { + t.Fatalf("redelivery: %v", err) + } + if got := bumpOutcomeSampleCount(t, "rebuilt"); got != beforeRebuilt+1 { + t.Fatalf("rebuilt samples = %d, want %d", got, beforeRebuilt+1) + } + ms.mu.Lock() + defer ms.mu.Unlock() + // The rebuild takes the first build's stamp off before overwriting the + // BUMP and puts it back once the expanded set is mined. + if len(ms.clearedBlocks) != 1 || ms.blockProc[0].ProcessedAt == nil { + t.Fatalf("rebuild must clear then re-stamp processed_at, cleared=%v processed_at=%v", ms.clearedBlocks, ms.blockProc[0].ProcessedAt) + } + leaves, err := levelZeroTxidsFromBUMP(ms.bumps[blockHash]) + if err != nil { + t.Fatal(err) + } + got := map[string]bool{} + for _, l := range leaves { + got[l] = true + } + for _, want := range []chainhash.Hash{cb, t1, l2, l3} { + if !got[want.String()] { + t.Fatalf("rebuilt BUMP leaves %v lack %s", leaves, want) + } + } +} + +// stampedBlock gives blockHash a block_processing row with processed_at set: +// an earlier delivery finalized the block, so its MINED rows were published. +func stampedBlock(ms *mockStore, blockHash string) { + now := time.Now() + ms.blockProc = append(ms.blockProc, &models.BlockProcessingStatus{BlockHash: blockHash, ProcessedAt: &now}) +} + +// A crash between SetMinedByTxIDs and PublishBulk leaves rows MINED but +// unpublished, and the block un-stamped. The redelivery must not take "already +// MINED" as "already published": with processed_at unset it republishes the +// whole stored set. +func TestBuilder_HandleMessage_RedeliveryOfUnstampedBlock_RepublishesMinedRows(t *testing.T) { + ms := newMockStore() + blockHash := testBlockHash + compound, _, err := bump.BuildCompoundBUMP( + []*models.Stump{{BlockHash: blockHash, SubtreeIndex: 0, StumpData: makeMinimalSTUMP(testTxidHex)}}, + []chainhash.Hash{mustHash(t, testTxidHex)}, nil, nil, + ) + if err != nil { + t.Fatalf("BuildCompoundBUMP: %v", err) + } + ms.bumps[blockHash] = compound.Bytes() + ms.bumpHeights[blockHash] = uint64(compound.BlockHeight) + ms.alreadyMined = map[string]string{testTxidHex: blockHash} + + pub := &recordingPublisher{} + b := &Builder{cfg: &config.Config{}, logger: zap.NewNop(), store: ms, publisher: pub} + if err := b.handleMessage(context.Background(), makeBlockProcessedMsg(blockHash)); err != nil { + t.Fatalf("handleMessage: %v", err) + } + emitted := pub.snapshot() + if len(emitted) != 1 || emitted[0].TxID != testTxidHex || emitted[0].Status != models.StatusMined { + t.Fatalf("an unstamped block's MINED rows must be republished, got %+v", emitted) + } +} + +// A rebuild whose mine fails must leave the block un-stamped: the first +// build's stamp comes off before the BUMP is overwritten, and goes back on +// only after the expanded set is mined, so the watchdog still re-drives the +// newly covered txs. +func TestBuilder_HandleMessage_RebuildMineFails_LeavesProcessedAtCleared(t *testing.T) { + f := newMultiSourceFixture(t) + ctx := context.Background() + f.ms.addStump(f.blockHash, 0, f.variantA) + if err := f.b.handleMessage(ctx, makeBlockProcessedMsg(f.blockHash)); err != nil { + t.Fatalf("first build: %v", err) + } + f.ms.mu.Lock() + if len(f.ms.blockProc) != 1 || f.ms.blockProc[0].ProcessedAt == nil { + f.ms.mu.Unlock() + t.Fatal("first build must stamp processed_at") + } + f.ms.setMinedErr = errors.New("store down") + f.ms.mu.Unlock() + + f.ms.addStump(f.blockHash, 0, f.variantB) + if err := f.b.handleMessage(ctx, makeBlockProcessedMsg(f.blockHash)); err != nil { + t.Fatalf("rebuild with a failed mine returns nil (the watchdog recovers), got %v", err) + } + f.ms.mu.Lock() + defer f.ms.mu.Unlock() + if len(f.ms.clearedBlocks) != 1 { + t.Fatalf("rebuild must clear the first build's stamp, cleared=%v", f.ms.clearedBlocks) + } + if f.ms.blockProc[0].ProcessedAt != nil { + t.Fatal("processed_at must stay cleared while the expanded set is not mined") + } +} + +// Service A built subtree 0 and stamped the block. Service B's delivery adds +// no retained leaves but expects subtree 1, whose STUMP is still retrying. +// That must not short-circuit (which would re-stamp the block and retire it +// from the watchdog): it goes through the completeness gate, which defers the +// block and clears the stamp. +func TestBuilder_HandleMessage_NoNewLeavesButExpectedMissing_DefersInsteadOfShortCircuit(t *testing.T) { + f := newMultiSourceFixture(t) + ctx := context.Background() + f.ms.addStump(f.blockHash, 0, f.variantA) + if err := f.b.handleMessage(ctx, makeBlockProcessedMsg(f.blockHash)); err != nil { + t.Fatalf("first build: %v", err) + } + f.b.cfg.BumpBuilder.GraceWindowMs = 0 + + beforeShort := bumpOutcomeSampleCount(t, "short_circuited") + beforeDeferred := bumpOutcomeSampleCount(t, "deferred_incomplete") + if err := f.b.handleMessage(ctx, makeBlockProcessedMsgWithExpected(f.blockHash, []int{0, 1})); err != nil { + t.Fatalf("delivery: %v", err) + } + if got := bumpOutcomeSampleCount(t, "short_circuited"); got != beforeShort { + t.Fatal("a delivery expecting an unretained subtree must not short-circuit") + } + if got := bumpOutcomeSampleCount(t, "deferred_incomplete"); got != beforeDeferred+1 { + t.Fatalf("deferred_incomplete samples = %d, want %d", got, beforeDeferred+1) + } + f.ms.mu.Lock() + defer f.ms.mu.Unlock() + if f.ms.blockProc[0].ProcessedAt != nil { + t.Fatal("processed_at must be cleared so the watchdog re-drives the block") + } +} + +// The interleaving from review: A builds subtree 0 (expects {0}); B supplies +// subtree 1 but expects {1,2} and defers for missing 2; then A's duplicate +// (expects {0}) arrives. Retained subtree 1 makes it a rebuild, and judged +// against A's set alone it would stamp the block with subtree 2 still owed. +// The deferral retained {1,2}, so A's duplicate defers too. Once the block +// is stamped, the retained set is dropped. +func TestBuilder_HandleMessage_RetainedExpectedSetOutlivesSmallerAnnouncements(t *testing.T) { + f := newMultiSourceFixture(t) + ctx := context.Background() + f.b.cfg.BumpBuilder.GraceWindowMs = 0 + + // A: subtree 0, expects {0}. + f.ms.addStump(f.blockHash, 0, f.variantA) + if err := f.b.handleMessage(ctx, makeBlockProcessedMsgWithExpected(f.blockHash, []int{0})); err != nil { + t.Fatalf("A: %v", err) + } + // B: subtree 1 retained, expects {1,2}: deferred. + const lateTx = "5555555555555555555555555555555555555555555555555555555555555555" + f.ms.addStump(f.blockHash, 1, makeMinimalSTUMP(lateTx)) + if err := f.b.handleMessage(ctx, makeBlockProcessedMsgWithExpected(f.blockHash, []int{1, 2})); err != nil { + t.Fatalf("B: %v", err) + } + f.ms.mu.Lock() + if got := f.ms.expectedSubtrees[f.blockHash]; len(got) != 2 || got[0] != 1 || got[1] != 2 { + f.ms.mu.Unlock() + t.Fatalf("B's deferral must retain its expected set, got %v", got) + } + f.ms.mu.Unlock() + + // A's duplicate, expects {0}: must defer, not stamp. + beforeDeferred := bumpOutcomeSampleCount(t, "deferred_incomplete") + if err := f.b.handleMessage(ctx, makeBlockProcessedMsgWithExpected(f.blockHash, []int{0})); err != nil { + t.Fatalf("A duplicate: %v", err) + } + if got := bumpOutcomeSampleCount(t, "deferred_incomplete"); got != beforeDeferred+1 { + t.Fatalf("A's duplicate must be held to the retained set and defer, deferred samples %d -> %d", beforeDeferred, got) + } + f.ms.mu.Lock() + if f.ms.blockProc[0].ProcessedAt != nil { + f.ms.mu.Unlock() + t.Fatal("block must stay un-stamped while subtree 2 is owed") + } + f.ms.mu.Unlock() + + // Stamping the block (whichever path finally completes it) drops the + // retained set: a finalized block owes nothing. + f.b.markBlockProcessed(ctx, zap.NewNop(), f.blockHash, 1) + f.ms.mu.Lock() + defer f.ms.mu.Unlock() + if got := f.ms.expectedSubtrees[f.blockHash]; len(got) != 0 { + t.Fatalf("stamping must drop the retained set, got %v", got) + } +} + +// failingBulkPublisher refuses every PublishBulk. +type failingBulkPublisher struct{ recordingPublisher } + +func (p *failingBulkPublisher) PublishBulk(context.Context, *models.TransactionStatus) error { + return errors.New("kafka down") +} + +// processed_at is the proof blockFinalized relies on that a block's MINED +// events went out, so a failed PublishBulk must withhold it: the watchdog's +// re-drive then sees an unstamped block and republishes in full. +func TestBuilder_HandleMessage_PublishFailure_LeavesBlockUnstamped(t *testing.T) { + ms := newMockStore() + blockHash := testBlockHash + compound, _, err := bump.BuildCompoundBUMP( + []*models.Stump{{BlockHash: blockHash, SubtreeIndex: 0, StumpData: makeMinimalSTUMP(testTxidHex)}}, + []chainhash.Hash{mustHash(t, testTxidHex)}, nil, nil, + ) + if err != nil { + t.Fatalf("BuildCompoundBUMP: %v", err) + } + ms.bumps[blockHash] = compound.Bytes() + ms.bumpHeights[blockHash] = uint64(compound.BlockHeight) + + b := &Builder{cfg: &config.Config{}, logger: zap.NewNop(), store: ms, publisher: &failingBulkPublisher{}} + if err := b.handleMessage(context.Background(), makeBlockProcessedMsg(blockHash)); err != nil { + t.Fatalf("handleMessage: %v", err) + } + ms.mu.Lock() + defer ms.mu.Unlock() + if len(ms.minedCalls) != 1 { + t.Fatalf("the rows must still be mined, SetMinedByTxIDs calls = %d", len(ms.minedCalls)) + } + if len(ms.processedCalls) != 0 { + t.Fatalf("a failed MINED publish must leave processed_at unset, got %d stamps", len(ms.processedCalls)) + } +} + +// Once the janitor has pruned a finalized block's STUMPs, a plain redelivery +// (Kafka retry, DLQ redrive) carrying an expected set must still +// short-circuit: with nothing retained every index would read as missing, +// and taking the completeness gate would un-stamp a complete block. +func TestBuilder_HandleMessage_RedeliveryAfterPrune_ShortCircuitsDespiteExpectedSet(t *testing.T) { + ms := newMockStore() + blockHash := testBlockHash + compound, _, err := bump.BuildCompoundBUMP( + []*models.Stump{{BlockHash: blockHash, SubtreeIndex: 0, StumpData: makeMinimalSTUMP(testTxidHex)}}, + []chainhash.Hash{mustHash(t, testTxidHex)}, nil, nil, + ) + if err != nil { + t.Fatalf("BuildCompoundBUMP: %v", err) + } + ms.bumps[blockHash] = compound.Bytes() + ms.bumpHeights[blockHash] = uint64(compound.BlockHeight) + stampedBlock(ms, blockHash) + + b := &Builder{cfg: &config.Config{}, logger: zap.NewNop(), store: ms, publisher: &recordingPublisher{}} + before := bumpOutcomeSampleCount(t, "short_circuited") + if err := b.handleMessage(context.Background(), makeBlockProcessedMsgWithExpected(blockHash, []int{0})); err != nil { + t.Fatalf("handleMessage: %v", err) + } + if got := bumpOutcomeSampleCount(t, "short_circuited"); got != before+1 { + t.Fatalf("short_circuited samples = %d, want %d", got, before+1) + } + ms.mu.Lock() + defer ms.mu.Unlock() + if ms.blockProc[0].ProcessedAt == nil { + t.Fatal("a complete block must stay stamped") + } +} + +// stampedShortCircuitFixture is a block whose BUMP is stored and whose +// processed_at an earlier delivery stamped, with no retained STUMPs: the next +// BLOCK_PROCESSED takes the short-circuit. The stored row is not yet MINED +// (e.g. registered after the first build), so the re-mine has work to do. +func stampedShortCircuitFixture(t *testing.T) *mockStore { + t.Helper() + ms := newMockStore() + compound, _, err := bump.BuildCompoundBUMP( + []*models.Stump{{BlockHash: testBlockHash, SubtreeIndex: 0, StumpData: makeMinimalSTUMP(testTxidHex)}}, + []chainhash.Hash{mustHash(t, testTxidHex)}, nil, nil, + ) + if err != nil { + t.Fatalf("BuildCompoundBUMP: %v", err) + } + ms.bumps[testBlockHash] = compound.Bytes() + ms.bumpHeights[testBlockHash] = uint64(compound.BlockHeight) + stampedBlock(ms, testBlockHash) + return ms +} + +// On an already-stamped block, a short-circuit whose re-mine fails must take +// the stamp off, or the watchdog never re-drives the block and the rows the +// mine missed stay SEEN_*. +func TestBuilder_ShortCircuit_StampedBlockMineFails_ClearsProcessedAt(t *testing.T) { + ms := stampedShortCircuitFixture(t) + ms.setMinedErr = errors.New("store down") + b := &Builder{cfg: &config.Config{}, logger: zap.NewNop(), store: ms, publisher: &recordingPublisher{}} + + before := bumpOutcomeSampleCount(t, "store_failed") + if err := b.handleMessage(context.Background(), makeBlockProcessedMsg(testBlockHash)); err != nil { + t.Fatalf("a failed mine returns nil (the watchdog recovers), got %v", err) + } + if got := bumpOutcomeSampleCount(t, "store_failed"); got != before+1 { + t.Fatalf("store_failed samples = %d, want %d", got, before+1) + } + ms.mu.Lock() + defer ms.mu.Unlock() + if ms.blockProc[0].ProcessedAt != nil { + t.Fatal("processed_at must be cleared so the watchdog re-drives the block") + } +} + +// On an already-stamped block, a short-circuit whose MINED publish fails must +// take the stamp off too: left stamped, the next delivery reads the block as +// finalized and filters the now-MINED rows out of the publish for good. +func TestBuilder_ShortCircuit_StampedBlockPublishFails_ClearsProcessedAtAndRepublishes(t *testing.T) { + ms := stampedShortCircuitFixture(t) + b := &Builder{cfg: &config.Config{}, logger: zap.NewNop(), store: ms, publisher: &failingBulkPublisher{}} + if err := b.handleMessage(context.Background(), makeBlockProcessedMsg(testBlockHash)); err != nil { + t.Fatalf("handleMessage: %v", err) + } + ms.mu.Lock() + if ms.blockProc[0].ProcessedAt != nil { + ms.mu.Unlock() + t.Fatal("processed_at must be cleared after a failed MINED publish") + } + // The row is MINED in the store now; the watchdog's re-drive must still + // publish it. + ms.alreadyMined = map[string]string{testTxidHex: testBlockHash} + ms.mu.Unlock() + + pub := &recordingPublisher{} + b.publisher = pub + if err := b.handleMessage(context.Background(), makeBlockProcessedMsg(testBlockHash)); err != nil { + t.Fatalf("re-drive: %v", err) + } + if emitted := pub.snapshot(); len(emitted) != 1 || emitted[0].TxID != testTxidHex { + t.Fatalf("the re-drive must republish the MINED row, got %+v", emitted) + } + ms.mu.Lock() + defer ms.mu.Unlock() + if ms.blockProc[0].ProcessedAt == nil { + t.Fatal("a successful re-drive re-stamps the block") + } +} + +// If the stamp cannot be taken off, the message fails so Kafka retries +// rather than committing a stamped block the watchdog will never re-drive. +func TestBuilder_ShortCircuit_StampedBlockMineFails_ClearFailureFailsMessage(t *testing.T) { + ms := stampedShortCircuitFixture(t) + ms.setMinedErr = errors.New("store down") + ms.clearProcessedErr = errors.New("still down") + b := &Builder{cfg: &config.Config{}, logger: zap.NewNop(), store: ms, publisher: &recordingPublisher{}} + if err := b.handleMessage(context.Background(), makeBlockProcessedMsg(testBlockHash)); err == nil { + t.Fatal("a failed ClearBlockProcessed must fail the message") + } +} + +// A rebuild waits the grace window even when the callback's expected index +// set already looks complete: the index can be covered by the earlier +// service's STUMP while the later service's own variant for it is still +// retrying. Here subtree 0 is covered on arrival, so without the rebuild case +// the builder would skip the window (finalized_complete_no_grace). +func TestBuilder_HandleMessage_RebuildWaitsGraceEvenWhenExpectedSetComplete(t *testing.T) { + f := newMultiSourceFixture(t) + ctx := context.Background() + f.ms.addStump(f.blockHash, 0, f.variantA) + if err := f.b.handleMessage(ctx, makeBlockProcessedMsgWithExpected(f.blockHash, []int{0})); err != nil { + t.Fatalf("first build: %v", err) + } + + const grace = 150 * time.Millisecond + f.b.cfg.BumpBuilder.GraceWindowMs = int(grace / time.Millisecond) + f.ms.addStump(f.blockHash, 0, f.variantB) // new leaves ⇒ rebuild; {0} already covered + waitsBefore := testutil.ToFloat64(metrics.BumpBuilderGraceWaitTotal) + noGraceBefore := bumpOutcomeSampleCount(t, "finalized_complete_no_grace") + start := time.Now() + if err := f.b.handleMessage(ctx, makeBlockProcessedMsgWithExpected(f.blockHash, []int{0})); err != nil { + t.Fatalf("rebuild: %v", err) + } + if elapsed := time.Since(start); elapsed < grace { + t.Fatalf("rebuild returned after %v, before the %v grace window", elapsed, grace) + } + if got := testutil.ToFloat64(metrics.BumpBuilderGraceWaitTotal) - waitsBefore; got != 1 { + t.Fatalf("grace waits = %v, want 1", got) + } + if got := bumpOutcomeSampleCount(t, "finalized_complete_no_grace"); got != noGraceBefore { + t.Fatal("a rebuild must not take the complete-set fast path") + } + if got := f.storedLeafCount(t); got != 4 { + t.Fatalf("rebuilt BUMP leaves = %d, want 4", got) + } +} diff --git a/services/bump_builder/reconciler.go b/services/bump_builder/reconciler.go index 7f76596d..6300e8e0 100644 --- a/services/bump_builder/reconciler.go +++ b/services/bump_builder/reconciler.go @@ -94,7 +94,7 @@ type Reconciler struct { store store.Store publisher events.Publisher chainHeader ChainHeaderReader - merkle *merkleservice.Client // nil ⇒ the /reprocess defer path is disabled + merkle merkleservice.Service // nil ⇒ the /reprocess defer path is disabled leaser store.Leaser // nil ⇒ single-replica mode, no lease gating holderID string @@ -154,7 +154,7 @@ func NewReconciler( st store.Store, publisher events.Publisher, chainHeader ChainHeaderReader, - merkle *merkleservice.Client, + merkle merkleservice.Service, leaser store.Leaser, ) *Reconciler { if !cfg.BumpBuilder.Reconciler.Enabled { diff --git a/services/bump_builder/short_circuit_helper_test.go b/services/bump_builder/short_circuit_helper_test.go new file mode 100644 index 00000000..40b937b5 --- /dev/null +++ b/services/bump_builder/short_circuit_helper_test.go @@ -0,0 +1,21 @@ +package bump_builder + +import ( + "context" + + "go.uber.org/zap" +) + +// tryShortCircuit is the pre-multi-source redelivery entry point, kept for +// the reorg-guard tests that drive the short-circuit path directly. It is +// loadStoredBUMP followed by shortCircuit; production goes through +// handleRedelivery, which also compares the retained STUMPs against the +// stored BUMP. +func (b *Builder) tryShortCircuit(ctx context.Context, logger *zap.Logger, blockHash string) (handled bool, mineFailed bool) { + height, txids, ok, err := b.loadStoredBUMP(ctx, logger, blockHash) + if err != nil || !ok { + return false, false + } + mineFailed, _ = b.shortCircuit(ctx, logger, blockHash, height, txids) + return true, mineFailed +} diff --git a/services/propagation/lifecycle_log_test.go b/services/propagation/lifecycle_log_test.go index f414834e..a5e9f270 100644 --- a/services/propagation/lifecycle_log_test.go +++ b/services/propagation/lifecycle_log_test.go @@ -165,7 +165,7 @@ func TestRegisterBatch_ClaimRevoked_SuppressesPartialFailureWarn(t *testing.T) { core, recorded := observer.New(zapcore.WarnLevel) logger := zap.New(core) - cfg := &config.Config{CallbackURL: "http://arcade/cb", CallbackToken: "tok"} + cfg := &config.Config{CallbackURL: testArcadeCallbackURL, CallbackToken: testArcadeCallbackToken} cfg.Propagation.MerkleConcurrency = 4 mc := merkleservice.NewClient(merkleSrv.URL, "", 5*time.Second) p := New(cfg, logger, nil, nil, newMockStore(), nil, nil, mc) diff --git a/services/propagation/propagator.go b/services/propagation/propagator.go index 92a30968..57f73468 100644 --- a/services/propagation/propagator.go +++ b/services/propagation/propagator.go @@ -112,7 +112,7 @@ type Propagator struct { store store.Store leaser store.Leaser teranodeClient *teranode.Client - merkleClient *merkleservice.Client + merkleClient merkleservice.Service // nil when no merkle_service endpoint is configured // consumer is written by Start (after kafka.NewConsumerGroup returns) and // read by Stop. atomic.Pointer makes that handoff race-free without // needing a mutex: in tests the harness can call Stop concurrently with @@ -208,13 +208,21 @@ type Propagator struct { inflightDepth atomic.Int64 // backgroundWG tracks the long-running Start-spawned goroutines — - // runReaper, runMerkleReplay — so Stop can wait for them to exit + // runReaper, runMerkleReplay, and any resync replay spawned by the + // merkle pool's overflow hook — so Stop can wait for them to exit // before the surrounding app cleanup closes the store backing them. // Without this, a reaper mid-lease-release or mid-status-scan // races with store.Close and the test framework attributes the // resulting goroutine panic to the test's Cleanup callback. backgroundWG sync.WaitGroup + // resync coalesces per-endpoint resync replays (see scheduleResync). + // resyncStopped is set by Stop before it waits on backgroundWG so a late + // hook cannot Add to it. + resyncMu sync.Mutex + resync map[string]*resyncState + resyncStopped bool + // initDone is closed once Start has finished its init phase (all // wg.Add()s done, all goroutines spawned) OR Start has returned with // an error before reaching that point. Stop blocks on it before @@ -371,7 +379,7 @@ const ( // runs unguarded — appropriate for tests and single-process deployments that // don't need coordination. In production every replica should receive a // non-nil Leaser so only one reaper is active at a time across the cluster. -func New(cfg *config.Config, logger *zap.Logger, producer *kafka.Producer, publisher events.Publisher, st store.Store, leaser store.Leaser, tc *teranode.Client, mc *merkleservice.Client) *Propagator { +func New(cfg *config.Config, logger *zap.Logger, producer *kafka.Producer, publisher events.Publisher, st store.Store, leaser store.Leaser, tc *teranode.Client, mc merkleservice.Service) *Propagator { // Export every series this service can emit at 0 from the first scrape — // a series born mid-burst is invisible to increase() until its second // sample, which zeroed the REJECTED count after every rollout. @@ -1171,6 +1179,15 @@ func (p *Propagator) Start(ctx context.Context) error { defer p.backgroundWG.Done() p.runMerkleReplay(ctx) }() + // With several merkle-services the pool re-sends every registration an + // endpoint missed from a bounded per-endpoint queue. If that queue + // overflowed, the dropped registrations are unknown and the endpoint + // needs a full lookback replay (see runResyncReplay). + if rp, ok := p.merkleClient.(merkleservice.Recoverable); ok { + rp.OnResyncNeeded(func(endpoint string) { + p.scheduleResync(ctx, rp, endpoint) + }) + } go func() { defer p.backgroundWG.Done() p.runReaper(ctx) @@ -1277,6 +1294,9 @@ func (p *Propagator) Stop() error { // attributes the failure to t.Cleanup. Their parent ctx has // already been canceled by the caller, so this only blocks on // in-flight scan/release work. + p.resyncMu.Lock() + p.resyncStopped = true + p.resyncMu.Unlock() p.backgroundWG.Wait() return consumerErr } diff --git a/services/propagation/propagator_test.go b/services/propagation/propagator_test.go index a7cb850e..0f8fa137 100644 --- a/services/propagation/propagator_test.go +++ b/services/propagation/propagator_test.go @@ -586,7 +586,7 @@ func newPropagator(merkleSrvURL, teranodeSrvURL string, st store.Store) *Propaga } cfg.Propagation.MerkleConcurrency = 10 - var mc *merkleservice.Client + var mc merkleservice.Service if merkleSrvURL != "" { mc = merkleservice.NewClient(merkleSrvURL, "", 5*time.Second) } @@ -828,7 +828,7 @@ func TestRunMerkleReplay_RegistersOnlyNonTerminal(t *testing.T) { merkleSrv := newMerkleServer(log, http.StatusOK) defer merkleSrv.Close() - cfg := &config.Config{CallbackURL: "http://arcade/cb", CallbackToken: "tok"} + cfg := &config.Config{CallbackURL: testArcadeCallbackURL, CallbackToken: testArcadeCallbackToken} cfg.Propagation.MerkleConcurrency = 4 cfg.Propagation.RegisterReplayLookbackHours = 24 enabled := true @@ -866,7 +866,7 @@ func TestRunMerkleReplay_DisabledByConfig(t *testing.T) { merkleSrv := newMerkleServer(log, http.StatusOK) defer merkleSrv.Close() - cfg := &config.Config{CallbackURL: "http://arcade/cb", CallbackToken: "tok"} + cfg := &config.Config{CallbackURL: testArcadeCallbackURL, CallbackToken: testArcadeCallbackToken} disabled := false cfg.Propagation.RegisterReplayOnStart = &disabled @@ -893,7 +893,7 @@ func TestRegisterBatch_AuthFailureClassifiedAndRequeued(t *testing.T) { defer merkleSrv.Close() ms := newMockStore() - cfg := &config.Config{CallbackURL: "http://arcade/cb", CallbackToken: "tok"} + cfg := &config.Config{CallbackURL: testArcadeCallbackURL, CallbackToken: testArcadeCallbackToken} cfg.Propagation.MerkleConcurrency = 4 mc := merkleservice.NewClient(merkleSrv.URL, "", 5*time.Second) p := New(cfg, zap.NewNop(), nil, nil, ms, nil, nil, mc) @@ -931,7 +931,7 @@ func TestRegisterBatch_ClaimRevokedClassifiedDistinctly(t *testing.T) { defer merkleSrv.Close() ms := newMockStore() - cfg := &config.Config{CallbackURL: "http://arcade/cb", CallbackToken: "tok"} + cfg := &config.Config{CallbackURL: testArcadeCallbackURL, CallbackToken: testArcadeCallbackToken} cfg.Propagation.MerkleConcurrency = 4 mc := merkleservice.NewClient(merkleSrv.URL, "", 5*time.Second) p := New(cfg, zap.NewNop(), nil, nil, ms, nil, nil, mc) @@ -964,7 +964,7 @@ func TestRegisterBatch_ClaimRevokedClassifiedDistinctly(t *testing.T) { // per-test setup boilerplate small. func replayPropagator(t *testing.T, ms *mockStore, merkleURL string, configure func(*config.Config)) *Propagator { t.Helper() - cfg := &config.Config{CallbackURL: "http://arcade/cb", CallbackToken: "tok"} + cfg := &config.Config{CallbackURL: testArcadeCallbackURL, CallbackToken: testArcadeCallbackToken} cfg.Propagation.MerkleConcurrency = 4 cfg.Propagation.RegisterReplayLookbackHours = 24 enabled := true diff --git a/services/propagation/replay.go b/services/propagation/replay.go index ff8e7595..71264263 100644 --- a/services/propagation/replay.go +++ b/services/propagation/replay.go @@ -2,7 +2,6 @@ package propagation import ( "context" - "errors" "time" "go.uber.org/zap" @@ -20,15 +19,52 @@ import ( const defaultReplayLookback = 24 * time.Hour // defaultReplaySkipRecent is the merkle_registered_at recency window used -// when MerkleReplaySkipRecentMinutes isn't set. Matches merkle-service's -// postMineTTLSec (1800s = 30min): if we registered within this window, -// merkle-service almost certainly still has the row. +// when MerkleReplaySkipRecentMinutes isn't set and a single merkle-service is +// configured. Matches merkle-service's postMineTTLSec (1800s = 30min): if we +// registered within this window, merkle-service almost certainly still has +// the row. const defaultReplaySkipRecent = 30 * time.Minute +// resyncRetryBackoff is the first wait before a resync pass whose store scan +// failed is run again; it doubles up to resyncRetryMaxBackoff. Variables so +// tests can shorten them. +var ( + resyncRetryBackoff = 5 * time.Second + resyncRetryMaxBackoff = 5 * time.Minute +) + // defaultReplayRPS caps the average requests-per-second the replay loop // issues against merkle-service when MerkleReplayRPS isn't set. const defaultReplayRPS = 50 +// replayParams parameterizes one replay pass. The startup pass and the +// per-endpoint resync pass differ only in target, label and the +// merkle_registered_at skip. +type replayParams struct { + // target receives the /watch calls: the whole pool on startup, a single + // endpoint on resync. + target merkleservice.Service + // since bounds IterateStatusesSince. + since time.Time + // skipRecent skips rows whose merkle_registered_at is within this + // window; 0 disables the skip. The stamp is pool-wide (set when any + // endpoint accepted), so the resync pass must pass 0: it cannot tell + // which endpoint a recent stamp came from. + skipRecent time.Duration + // label tags the log lines ("startup" or "resync"). + label string + // endpoint is the resynced endpoint's base URL; empty on startup. + endpoint string +} + +// replayLookback is the IterateStatusesSince window for a full replay. +func (p *Propagator) replayLookback() time.Duration { + if lookback := time.Duration(p.cfg.Propagation.RegisterReplayLookbackHours) * time.Hour; lookback > 0 { + return lookback + } + return defaultReplayLookback +} + // runMerkleReplay re-registers every non-terminal tx in the store with // merkle-service /watch. Runs once on startup and exits. // @@ -53,21 +89,17 @@ func (p *Propagator) runMerkleReplay(ctx context.Context) { return } - lookback := time.Duration(p.cfg.Propagation.RegisterReplayLookbackHours) * time.Hour - if lookback <= 0 { - lookback = defaultReplayLookback - } - since := time.Now().Add(-lookback) - - concurrency := p.merkleConcurrency - if concurrency <= 0 { - concurrency = 10 - } // skipRecent: rows registered within this window are skipped. 0 disables // the skip — useful for forcing a full re-sync after a known - // merkle-service wipe (issue #145). + // merkle-service wipe (issue #145). Unset (negative) means no skip with + // several merkle-services: the stamp is set once any endpoint accepted, + // so a recent stamp does not mean every endpoint has the tx, and the + // pool's queue of registrations owed to a lagging endpoint does not + // survive the restart. var skipRecent time.Duration switch m := p.cfg.Propagation.MerkleReplaySkipRecentMinutes; { + case m < 0 && p.multiEndpoint(): + skipRecent = 0 case m < 0: skipRecent = defaultReplaySkipRecent case m == 0: @@ -75,6 +107,53 @@ func (p *Propagator) runMerkleReplay(ctx context.Context) { default: skipRecent = time.Duration(m) * time.Minute } + + _ = p.replayTo(ctx, replayParams{ + target: p.merkleClient, + since: time.Now().Add(-p.replayLookback()), + skipRecent: skipRecent, + label: "startup", + }) +} + +// multiEndpoint reports whether the merkle client fans out to more than one +// merkle-service. +func (p *Propagator) multiEndpoint() bool { + rp, ok := p.merkleClient.(merkleservice.Recoverable) + return ok && len(rp.Endpoints()) > 1 +} + +// runResyncReplay re-registers, with ONE endpoint, every non-terminal tx in +// the lookback window. The pool normally keeps an endpoint's watch set +// complete on its own: a tx another endpoint accepted but this one refused, +// timed out on, or was skipped for (breaker open) sits in the endpoint's +// catch-up queue until it lands. That queue is bounded; when it overflowed, +// the dropped registrations are unknown and only a full replay of the +// in-flight set restores the endpoint's watch set. The target is +// merkleservice.Recoverable.Endpoint, so a /watch that fails during the pass +// goes back to that endpoint's queue instead of being lost. The returned +// error is the store scan's; the caller retries the pass on it. +func (p *Propagator) runResyncReplay(ctx context.Context, rp merkleservice.Recoverable, endpoint string) error { + target := rp.Endpoint(endpoint) + if target == nil || p.cfg.CallbackURL == "" { + return nil + } + return p.replayTo(ctx, replayParams{ + target: target, + since: time.Now().Add(-p.replayLookback()), + label: "resync", + endpoint: endpoint, + }) +} + +// replayTo is the shared body of the startup and recovery replays. It returns +// the store scan's error (nil when the scan completed or ctx ended it); +// per-tx /watch failures are not errors here, see flush. +func (p *Propagator) replayTo(ctx context.Context, rp replayParams) error { + concurrency := p.merkleConcurrency + if concurrency <= 0 { + concurrency = 10 + } // rps: average rate cap on RegisterBatch calls. 0 disables throttling. rps := p.cfg.Propagation.MerkleReplayRPS if rps < 0 { @@ -86,16 +165,21 @@ func (p *Propagator) runMerkleReplay(ctx context.Context) { // overhead amortizes well. const batchSize = 1000 + logger := p.logger.With(zap.String("replay", rp.label)) + if rp.endpoint != "" { + logger = logger.With(zap.String("endpoint", rp.endpoint)) + } + start := time.Now() - p.logger.Info( + logger.Info( "merkle-service replay starting", - zap.Time("since", since), + zap.Time("since", rp.since), zap.Int("concurrency", concurrency), - zap.Duration("skip_recent", skipRecent), + zap.Duration("skip_recent", rp.skipRecent), zap.Int("rps", rps), ) - var scanned, queued, failures, skippedRecent int + var scanned, queued, failures, skippedRecent, skippedOld int var throttled time.Duration batch := make([]merkleservice.Registration, 0, batchSize) now := time.Now() @@ -118,34 +202,57 @@ func (p *Propagator) runMerkleReplay(ctx context.Context) { } } } - if err := p.merkleClient.RegisterBatch(ctx, batch, concurrency); err != nil { - failures += len(batch) - p.logger.Warn( - "merkle-service replay batch failed", - zap.Int("batch_size", len(batch)), - zap.Error(err), - ) - batch = batch[:0] - return + // Per-tx results, not fail-fast: one refused /watch must not drop + // the merkle_registered_at stamp for the rest of the batch, or the + // next boot re-registers them all again. A failed entry is not + // retried here; the pool keeps it in a catch-up queue instead. The + // resync target (one endpoint) queues its own failures. On startup + // the pool only queues the endpoints that missed a tx some other + // endpoint accepted, so a tx no endpoint accepted is handed back + // through Retain, or it would wait for the next restart. + errs := rp.target.RegisterBatchWithResults(ctx, batch, concurrency) + if rp.endpoint == "" { + if r, ok := rp.target.(merkleservice.Recoverable); ok { + r.Retain(batch, errs) + } } - // On success, stamp merkle_registered_at so future replays can skip - // these rows. RegisterBatch is fail-fast, so reaching here means - // every entry succeeded — mark them all in one round-trip. - txids := make([]string, len(batch)) - for i := range batch { - txids[i] = batch[i].TxID + okTxIDs := make([]string, 0, len(batch)) + var sampleErr error + for i, err := range errs { + if err != nil { + failures++ + if sampleErr == nil { + sampleErr = err + } + continue + } + okTxIDs = append(okTxIDs, batch[i].TxID) } - if err := p.store.MarkMerkleRegisteredByTxIDs(ctx, txids, time.Now()); err != nil { - p.logger.Warn( - "merkle-service replay mark failed", - zap.Int("count", len(txids)), - zap.Error(err), + if sampleErr != nil { + logger.Warn( + "merkle-service replay batch partially failed", + zap.Int("batch_size", len(batch)), + zap.Int("failed", len(batch)-len(okTxIDs)), + zap.Error(sampleErr), ) } + // Stamp merkle_registered_at on the rows that landed so future + // startup replays can skip them. The stamp is pool-wide: a resync + // pass stamping it is harmless (the row IS registered with every + // endpoint that is up). + if len(okTxIDs) > 0 { + if err := p.store.MarkMerkleRegisteredByTxIDs(ctx, okTxIDs, time.Now()); err != nil { + logger.Warn( + "merkle-service replay mark failed", + zap.Int("count", len(okTxIDs)), + zap.Error(err), + ) + } + } batch = batch[:0] } - err := p.store.IterateStatusesSince(ctx, since, func(status *models.TransactionStatus) error { + err := p.store.IterateStatusesSince(ctx, rp.since, func(status *models.TransactionStatus) error { scanned++ // Skip rows that are already terminal — re-registering MINED txs is // wasted bandwidth and risks resurrecting watches merkle-service may @@ -156,10 +263,16 @@ func (p *Propagator) runMerkleReplay(ctx context.Context) { if status.TxID == "" { return nil } + // Not every backend honors since (Aerospike scans the whole set), + // so bound the window here too. + if !status.Timestamp.IsZero() && status.Timestamp.Before(rp.since) { + skippedOld++ + return nil + } // Skip rows we registered recently — merkle-service almost certainly // still has them, and POST /watch doesn't refresh expires_at anyway // (issue #145). skipRecent == 0 disables. - if skipRecent > 0 && !status.MerkleRegisteredAt.IsZero() && now.Sub(status.MerkleRegisteredAt) < skipRecent { + if rp.skipRecent > 0 && !status.MerkleRegisteredAt.IsZero() && now.Sub(status.MerkleRegisteredAt) < rp.skipRecent { skippedRecent++ return nil } @@ -183,21 +296,105 @@ func (p *Propagator) runMerkleReplay(ctx context.Context) { }) flush() - if err != nil && !errors.Is(err, ctx.Err()) { - p.logger.Warn( + if err != nil && ctx.Err() != nil { + err = nil // shutdown, not a scan failure + } + if err != nil { + logger.Warn( "merkle-service replay scan ended with error", zap.Error(err), zap.Int("scanned", scanned), zap.Int("queued", queued), ) } - p.logger.Info( + logger.Info( "merkle-service replay complete", zap.Duration("elapsed", time.Since(start)), zap.Int("scanned", scanned), zap.Int("queued", queued), zap.Int("skipped_recent", skippedRecent), + zap.Int("skipped_old", skippedOld), zap.Int("failures", failures), zap.Duration("throttled", throttled), ) + return err +} + +// resyncState coalesces resync requests per endpoint: while a resync for an +// endpoint is running, a second request for the same endpoint only asks the +// running pass to go again, so a flapping endpoint never stacks concurrent +// replays. +type resyncState struct { + running bool + again bool +} + +// scheduleResync is the merkleservice.Recoverable hook. It runs on the +// pool's background goroutine and must not block, so the replay itself is +// spawned under backgroundWG (Stop waits for it) and guarded by +// resyncStopped so a hook firing during shutdown cannot Add to a WaitGroup +// that Stop is already waiting on. +func (p *Propagator) scheduleResync(ctx context.Context, rp merkleservice.Recoverable, endpoint string) { + p.resyncMu.Lock() + defer p.resyncMu.Unlock() + if p.resyncStopped || ctx.Err() != nil { + return + } + if p.resync == nil { + p.resync = make(map[string]*resyncState) + } + st := p.resync[endpoint] + if st == nil { + st = &resyncState{} + p.resync[endpoint] = st + } + if st.running { + st.again = true + return + } + st.running = true + p.backgroundWG.Add(1) + go p.resyncLoop(ctx, rp, endpoint, st) +} + +// resyncLoop runs resync passes for one endpoint until no further request +// arrived while the last pass was running. A pass whose store scan failed is +// run again after a backoff: the request it served is not consumed until a +// scan completes, since the endpoint is still missing whatever the +// overflowed queue dropped. +func (p *Propagator) resyncLoop(ctx context.Context, rp merkleservice.Recoverable, endpoint string, st *resyncState) { + defer p.backgroundWG.Done() + backoff := resyncRetryBackoff + for { + p.resyncMu.Lock() + st.again = false + p.resyncMu.Unlock() + + if err := p.runResyncReplay(ctx, rp, endpoint); err != nil { + p.logger.Warn( + "merkle-service resync scan failed; retrying", + zap.String("endpoint", endpoint), + zap.Duration("backoff", backoff), + zap.Error(err), + ) + select { + case <-ctx.Done(): + case <-time.After(backoff): + } + backoff = min(backoff*2, resyncRetryMaxBackoff) + p.resyncMu.Lock() + st.again = true + p.resyncMu.Unlock() + } else { + backoff = resyncRetryBackoff + } + + p.resyncMu.Lock() + if !st.again || ctx.Err() != nil || p.resyncStopped { + st.running = false + p.resyncMu.Unlock() + return + } + p.resyncMu.Unlock() + } } diff --git a/services/propagation/replay_multi_test.go b/services/propagation/replay_multi_test.go new file mode 100644 index 00000000..9cb13037 --- /dev/null +++ b/services/propagation/replay_multi_test.go @@ -0,0 +1,292 @@ +package propagation + +import ( + "context" + "errors" + "net/http" + "sync/atomic" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "go.uber.org/zap" + + "github.com/bsv-blockchain/arcade/config" + "github.com/bsv-blockchain/arcade/merkleservice" + "github.com/bsv-blockchain/arcade/metrics" + "github.com/bsv-blockchain/arcade/models" +) + +// testArcadeCallbackURL and testArcadeCallbackToken are the callback_url and +// callback_token the propagation tests configure. +const ( + testArcadeCallbackURL = "http://arcade/cb" + testArcadeCallbackToken = "tok" +) + +// A tx is registered once any endpoint accepts it: with one merkle-service +// unreachable the batch still lands as fully_ok and every tx is broadcast +// (F-024 is satisfied by the surviving endpoint). +func TestRegisterBatch_PoolWithOneDeadEndpoint_FullyOK(t *testing.T) { + log := &eventLog{} + live := newMerkleServer(log, http.StatusOK) + defer live.Close() + + pool := merkleservice.NewPool([]string{live.URL, "http://127.0.0.1:1"}, "", 2*time.Second) + cfg := &config.Config{CallbackURL: testArcadeCallbackURL, CallbackToken: testArcadeCallbackToken} + cfg.Propagation.MerkleConcurrency = 4 + ms := newMockStore() + p := New(cfg, zap.NewNop(), nil, nil, ms, nil, nil, pool) + + before := testutil.ToFloat64(metrics.PropagationMerkleRegisterBatchOutcomeTotal.WithLabelValues("fully_ok")) + batch := []propagationMsg{{TXID: "tx-a"}, {TXID: "tx-b"}} + registered, failed := p.registerBatch(context.Background(), batch) + if len(registered) != 2 || len(failed) != 0 { + t.Fatalf("registered=%d failed=%d want 2/0", len(registered), len(failed)) + } + if got := testutil.ToFloat64(metrics.PropagationMerkleRegisterBatchOutcomeTotal.WithLabelValues("fully_ok")) - before; got != 1 { + t.Fatalf("fully_ok delta=%v want 1", got) + } + if log.count("register:") != 2 { + t.Fatalf("live endpoint registrations=%d want 2", log.count("register:")) + } + ms.mu.Lock() + defer ms.mu.Unlock() + if len(ms.merkleMarks) != 1 || len(ms.merkleMarks[0]) != 2 { + t.Fatalf("merkle_registered_at must be stamped for both txs, marks=%v", ms.merkleMarks) + } + // The dead endpoint still owes both registrations; the pool re-sends + // them once it answers again. + for _, st := range pool.EndpointStatuses() { + if st.URL == live.URL && st.CatchupPending != 0 { + t.Fatalf("live endpoint must owe nothing, got %+v", st) + } + if st.URL != live.URL && st.CatchupPending != 2 { + t.Fatalf("dead endpoint must owe both txs, got %+v", st) + } + } +} + +// When an endpoint's catch-up queue overflowed, the resync replay +// re-registers the whole lookback window with THAT endpoint only, ignoring +// merkle_registered_at (the stamp is pool-wide and cannot say which endpoint +// missed the row). +func TestRunResyncReplay_TargetsOnlyThatEndpoint(t *testing.T) { + logA, logB := &eventLog{}, &eventLog{} + srvA := newMerkleServer(logA, http.StatusOK) + defer srvA.Close() + srvB := newMerkleServer(logB, http.StatusOK) + defer srvB.Close() + + pool := merkleservice.NewPool([]string{srvA.URL, srvB.URL}, "", 2*time.Second) + cfg := &config.Config{CallbackURL: testArcadeCallbackURL, CallbackToken: testArcadeCallbackToken} + cfg.Propagation.MerkleConcurrency = 4 + ms := newMockStore() + ms.replayRows = []*models.TransactionStatus{ + {TxID: "tx-recent", Status: models.StatusSeenOnNetwork, MerkleRegisteredAt: time.Now()}, + {TxID: "tx-older", Status: models.StatusAcceptedByNetwork}, + {TxID: "tx-mined", Status: models.StatusMined}, // terminal: skipped + } + p := New(cfg, zap.NewNop(), nil, nil, ms, nil, nil, pool) + + if err := p.runResyncReplay(context.Background(), pool, srvB.URL); err != nil { + t.Fatalf("resync: %v", err) + } + + if n := logB.count("register:"); n != 2 { + t.Fatalf("resynced endpoint registrations=%d want 2 (recent stamp must NOT skip)", n) + } + if n := logA.count("register:"); n != 0 { + t.Fatalf("other endpoint must not be re-registered, got %d", n) + } +} + +// scheduleResync coalesces: a second request for the same endpoint while a +// pass is running triggers exactly one more pass, and nothing runs after +// Stop flagged the propagator. +func TestScheduleResync_CoalescesAndHonorsStop(t *testing.T) { + logB := &eventLog{} + srvB := newMerkleServer(logB, http.StatusOK) + defer srvB.Close() + logA := &eventLog{} + srvA := newMerkleServer(logA, http.StatusOK) + defer srvA.Close() + + pool := merkleservice.NewPool([]string{srvA.URL, srvB.URL}, "", 2*time.Second) + cfg := &config.Config{CallbackURL: testArcadeCallbackURL, CallbackToken: testArcadeCallbackToken} + ms := newMockStore() + ms.replayRows = []*models.TransactionStatus{{TxID: "tx-1", Status: models.StatusSeenOnNetwork}} + p := New(cfg, zap.NewNop(), nil, nil, ms, nil, nil, pool) + + ctx := context.Background() + p.scheduleResync(ctx, pool, srvB.URL) + p.scheduleResync(ctx, pool, srvB.URL) + p.scheduleResync(ctx, pool, srvB.URL) + p.backgroundWG.Wait() + + // At most two passes (the running one, plus one coalesced rerun), never + // three; at least one. + if n := logB.count("register:"); n < 1 || n > 2 { + t.Fatalf("resync passes registered tx-1 %d times, want 1..2", n) + } + + p.resyncMu.Lock() + p.resyncStopped = true + p.resyncMu.Unlock() + before := logB.count("register:") + p.scheduleResync(ctx, pool, srvB.URL) + p.backgroundWG.Wait() + if logB.count("register:") != before { + t.Fatal("no resync pass may start after Stop") + } +} + +// With several merkle-services an unset skip window (-1) means no skip: the +// stamp is set once any endpoint accepted, and the queue of registrations +// owed to a lagging endpoint did not survive the restart, so every +// non-terminal row must go to every endpoint again. One merkle-service keeps +// the 30-minute default. +func TestRunMerkleReplay_DefaultSkipDependsOnEndpointCount(t *testing.T) { + rows := func() []*models.TransactionStatus { + return []*models.TransactionStatus{ + {TxID: "tx-recent", Status: models.StatusSeenOnNetwork, MerkleRegisteredAt: time.Now().Add(-time.Minute)}, + {TxID: "tx-unstamped", Status: models.StatusAcceptedByNetwork}, + } + } + + t.Run("several endpoints", func(t *testing.T) { + logA, logB := &eventLog{}, &eventLog{} + srvA := newMerkleServer(logA, http.StatusOK) + defer srvA.Close() + srvB := newMerkleServer(logB, http.StatusOK) + defer srvB.Close() + pool := merkleservice.NewPool([]string{srvA.URL, srvB.URL}, "", 2*time.Second) + cfg := &config.Config{CallbackURL: testArcadeCallbackURL, CallbackToken: testArcadeCallbackToken} + cfg.Propagation.MerkleReplaySkipRecentMinutes = -1 + ms := newMockStore() + ms.replayRows = rows() + New(cfg, zap.NewNop(), nil, nil, ms, nil, nil, pool).runMerkleReplay(context.Background()) + + if a, b := logA.count("register:"), logB.count("register:"); a != 2 || b != 2 { + t.Fatalf("registrations a=%d b=%d, want 2 each (recent stamp must not skip)", a, b) + } + }) + + t.Run("one endpoint", func(t *testing.T) { + log := &eventLog{} + srv := newMerkleServer(log, http.StatusOK) + defer srv.Close() + ms := newMockStore() + ms.replayRows = rows() + p := replayPropagator(t, ms, srv.URL, func(cfg *config.Config) { + cfg.Propagation.MerkleReplaySkipRecentMinutes = -1 + }) + p.runMerkleReplay(context.Background()) + + if got := log.count("register:"); got != 1 { + t.Fatalf("registrations=%d, want 1 (recent stamp skipped by the 30m default)", got) + } + }) +} + +// sinceIgnoringStore returns every replay row regardless of since, as the +// Aerospike backend does. +type sinceIgnoringStore struct{ *mockStore } + +func (s sinceIgnoringStore) IterateStatusesSince(ctx context.Context, _ time.Time, fn func(*models.TransactionStatus) error) error { + return s.mockStore.IterateStatusesSince(ctx, time.Time{}, fn) +} + +// The replay bounds the lookback window itself, so a backend that ignores +// since does not turn a resync into a re-registration of the whole history. +func TestReplay_BoundsWindowWhenStoreIgnoresSince(t *testing.T) { + log := &eventLog{} + srv := newMerkleServer(log, http.StatusOK) + defer srv.Close() + ms := newMockStore() + ms.replayRows = []*models.TransactionStatus{ + {TxID: "tx-in-window", Status: models.StatusSeenOnNetwork, Timestamp: time.Now().Add(-time.Hour)}, + {TxID: "tx-ancient", Status: models.StatusSeenOnNetwork, Timestamp: time.Now().Add(-30 * 24 * time.Hour)}, + } + cfg := &config.Config{CallbackURL: testArcadeCallbackURL, CallbackToken: testArcadeCallbackToken} + cfg.Propagation.MerkleReplaySkipRecentMinutes = 0 + p := New(cfg, zap.NewNop(), nil, nil, sinceIgnoringStore{ms}, nil, nil, merkleservice.NewClient(srv.URL, "", 2*time.Second)) + p.runMerkleReplay(context.Background()) + + if got := log.count("register:"); got != 1 { + t.Fatalf("registrations=%d, want 1 (row older than the lookback must be skipped)", got) + } +} + +// flakyScanStore fails the first failures IterateStatusesSince calls. +type flakyScanStore struct { + *mockStore + + failures atomic.Int32 +} + +func (s *flakyScanStore) IterateStatusesSince(ctx context.Context, since time.Time, fn func(*models.TransactionStatus) error) error { + if s.failures.Add(-1) >= 0 { + return errors.New("store unavailable") + } + return s.mockStore.IterateStatusesSince(ctx, since, fn) +} + +// A resync pass whose store scan failed is not consumed: it runs again after +// a backoff until a scan completes. +func TestResyncLoop_RetriesFailedScan(t *testing.T) { + prev := resyncRetryBackoff + resyncRetryBackoff = 10 * time.Millisecond + t.Cleanup(func() { resyncRetryBackoff = prev }) + + logA, logB := &eventLog{}, &eventLog{} + srvA := newMerkleServer(logA, http.StatusOK) + defer srvA.Close() + srvB := newMerkleServer(logB, http.StatusOK) + defer srvB.Close() + pool := merkleservice.NewPool([]string{srvA.URL, srvB.URL}, "", 2*time.Second) + cfg := &config.Config{CallbackURL: testArcadeCallbackURL, CallbackToken: testArcadeCallbackToken} + ms := newMockStore() + ms.replayRows = []*models.TransactionStatus{{TxID: "tx-1", Status: models.StatusSeenOnNetwork}} + st := &flakyScanStore{mockStore: ms} + st.failures.Store(2) + p := New(cfg, zap.NewNop(), nil, nil, st, nil, nil, pool) + + p.scheduleResync(context.Background(), pool, srvB.URL) + p.backgroundWG.Wait() + + if n := logB.count("register:"); n != 1 { + t.Fatalf("resynced endpoint registrations=%d, want 1 after two failed scans", n) + } +} + +// A startup-replay tx that no endpoint accepted is handed back to the pool, +// which queues it for every endpoint and re-sends it once they answer, +// instead of dropping it until the next restart. It is not stamped +// merkle_registered_at: it has not landed anywhere yet. +func TestRunMerkleReplay_RetainsTxsEveryEndpointRefused(t *testing.T) { + logA, logB := &eventLog{}, &eventLog{} + srvA := newMerkleServer(logA, http.StatusServiceUnavailable) + defer srvA.Close() + srvB := newMerkleServer(logB, http.StatusServiceUnavailable) + defer srvB.Close() + pool := merkleservice.NewPool([]string{srvA.URL, srvB.URL}, "", 2*time.Second, merkleservice.WithFailureThreshold(100)) + cfg := &config.Config{CallbackURL: testArcadeCallbackURL, CallbackToken: testArcadeCallbackToken} + ms := newMockStore() + ms.replayRows = []*models.TransactionStatus{ + {TxID: "tx-a", Status: models.StatusSeenOnNetwork}, + {TxID: "tx-b", Status: models.StatusAcceptedByNetwork}, + } + New(cfg, zap.NewNop(), nil, nil, ms, nil, nil, pool).runMerkleReplay(context.Background()) + + for _, st := range pool.EndpointStatuses() { + if st.CatchupPending != 2 { + t.Fatalf("endpoint %s must owe both refused txs, got %+v", st.URL, st) + } + } + ms.mu.Lock() + defer ms.mu.Unlock() + if len(ms.merkleMarks) != 0 { + t.Fatalf("refused txs must not be stamped merkle_registered_at, marks=%v", ms.merkleMarks) + } +} diff --git a/services/watchdog/service.go b/services/watchdog/service.go index 03f13713..5cb1f49f 100644 --- a/services/watchdog/service.go +++ b/services/watchdog/service.go @@ -34,11 +34,11 @@ func NewService( logger *zap.Logger, st store.Store, leaser store.Leaser, - mc *merkleservice.Client, + mc merkleservice.Service, ) *Service { if mc == nil || leaser == nil { // Defensive: app.BuildServices already gates this, but a misconfig - // (merkle_service.url unset) should produce a no-op Service rather + // (no merkle_service endpoint) should produce a no-op Service rather // than a nil-deref at the first tick. return nil } diff --git a/services/watchdog/watchdog.go b/services/watchdog/watchdog.go index b867c383..213b6f3e 100644 --- a/services/watchdog/watchdog.go +++ b/services/watchdog/watchdog.go @@ -107,7 +107,7 @@ type Config struct { type Watchdog struct { store store.Store leaser store.Leaser - merkleClient *merkleservice.Client + merkleClient merkleservice.Service cfg Config callbackURL string callbackTok string @@ -132,7 +132,7 @@ type Watchdog struct { func New( st store.Store, leaser store.Leaser, - mc *merkleservice.Client, + mc merkleservice.Service, callbackURL, callbackToken string, cfg Config, logger *zap.Logger, diff --git a/services/watchdog/watchdog_test.go b/services/watchdog/watchdog_test.go index 8fe2c0d9..c37545a5 100644 --- a/services/watchdog/watchdog_test.go +++ b/services/watchdog/watchdog_test.go @@ -218,7 +218,7 @@ func (r *recordingReprocessServer) capturedCalls() []reprocessCall { // --- tests --- -func newTestWatchdog(t *testing.T, st *watchdogStore, leaser *watchdogLeaser, mc *merkleservice.Client) *Watchdog { +func newTestWatchdog(t *testing.T, st *watchdogStore, leaser *watchdogLeaser, mc merkleservice.Service) *Watchdog { t.Helper() cfg := Config{ Interval: 10 * time.Millisecond, diff --git a/store/aerospike/aerospike.go b/store/aerospike/aerospike.go index e207aafa..d8f6dbad 100644 --- a/store/aerospike/aerospike.go +++ b/store/aerospike/aerospike.go @@ -65,8 +65,13 @@ const ( // the manifest/chunk layout. The chunk size is shared with BUMP (bumpChunkSize) // since the same namespace write-block-size bounds both record types. const ( - stumpFormatVersion = 1 - stumpChunkKeyFormat = "%s:%d:c:%08d" + stumpFormatVersion = 1 + // stumpChunkKeyFormat is the chunk key of a content-addressed STUMP + // (block:subtree:content_hash:c:index); stumpChunkKeyFormatLegacy is the + // pre-content-addressing layout, still readable so a STUMP written before + // the upgrade can be assembled and deleted. + stumpChunkKeyFormat = "%s:%d:%s:c:%08d" + stumpChunkKeyFormatLegacy = "%s:%d:c:%08d" ) // Ensure Store implements the store interfaces. @@ -1659,15 +1664,24 @@ func (s *Store) DeleteBUMPByBlockHash(ctx context.Context, blockHash string) err // which is the linearization point a concurrent reader keys off. // stumpManifestKey builds the primary key for a STUMP manifest record. -func (s *Store) stumpManifestKey(blockHash string, subtreeIndex int) (*aero.Key, error) { - return s.key(setStumps, fmt.Sprintf("%s:%d", blockHash, subtreeIndex)) +// stumpManifestKey is content-addressed (see models.Stump); an empty +// contentHash selects the legacy (block:subtree) key for rows written before +// the upgrade. +func (s *Store) stumpManifestKey(blockHash string, subtreeIndex int, contentHash string) (*aero.Key, error) { + if contentHash == "" { + return s.key(setStumps, fmt.Sprintf("%s:%d", blockHash, subtreeIndex)) + } + return s.key(setStumps, fmt.Sprintf("%s:%d:%s", blockHash, subtreeIndex, contentHash)) } // stumpChunkKey builds the primary key for one chunk of a STUMP. The index is // zero-padded so chunk keys sort lexicographically and never collide with a // manifest key (which has no ":c:" segment). -func (s *Store) stumpChunkKey(blockHash string, subtreeIndex, idx int) (*aero.Key, error) { - return s.key(setStumps, fmt.Sprintf(stumpChunkKeyFormat, blockHash, subtreeIndex, idx)) +func (s *Store) stumpChunkKey(blockHash string, subtreeIndex int, contentHash string, idx int) (*aero.Key, error) { + if contentHash == "" { + return s.key(setStumps, fmt.Sprintf(stumpChunkKeyFormatLegacy, blockHash, subtreeIndex, idx)) + } + return s.key(setStumps, fmt.Sprintf(stumpChunkKeyFormat, blockHash, subtreeIndex, contentHash, idx)) } // InsertStump stores a STUMP as a manifest plus chunk records. Chunks are @@ -1678,52 +1692,41 @@ func (s *Store) InsertStump(ctx context.Context, stump *models.Stump) error { return fmt.Errorf("insert stump: empty block hash") } - // Read the previous manifest's chunk count so we can clean up orphan - // chunks if this rewrite produces fewer of them. Failure here is non-fatal - // — without it we skip cleanup and orphans only waste disk. - oldChunkCount := 0 - if oldKey, err := s.stumpManifestKey(stump.BlockHash, stump.SubtreeIndex); err == nil { - if rec, err := s.client.Get(s.readPolicy(ctx), oldKey, "chunk_count"); err == nil && rec != nil { - if v, ok := rec.Bins["chunk_count"].(int); ok && v > 0 { - oldChunkCount = v - } - } + hash := stump.EnsureContentHash() + manifestKey, err := s.stumpManifestKey(stump.BlockHash, stump.SubtreeIndex, hash) + if err != nil { + return err + } + // Content-addressed: a manifest under this key already references + // exactly these bytes, so a redelivery from another merkle-service is + // one read instead of a chunk rewrite. A key's content never changes, + // which is also why no orphan-chunk cleanup is needed here any more. + if exists, err := s.client.Exists(s.readPolicy(ctx), manifestKey); err == nil && exists { + return nil } - newChunkCount := 1 + chunkCount := 1 if len(stump.StumpData) > s.bumpChunkSize { - newChunkCount = (len(stump.StumpData) + s.bumpChunkSize - 1) / s.bumpChunkSize + chunkCount = (len(stump.StumpData) + s.bumpChunkSize - 1) / s.bumpChunkSize } - if err := s.writeStumpChunks(ctx, stump.BlockHash, stump.SubtreeIndex, stump.StumpData, newChunkCount); err != nil { + if err := s.writeStumpChunks(ctx, stump.BlockHash, stump.SubtreeIndex, hash, stump.StumpData, chunkCount); err != nil { return fmt.Errorf("write stump chunks for %s:%d: %w", stump.BlockHash, stump.SubtreeIndex, err) } - manifestKey, err := s.stumpManifestKey(stump.BlockHash, stump.SubtreeIndex) - if err != nil { - return err - } wp := s.writePolicy(ctx) wp.RecordExistsAction = aero.REPLACE bins := aero.BinMap{ "block_hash": stump.BlockHash, "subtree_index": stump.SubtreeIndex, - "chunk_count": newChunkCount, + "content_hash": hash, + "chunk_count": chunkCount, "total_size": len(stump.StumpData), "format_version": stumpFormatVersion, } if err := s.client.Put(wp, manifestKey, bins); err != nil { return fmt.Errorf("write stump manifest for %s:%d: %w", stump.BlockHash, stump.SubtreeIndex, err) } - - // Best-effort cleanup of orphan chunks left by a prior larger write for - // the same (blockHash, subtreeIndex). They are unreferenced now that the - // manifest caps at newChunkCount, so a failure here only wastes disk. - if oldChunkCount > newChunkCount { - if err := s.deleteStumpChunkRange(ctx, stump.BlockHash, stump.SubtreeIndex, newChunkCount, oldChunkCount); err != nil { - _ = err - } - } return nil } @@ -1732,6 +1735,7 @@ func (s *Store) InsertStump(ctx context.Context, stump *models.Stump) error { // DeleteStumpsByBlockHash. type stumpManifest struct { subtreeIndex int + contentHash string // empty on manifests written before content addressing chunkCount int totalSize int } @@ -1776,6 +1780,9 @@ loop: if v, ok := rec.Record.Bins["subtree_index"].(int); ok { m.subtreeIndex = v } + if v, ok := rec.Record.Bins["content_hash"].(string); ok { + m.contentHash = v + } if v, ok := rec.Record.Bins["chunk_count"].(int); ok { m.chunkCount = v } @@ -1808,17 +1815,29 @@ func (s *Store) GetStumpsByBlockHash(ctx context.Context, blockHash string) ([]* return nil, err } + // A secondary-index query returns records in no particular order; the + // store contract is subtree-index order, with variants of one subtree + // ordered by content hash so repeated reads agree. + sort.Slice(manifests, func(i, j int) bool { + if manifests[i].subtreeIndex != manifests[j].subtreeIndex { + return manifests[i].subtreeIndex < manifests[j].subtreeIndex + } + return manifests[i].contentHash < manifests[j].contentHash + }) stumps := make([]*models.Stump, 0, len(manifests)) for _, m := range manifests { - data, err := s.readStumpChunks(ctx, blockHash, m.subtreeIndex, m.chunkCount, m.totalSize) + data, err := s.readStumpChunks(ctx, blockHash, m.subtreeIndex, m.contentHash, m.chunkCount, m.totalSize) if err != nil { return nil, fmt.Errorf("get stumps %s: subtree %d: %w", blockHash, m.subtreeIndex, err) } - stumps = append(stumps, &models.Stump{ + st := &models.Stump{ BlockHash: blockHash, SubtreeIndex: m.subtreeIndex, StumpData: data, - }) + ContentHash: m.contentHash, + } + st.EnsureContentHash() + stumps = append(stumps, st) } return stumps, nil } @@ -1826,7 +1845,7 @@ func (s *Store) GetStumpsByBlockHash(ctx context.Context, blockHash string) ([]* // writeStumpChunks slices stumpData into chunkCount chunk records under the // (blockHash, subtreeIndex) namespace and writes them with REPLACE semantics // so stale state from a previously failed write at the same key is dropped. -func (s *Store) writeStumpChunks(ctx context.Context, blockHash string, subtreeIndex int, stumpData []byte, chunkCount int) error { +func (s *Store) writeStumpChunks(ctx context.Context, blockHash string, subtreeIndex int, contentHash string, stumpData []byte, chunkCount int) error { bwp := aero.NewBatchWritePolicy() bwp.RecordExistsAction = aero.REPLACE @@ -1846,7 +1865,7 @@ func (s *Store) writeStumpChunks(ctx context.Context, blockHash string, subtreeI if tail > len(stumpData) { tail = len(stumpData) } - key, err := s.stumpChunkKey(blockHash, subtreeIndex, i) + key, err := s.stumpChunkKey(blockHash, subtreeIndex, contentHash, i) if err != nil { return err } @@ -1874,7 +1893,7 @@ func (s *Store) writeStumpChunks(ctx context.Context, blockHash string, subtreeI // readStumpChunks batch-reads chunkCount chunk records and assembles them into // a single byte slice of length totalSize, rejecting any missing chunk, index // mismatch, or length overflow. -func (s *Store) readStumpChunks(ctx context.Context, blockHash string, subtreeIndex, chunkCount, totalSize int) ([]byte, error) { +func (s *Store) readStumpChunks(ctx context.Context, blockHash string, subtreeIndex int, contentHash string, chunkCount, totalSize int) ([]byte, error) { out := make([]byte, totalSize) written := 0 @@ -1889,7 +1908,7 @@ func (s *Store) readStumpChunks(ctx context.Context, blockHash string, subtreeIn keys := make([]*aero.Key, 0, end-start) for i := start; i < end; i++ { - key, err := s.stumpChunkKey(blockHash, subtreeIndex, i) + key, err := s.stumpChunkKey(blockHash, subtreeIndex, contentHash, i) if err != nil { return nil, err } @@ -1934,7 +1953,7 @@ func (s *Store) readStumpChunks(ctx context.Context, blockHash string, subtreeIn // deleteStumpChunkRange batch-deletes chunk records at indices [fromIdx, // toExcl) for one (blockHash, subtreeIndex) STUMP. -func (s *Store) deleteStumpChunkRange(ctx context.Context, blockHash string, subtreeIndex, fromIdx, toExcl int) error { +func (s *Store) deleteStumpChunkRange(ctx context.Context, blockHash string, subtreeIndex int, contentHash string, fromIdx, toExcl int) error { if toExcl <= fromIdx { return nil } @@ -1948,7 +1967,7 @@ func (s *Store) deleteStumpChunkRange(ctx context.Context, blockHash string, sub } records := make([]aero.BatchRecordIfc, 0, end-start) for i := start; i < end; i++ { - key, err := s.stumpChunkKey(blockHash, subtreeIndex, i) + key, err := s.stumpChunkKey(blockHash, subtreeIndex, contentHash, i) if err != nil { return err } @@ -1973,14 +1992,14 @@ func (s *Store) DeleteStumpsByBlockHash(ctx context.Context, blockHash string) e } for _, m := range manifests { - manifestKey, err := s.stumpManifestKey(blockHash, m.subtreeIndex) + manifestKey, err := s.stumpManifestKey(blockHash, m.subtreeIndex, m.contentHash) if err != nil { return err } if _, err := s.client.Delete(s.writePolicy(ctx), manifestKey); err != nil { return fmt.Errorf("delete stump manifest %s:%d: %w", blockHash, m.subtreeIndex, err) } - if err := s.deleteStumpChunkRange(ctx, blockHash, m.subtreeIndex, 0, m.chunkCount); err != nil { + if err := s.deleteStumpChunkRange(ctx, blockHash, m.subtreeIndex, m.contentHash, 0, m.chunkCount); err != nil { return fmt.Errorf("delete stump chunks %s:%d: %w", blockHash, m.subtreeIndex, err) } } @@ -2379,6 +2398,10 @@ const ( binStatus = "status" binOrphanedAt = "orphaned_at" binReconciledAt = "reconciled_at" + // binExpectedSubtrees holds the retained expected-subtree set + // (store.Store.AddBlockExpectedSubtrees) as an ordered unique list. + // Bin names are capped at 15 bytes. + binExpectedSubtrees = "exp_subtrees" ) func (s *Store) UpsertBlockHeaderSeen(ctx context.Context, blockHash string, blockHeight uint64, seenAt time.Time) error { @@ -2423,6 +2446,75 @@ func (s *Store) UpsertBlockHeaderSeen(ctx context.Context, blockHash string, blo return nil } +// AddBlockExpectedSubtrees implements store.Store with an ordered, unique +// list append, so the union is applied atomically on the server. UPDATE_ONLY +// keeps a missing row a no-op. +func (s *Store) AddBlockExpectedSubtrees(ctx context.Context, blockHash string, indices []int) error { + if len(indices) == 0 { + return nil + } + key, err := s.key(setBlockProcessing, blockHash) + if err != nil { + return err + } + vals := make([]interface{}, len(indices)) + for i, v := range indices { + vals[i] = v + } + policy := aero.NewListPolicy(aero.ListOrderOrdered, aero.ListWriteFlagsAddUnique|aero.ListWriteFlagsNoFail|aero.ListWriteFlagsPartial) + wp := s.writePolicy(ctx) + wp.RecordExistsAction = aero.UPDATE_ONLY + _, err = s.client.Operate(wp, key, aero.ListAppendWithPolicyOp(policy, binExpectedSubtrees, vals...)) + if err != nil && !isKeyNotFound(err) { + return fmt.Errorf("add block expected subtrees %s: %w", blockHash, err) + } + return nil +} + +// GetBlockExpectedSubtrees implements store.Store. +func (s *Store) GetBlockExpectedSubtrees(ctx context.Context, blockHash string) ([]int, error) { + key, err := s.key(setBlockProcessing, blockHash) + if err != nil { + return nil, err + } + rec, err := s.client.Get(s.readPolicy(ctx), key, binExpectedSubtrees) + if err != nil { + if isKeyNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("get block expected subtrees %s: %w", blockHash, err) + } + raw, _ := rec.Bins[binExpectedSubtrees].([]interface{}) + if len(raw) == 0 { + return nil, nil + } + out := make([]int, 0, len(raw)) + for _, v := range raw { + switch n := v.(type) { + case int: + out = append(out, n) + case int64: + out = append(out, int(n)) + } + } + return out, nil +} + +// ClearBlockExpectedSubtrees implements store.Store. +func (s *Store) ClearBlockExpectedSubtrees(ctx context.Context, blockHash string) error { + key, err := s.key(setBlockProcessing, blockHash) + if err != nil { + return err + } + wp := s.writePolicy(ctx) + wp.RecordExistsAction = aero.UPDATE_ONLY + _, err = s.client.Operate(wp, key, aero.PutOp(aero.NewBin(binExpectedSubtrees, nil))) + if err != nil && !isKeyNotFound(err) { + return fmt.Errorf("clear block expected subtrees %s: %w", blockHash, err) + } + return nil +} + func (s *Store) MarkBlockProcessed(ctx context.Context, blockHash string, blockHeight uint64, processedAt time.Time) error { return s.markBlockMilestone(ctx, blockHash, blockHeight, processedAt, binProcessedAt) } @@ -2431,6 +2523,22 @@ func (s *Store) MarkBlockBUMPBuilt(ctx context.Context, blockHash string, blockH return s.markBlockMilestone(ctx, blockHash, blockHeight, builtAt, binBUMPBuiltAt) } +// ClearBlockProcessed drops the processed_at bin on an existing row (a nil +// bin value deletes the bin); a missing row is a no-op, never an upsert. +func (s *Store) ClearBlockProcessed(ctx context.Context, blockHash string) error { + key, err := s.key(setBlockProcessing, blockHash) + if err != nil { + return err + } + wp := s.writePolicy(ctx) + wp.RecordExistsAction = aero.UPDATE_ONLY + _, err = s.client.Operate(wp, key, aero.PutOp(aero.NewBin(binProcessedAt, nil))) + if err != nil && !isKeyNotFound(err) { + return fmt.Errorf("clear block processed %s: %w", blockHash, err) + } + return nil +} + // markBlockMilestone is the shared upsert path for processed_at and // bump_built_at. When the row exists, only the milestone bin is written // (block_height and other bins are preserved). When it doesn't, the row diff --git a/store/aerospike/conformance_test.go b/store/aerospike/conformance_test.go index 44a2a258..656fd4a0 100644 --- a/store/aerospike/conformance_test.go +++ b/store/aerospike/conformance_test.go @@ -31,6 +31,15 @@ func TestBlockStatus_Conformance(t *testing.T) { }) } +// STUMP rows are content-addressed so several merkle-services can deliver +// the same subtree: identical bytes collapse, divergent variants coexist. +func TestStumps_Conformance(t *testing.T) { + storetest.RunStumpSuite(t, func(t *testing.T) storetest.StumpBackend { + t.Helper() + return integrationStore(t) + }) +} + // A double spend is discovered after the row exists, so the competing // spender must persist through the status-update paths, not only on insert. func TestCompetingTxs_Conformance(t *testing.T) { diff --git a/store/aerospike/stump_chunk_test.go b/store/aerospike/stump_chunk_test.go index 5e283b08..c12982bf 100644 --- a/store/aerospike/stump_chunk_test.go +++ b/store/aerospike/stump_chunk_test.go @@ -13,8 +13,10 @@ import ( // TestInsertGetStump_Chunked exercises the manifest+chunk STUMP layout: a STUMP // larger than a single Aerospike record (the case that previously failed with -// RECORD_TOO_BIG) must round-trip byte-identical, alongside a small one, and a -// shrinking rewrite must not leave stale chunk data behind. +// RECORD_TOO_BIG) must round-trip byte-identical, alongside a small one. Rows +// are content-addressed, so a second, different STUMP for the same subtree is +// a second variant with its own chunks (never a rewrite of the first), an +// identical re-insert is a no-op, and the block delete removes every variant. func TestInsertGetStump_Chunked(t *testing.T) { s := integrationStore(t) ctx := context.Background() @@ -58,19 +60,34 @@ func TestInsertGetStump_Chunked(t *testing.T) { } } - // Rewrite the large STUMP smaller — orphan chunks from the first write - // must be cleaned up so the reassembled payload is exactly the new data. + // An identical re-insert of the large STUMP is a no-op; a different + // STUMP for subtree 0 is a second variant. Both must read back + // byte-identical: the big one's chunks are not disturbed by the small + // variant's write. + if err := s.InsertStump(ctx, &models.Stump{BlockHash: blockHash, SubtreeIndex: 0, StumpData: big}); err != nil { + t.Fatalf("InsertStump identical re-insert: %v", err) + } if err := s.InsertStump(ctx, &models.Stump{BlockHash: blockHash, SubtreeIndex: 0, StumpData: small}); err != nil { - t.Fatalf("InsertStump rewrite: %v", err) + t.Fatalf("InsertStump variant: %v", err) } got, err = s.GetStumpsByBlockHash(ctx, blockHash) if err != nil { - t.Fatalf("GetStumpsByBlockHash after rewrite: %v", err) + t.Fatalf("GetStumpsByBlockHash after variant insert: %v", err) + } + if len(got) != 3 { + t.Fatalf("after variant insert got %d stumps, want 3 (big@0, small@0, small@1)", len(got)) } + variants0 := map[string]bool{} for _, st := range got { - if st.SubtreeIndex == 0 && !bytes.Equal(st.StumpData, small) { - t.Fatalf("subtree 0 after rewrite: got %d bytes, want %d", len(st.StumpData), len(small)) + if st.ContentHash != models.StumpContentHash(st.StumpData) { + t.Fatalf("subtree %d: content hash does not match the bytes", st.SubtreeIndex) } + if st.SubtreeIndex == 0 { + variants0[st.ContentHash] = bytes.Equal(st.StumpData, big) || bytes.Equal(st.StumpData, small) + } + } + if len(variants0) != 2 || !variants0[models.StumpContentHash(big)] || !variants0[models.StumpContentHash(small)] { + t.Fatalf("subtree 0 must hold both variants byte-identical, got %v", variants0) } if err := s.DeleteStumpsByBlockHash(ctx, blockHash); err != nil { diff --git a/store/bumpcache/bumpcache.go b/store/bumpcache/bumpcache.go index 251a5e75..9031d01b 100644 --- a/store/bumpcache/bumpcache.go +++ b/store/bumpcache/bumpcache.go @@ -14,6 +14,7 @@ package bumpcache import ( "sync" + "time" "github.com/hashicorp/golang-lru/v2/simplelru" "golang.org/x/sync/singleflight" @@ -34,6 +35,14 @@ const maxEntries = 16 // floor the process pays anyway while serving it. const maxTotalLeaves = 512 * 1024 +// refreshInterval bounds how often a block's cached index is re-fetched +// because a lookup missed a leaf (see pathFor). +const refreshInterval = 5 * time.Second + +// maxRefreshTracked bounds the per-block refresh timestamps kept for that +// throttle. +const maxRefreshTracked = 256 + // Cache is a bounded, singleflight-guarded cache of bump.CompoundIndex per // block hash. The zero value is not usable; construct with New. type Cache struct { @@ -69,6 +78,12 @@ type Cache struct { // leaving the cache permanently empty exactly when it is load-bearing. gen map[string]uint64 inflight map[string]int + + // refreshed records when a block's index was last re-fetched for a + // missing leaf, so a txid that is legitimately not in the block cannot + // turn every lookup into a fetch. Guarded by mu. + refreshed *simplelru.LRU[string, time.Time] + now func() time.Time } // New constructs an empty cache with the package's production bounds. @@ -79,10 +94,16 @@ func New() *Cache { // newWithLimits is the constructor proper, split out so tests can exercise // eviction without building half-million-leaf compounds. func newWithLimits(entries, leaves int) *Cache { + refreshed, err := simplelru.NewLRU[string, time.Time](maxRefreshTracked, nil) + if err != nil { + panic(err) // unreachable: positive constant size + } c := &Cache{ maxLeaves: leaves, gen: make(map[string]uint64), inflight: make(map[string]int), + refreshed: refreshed, + now: time.Now, } l, err := simplelru.NewLRU(entries, func(_ string, v *bump.CompoundIndex) { c.totalLeaves -= v.Leaves() @@ -109,11 +130,7 @@ func (c *Cache) Enrich(status *models.TransactionStatus, fetch func() ([]byte, e if status.Status != models.StatusMined && status.Status != models.StatusImmutable { return } - idx := c.index(status.BlockHash, fetch) - if idx == nil { - return - } - status.MerklePath = idx.MinimalPathBytes(status.TxID) + status.MerklePath = c.pathFor(status.BlockHash, status.TxID, fetch) } // MinimalPath returns txid's BRC-74 minimal path from blockHash's compound @@ -126,13 +143,51 @@ func (c *Cache) MinimalPath(blockHash, txid string, fetch func() ([]byte, error) if blockHash == "" || txid == "" { return nil } - idx := c.index(blockHash, fetch) + return c.pathFor(blockHash, txid, fetch) +} + +// pathFor extracts txid's minimal path from blockHash's index. A cached +// index that lacks the leaf may be stale rather than final: a bump-builder +// rebuild (a later merkle-service's STUMPs adding leaves) overwrites the +// stored compound, and InsertBUMP only invalidates the WRITER's cache. Every +// other process keeps serving the earlier compound, so a newly covered tx +// would stay MINED without a proof until eviction. So a miss on a cached +// index drops it and reads the store once more, at most once per block per +// refreshInterval so an absent txid cannot turn into a fetch per lookup. +func (c *Cache) pathFor(blockHash, txid string, fetch func() ([]byte, error)) []byte { + idx, cached := c.lookup(blockHash) + if !cached { + idx = c.index(blockHash, fetch) + } if idx == nil { return nil } + if path := idx.MinimalPathBytes(txid); path != nil || !cached { + return path + } + if !c.claimRefresh(blockHash) { + return nil + } + c.Remove(blockHash) + if idx = c.index(blockHash, fetch); idx == nil { + return nil + } return idx.MinimalPathBytes(txid) } +// claimRefresh reports whether blockHash may be re-fetched for a missing +// leaf now, and if so records the attempt. +func (c *Cache) claimRefresh(blockHash string) bool { + c.mu.Lock() + defer c.mu.Unlock() + now := c.now() + if last, ok := c.refreshed.Get(blockHash); ok && now.Sub(last) < refreshInterval { + return false + } + c.refreshed.Add(blockHash, now) + return true +} + // Remove invalidates a block's cached index. Called by InsertBUMP: a rebuild // can overwrite the stored compound for an existing block (e.g. late STUMP // callbacks), so the next enrichment must re-fetch and re-parse. diff --git a/store/bumpcache/bumpcache_test.go b/store/bumpcache/bumpcache_test.go index 12ebb886..9b397688 100644 --- a/store/bumpcache/bumpcache_test.go +++ b/store/bumpcache/bumpcache_test.go @@ -5,6 +5,7 @@ import ( "sync" "sync/atomic" "testing" + "time" "github.com/bsv-blockchain/arcade/bump" "github.com/bsv-blockchain/arcade/internal/synthblock" @@ -495,3 +496,71 @@ func TestFill_PanicDoesNotPinTheBlock(t *testing.T) { t.Fatalf("panicking fetch leaked bookkeeping: inflight=%d gen=%d", inflight, gens) } } + +// A rebuild in another process (the bump-builder writing an expanded +// compound) does not invalidate this process's cache. A lookup that misses a +// leaf on a cached index re-reads the store once, so the newly covered tx +// gets its proof; a txid that is not in the block either way is throttled to +// one re-read per refreshInterval. +func TestPathFor_StaleCachedIndexIsRefreshedOnLeafMiss(t *testing.T) { + // Txids depend only on the leaf index, so the 4-leaf compound is the + // 8-leaf one before a rebuild added leaves 4..7. + before, err := synthblock.Build(4, 900300) + if err != nil { + t.Fatalf("synthblock.Build: %v", err) + } + after, err := synthblock.Build(8, 900300) + if err != nil { + t.Fatalf("synthblock.Build: %v", err) + } + var ( + mu sync.Mutex + stored = before.BumpBytes + calls atomic.Int64 + ) + fetch := func() ([]byte, error) { + calls.Add(1) + mu.Lock() + defer mu.Unlock() + return stored, nil + } + + reader := New() + clock := time.Unix(1_700_000_000, 0) + reader.now = func() time.Time { return clock } + reader.Enrich(minedStatus(before.Txids[0]), fetch) + if !reader.Contains(testBlockHash) || calls.Load() != 1 { + t.Fatalf("first lookup must fill the cache once, fetches=%d", calls.Load()) + } + + // The writer rebuilds the stored compound; nothing invalidates reader. + mu.Lock() + stored = after.BumpBytes + mu.Unlock() + + st := minedStatus(after.Txids[5]) + reader.Enrich(st, fetch) + if len(st.MerklePath) == 0 { + t.Fatal("a leaf missing from the cached index must trigger one re-read of the store") + } + if vErr := synthblock.VerifyMerklePath(st.MerklePath, after.Txids[5], after.Root); vErr != nil { + t.Fatalf("refreshed path does not verify: %v", vErr) + } + if got := calls.Load(); got != 2 { + t.Fatalf("fetches = %d, want 2", got) + } + + // An absent txid within refreshInterval: no further fetch. + const absent = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + if p := reader.MinimalPath(testBlockHash, absent, fetch); p != nil { + t.Fatal("absent txid must have no path") + } + if got := calls.Load(); got != 2 { + t.Fatalf("a refresh within refreshInterval must be throttled, fetches = %d", got) + } + clock = clock.Add(refreshInterval) + _ = reader.MinimalPath(testBlockHash, absent, fetch) + if got := calls.Load(); got != 3 { + t.Fatalf("after refreshInterval the miss may re-read once more, fetches = %d", got) + } +} diff --git a/store/mongodb/blobs.go b/store/mongodb/blobs.go index 44aa65fe..91c586ae 100644 --- a/store/mongodb/blobs.go +++ b/store/mongodb/blobs.go @@ -88,6 +88,7 @@ const ( metaBlockHash = "block_hash" metaBlockHeight = "block_height" metaSubtreeIndex = "subtree_index" + metaContentHash = "content_hash" // staleUploadAge is how long an upload must have been complete and // unreferenced before a later write for the same key sweeps it as a @@ -130,6 +131,7 @@ type stumpManifest struct { Key string `bson:"_id"` BlockHash string `bson:"block_hash"` SubtreeIndex int64 `bson:"subtree_index"` + ContentHash string `bson:"content_hash,omitempty"` FileID bson.ObjectID `bson:"file_id"` Length int64 `bson:"length"` UpdatedAt time.Time `bson:"updated_at"` @@ -148,8 +150,11 @@ type blobRef struct { var projBlobRef = doc(kv(fFileID, 1), kv(fLength, 1), kv(fBlockHeight, 1)) -func stumpKey(blockHash string, subtreeIndex int64) string { - return fmt.Sprintf("%s:%d", blockHash, subtreeIndex) +// stumpKey is content-addressed (see models.Stump): identical STUMP bytes +// from several merkle-services share one manifest; a divergent variant for +// the same subtree gets its own. +func stumpKey(blockHash string, subtreeIndex int64, contentHash string) string { + return fmt.Sprintf("%s:%d:%s", blockHash, subtreeIndex, contentHash) } // --- GridFS primitives --- @@ -600,9 +605,23 @@ func (s *Store) InsertStump(ctx context.Context, stump *models.Stump) error { return errors.New("insert stump: empty block hash") } idx := int64(stump.SubtreeIndex) - key := stumpKey(stump.BlockHash, idx) - meta := doc(kv(metaBlockHash, stump.BlockHash), kv(metaSubtreeIndex, idx)) - manifest := doc(kv(fBlockHash, stump.BlockHash), kv(fSubtreeIndex, idx)) + hash := stump.EnsureContentHash() + key := stumpKey(stump.BlockHash, idx, hash) + // Content-addressed: a manifest under this key already holds exactly + // these bytes, so a redelivery from another merkle-service is one cheap + // read instead of a GridFS upload, swap and delete. + octx, cancel := s.opCtx(ctx) + err := s.stumps.manifests.FindOne(octx, idFilter(key), options.FindOne().SetProjection(doc(kv(fID, 1)))).Err() + cancel() + switch { + case err == nil: + return nil + case errors.Is(err, mongo.ErrNoDocuments): + default: + return fmt.Errorf("insert stump %s/%d: %w", stump.BlockHash, stump.SubtreeIndex, err) + } + meta := doc(kv(metaBlockHash, stump.BlockHash), kv(metaSubtreeIndex, idx), kv(metaContentHash, hash)) + manifest := doc(kv(fBlockHash, stump.BlockHash), kv(fSubtreeIndex, idx), kv(fContentHash, hash)) if err := s.replaceBlob(ctx, &s.stumps, key, key, meta, manifest, stump.StumpData); err != nil { return fmt.Errorf("insert stump %s/%d: %w", stump.BlockHash, stump.SubtreeIndex, err) } @@ -633,11 +652,14 @@ func (s *Store) GetStumpsByBlockHash(ctx context.Context, blockHash string) ([]* if ferr != nil { return fmt.Errorf("get stumps %s/%d: %w", blockHash, m.SubtreeIndex, ferr) } - out[i] = &models.Stump{ + st := &models.Stump{ BlockHash: blockHash, SubtreeIndex: int(m.SubtreeIndex), StumpData: data, + ContentHash: m.ContentHash, } + st.EnsureContentHash() + out[i] = st return nil }); err != nil { return nil, err @@ -685,7 +707,7 @@ func (s *Store) DeleteStumpsByBlockHash(ctx context.Context, blockHash string) e // reorg cleanup every time it is attempted, which is the failure mode the // GetTxIDsByBlockHash and CensusStatusesSince changes already removed. func (s *Store) stumpManifestsFor(ctx context.Context, blockHash string) ([]stumpManifest, error) { - cur, err := s.stumps.manifests.Find(ctx, doc(kv(fBlockHash, blockHash)), options.Find().SetSort(doc(kv(fSubtreeIndex, 1)))) + cur, err := s.stumps.manifests.Find(ctx, doc(kv(fBlockHash, blockHash)), options.Find().SetSort(doc(kv(fSubtreeIndex, 1), kv(fContentHash, 1)))) if err != nil { return nil, err } diff --git a/store/mongodb/blocks.go b/store/mongodb/blocks.go index c7644b37..0ae89402 100644 --- a/store/mongodb/blocks.go +++ b/store/mongodb/blocks.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "slices" "time" "go.mongodb.org/mongo-driver/v2/bson" @@ -67,6 +68,69 @@ func (s *Store) MarkBlockBUMPBuilt(ctx context.Context, blockHash string, blockH return s.markBlockMilestone(ctx, blockHash, blockHeight, builtAt, fBUMPBuiltAt) } +// ClearBlockProcessed implements store.Store: unsets processed_at on an +// existing row; a missing row is a no-op (no upsert). +func (s *Store) ClearBlockProcessed(ctx context.Context, blockHash string) error { + octx, cancel := s.opCtx(ctx) + defer cancel() + _, err := s.blocks.UpdateOne(octx, idFilter(blockHash), doc(kv(opUnset, doc(kv(fProcessedAt, ""))))) + if err != nil { + return fmt.Errorf("clear block processed %s: %w", blockHash, err) + } + return nil +} + +// AddBlockExpectedSubtrees implements store.Store with $addToSet, so the +// union is applied atomically on the server; a missing row is a no-op. +func (s *Store) AddBlockExpectedSubtrees(ctx context.Context, blockHash string, indices []int) error { + if len(indices) == 0 { + return nil + } + octx, cancel := s.opCtx(ctx) + defer cancel() + each := make(bson.A, len(indices)) + for i, v := range indices { + each[i] = int32(v) //nolint:gosec // subtree indices are small + } + _, err := s.blocks.UpdateOne(octx, idFilter(blockHash), doc(kv(opAddToSet, doc(kv(fExpectedSubtrees, doc(kv(opEach, each))))))) + if err != nil { + return fmt.Errorf("add block expected subtrees %s: %w", blockHash, err) + } + return nil +} + +// GetBlockExpectedSubtrees implements store.Store. +func (s *Store) GetBlockExpectedSubtrees(ctx context.Context, blockHash string) ([]int, error) { + octx, cancel := s.opCtx(ctx) + defer cancel() + var d struct { + Expected []int `bson:"expected_subtrees"` + } + err := s.blocks.FindOne(octx, idFilter(blockHash), options.FindOne().SetProjection(doc(kv(fExpectedSubtrees, 1)))).Decode(&d) + if errors.Is(err, mongo.ErrNoDocuments) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("get block expected subtrees %s: %w", blockHash, err) + } + if len(d.Expected) == 0 { + return nil, nil + } + slices.Sort(d.Expected) + return d.Expected, nil +} + +// ClearBlockExpectedSubtrees implements store.Store. +func (s *Store) ClearBlockExpectedSubtrees(ctx context.Context, blockHash string) error { + octx, cancel := s.opCtx(ctx) + defer cancel() + _, err := s.blocks.UpdateOne(octx, idFilter(blockHash), doc(kv(opUnset, doc(kv(fExpectedSubtrees, ""))))) + if err != nil { + return fmt.Errorf("clear block expected subtrees %s: %w", blockHash, err) + } + return nil +} + // MarkBlocksOrphaned implements store.Store; hashes without a row are skipped. // // It clears reconciled_at as well as setting the status. A block that was diff --git a/store/mongodb/conformance_test.go b/store/mongodb/conformance_test.go index 830e7cf0..f0bb63e0 100644 --- a/store/mongodb/conformance_test.go +++ b/store/mongodb/conformance_test.go @@ -27,6 +27,12 @@ func TestBlockStatus_Conformance(t *testing.T) { storetest.RunBlockStatusSuite(t, func(t *testing.T) storetest.BlockStatusBackend { return newTestStore(t) }) } +// STUMP rows are content-addressed so several merkle-services can deliver +// the same subtree: identical bytes collapse, divergent variants coexist. +func TestStumps_Conformance(t *testing.T) { + storetest.RunStumpSuite(t, func(t *testing.T) storetest.StumpBackend { return newTestStore(t) }) +} + // A double spend is discovered after the row exists, so the competing // spender must persist through the status-update paths, not only on insert. func TestCompetingTxs_Conformance(t *testing.T) { diff --git a/store/mongodb/docs.go b/store/mongodb/docs.go index 557cc858..413ef321 100644 --- a/store/mongodb/docs.go +++ b/store/mongodb/docs.go @@ -57,10 +57,14 @@ const ( fHeaderSeenAt = "header_seen_at" fProcessedAt = "processed_at" + fContentHash = "content_hash" fBUMPBuiltAt = "bump_built_at" fOrphanedAt = "orphaned_at" fOrphanedGen = "orphaned_gen" fReconciledAt = "reconciled_at" + // fExpectedSubtrees is the union of announced expected subtree indices + // (store.Store.AddBlockExpectedSubtrees). + fExpectedSubtrees = "expected_subtrees" fHolder = "holder" fExpiresAt = "expires_at" @@ -92,6 +96,8 @@ const ( opUnset = "$unset" opInc = "$inc" opSetOnInsert = "$setOnInsert" + opAddToSet = "$addToSet" + opEach = "$each" opIn = "$in" opNin = "$nin" opNe = "$ne" diff --git a/store/mongodb/mongo_specific_test.go b/store/mongodb/mongo_specific_test.go index 39b5e9d3..58bd573c 100644 --- a/store/mongodb/mongo_specific_test.go +++ b/store/mongodb/mongo_specific_test.go @@ -218,8 +218,11 @@ func TestInsertBUMP_OverwriteNeverGaps(t *testing.T) { } // GridFS is load-bearing: a STUMP larger than the 16 MB document cap must -// round-trip, and a re-insert for the same subtree supersedes the old copy. -func TestStump_LargePayloadAndSupersede(t *testing.T) { +// round-trip. STUMP rows are content-addressed: re-inserting the same bytes +// for a subtree is a no-op (no second file either), while a different STUMP +// for the same subtree — another merkle-service with a different registered +// set — is kept as a second variant. +func TestStump_LargePayloadAndVariants(t *testing.T) { s := newTestStore(t) ctx := context.Background() const hash = "stump-big" @@ -233,7 +236,8 @@ func TestStump_LargePayloadAndSupersede(t *testing.T) { if err := s.InsertStump(ctx, &models.Stump{BlockHash: hash, SubtreeIndex: 1, StumpData: []byte("small")}); err != nil { t.Fatal(err) } - if err := s.InsertStump(ctx, &models.Stump{BlockHash: hash, SubtreeIndex: 1, StumpData: []byte("small-v2")}); err != nil { + // Same bytes again: still one row, one file. + if err := s.InsertStump(ctx, &models.Stump{BlockHash: hash, SubtreeIndex: 1, StumpData: []byte("small")}); err != nil { t.Fatal(err) } got, err := s.GetStumpsByBlockHash(ctx, hash) @@ -243,15 +247,36 @@ func TestStump_LargePayloadAndSupersede(t *testing.T) { if len(got) != 2 || got[0].SubtreeIndex != 1 || got[1].SubtreeIndex != 3 { t.Fatalf("expected subtrees [1 3], got %d stumps", len(got)) } - if string(got[0].StumpData) != "small-v2" { - t.Fatalf("re-insert must supersede: got %q", got[0].StumpData) - } if !bytes.Equal(got[1].StumpData, big) { t.Fatalf("17 MB stump did not round-trip (len %d)", len(got[1].StumpData)) } n, err := s.stumps.bucket.GetFilesCollection().CountDocuments(ctx, doc(kv(fMetaBlockHash, hash))) if err != nil || n != 2 { - t.Fatalf("expected 2 surviving files, got %d (%v)", n, err) + t.Fatalf("expected 2 surviving files after an identical re-insert, got %d (%v)", n, err) + } + // A different STUMP for subtree 1 is a second variant, not a replacement. + if err := s.InsertStump(ctx, &models.Stump{BlockHash: hash, SubtreeIndex: 1, StumpData: []byte("small-v2")}); err != nil { + t.Fatal(err) + } + got, err = s.GetStumpsByBlockHash(ctx, hash) + if err != nil { + t.Fatal(err) + } + if len(got) != 3 || got[0].SubtreeIndex != 1 || got[1].SubtreeIndex != 1 || got[2].SubtreeIndex != 3 { + t.Fatalf("expected subtrees [1 1 3], got %d stumps", len(got)) + } + variants := map[string]bool{string(got[0].StumpData): true, string(got[1].StumpData): true} + if !variants["small"] || !variants["small-v2"] { + t.Fatalf("both variants must be kept, got %q / %q", got[0].StumpData, got[1].StumpData) + } + for _, st := range got { + if st.ContentHash != models.StumpContentHash(st.StumpData) { + t.Fatalf("content hash mismatch on subtree %d", st.SubtreeIndex) + } + } + n, err = s.stumps.bucket.GetFilesCollection().CountDocuments(ctx, doc(kv(fMetaBlockHash, hash))) + if err != nil || n != 3 { + t.Fatalf("expected 3 surviving files, got %d (%v)", n, err) } if err := s.DeleteStumpsByBlockHash(ctx, hash); err != nil { t.Fatal(err) @@ -263,7 +288,10 @@ func TestStump_LargePayloadAndSupersede(t *testing.T) { // Concurrent rebuilds of one block must converge on exactly one surviving // file that the manifest references, never on none: each writer deletes only -// the file its own manifest swap replaced. Same for one STUMP subtree. +// the file its own manifest swap replaced. Same for one STUMP row: STUMPs are +// content-addressed, so the race is several merkle-services delivering the +// SAME bytes at once (all miss the existence check and all publish), which +// must still leave one manifest and one file. func TestInsertBlob_ConcurrentOverwritesKeepExactlyOne(t *testing.T) { s := newTestStore(t) ctx := context.Background() @@ -326,17 +354,31 @@ func TestInsertBlob_ConcurrentOverwritesKeepExactlyOne(t *testing.T) { t.Fatalf("manifest references %s but the surviving file is %s", m.FileID.Hex(), surviving.ID.Hex()) } - run(func(i int) error { - return s.InsertStump(ctx, &models.Stump{BlockHash: hash, SubtreeIndex: 4, StumpData: payloads[i]}) + run(func(_ int) error { + return s.InsertStump(ctx, &models.Stump{BlockHash: hash, SubtreeIndex: 4, StumpData: payloads[0]}) }) stumps, err := s.GetStumpsByBlockHash(ctx, hash) - if err != nil || len(stumps) != 1 || stumps[0].SubtreeIndex != 4 || !isPayload(stumps[0].StumpData) { - t.Fatalf("after concurrent stump inserts: %d stumps, %v", len(stumps), err) + if err != nil || len(stumps) != 1 || stumps[0].SubtreeIndex != 4 || !bytes.Equal(stumps[0].StumpData, payloads[0]) { + t.Fatalf("after concurrent identical stump inserts: %d stumps, %v", len(stumps), err) } n, err = s.stumps.bucket.GetFilesCollection().CountDocuments(ctx, doc(kv(fMetaBlockHash, hash))) if err != nil || n != 1 { t.Fatalf("expected exactly one surviving STUMP file, got %d (%v)", n, err) } + + // Distinct payloads for the same subtree are distinct variants: one row + // and one file each, nothing orphaned. + run(func(i int) error { + return s.InsertStump(ctx, &models.Stump{BlockHash: hash, SubtreeIndex: 5, StumpData: payloads[i]}) + }) + stumps, err = s.GetStumpsByBlockHash(ctx, hash) + if err != nil || len(stumps) != 1+writers { + t.Fatalf("after concurrent variant inserts: %d stumps, %v (want %d)", len(stumps), err, 1+writers) + } + n, err = s.stumps.bucket.GetFilesCollection().CountDocuments(ctx, doc(kv(fMetaBlockHash, hash))) + if err != nil || n != 1+writers { + t.Fatalf("expected %d surviving STUMP files, got %d (%v)", 1+writers, n, err) + } } // fileMissing must distinguish "definitively gone" from "could not tell": diff --git a/store/pebble/conformance_test.go b/store/pebble/conformance_test.go index 1fabcad7..927f4313 100644 --- a/store/pebble/conformance_test.go +++ b/store/pebble/conformance_test.go @@ -28,6 +28,15 @@ func TestBlockStatus_Conformance(t *testing.T) { }) } +// STUMP rows are content-addressed so several merkle-services can deliver +// the same subtree: identical bytes collapse, divergent variants coexist. +func TestStumps_Conformance(t *testing.T) { + storetest.RunStumpSuite(t, func(t *testing.T) storetest.StumpBackend { + t.Helper() + return newTestStore(t) + }) +} + // A double spend is discovered after the row exists, so the competing // spender must persist through the status-update paths, not only on insert. func TestCompetingTxs_Conformance(t *testing.T) { diff --git a/store/pebble/keys.go b/store/pebble/keys.go index 58f1023d..8c1a6879 100644 --- a/store/pebble/keys.go +++ b/store/pebble/keys.go @@ -30,6 +30,7 @@ const ( prefixIdxSubToken = "idx:sub:token:" //nolint:gosec // index-prefix string, not a credential prefixIdxStumpBlock = "idx:stump:block:" prefixBlockProc = "blkproc:" + prefixBlockExpected = "blkexp:" prefixIdxBlockProcHeight = "idx:blkproc:height:" ) @@ -43,8 +44,12 @@ func datahubEndpointPrefix() []byte { return []byte(prefixDatahub) } func peerPolicyKey(peerID string) []byte { return []byte(prefixPeerPolicy + peerID) } func peerPolicyPrefix() []byte { return []byte(prefixPeerPolicy) } -func stumpKey(blockHash string, idx int) []byte { - return []byte(fmt.Sprintf("%s%s:%010d", prefixStump, blockHash, idx)) +// stumpKey is content-addressed: several merkle-services may deliver the +// same subtree's STUMP, and a service that missed registrations delivers a +// different one. Identical bytes map to the same key; variants coexist under +// the block prefix for the bump-builder to merge. +func stumpKey(blockHash string, idx int, contentHash string) []byte { + return []byte(fmt.Sprintf("%s%s:%010d:%s", prefixStump, blockHash, idx, contentHash)) } // stumpBlockPrefix returns the prefix used to iterate all STUMP rows for a block. @@ -108,13 +113,24 @@ func idxSubTokenPrefix(token string) []byte { return []byte(prefixIdxSubToken + token + ":") } -func idxStumpBlockKey(blockHash string, subtreeIndex int) []byte { - return []byte(fmt.Sprintf("%s%s:%010d", prefixIdxStumpBlock, blockHash, subtreeIndex)) +func idxStumpBlockKey(blockHash string, subtreeIndex int, contentHash string) []byte { + return []byte(fmt.Sprintf("%s%s:%010d:%s", prefixIdxStumpBlock, blockHash, subtreeIndex, contentHash)) +} + +// idxStumpBlockPrefix is the prefix of every idxStumpBlockKey for a block. +func idxStumpBlockPrefix(blockHash string) []byte { + return []byte(prefixIdxStumpBlock + blockHash + ":") } // blockProcKey is the primary record key for a block-processing row. func blockProcKey(blockHash string) []byte { return []byte(prefixBlockProc + blockHash) } +// blockExpectedKey holds a block's retained expected-subtree set +// (store.Store.AddBlockExpectedSubtrees) as a JSON int array. It lives beside +// the block-processing row rather than in it so the row's many rebuild-from- +// scratch writers cannot drop it. +func blockExpectedKey(blockHash string) []byte { return []byte(prefixBlockExpected + blockHash) } + // idxBlockProcHeightKey encodes block_height as the bitwise-inverted uint64 // rendered as 16 hex digits so a forward iterator over the index prefix // returns rows in descending-height order. Same trick the retry index uses diff --git a/store/pebble/pebble.go b/store/pebble/pebble.go index f71a78a0..13e4c582 100644 --- a/store/pebble/pebble.go +++ b/store/pebble/pebble.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "hash/fnv" + "slices" "sort" "sync" "time" @@ -1293,16 +1294,17 @@ func (s *Store) InsertStump(ctx context.Context, stump *models.Stump) error { if err := ctx.Err(); err != nil { return err } + hash := stump.EnsureContentHash() payload, err := json.Marshal(stump) if err != nil { return err } b := s.db.NewBatch() defer func() { _ = b.Close() }() - if err := b.Set(stumpKey(stump.BlockHash, stump.SubtreeIndex), payload, nil); err != nil { + if err := b.Set(stumpKey(stump.BlockHash, stump.SubtreeIndex, hash), payload, nil); err != nil { return err } - if err := b.Set(idxStumpBlockKey(stump.BlockHash, stump.SubtreeIndex), nil, nil); err != nil { + if err := b.Set(idxStumpBlockKey(stump.BlockHash, stump.SubtreeIndex, hash), nil, nil); err != nil { return err } return b.Commit(s.writeOpts) @@ -1329,8 +1331,13 @@ func (s *Store) GetStumpsByBlockHash(ctx context.Context, blockHash string) ([]* } var st models.Stump if err := json.Unmarshal(iter.Value(), &st); err != nil { - continue + // An unreadable row means the set is incomplete. Skipping it + // would let the bump-builder conclude a redelivery adds nothing + // and stamp a block it never fully built, so fail the read like + // the other backends do and let the caller retry. + return nil, fmt.Errorf("get stumps %s: decode %q: %w", blockHash, iter.Key(), err) } + st.EnsureContentHash() stumps = append(stumps, &st) } return stumps, nil @@ -1496,6 +1503,94 @@ func (s *Store) MarkBlockProcessed(ctx context.Context, blockHash string, blockH return s.writeBlockProc(prev, &cur) } +// ClearBlockProcessed unsets processed_at on an existing row; a missing row +// is a no-op. +func (s *Store) ClearBlockProcessed(ctx context.Context, blockHash string) error { + if err := ctx.Err(); err != nil { + return err + } + mu := s.shardFor(blockHash) + mu.Lock() + defer mu.Unlock() + + prev, err := s.readBlockProc(blockHash) + if err != nil || prev == nil { + return err + } + cur := *prev + cur.ProcessedUnixNs = 0 + return s.writeBlockProc(prev, &cur) +} + +// AddBlockExpectedSubtrees implements store.Store. Read-modify-write under +// the block's shard lock; a missing block-processing row is a no-op. +func (s *Store) AddBlockExpectedSubtrees(ctx context.Context, blockHash string, indices []int) error { + if err := ctx.Err(); err != nil { + return err + } + if len(indices) == 0 { + return nil + } + mu := s.shardFor(blockHash) + mu.Lock() + defer mu.Unlock() + + row, err := s.readBlockProc(blockHash) + if err != nil || row == nil { + return err + } + cur, err := s.readBlockExpected(blockHash) + if err != nil { + return err + } + merged := append(cur, indices...) + slices.Sort(merged) + merged = slices.Compact(merged) + payload, err := json.Marshal(merged) + if err != nil { + return err + } + return s.db.Set(blockExpectedKey(blockHash), payload, s.writeOpts) +} + +// GetBlockExpectedSubtrees implements store.Store. +func (s *Store) GetBlockExpectedSubtrees(ctx context.Context, blockHash string) ([]int, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return s.readBlockExpected(blockHash) +} + +// ClearBlockExpectedSubtrees implements store.Store. +func (s *Store) ClearBlockExpectedSubtrees(ctx context.Context, blockHash string) error { + if err := ctx.Err(); err != nil { + return err + } + mu := s.shardFor(blockHash) + mu.Lock() + defer mu.Unlock() + return s.db.Delete(blockExpectedKey(blockHash), s.writeOpts) +} + +func (s *Store) readBlockExpected(blockHash string) ([]int, error) { + v, closer, err := s.db.Get(blockExpectedKey(blockHash)) + if errors.Is(err, pebbledb.ErrNotFound) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("get block expected subtrees %s: %w", blockHash, err) + } + defer func() { _ = closer.Close() }() + var out []int + if err := json.Unmarshal(v, &out); err != nil { + return nil, fmt.Errorf("unmarshal block expected subtrees %s: %w", blockHash, err) + } + if len(out) == 0 { + return nil, nil + } + return out, nil +} + func (s *Store) MarkBlockBUMPBuilt(ctx context.Context, blockHash string, blockHeight uint64, builtAt time.Time) error { if err := ctx.Err(); err != nil { return err @@ -2007,23 +2102,35 @@ func (s *Store) DeleteStumpsByBlockHash(ctx context.Context, blockHash string) e return err } - var toDelete []*models.Stump + // Delete the keys as found rather than recomputing them: rows written + // before content addressing carry the old two-part key, and a block's + // rows must all go regardless of which layout wrote them. + var toDelete [][]byte for iter.First(); iter.Valid(); iter.Next() { - var st models.Stump - if err := json.Unmarshal(iter.Value(), &st); err != nil { - continue - } - toDelete = append(toDelete, &st) + toDelete = append(toDelete, append([]byte(nil), iter.Key()...)) } - if err := iter.Close(); err != nil { + if cerr := iter.Close(); cerr != nil { + return cerr + } + idxPrefix := idxStumpBlockPrefix(blockHash) + idxIter, err := s.db.NewIter(&pebbledb.IterOptions{ + LowerBound: idxPrefix, + UpperBound: endOfPrefix(idxPrefix), + }) + if err != nil { return err } + for idxIter.First(); idxIter.Valid(); idxIter.Next() { + toDelete = append(toDelete, append([]byte(nil), idxIter.Key()...)) + } + if cerr := idxIter.Close(); cerr != nil { + return cerr + } b := s.db.NewBatch() defer func() { _ = b.Close() }() - for _, st := range toDelete { - _ = b.Delete(stumpKey(st.BlockHash, st.SubtreeIndex), nil) - _ = b.Delete(idxStumpBlockKey(st.BlockHash, st.SubtreeIndex), nil) + for _, key := range toDelete { + _ = b.Delete(key, nil) } return b.Commit(s.writeOpts) } diff --git a/store/postgres/conformance_test.go b/store/postgres/conformance_test.go index 7f6041dd..0041b35c 100644 --- a/store/postgres/conformance_test.go +++ b/store/postgres/conformance_test.go @@ -28,6 +28,15 @@ func TestBlockStatus_Conformance(t *testing.T) { }) } +// STUMP rows are content-addressed so several merkle-services can deliver +// the same subtree: identical bytes collapse, divergent variants coexist. +func TestStumps_Conformance(t *testing.T) { + storetest.RunStumpSuite(t, func(t *testing.T) storetest.StumpBackend { + t.Helper() + return newTestStore(t) + }) +} + // A double spend is discovered after the row exists, so the competing // spender must persist through the status-update paths, not only on insert. func TestCompetingTxs_Conformance(t *testing.T) { diff --git a/store/postgres/postgres.go b/store/postgres/postgres.go index a77565e2..be532b78 100644 --- a/store/postgres/postgres.go +++ b/store/postgres/postgres.go @@ -1193,10 +1193,13 @@ func (s *Store) GetBUMP(ctx context.Context, blockHash string) (uint64, []byte, } func (s *Store) InsertStump(ctx context.Context, stump *models.Stump) error { + // Content-addressed: the same bytes from a second merkle-service hit the + // conflict and do nothing; different bytes for the same subtree land as + // a second row for the bump-builder to merge. const q = ` -INSERT INTO stumps (block_hash, subtree_index, stump_data) VALUES ($1,$2,$3) -ON CONFLICT (block_hash, subtree_index) DO UPDATE SET stump_data=EXCLUDED.stump_data` - _, err := s.pool.Exec(ctx, q, stump.BlockHash, stump.SubtreeIndex, stump.StumpData) +INSERT INTO stumps (block_hash, subtree_index, content_hash, stump_data) VALUES ($1,$2,$3,$4) +ON CONFLICT (block_hash, subtree_index, content_hash) DO NOTHING` + _, err := s.pool.Exec(ctx, q, stump.BlockHash, stump.SubtreeIndex, stump.EnsureContentHash(), stump.StumpData) if err != nil { return fmt.Errorf("insert stump: %w", err) } @@ -1204,7 +1207,7 @@ ON CONFLICT (block_hash, subtree_index) DO UPDATE SET stump_data=EXCLUDED.stump_ } func (s *Store) GetStumpsByBlockHash(ctx context.Context, blockHash string) ([]*models.Stump, error) { - const q = `SELECT block_hash, subtree_index, stump_data FROM stumps WHERE block_hash = $1 ORDER BY subtree_index` + const q = `SELECT block_hash, subtree_index, content_hash, stump_data FROM stumps WHERE block_hash = $1 ORDER BY subtree_index, content_hash` rows, err := s.pool.Query(ctx, q, blockHash) if err != nil { return nil, err @@ -1213,9 +1216,10 @@ func (s *Store) GetStumpsByBlockHash(ctx context.Context, blockHash string) ([]* var out []*models.Stump for rows.Next() { st := &models.Stump{} - if err := rows.Scan(&st.BlockHash, &st.SubtreeIndex, &st.StumpData); err != nil { + if err := rows.Scan(&st.BlockHash, &st.SubtreeIndex, &st.ContentHash, &st.StumpData); err != nil { return out, err } + st.EnsureContentHash() out = append(out, st) } return out, rows.Err() @@ -1263,6 +1267,66 @@ ON CONFLICT (block_hash) DO UPDATE SET return nil } +// ClearBlockProcessed unsets processed_at so the watchdog's stale scan +// (processed_at IS NULL) picks the block up again. +func (s *Store) ClearBlockProcessed(ctx context.Context, blockHash string) error { + const q = `UPDATE block_processing SET processed_at = NULL WHERE block_hash = $1` + if _, err := s.pool.Exec(ctx, q, blockHash); err != nil { + return fmt.Errorf("clear block processed %s: %w", blockHash, err) + } + return nil +} + +// AddBlockExpectedSubtrees implements store.Store in one UPDATE, so the +// union is computed against the row's current set server-side. +func (s *Store) AddBlockExpectedSubtrees(ctx context.Context, blockHash string, indices []int) error { + if len(indices) == 0 { + return nil + } + const q = ` +UPDATE block_processing SET expected_subtrees = ( + SELECT array_agg(DISTINCT i ORDER BY i) + FROM unnest(COALESCE(expected_subtrees, '{}'::int[]) || $2::int[]) AS i +) +WHERE block_hash = $1` + vals := make([]int32, len(indices)) + for i, v := range indices { + vals[i] = int32(v) //nolint:gosec // subtree indices are small + } + if _, err := s.pool.Exec(ctx, q, blockHash, vals); err != nil { + return fmt.Errorf("add block expected subtrees %s: %w", blockHash, err) + } + return nil +} + +// GetBlockExpectedSubtrees implements store.Store. +func (s *Store) GetBlockExpectedSubtrees(ctx context.Context, blockHash string) ([]int, error) { + var vals []int32 + err := s.pool.QueryRow(ctx, `SELECT expected_subtrees FROM block_processing WHERE block_hash = $1`, blockHash).Scan(&vals) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("get block expected subtrees %s: %w", blockHash, err) + } + if len(vals) == 0 { + return nil, nil + } + out := make([]int, len(vals)) + for i, v := range vals { + out[i] = int(v) + } + return out, nil +} + +// ClearBlockExpectedSubtrees implements store.Store. +func (s *Store) ClearBlockExpectedSubtrees(ctx context.Context, blockHash string) error { + if _, err := s.pool.Exec(ctx, `UPDATE block_processing SET expected_subtrees = NULL WHERE block_hash = $1`, blockHash); err != nil { + return fmt.Errorf("clear block expected subtrees %s: %w", blockHash, err) + } + return nil +} + func (s *Store) MarkBlockBUMPBuilt(ctx context.Context, blockHash string, blockHeight uint64, builtAt time.Time) error { const q = ` INSERT INTO block_processing (block_hash, block_height, header_seen_at, bump_built_at, status) diff --git a/store/postgres/schema.sql b/store/postgres/schema.sql index f82b3c7a..67fb5206 100644 --- a/store/postgres/schema.sql +++ b/store/postgres/schema.sql @@ -66,12 +66,30 @@ CREATE TABLE IF NOT EXISTS bumps ( bump_data BYTEA NOT NULL ); +-- STUMP rows are content-addressed: (block_hash, subtree_index, content_hash). +-- Several merkle-services may deliver the same subtree's STUMP; identical +-- bytes collapse to one row, divergent variants (a service that missed +-- registrations during an outage) coexist and are merged at build time. CREATE TABLE IF NOT EXISTS stumps ( block_hash TEXT NOT NULL, subtree_index INT NOT NULL, + content_hash TEXT NOT NULL DEFAULT '', stump_data BYTEA NOT NULL, - PRIMARY KEY (block_hash, subtree_index) + PRIMARY KEY (block_hash, subtree_index, content_hash) ); +-- Idempotent upgrade for stores created with the (block_hash, subtree_index) +-- key: add the column, then swap the primary key once. +ALTER TABLE stumps ADD COLUMN IF NOT EXISTS content_hash TEXT NOT NULL DEFAULT ''; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'stumps'::regclass AND contype = 'p' AND array_length(conkey, 1) = 3 + ) THEN + ALTER TABLE stumps DROP CONSTRAINT IF EXISTS stumps_pkey; + ALTER TABLE stumps ADD PRIMARY KEY (block_hash, subtree_index, content_hash); + END IF; +END $$; CREATE INDEX IF NOT EXISTS idx_stump_block_hash ON stumps(block_hash); -- Per-block processing status. One row per block hash; tracks the milestones @@ -100,6 +118,9 @@ CREATE INDEX IF NOT EXISTS idx_bp_stale_seen -- reconciler once every tx anchored to this orphaned block has been -- re-anchored or reverted; reset to NULL when the block is resurrected. ALTER TABLE block_processing ADD COLUMN IF NOT EXISTS reconciled_at TIMESTAMPTZ; +-- Union of the subtree indices merkle-services announced STUMPs for while +-- the block was incomplete (multi merkle-service). NULL when none retained. +ALTER TABLE block_processing ADD COLUMN IF NOT EXISTS expected_subtrees INT[]; -- Partial index over the reconciler's work queue — orphaned rows still -- awaiting tx reconciliation. Stays proportional to the (tiny) backlog. CREATE INDEX IF NOT EXISTS idx_bp_orphaned_unreconciled diff --git a/store/postgres/stump_upgrade_test.go b/store/postgres/stump_upgrade_test.go new file mode 100644 index 00000000..955b13b9 --- /dev/null +++ b/store/postgres/stump_upgrade_test.go @@ -0,0 +1,81 @@ +//go:build postgres + +package postgres + +import ( + "context" + "testing" + "time" + + "github.com/bsv-blockchain/arcade/models" +) + +// A store created before STUMP rows were content-addressed has a +// (block_hash, subtree_index) primary key and no content_hash column. The +// schema's idempotent upgrade must add the column, swap the primary key, keep +// the legacy rows readable, and from then on accept two variants for the +// same subtree. +func TestSchema_UpgradesStumpsToContentAddressedKey(t *testing.T) { + s := newTestStore(t) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + const legacy = ` +DROP TABLE stumps; +CREATE TABLE stumps ( + block_hash TEXT NOT NULL, + subtree_index INT NOT NULL, + stump_data BYTEA NOT NULL, + PRIMARY KEY (block_hash, subtree_index) +); +INSERT INTO stumps (block_hash, subtree_index, stump_data) VALUES ('legacyblock', 4, '\x0102'); +DELETE FROM schema_info;` + if _, err := s.pool.Exec(ctx, legacy); err != nil { + t.Fatalf("seed legacy shape: %v", err) + } + if err := s.EnsureIndexes(ctx); err != nil { + t.Fatalf("EnsureIndexes on legacy shape: %v", err) + } + + var pkCols int + if err := s.pool.QueryRow(ctx, + `SELECT array_length(conkey, 1) FROM pg_constraint WHERE conrelid = 'stumps'::regclass AND contype = 'p'`, + ).Scan(&pkCols); err != nil { + t.Fatalf("read primary key: %v", err) + } + if pkCols != 3 { + t.Fatalf("stumps primary key has %d columns after upgrade, want 3", pkCols) + } + + // The legacy row survives with an empty content_hash and reads back with + // the hash computed from its bytes. + got, err := s.GetStumpsByBlockHash(ctx, "legacyblock") + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].SubtreeIndex != 4 || got[0].ContentHash != models.StumpContentHash([]byte{1, 2}) { + t.Fatalf("legacy row after upgrade = %+v", got) + } + + // Re-running the apply is a no-op (the DO block sees the 3-column key). + if _, err := s.pool.Exec(ctx, `DELETE FROM schema_info`); err != nil { + t.Fatal(err) + } + if err := s.EnsureIndexes(ctx); err != nil { + t.Fatalf("second EnsureIndexes: %v", err) + } + + // Two variants for the same subtree now coexist. + for _, data := range [][]byte{[]byte("variant-a"), []byte("variant-b"), []byte("variant-a")} { + if err := s.InsertStump(ctx, models.NewStump("legacyblock", 4, data)); err != nil { + t.Fatalf("insert %q: %v", data, err) + } + } + got, err = s.GetStumpsByBlockHash(ctx, "legacyblock") + if err != nil { + t.Fatal(err) + } + if len(got) != 3 { + t.Fatalf("rows = %d, want 3 (legacy + two variants): %+v", len(got), got) + } +} diff --git a/store/store.go b/store/store.go index a7c03b22..72b694d7 100644 --- a/store/store.go +++ b/store/store.go @@ -492,6 +492,31 @@ type Store interface { // MarkBlockProcessed. MarkBlockBUMPBuilt(ctx context.Context, blockHash string, blockHeight uint64, builtAt time.Time) error + // ClearBlockProcessed unsets processed_at on an existing row so the block + // re-enters ListStaleBlockProcessingStatus and the watchdog re-drives it. + // The bump-builder calls it when a later merkle-service's BLOCK_PROCESSED + // turns out incomplete for a block an earlier service's build already + // finalized. No-op (nil) when the row does not exist; every other column + // is left alone. + ClearBlockProcessed(ctx context.Context, blockHash string) error + + // AddBlockExpectedSubtrees unions indices into the block's retained set + // of expected subtree indices: every subtree index some merkle-service's + // BLOCK_PROCESSED announced a STUMP for while the block was incomplete. + // Each service only announces the subtrees IT tracks, so the bump-builder + // keeps the union to stop a later, smaller announcement from finalizing + // a block another service is still owed a STUMP for. No-op (nil) when + // the row does not exist. + AddBlockExpectedSubtrees(ctx context.Context, blockHash string, indices []int) error + + // GetBlockExpectedSubtrees returns the retained set, ascending; empty + // (nil, nil) when none is retained or the row does not exist. + GetBlockExpectedSubtrees(ctx context.Context, blockHash string) ([]int, error) + + // ClearBlockExpectedSubtrees drops the retained set once a build has + // satisfied it. No-op (nil) when the row does not exist. + ClearBlockExpectedSubtrees(ctx context.Context, blockHash string) error + // MarkBlocksOrphaned transitions every named block to status='orphaned', // stamps orphaned_at and CLEARS reconciled_at. Clearing is required, not // cosmetic: orphaned rows with reconciled_at IS NULL form the anchor @@ -686,10 +711,16 @@ type Store interface { // STUMP operations for Merkle Service integration - // InsertStump stores a STUMP for a subtree in a specific block. + // InsertStump stores a STUMP keyed by (block_hash, subtree_index, + // content_hash). Inserting identical bytes again is a no-op; a different + // STUMP for the same subtree (another merkle-service with a different + // registered set) is stored as a second row. ContentHash is computed from + // StumpData when the caller left it empty. InsertStump(ctx context.Context, stump *models.Stump) error - // GetStumpsByBlockHash retrieves all STUMPs for a given block hash. + // GetStumpsByBlockHash retrieves every STUMP row for a block, ordered by + // subtree index. There may be several rows per subtree index; callers + // merge them (bump.BuildCompoundBUMP does). GetStumpsByBlockHash(ctx context.Context, blockHash string) ([]*models.Stump, error) // DeleteStumpsByBlockHash removes all STUMPs for a given block hash (used during reorg cleanup). diff --git a/store/storetest/blockstatus.go b/store/storetest/blockstatus.go index d9ae80dd..3a060f33 100644 --- a/store/storetest/blockstatus.go +++ b/store/storetest/blockstatus.go @@ -16,6 +16,10 @@ import ( type BlockStatusBackend interface { UpsertBlockHeaderSeen(ctx context.Context, blockHash string, blockHeight uint64, seenAt time.Time) error MarkBlockProcessed(ctx context.Context, blockHash string, blockHeight uint64, processedAt time.Time) error + ClearBlockProcessed(ctx context.Context, blockHash string) error + AddBlockExpectedSubtrees(ctx context.Context, blockHash string, indices []int) error + GetBlockExpectedSubtrees(ctx context.Context, blockHash string) ([]int, error) + ClearBlockExpectedSubtrees(ctx context.Context, blockHash string) error MarkBlocksOrphaned(ctx context.Context, blockHashes []string, orphanedAt time.Time) (int, error) MarkBlocksParked(ctx context.Context, blockHashes []string) error MarkBlockReconciled(ctx context.Context, blockHash string, orphanedAt, at time.Time) (bool, error) @@ -140,6 +144,102 @@ func RunBlockStatusSuite(t *testing.T, newBackend func(t *testing.T) BlockStatus return false } + // ClearBlockProcessed unsets processed_at only: the bump-builder uses + // it to hand a block back to the watchdog when a later merkle-service's + // BLOCK_PROCESSED proves the finalized build incomplete. Height, status + // and the other milestones stay; a missing row is a no-op. + t.Run("ClearBlockProcessed", func(t *testing.T) { + b := newBackend(t) + const hash = "cbp-cbp-cbp-cbp-cbp-cbp-cbp-cbp-cbp-cbp-cbp-cbp-cbp-cbp-cbp-cbp-" + height := blockSuiteHeight + 7 + if err := b.UpsertBlockHeaderSeen(ctx, hash, height, t0); err != nil { + t.Fatal(err) + } + if err := b.MarkBlockProcessed(ctx, hash, height, t0.Add(30*time.Second)); err != nil { + t.Fatal(err) + } + if got := row(t, b, hash); got.ProcessedAt == nil { + t.Fatalf("seed: processed_at must be set, got %+v", got) + } + if err := b.ClearBlockProcessed(ctx, hash); err != nil { + t.Fatalf("ClearBlockProcessed: %v", err) + } + got := row(t, b, hash) + if got.ProcessedAt != nil { + t.Fatalf("processed_at must be cleared, got %+v", got) + } + if got.BlockHeight != height || got.Status != models.BlockStatusActive || !got.HeaderSeenAt.Equal(t0) { + t.Fatalf("only processed_at may change, got %+v", got) + } + if err := b.ClearBlockProcessed(ctx, hash); err != nil { + t.Fatalf("clearing twice must be a no-op: %v", err) + } + if err := b.ClearBlockProcessed(ctx, "missing-"+hash[8:]); err != nil { + t.Fatalf("clearing a missing row must be a no-op: %v", err) + } + if _, err := b.GetBlockProcessingStatus(ctx, "missing-"+hash[8:]); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("clearing a missing row must not create it: err=%v", err) + } + // Re-stamping works after a clear. + if err := b.MarkBlockProcessed(ctx, hash, height, t0.Add(time.Minute)); err != nil { + t.Fatal(err) + } + if got := row(t, b, hash); got.ProcessedAt == nil { + t.Fatalf("re-stamp after clear must set processed_at, got %+v", got) + } + }) + + // The expected-subtree set is a union across merkle-services' + // announcements: adds merge (deduplicated, ascending), the block's other + // columns are untouched, a clear drops it, and a missing row is a no-op + // that is not created. + t.Run("BlockExpectedSubtrees", func(t *testing.T) { + b := newBackend(t) + const hash = "bes-bes-bes-bes-bes-bes-bes-bes-bes-bes-bes-bes-bes-bes-bes-bes-" + height := blockSuiteHeight + 8 + if err := b.UpsertBlockHeaderSeen(ctx, hash, height, t0); err != nil { + t.Fatal(err) + } + if got, err := b.GetBlockExpectedSubtrees(ctx, hash); err != nil || len(got) != 0 { + t.Fatalf("fresh row: got %v, %v; want empty", got, err) + } + if err := b.AddBlockExpectedSubtrees(ctx, hash, []int{2, 0}); err != nil { + t.Fatalf("add: %v", err) + } + if err := b.AddBlockExpectedSubtrees(ctx, hash, []int{1, 2}); err != nil { + t.Fatalf("add: %v", err) + } + got, err := b.GetBlockExpectedSubtrees(ctx, hash) + if err != nil { + t.Fatal(err) + } + if len(got) != 3 || got[0] != 0 || got[1] != 1 || got[2] != 2 { + t.Fatalf("union = %v, want [0 1 2]", got) + } + if r := row(t, b, hash); r.BlockHeight != height || r.Status != models.BlockStatusActive || !r.HeaderSeenAt.Equal(t0) || r.ProcessedAt != nil { + t.Fatalf("adding expected subtrees must not touch the row's other columns, got %+v", r) + } + if err := b.ClearBlockExpectedSubtrees(ctx, hash); err != nil { + t.Fatalf("clear: %v", err) + } + if got, err := b.GetBlockExpectedSubtrees(ctx, hash); err != nil || len(got) != 0 { + t.Fatalf("after clear: got %v, %v; want empty", got, err) + } + missing := "missing-" + hash[8:] + if err := b.AddBlockExpectedSubtrees(ctx, missing, []int{1}); err != nil { + t.Fatalf("add on a missing row must be a no-op: %v", err) + } + if got, err := b.GetBlockExpectedSubtrees(ctx, missing); err != nil || len(got) != 0 { + t.Fatalf("missing row: got %v, %v; want empty", got, err) + } + if err := b.ClearBlockExpectedSubtrees(ctx, missing); err != nil { + t.Fatalf("clear on a missing row must be a no-op: %v", err) + } + if _, err := b.GetBlockProcessingStatus(ctx, missing); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("expected-subtree ops must not create a row: err=%v", err) + } + }) + t.Run("ReactivateBlock", func(t *testing.T) { t.Run("AppliesOnMatchingGeneration", func(t *testing.T) { b := newBackend(t) diff --git a/store/storetest/stumps.go b/store/storetest/stumps.go new file mode 100644 index 00000000..79887d82 --- /dev/null +++ b/store/storetest/stumps.go @@ -0,0 +1,168 @@ +package storetest + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "testing" + + "github.com/bsv-blockchain/arcade/models" +) + +// StumpBackend is the slice of store.Store the STUMP suite drives. +type StumpBackend interface { + InsertStump(ctx context.Context, stump *models.Stump) error + GetStumpsByBlockHash(ctx context.Context, blockHash string) ([]*models.Stump, error) + DeleteStumpsByBlockHash(ctx context.Context, blockHash string) error +} + +// RunStumpSuite asserts the content-addressed STUMP contract every backend +// shares: rows key on (block_hash, subtree_index, content_hash), so the same +// bytes inserted twice (two merkle-services delivering an identical STUMP) +// are one row, a different STUMP for the same subtree (a merkle-service that +// missed registrations during an outage) is a second row, reads come back +// ordered by subtree index with ContentHash set, and the block delete removes +// every row. newBackend is called once per subtest. +// +// Block hashes are derived from the subtest name: Aerospike runs this suite +// in a namespace shared with every other integration test and never clears +// it, so each subtest must only ever read back rows it wrote itself. +func RunStumpSuite(t *testing.T, newBackend func(t *testing.T) StumpBackend) { + t.Helper() + ctx := context.Background() + variantA := []byte("stump-subtree-3-variant-a") + variantB := []byte("stump-subtree-3-variant-b") + subtree0 := []byte("stump-subtree-0") + + t.Run("identical bytes collapse to one row", func(t *testing.T) { + b := newBackend(t) + block := suiteBlockHash(t, "block") + for i := 0; i < 3; i++ { + if err := b.InsertStump(ctx, models.NewStump(block, 3, variantA)); err != nil { + t.Fatalf("insert %d: %v", i, err) + } + } + // A caller that left ContentHash empty still lands on the same row. + if err := b.InsertStump(ctx, &models.Stump{BlockHash: block, SubtreeIndex: 3, StumpData: variantA}); err != nil { + t.Fatalf("insert without hash: %v", err) + } + got, err := b.GetStumpsByBlockHash(ctx, block) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("rows = %d, want 1: %s", len(got), describe(got)) + } + if got[0].SubtreeIndex != 3 || !bytes.Equal(got[0].StumpData, variantA) || got[0].ContentHash != models.StumpContentHash(variantA) { + t.Fatalf("row = %s", describe(got)) + } + }) + + t.Run("divergent variants coexist", func(t *testing.T) { + b := newBackend(t) + block, other := suiteBlockHash(t, "block"), suiteBlockHash(t, "other") + for _, st := range []*models.Stump{ + models.NewStump(block, 3, variantA), + models.NewStump(block, 3, variantB), + models.NewStump(block, 0, subtree0), + models.NewStump(block, 3, variantA), // redelivery + models.NewStump(other, 3, variantA), // another block, same bytes + } { + if err := b.InsertStump(ctx, st); err != nil { + t.Fatalf("insert %d/%s: %v", st.SubtreeIndex, st.ContentHash[:8], err) + } + } + got, err := b.GetStumpsByBlockHash(ctx, block) + if err != nil { + t.Fatal(err) + } + if len(got) != 3 { + t.Fatalf("rows = %d, want 3: %s", len(got), describe(got)) + } + if got[0].SubtreeIndex != 0 || got[1].SubtreeIndex != 3 || got[2].SubtreeIndex != 3 { + t.Fatalf("rows must be ordered by subtree index: %s", describe(got)) + } + seen := map[string]bool{} + for _, st := range got { + if st.BlockHash != block { + t.Fatalf("row from another block leaked: %s", describe(got)) + } + if st.ContentHash != models.StumpContentHash(st.StumpData) { + t.Fatalf("ContentHash must match the bytes: %s", describe(got)) + } + seen[st.ContentHash] = true + } + for _, want := range [][]byte{variantA, variantB, subtree0} { + if !seen[models.StumpContentHash(want)] { + t.Fatalf("variant %q missing: %s", want, describe(got)) + } + } + }) + + t.Run("delete removes every variant and nothing else", func(t *testing.T) { + b := newBackend(t) + block, other := suiteBlockHash(t, "block"), suiteBlockHash(t, "other") + for _, st := range []*models.Stump{ + models.NewStump(block, 3, variantA), + models.NewStump(block, 3, variantB), + models.NewStump(other, 1, subtree0), + } { + if err := b.InsertStump(ctx, st); err != nil { + t.Fatal(err) + } + } + if err := b.DeleteStumpsByBlockHash(ctx, block); err != nil { + t.Fatal(err) + } + got, err := b.GetStumpsByBlockHash(ctx, block) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Fatalf("rows after delete = %s", describe(got)) + } + kept, err := b.GetStumpsByBlockHash(ctx, other) + if err != nil { + t.Fatal(err) + } + if len(kept) != 1 { + t.Fatalf("other block's rows must survive, got %s", describe(kept)) + } + // Deleting again is a no-op. + if err := b.DeleteStumpsByBlockHash(ctx, block); err != nil { + t.Fatalf("second delete: %v", err) + } + }) + + t.Run("unknown block reads empty", func(t *testing.T) { + b := newBackend(t) + got, err := b.GetStumpsByBlockHash(ctx, suiteBlockHash(t, "never-written")) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Fatalf("rows = %s", describe(got)) + } + }) +} + +// suiteBlockHash derives a 64-hex block hash unique to the calling subtest. +func suiteBlockHash(t *testing.T, tag string) string { + t.Helper() + sum := sha256.Sum256([]byte(t.Name() + "/" + tag)) + return hex.EncodeToString(sum[:]) +} + +func describe(stumps []*models.Stump) string { + var buf bytes.Buffer + for _, st := range stumps { + hash := st.ContentHash + if len(hash) > 8 { + hash = hash[:8] + } + fmt.Fprintf(&buf, "[%s idx=%d hash=%s %q]", st.BlockHash[:8], st.SubtreeIndex, hash, st.StumpData) + } + return buf.String() +}